2023
|
| NorBench – A Benchmark for Norwegian Language Models Conference Samuel, David; Kutuzov, Andrey; Touileb, Samia; Velldal, Erik; Øvrelid, Lilja; Rønningstad, Egil; Sigdel, Elina; Palatkina, Anna 2023. @conference{Samuel2023,
title = {NorBench – A Benchmark for Norwegian Language Models},
author = {David Samuel and Andrey Kutuzov and Samia Touileb and Erik Velldal and Lilja Øvrelid and Egil Rønningstad and Elina Sigdel and Anna Palatkina},
url = {https://mediafutures.no/2023_nodalida-1_61/},
year = {2023},
date = {2023-05-24},
urldate = {2023-05-24},
abstract = {We present NorBench: a streamlined suite of NLP tasks and probes for evaluating Norwegian language models (LMs) on standardized data splits and evaluation metrics. We also introduce a range of new Norwegian language models (both encoder and encoder-decoder based). Finally, we compare and analyze their performance, along with other existing LMs, across the different benchmark tests of NorBench.},
keywords = {},
pubstate = {published},
tppubtype = {conference}
}
We present NorBench: a streamlined suite of NLP tasks and probes for evaluating Norwegian language models (LMs) on standardized data splits and evaluation metrics. We also introduce a range of new Norwegian language models (both encoder and encoder-decoder based). Finally, we compare and analyze their performance, along with other existing LMs, across the different benchmark tests of NorBench. |
| Measuring Normative and Descriptive Biases in Language Models Using Census Data Conference Touileb, Samia; Øvrelid, Lilja; Velldal, Erik 2023. @conference{Touileb2023,
title = {Measuring Normative and Descriptive Biases in Language Models Using Census Data},
author = {Samia Touileb and Lilja Øvrelid and Erik Velldal},
url = {https://mediafutures.no/2023_eacl-main_164/},
year = {2023},
date = {2023-05-02},
abstract = {We investigate in this paper how distributions of occupations with respect to gender is reflected
in pre-trained language models. Such distributions are not always aligned to normative ideals, nor do they necessarily reflect a descriptive assessment of reality. In this paper, we introduce an approach for measuring to what degree pre-trained language models are aligned to normative and descriptive occupational distributions. To this end, we use official demographic information about gender–occupation distributions provided by the national statistics agencies of France, Norway, United Kingdom, and the United States. We manually generate template-based sentences combining gendered pronouns and nouns with occupations,
and subsequently probe a selection of ten language models covering the English, French, and Norwegian languages. The scoring system we introduce in this work is language independent, and can be used on any combination of
template-based sentences, occupations, and languages. The approach could also be extended to other dimensions of national census data and other demographic variables.},
keywords = {},
pubstate = {published},
tppubtype = {conference}
}
We investigate in this paper how distributions of occupations with respect to gender is reflected
in pre-trained language models. Such distributions are not always aligned to normative ideals, nor do they necessarily reflect a descriptive assessment of reality. In this paper, we introduce an approach for measuring to what degree pre-trained language models are aligned to normative and descriptive occupational distributions. To this end, we use official demographic information about gender–occupation distributions provided by the national statistics agencies of France, Norway, United Kingdom, and the United States. We manually generate template-based sentences combining gendered pronouns and nouns with occupations,
and subsequently probe a selection of ten language models covering the English, French, and Norwegian languages. The scoring system we introduce in this work is language independent, and can be used on any combination of
template-based sentences, occupations, and languages. The approach could also be extended to other dimensions of national census data and other demographic variables. |
2022
|
| Occupational Biases in Norwegian and Multilingual Language Models Workshop Touileb, Samia; Øvrelid, Lilja; Velldal, Erik 2022. @workshop{Touileb2022,
title = {Occupational Biases in Norwegian and Multilingual Language Models},
author = {Samia Touileb and Lilja Øvrelid and Erik Velldal },
url = {https://mediafutures.no/2022-gebnlp-1-21/},
year = {2022},
date = {2022-07-01},
abstract = {In this paper we explore how a demographic distribution of occupations, along gender dimensions, is reflected in pre-trained language models. We give a descriptive assessment of the distribution of occupations, and investigate to what extent these are reflected in four Norwegian and two multilingual models. To this end, we introduce a set of simple bias probes, and perform five different tasks combining gendered pronouns, first names, and a set of occupations from the Norwegian statistics bureau. We show that language specific models obtain more accurate results, and are much closer to the real-world distribution of clearly gendered occupations. However, we see that none of the models have correct representations of the occupations that are demographically balanced between genders. We also discuss the importance of the training data on which the models were trained on, and argue that template-based bias probes can sometimes be fragile, and a simple alteration in a template can change a model’s behavior.},
keywords = {},
pubstate = {published},
tppubtype = {workshop}
}
In this paper we explore how a demographic distribution of occupations, along gender dimensions, is reflected in pre-trained language models. We give a descriptive assessment of the distribution of occupations, and investigate to what extent these are reflected in four Norwegian and two multilingual models. To this end, we introduce a set of simple bias probes, and perform five different tasks combining gendered pronouns, first names, and a set of occupations from the Norwegian statistics bureau. We show that language specific models obtain more accurate results, and are much closer to the real-world distribution of clearly gendered occupations. However, we see that none of the models have correct representations of the occupations that are demographically balanced between genders. We also discuss the importance of the training data on which the models were trained on, and argue that template-based bias probes can sometimes be fragile, and a simple alteration in a template can change a model’s behavior. |
2021
|
| Responsible media technology and AI: challenges and research directions Journal Article Trattner, Christoph; Jannach, Dietmar; Motta, Enrico; Meijer, Irene Costera; Diakopoulos, Nicholas; Elahi, Mehdi; Opdahl, Andreas L.; Tessem, Bjørnar; Borch, Njål; Fjeld, Morten; Øvrelid, Lilja; Smedt, Koenraad De; Moe, Hallvard In: AI and Ethics, 2021. @article{cristin2000622,
title = {Responsible media technology and AI: challenges and research directions},
author = {Christoph Trattner and Dietmar Jannach and Enrico Motta and Irene Costera Meijer and Nicholas Diakopoulos and Mehdi Elahi and Andreas L. Opdahl and Bjørnar Tessem and Njål Borch and Morten Fjeld and Lilja Øvrelid and Koenraad De Smedt and Hallvard Moe},
url = {https://app.cristin.no/results/show.jsf?id=2000622, Cristin
https://link.springer.com/content/pdf/10.1007/s43681-021-00126-4.pdf},
doi = {https://doi.org/10.1007/s43681-021-00126-4},
year = {2021},
date = {2021-12-20},
urldate = {2021-12-20},
journal = {AI and Ethics},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
|
| Using Gender- and Polarity-informed Models to Investigate Bias Working paper Touileb, Samia; Øvrelid, Lilja; Velldal, Erik 2021. @workingpaper{cristin1958571,
title = {Using Gender- and Polarity-informed Models to Investigate Bias},
author = {Samia Touileb and Lilja Øvrelid and Erik Velldal},
url = {https://app.cristin.no/results/show.jsf?id=1958571, Cristin},
year = {2021},
date = {2021-01-01},
keywords = {},
pubstate = {published},
tppubtype = {workingpaper}
}
|
2020
|
| Gender and sentiment, critics and authors: a dataset of Norwegian book reviews Journal Article Touileb, Samia; Øvrelid, Lilja; Velldal, Erik In: Gender Bias in Natural Language Processing. Association for Computational Linguistics, 2020, (Pre SFI). @article{Touileb2020,
title = {Gender and sentiment, critics and authors: a dataset of Norwegian book reviews},
author = {Samia Touileb and Lilja Øvrelid and Erik Velldal},
url = {https://www.aclweb.org/anthology/2020.gebnlp-1.11.pdf},
year = {2020},
date = {2020-12-01},
journal = {Gender Bias in Natural Language Processing. Association for Computational Linguistics},
abstract = {Gender bias in models and datasets is widely studied in NLP. The focus has usually been on analysing how females and males express themselves, or how females and males are described. However, a less studied aspect is the combination of these two perspectives, how female and male describe the same or opposite gender. In this paper, we present a new gender annotated sentiment dataset of critics reviewing the works of female and male authors. We investigate if this newly annotated dataset contains differences in how the works of male and female authors are critiqued, in particular in terms of positive and negative sentiment. We also explore the differences in how this is done by male and female critics. We show that there are differences in how critics assess the works of authors of the same or opposite gender. For example, male critics rate crime novels written by females, and romantic and sentimental works written by males, more negatively.},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Gender bias in models and datasets is widely studied in NLP. The focus has usually been on analysing how females and males express themselves, or how females and males are described. However, a less studied aspect is the combination of these two perspectives, how female and male describe the same or opposite gender. In this paper, we present a new gender annotated sentiment dataset of critics reviewing the works of female and male authors. We investigate if this newly annotated dataset contains differences in how the works of male and female authors are critiqued, in particular in terms of positive and negative sentiment. We also explore the differences in how this is done by male and female critics. We show that there are differences in how critics assess the works of authors of the same or opposite gender. For example, male critics rate crime novels written by females, and romantic and sentimental works written by males, more negatively. |
| Improving sentiment analysis with multi-task learning of negation Journal Article Barnes, J; Velldal, Erik; Øvrelid, Lilja In: 2020, (Pre SFI). @article{Barnes2020,
title = {Improving sentiment analysis with multi-task learning of negation},
author = {J Barnes and Erik Velldal and Lilja Øvrelid},
url = {https://www.cambridge.org/core/journals/natural-language-engineering/article/abs/improving-sentiment-analysis-with-multitask-learning-of-negation/14EF2B829EC4B8EC29E7C0C5C77B95B0},
year = {2020},
date = {2020-11-11},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
|
| Sentiment analysis is not solved! Assessing and probing sentiment classification Proceeding Barnes, J; Øvrelid, Lilja; Velldal, Erik 2020, (Pre SFI). @proceedings{Barnes2020b,
title = {Sentiment analysis is not solved! Assessing and probing sentiment classification},
author = {J Barnes and Lilja Øvrelid and Erik Velldal},
url = {https://www.aclweb.org/anthology/W19-4802/},
year = {2020},
date = {2020-08-01},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {proceedings}
}
|
| NorNE: Annotating Named Entities for Norwegian Proceeding Jørgensen, F; Aasmoe, T; Husevåg, ASR; Øvrelid, Lilja; Velldal, Erik (Ed.) 2020, (Pre SFI). @proceedings{Jørgensen2020,
title = {NorNE: Annotating Named Entities for Norwegian},
editor = {F Jørgensen and T Aasmoe and ASR Husevåg and Lilja Øvrelid and Erik Velldal},
url = {https://oda.oslomet.no/handle/10642/8830},
year = {2020},
date = {2020-05-01},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {proceedings}
}
|
| A Fine-Grained Sentiment Dataset for Norwegian Proceeding Øvrelid, Lilja; Mæhlum, Petter; Barnes, Jeremy; Velldal, Erik 2020, (Pre SFI). @proceedings{Øvrelid2020,
title = {A Fine-Grained Sentiment Dataset for Norwegian},
author = {Lilja Øvrelid and Petter Mæhlum and Jeremy Barnes and Erik Velldal},
url = {https://www.aclweb.org/anthology/2020.lrec-1.618/},
year = {2020},
date = {2020-05-01},
urldate = {2020-05-01},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {proceedings}
}
|
2019
|
| Lexicon information in neural sentiment analysis: a multi-task learning approach Conference Barnes, Jeremy; Touileb, Samia; Øvrelid, Lilja; Velldal, Erik Linköping University Electronic Press, 2019, (Pre SFI). @conference{Barnes2019,
title = {Lexicon information in neural sentiment analysis: a multi-task learning approach},
author = {Jeremy Barnes and Samia Touileb and Lilja Øvrelid and Erik Velldal},
url = {https://www.aclweb.org/anthology/W19-6119.pdf},
year = {2019},
date = {2019-10-01},
journal = {Proceedings of the 22nd Nordic Conference on Computational Linguistics (NoDaLiDa)},
pages = {175–186},
publisher = {Linköping University Electronic Press},
abstract = {This paper explores the use of multi-task learning (MTL) for incorporating external knowledge in neural models. Specifically, we show how MTL can enable a BiLSTM sentiment classifier to incorporate information from sentiment lexicons. Our MTL set-up is shown to improve model performance (compared to a single-task set-up) on both English and Norwegian sentence-level sentiment datasets. The paper also introduces a new sentiment lexicon for Norwegian.},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {conference}
}
This paper explores the use of multi-task learning (MTL) for incorporating external knowledge in neural models. Specifically, we show how MTL can enable a BiLSTM sentiment classifier to incorporate information from sentiment lexicons. Our MTL set-up is shown to improve model performance (compared to a single-task set-up) on both English and Norwegian sentence-level sentiment datasets. The paper also introduces a new sentiment lexicon for Norwegian. |
2018
|
| Diachronic word embeddings and semantic shifts: a survey Proceeding Kutuzov, Andrey; Øvrelid, Lilja; Szymanski, Terrence; Velldal, Erik 2018, (Pre SFI). @proceedings{Kutuzov2018,
title = {Diachronic word embeddings and semantic shifts: a survey},
author = {Andrey Kutuzov and Lilja Øvrelid and Terrence Szymanski and Erik Velldal},
url = {https://www.aclweb.org/anthology/C18-1117/},
year = {2018},
date = {2018-08-01},
urldate = {2018-08-01},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {proceedings}
}
|
NoReC: The Norwegian Review Corpus Proceeding Velldal, Erik; Øvrelid, Lilja; Bergem, Eivind Alexander; Stadsnes, Cathrine; Touileb, Samia; Jørgensen, Fredrik 2018, (Pre SFI). @proceedings{Velldal2018,
title = {NoReC: The Norwegian Review Corpus},
author = {Erik Velldal and Lilja Øvrelid and Eivind Alexander Bergem and Cathrine Stadsnes and Samia Touileb and Fredrik Jørgensen},
year = {2018},
date = {2018-05-12},
abstract = {https://repo.clarino.uib.no/xmlui/handle/11509/124},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {proceedings}
}
https://repo.clarino.uib.no/xmlui/handle/11509/124 |
2016
|
| Universal dependencies for Norwegian Proceeding Øvrelid, Lilja; Hohle, Petter 2016, (Pre SFI). @proceedings{Øvrelid2016,
title = { Universal dependencies for Norwegian},
author = {Lilja Øvrelid and Petter Hohle},
url = {https://www.aclweb.org/anthology/L16-1250/},
year = {2016},
date = {2016-05-01},
urldate = {2016-05-01},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {proceedings}
}
|
2012
|
| Representing and resolving negation for sentiment analysis Proceeding Lapponi, Emanuele; Read, Jonathon; Øvrelid, Lilja 2012, (Pre SFI). @proceedings{Lapponi2012,
title = {Representing and resolving negation for sentiment analysis},
author = {Emanuele Lapponi and Jonathon Read and Lilja Øvrelid},
url = {https://ieeexplore.ieee.org/document/6406506},
year = {2012},
date = {2012-12-10},
urldate = {2012-12-10},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {proceedings}
}
|
| Speculation and negation: Rules, rankers, and the role of syntax Journal Article Velldal, Erik; Øvrelid, Lilja; Read, Jonathon; Oepen, Stephan In: 2012, (Pre SFI). @article{Velldal2012,
title = {Speculation and negation: Rules, rankers, and the role of syntax},
author = {Erik Velldal and Lilja Øvrelid and Jonathon Read and Stephan Oepen},
url = {https://www.mitpressjournals.org/doi/pdf/10.1162/COLI_a_00126},
year = {2012},
date = {2012-01-01},
urldate = {2012-01-01},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
|
