Erik Velldal
UiO
Erik Velldal is a professor at the Language Technology Group (LTG) in the Machine Learning section at the Department of Informatics (IFI). He has over 25 years of experience in research and teaching in the use of machine learning for the analysis of natural (i.e., human) languages. Currently, he leads the SANT project, which focuses on sentiment analysis. Velldal is also involved in the Centres for Research-based Innovation (SFI) MediaFutures and NorwAI, as well as the Centre for Excellence (SFF) Integreat – Norwegian Centre for Knowledge-Driven Machine Learning. He is a member of the advisory council for the University of Oslo's Centre for Data Science, dScience.
2023
Samuel, David; Kutuzov, Andrey; Touileb, Samia; Velldal, Erik; Øvrelid, Lilja; Rønningstad, Egil; Sigdel, Elina; Palatkina, Anna
NorBench – A Benchmark for Norwegian Language Models Conference
2023.
@conference{Samuel2023,
title = {NorBench – A Benchmark for Norwegian Language Models},
author = {David Samuel and Andrey Kutuzov and Samia Touileb and Erik Velldal and Lilja Øvrelid and Egil Rønningstad and Elina Sigdel and Anna Palatkina},
url = {https://mediafutures.no/2023_nodalida-1_61/},
year = {2023},
date = {2023-05-24},
urldate = {2023-05-24},
abstract = {We present NorBench: a streamlined suite of NLP tasks and probes for evaluating Norwegian language models (LMs) on standardized data splits and evaluation metrics. We also introduce a range of new Norwegian language models (both encoder and encoder-decoder based). Finally, we compare and analyze their performance, along with other existing LMs, across the different benchmark tests of NorBench.},
keywords = {},
pubstate = {published},
tppubtype = {conference}
}
We present NorBench: a streamlined suite of NLP tasks and probes for evaluating Norwegian language models (LMs) on standardized data splits and evaluation metrics. We also introduce a range of new Norwegian language models (both encoder and encoder-decoder based). Finally, we compare and analyze their performance, along with other existing LMs, across the different benchmark tests of NorBench.
Touileb, Samia; Øvrelid, Lilja; Velldal, Erik
Measuring Normative and Descriptive Biases in Language Models Using Census Data Conference
2023.
@conference{Touileb2023,
title = {Measuring Normative and Descriptive Biases in Language Models Using Census Data},
author = {Samia Touileb and Lilja Øvrelid and Erik Velldal},
url = {https://mediafutures.no/2023_eacl-main_164/},
year = {2023},
date = {2023-05-02},
abstract = {We investigate in this paper how distributions of occupations with respect to gender is reflected
in pre-trained language models. Such distributions are not always aligned to normative ideals, nor do they necessarily reflect a descriptive assessment of reality. In this paper, we introduce an approach for measuring to what degree pre-trained language models are aligned to normative and descriptive occupational distributions. To this end, we use official demographic information about gender–occupation distributions provided by the national statistics agencies of France, Norway, United Kingdom, and the United States. We manually generate template-based sentences combining gendered pronouns and nouns with occupations,
and subsequently probe a selection of ten language models covering the English, French, and Norwegian languages. The scoring system we introduce in this work is language independent, and can be used on any combination of
template-based sentences, occupations, and languages. The approach could also be extended to other dimensions of national census data and other demographic variables.},
keywords = {},
pubstate = {published},
tppubtype = {conference}
}
We investigate in this paper how distributions of occupations with respect to gender is reflected
in pre-trained language models. Such distributions are not always aligned to normative ideals, nor do they necessarily reflect a descriptive assessment of reality. In this paper, we introduce an approach for measuring to what degree pre-trained language models are aligned to normative and descriptive occupational distributions. To this end, we use official demographic information about gender–occupation distributions provided by the national statistics agencies of France, Norway, United Kingdom, and the United States. We manually generate template-based sentences combining gendered pronouns and nouns with occupations,
and subsequently probe a selection of ten language models covering the English, French, and Norwegian languages. The scoring system we introduce in this work is language independent, and can be used on any combination of
template-based sentences, occupations, and languages. The approach could also be extended to other dimensions of national census data and other demographic variables.
in pre-trained language models. Such distributions are not always aligned to normative ideals, nor do they necessarily reflect a descriptive assessment of reality. In this paper, we introduce an approach for measuring to what degree pre-trained language models are aligned to normative and descriptive occupational distributions. To this end, we use official demographic information about gender–occupation distributions provided by the national statistics agencies of France, Norway, United Kingdom, and the United States. We manually generate template-based sentences combining gendered pronouns and nouns with occupations,
and subsequently probe a selection of ten language models covering the English, French, and Norwegian languages. The scoring system we introduce in this work is language independent, and can be used on any combination of
template-based sentences, occupations, and languages. The approach could also be extended to other dimensions of national census data and other demographic variables.
2022
Touileb, Samia; Øvrelid, Lilja; Velldal, Erik
Occupational Biases in Norwegian and Multilingual Language Models Workshop
2022.
@workshop{Touileb2022,
title = {Occupational Biases in Norwegian and Multilingual Language Models},
author = {Samia Touileb and Lilja Øvrelid and Erik Velldal },
url = {https://mediafutures.no/2022-gebnlp-1-21/},
year = {2022},
date = {2022-07-01},
abstract = {In this paper we explore how a demographic distribution of occupations, along gender dimensions, is reflected in pre-trained language models. We give a descriptive assessment of the distribution of occupations, and investigate to what extent these are reflected in four Norwegian and two multilingual models. To this end, we introduce a set of simple bias probes, and perform five different tasks combining gendered pronouns, first names, and a set of occupations from the Norwegian statistics bureau. We show that language specific models obtain more accurate results, and are much closer to the real-world distribution of clearly gendered occupations. However, we see that none of the models have correct representations of the occupations that are demographically balanced between genders. We also discuss the importance of the training data on which the models were trained on, and argue that template-based bias probes can sometimes be fragile, and a simple alteration in a template can change a model’s behavior.},
keywords = {},
pubstate = {published},
tppubtype = {workshop}
}
In this paper we explore how a demographic distribution of occupations, along gender dimensions, is reflected in pre-trained language models. We give a descriptive assessment of the distribution of occupations, and investigate to what extent these are reflected in four Norwegian and two multilingual models. To this end, we introduce a set of simple bias probes, and perform five different tasks combining gendered pronouns, first names, and a set of occupations from the Norwegian statistics bureau. We show that language specific models obtain more accurate results, and are much closer to the real-world distribution of clearly gendered occupations. However, we see that none of the models have correct representations of the occupations that are demographically balanced between genders. We also discuss the importance of the training data on which the models were trained on, and argue that template-based bias probes can sometimes be fragile, and a simple alteration in a template can change a model’s behavior.
2021
Touileb, Samia; Øvrelid, Lilja; Velldal, Erik
Using Gender- and Polarity-informed Models to Investigate Bias Working paper
2021.
@workingpaper{cristin1958571,
title = {Using Gender- and Polarity-informed Models to Investigate Bias},
author = {Samia Touileb and Lilja Øvrelid and Erik Velldal},
url = {https://app.cristin.no/results/show.jsf?id=1958571, Cristin},
year = {2021},
date = {2021-01-01},
keywords = {},
pubstate = {published},
tppubtype = {workingpaper}
}
2020
Touileb, Samia; Øvrelid, Lilja; Velldal, Erik
Gender and sentiment, critics and authors: a dataset of Norwegian book reviews Journal Article
In: Gender Bias in Natural Language Processing. Association for Computational Linguistics, 2020, (Pre SFI).
@article{Touileb2020,
title = {Gender and sentiment, critics and authors: a dataset of Norwegian book reviews},
author = {Samia Touileb and Lilja Øvrelid and Erik Velldal},
url = {https://www.aclweb.org/anthology/2020.gebnlp-1.11.pdf},
year = {2020},
date = {2020-12-01},
journal = {Gender Bias in Natural Language Processing. Association for Computational Linguistics},
abstract = {Gender bias in models and datasets is widely studied in NLP. The focus has usually been on analysing how females and males express themselves, or how females and males are described. However, a less studied aspect is the combination of these two perspectives, how female and male describe the same or opposite gender. In this paper, we present a new gender annotated sentiment dataset of critics reviewing the works of female and male authors. We investigate if this newly annotated dataset contains differences in how the works of male and female authors are critiqued, in particular in terms of positive and negative sentiment. We also explore the differences in how this is done by male and female critics. We show that there are differences in how critics assess the works of authors of the same or opposite gender. For example, male critics rate crime novels written by females, and romantic and sentimental works written by males, more negatively.},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Gender bias in models and datasets is widely studied in NLP. The focus has usually been on analysing how females and males express themselves, or how females and males are described. However, a less studied aspect is the combination of these two perspectives, how female and male describe the same or opposite gender. In this paper, we present a new gender annotated sentiment dataset of critics reviewing the works of female and male authors. We investigate if this newly annotated dataset contains differences in how the works of male and female authors are critiqued, in particular in terms of positive and negative sentiment. We also explore the differences in how this is done by male and female critics. We show that there are differences in how critics assess the works of authors of the same or opposite gender. For example, male critics rate crime novels written by females, and romantic and sentimental works written by males, more negatively.
Barnes, J; Velldal, Erik; Øvrelid, Lilja
Improving sentiment analysis with multi-task learning of negation Journal Article
In: 2020, (Pre SFI).
@article{Barnes2020,
title = {Improving sentiment analysis with multi-task learning of negation},
author = {J Barnes and Erik Velldal and Lilja Øvrelid},
url = {https://www.cambridge.org/core/journals/natural-language-engineering/article/abs/improving-sentiment-analysis-with-multitask-learning-of-negation/14EF2B829EC4B8EC29E7C0C5C77B95B0},
year = {2020},
date = {2020-11-11},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Barnes, J; Øvrelid, Lilja; Velldal, Erik
Sentiment analysis is not solved! Assessing and probing sentiment classification Proceedings
2020, (Pre SFI).
@proceedings{Barnes2020b,
title = {Sentiment analysis is not solved! Assessing and probing sentiment classification},
author = {J Barnes and Lilja Øvrelid and Erik Velldal},
url = {https://www.aclweb.org/anthology/W19-4802/},
year = {2020},
date = {2020-08-01},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {proceedings}
}
Jørgensen, F; Aasmoe, T; Husevåg, ASR; Øvrelid, Lilja; Velldal, Erik (Ed.)
NorNE: Annotating Named Entities for Norwegian Proceedings
2020, (Pre SFI).
@proceedings{Jørgensen2020,
title = {NorNE: Annotating Named Entities for Norwegian},
editor = {F Jørgensen and T Aasmoe and ASR Husevåg and Lilja Øvrelid and Erik Velldal},
url = {https://oda.oslomet.no/handle/10642/8830},
year = {2020},
date = {2020-05-01},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {proceedings}
}
Øvrelid, Lilja; Mæhlum, Petter; Barnes, Jeremy; Velldal, Erik
A Fine-Grained Sentiment Dataset for Norwegian Proceedings
2020, (Pre SFI).
@proceedings{Øvrelid2020,
title = {A Fine-Grained Sentiment Dataset for Norwegian},
author = {Lilja Øvrelid and Petter Mæhlum and Jeremy Barnes and Erik Velldal},
url = {https://www.aclweb.org/anthology/2020.lrec-1.618/},
year = {2020},
date = {2020-05-01},
urldate = {2020-05-01},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {proceedings}
}
2019
Barnes, Jeremy; Touileb, Samia; Øvrelid, Lilja; Velldal, Erik
Lexicon information in neural sentiment analysis: a multi-task learning approach Conference
Linköping University Electronic Press, 2019, (Pre SFI).
@conference{Barnes2019,
title = {Lexicon information in neural sentiment analysis: a multi-task learning approach},
author = {Jeremy Barnes and Samia Touileb and Lilja Øvrelid and Erik Velldal},
url = {https://www.aclweb.org/anthology/W19-6119.pdf},
year = {2019},
date = {2019-10-01},
journal = {Proceedings of the 22nd Nordic Conference on Computational Linguistics (NoDaLiDa)},
pages = {175–186},
publisher = {Linköping University Electronic Press},
abstract = {This paper explores the use of multi-task learning (MTL) for incorporating external knowledge in neural models. Specifically, we show how MTL can enable a BiLSTM sentiment classifier to incorporate information from sentiment lexicons. Our MTL set-up is shown to improve model performance (compared to a single-task set-up) on both English and Norwegian sentence-level sentiment datasets. The paper also introduces a new sentiment lexicon for Norwegian.},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {conference}
}
This paper explores the use of multi-task learning (MTL) for incorporating external knowledge in neural models. Specifically, we show how MTL can enable a BiLSTM sentiment classifier to incorporate information from sentiment lexicons. Our MTL set-up is shown to improve model performance (compared to a single-task set-up) on both English and Norwegian sentence-level sentiment datasets. The paper also introduces a new sentiment lexicon for Norwegian.
2018
Kutuzov, Andrey; Øvrelid, Lilja; Szymanski, Terrence; Velldal, Erik
Diachronic word embeddings and semantic shifts: a survey Proceedings
2018, (Pre SFI).
@proceedings{Kutuzov2018,
title = {Diachronic word embeddings and semantic shifts: a survey},
author = {Andrey Kutuzov and Lilja Øvrelid and Terrence Szymanski and Erik Velldal},
url = {https://www.aclweb.org/anthology/C18-1117/},
year = {2018},
date = {2018-08-01},
urldate = {2018-08-01},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {proceedings}
}
Velldal, Erik; Øvrelid, Lilja; Bergem, Eivind Alexander; Stadsnes, Cathrine; Touileb, Samia; Jørgensen, Fredrik
NoReC: The Norwegian Review Corpus Proceedings
2018, (Pre SFI).
@proceedings{Velldal2018,
title = {NoReC: The Norwegian Review Corpus},
author = {Erik Velldal and Lilja Øvrelid and Eivind Alexander Bergem and Cathrine Stadsnes and Samia Touileb and Fredrik Jørgensen},
year = {2018},
date = {2018-05-12},
abstract = {https://repo.clarino.uib.no/xmlui/handle/11509/124},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {proceedings}
}
https://repo.clarino.uib.no/xmlui/handle/11509/124
2017
Kutuzov, Andrei; Fares, Murhaf; Stephan, Oepen; Velldal, Erik
Word vectors, reuse, and replicability: Towards a community repository of large-text resources Proceedings
2017, (Pre SFI).
@proceedings{Fares2017,
title = { Word vectors, reuse, and replicability: Towards a community repository of large-text resources},
author = {Andrei Kutuzov and Murhaf Fares and Oepen Stephan and Erik Velldal},
url = {https://www.duo.uio.no/handle/10852/65205},
year = {2017},
date = {2017-05-22},
urldate = {2017-05-22},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {proceedings}
}
2012
Velldal, Erik; Øvrelid, Lilja; Read, Jonathon; Oepen, Stephan
Speculation and negation: Rules, rankers, and the role of syntax Journal Article
In: 2012, (Pre SFI).
@article{Velldal2012,
title = {Speculation and negation: Rules, rankers, and the role of syntax},
author = {Erik Velldal and Lilja Øvrelid and Jonathon Read and Stephan Oepen},
url = {https://www.mitpressjournals.org/doi/pdf/10.1162/COLI_a_00126},
year = {2012},
date = {2012-01-01},
urldate = {2012-01-01},
note = {Pre SFI},
keywords = {},
pubstate = {published},
tppubtype = {article}
}