@conference{incoed24,

title = {Incorporating Editorial Feedback in the Evaluation of News Recommender Systems},

author = {Bilal Mahmood and Mehdi Elahi and Samia Touileb and Lubos Steskal and Christoph Trattner},

url = {https://mediafutures.no/lbr_umap_editorial_component_in_nrs/},

year  = {2024},

date = {2024-07-01},

urldate = {2024-07-01},

booktitle = {ACM UMAP 2024},

abstract = {Research in the recommender systems field typically applies a rather traditional evaluation methodology when assessing the quality of recommendations. This methodology heavily relies on incorporating different forms of user feedback (e.g., clicks) representing the specific needs and interests of the users. While this methodology may offer various benefits, it may fail to comprehensively project the complexities of certain application domains, such as the news domain. This domain is distinct from other domains primarily due to the strong influence of editorial control in the news delivery process. Incorporation of this role can profoundly impact how the relevance of news articles is measured when recommended to the users. Despite its critical importance, there appears to be a research gap in investigating the dynamics between the roles of editorial control and personalization in the community of recommender systems. In this paper, we address this gap by conducting experiments where the relevance of recommendations is assessed from an editorial perspective. We received a real-world dataset from TV 2, one of the largest editor-managed commercial media houses in Norway, which includes editors’ feedback on how news articles are being related. In our experiment, we considered a scenario where algorithm-generated recommendations, using the K-Nearest Neighbor (KNN) model, employing various text embedding models to encode different sections of the news articles (e.g., title, lead title, body text, and full text), are compared against the editorial feedback. The results are promising, demonstrating the effectiveness of the recommendation in fulfilling the editorial prospects.},

keywords = {WP2: User Modeling Personalization and Engagement},

pubstate = {published},

tppubtype = {conference}

}

@conference{percepvsreal24,

title = {Perception versus Reality: Evaluating User Awareness of Political Selective Exposure in News Recommender Systems},

author = {Alain D. Starke and Anders Sandvik Bremnes and Erik Knudsen and Damian Trilling and Christoph Trattner},

url = {https://mediafutures.no/umap2024___erik_alain_damian_anders_christoph/},

year  = {2024},

date = {2024-07-01},

urldate = {2024-07-01},

booktitle = {ACM UMAP 2024},

abstract = {News Recommender Systems (NRSs) have become increasingly pivotal in shaping the news landscape, particularly in how news is disseminated. This has also led to concerns about information diversity, especially regarding selective exposure in the realm of political news. Users may not recognize that news content presented to them is subject to selective exposure, through users that incorporate political beliefs. Within the U.S. two-party system, our research explores the interactions between NRSs and users’ ability to discern news articles that align with their political biases. We performed an online experiment (N = 160) to address the issue of user awareness and self-recognition of selective exposure within NRSs. Users were asked to select any number of news articles that matched their political orientation (i.e., Democrat or Republican) from a list of 50 news articles (5 Democrat, 5 Republican, 40 filler articles), which were either ranked saliently towards their political orientation or randomly. Contrary to expectations, our findings reveal no significant difference in article selection between participants exposed to a baseline random order and those who where presented with the more salient and easy to select version. We did observe that Republicans performed worse than Democrats in identifying aligning articles, based on precision and recall metrics.},

keywords = {WP2: User Modeling Personalization and Engagement},

pubstate = {published},

tppubtype = {conference}

}

@conference{emorefram24,

title = {Emotional Reframing of Economic News using a Large Language Model},

author = {Jia Hua Jeng and Gloria Anne Babile Kasangu and Alain D. Starke and Christoph Trattner},

url = {https://mediafutures.no/umap2024___jeng_alain_gloria_christoph__workshop_-3/},

year  = {2024},

date = {2024-07-01},

urldate = {2024-07-01},

booktitle = {ACM UMAP 2024},

abstract = {News media framing can shape public perception and potentially polarize views. Emotional language can exacerbate these framing effects, as a user’s emotional state can be an important contextual factor to use in news recommendation. Our research explores the relation between emotional framing techniques and the emotional states of readers, as well as readers’ perceived trust in specific news articles. Users (N = 200) had to read three economic news articles from the Washington Post. We used ChatGPT-4 to reframe news articles with specific emotional languages (Anger, Fear, Hope), compared to a neutral baseline reframed by a human journalist. Our results revealed that negative framing (Anger, Fear) elicited stronger negative emotional states among users than the neutral baseline, while Hope led to little changes overall. In contrast, perceived trust levels varied little across the different conditions. We discuss the implications of our findings and how emotional framing could affect societal polarization issues},

keywords = {WP2: User Modeling Personalization and Engagement},

pubstate = {published},

tppubtype = {conference}

}

@conference{umap2024Daniel,

title = {Shaping the Future of Content-based News Recommenders: Insights from Evaluating Feature-Specific Similarity Metrics},

author = {Daniel Rosnes and Alain Starke and Christoph Trattner },

url = {https://mediafutures.no/umap2024/},

year  = {2024},

date = {2024-07-01},

booktitle = {ACM UMAP '24},

abstract = {In news media, recommender system technology faces several domain-specific challenges. The continuous stream of new content and users deems content-based recommendation strategies, based on similar-item retrieval, to remain popular. However, a persistent challenge is to select relevant features and corresponding similarity functions, and whether this depends on the specific context. We evaluated feature-specific similarity metrics using human similarity judgments across national and local news domains. We performed an online experiment ($N = 141$) where we asked participants to judge the similarity between pairs of randomly sampled news articles. We had three contributions: (1) comparing novel metrics based on large language models to ones traditionally used in news recommendations, (2) exploring differences in similarity judgments across national and local news domains, and (3) examining which content-based strategies were perceived as appropriate in the news domain. Our results showed that one of the novel large language model based metrics (SBERT) was highly correlated with human judgments, while there were only small, most non-significant differences across national and local news domains. Finally, we found that while it may be possible to automatically recommend similar news using feature-specific metrics, their representativeness and appropriateness varied. We explain how our findings can guide the design of future content-based and hybrid recommender strategies in the news domain.},

keywords = {WP2: User Modeling Personalization and Engagement},

pubstate = {published},

tppubtype = {conference}

}

@conference{designing_for_automated24,

title = {Designing for Automated Sports Commentary Systems},

author = {Peter Andrews and Oda Elise Nordberg and Frode Guribye and Morten Fjeld and Njål Borch },

url = {https://mediafutures.no/designing_for_automated_sports_commentary_systems-2/},

year  = {2024},

date = {2024-06-12},

booktitle = {IMX'24},

abstract = {Advancements in Natural Language Processing (NLP) and Computer Vision (CV) are revolutionizing how we experience sports broadcasting. Traditionally, sports commentary has played a crucial role in enhancing viewer understanding and engagement with live games. Yet, the prospects of automated commentary, especially in light of these technological advancements and their impact on viewers’ experience, remain largely unexplored. This paper elaborates upon an innovative automated commentary system that integrates NLP and CV to provide a multimodal experience, combining auditory feedback through text-to-speech and visual cues, known as italicizing, for real-time in-game commentary. The system supports color commentary, which aims to inform the viewer of information surrounding the game by pulling additional content from a database. Moreover, it also supports play-by-play commentary covering in-game developments derived from an event system based on CV. As the system reinvents the role of commentary in sports video, we must consider the design and implications of multimodal artificial commentators. A focused user study with eight participants aimed at understanding the design implications of such multimodal artificial commentators reveals critical insights. Key findings emphasize the importance of language precision, content relevance, and delivery style in automated commentary, underscoring the necessity for personalization to meet diverse viewer preferences. Our results validate the potential value and effectiveness of multimodal feedback and derive design considerations, particularly in personalizing content to revolutionize the role of commentary in sports broadcasts.},

keywords = {WP4: Media Content Interaction and Accessibility},

pubstate = {published},

tppubtype = {conference}

}

@conference{fisherycatch24,

title = {Analysing Unlabeled Data with Randomness and Noise: The Case of Fishery Catch Reports},

author = {Aida Ashrafi and Bjørnar Tessem and Katja Enberg},

url = {https://mediafutures.no/analysing_unlabeled_data_with_randomness_and_noise/},

year  = {2024},

date = {2024-06-10},

booktitle = {14th Scandinavian Conference on Artificial Intelligence SCAI 2024},

abstract = {Detecting violations within fishing activity reports is crucial for ensuring the sustainable utilization of fish resources, and employing machine learning methods holds promise for uncovering hidden patterns within this complex dataset. Given that these violations are infrequent occurrences, as fishermen generally adhere to regulations, identifying them becomes akin to an anomaly outlier detection task. Since labeled data distinguishing between normal and anomalous instances is not available for catch reports from Norwegian waters, we have opted for more conventional approaches, such as clustering methods, to identify potential clusters and outliers. Moreover, the catch reports inherently exhibit randomness and noise due to environmental factors and potential errors made by fishermen during report registration which complicates the processes of scaling, clustering, and anomaly detection. Through experimentation with various scaling and clustering techniques, we have observed that many of these methods tend to group the data based on the species caught, exhibiting a high level of agreement in cluster formation, indicating the stability of the clusters. Anomaly detection methods, however, yield varying potential outliers as it is a more challenging task.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{EDEN,

title = {EDEN: A Dataset for Event Detection in Norwegian News},

author = {Samia Touileb and Jeanett Murstad and Petter Mæhlum and Lubos Steskal and Lilja Charlotte Storset and Huiling You and Lilja Øvrelid},

url = {https://mediafutures.no/2024-lrec-main-488/},

year  = {2024},

date = {2024-05-23},

booktitle = {LREC-COLING 2024},

abstract = {We present EDEN, the first Norwegian dataset annotated with event information at the sentence level, adapting the

widely used ACE event schema to Norwegian. The paper describes the manual annotation of Norwegian text as well

as transcribed speech in the news domain, together with inter-annotator agreement and discussions of relevant

dataset statistics. We also present preliminary modeling results using a graph-based event parser. The resulting

dataset will be made freely available for download and use.

},

keywords = {WP5: Norwegian Language Technologies},

pubstate = {published},

tppubtype = {conference}

}

@conference{Footyvision1,

title = {FootyVision: Multi-Object Tracking, Localisation, and Augmentation of Players and Ball in Football Video},

author = {Peter Andrews and Njål Borch and Morten Fjeld},

url = {https://mediafutures.no/peterandrews-footyvision-icmip24-final/},

year  = {2024},

date = {2024-04-20},

booktitle = {ACM ICMIP},

abstract = {Football video content analysis is a rapidly evolving field aiming to enrich the viewing experience of football matches. Current research often focuses on specific tasks like player and/or ball detection, tracking, and localisation in top-down views. Our study strives to integrate these efforts into a comprehensive Multi-Object Tracking (MOT) model capable of handling perspective transformations. Our framework, FootyVision, employs a YOLOv7 backbone trained on an extended player and ball dataset. The MOT module builds a gallery and assigns identities via the Hungarian algorithm based on feature embeddings, bounding box intersection over union, distance, and velocity. A novel component of our model is the perspective transformation module that leverages activation maps from the YOLOv7 backbone to compute homographies using lines, intersection points, and ellipses. This method effectively adapts to dynamic and uncalibrated video data, even in viewpoints with limited visual information. In terms of performance, FootyVision sets new benchmarks. The model achieves a mean average precision (mAP) of 95.7% and an F1-score of 95.5% in object detection. For MOT, it demonstrates robust capabilities, with an IDF1 score of approximately 93% on both ISSIA and SoccerNet datasets. For SoccerNet, it reaches a MOTA of 94.04% and shows competitive results for ISSIA. Additionally, FootyVision scores a HOTA(0) of 93.1% and an overall HOTA of 72.16% for the SoccerNet dataset. Our ablation study confirms the effectiveness of the selected tracking features and identifies key attributes for further improvement. While the model excels in maintaining track accuracy throughout the testing dataset, we recognise the potential to enhance spatial-location accuracy.},

keywords = {WP4: Media Content Interaction and Accessibility},

pubstate = {published},

tppubtype = {conference}

}

@conference{CLIpingSohail24,

title = {CLIPping the Deception: Adapting Vision-Language Models for Universal Deepfake Detection},

author = {Sohail Ahmed Khan and Duc-Tien Dang-Nguyen},

url = {https://mediafutures.no/icmr24/},

year  = {2024},

date = {2024-04-07},

booktitle = {ACM International Conference on Multimedia Retrieval (ICMR)},

journal = {ACM International Conference on Multimedia Retrieval (ICMR)},

abstract = {The recent advancements in Generative Adversarial Networks (GANs) and the emergence of Diffusion models have significantly streamlined the production of highly realistic and widely accessible synthetic content. As a result, there is a pressing need for effective general purpose detection mechanisms to mitigate the potential risks posed by deepfakes. In this paper, we explore the effectiveness of pre-trained vision-language models (VLMs) when paired with recent adaptation methods for universal deepfake detection. Following previous studies in this domain, we employ only a single dataset (ProGAN) in order to adapt CLIP for deepfake detection. However, in contrast to prior research, which rely solely on the visual part of CLIP while ignoring its textual component, our analysis reveals that retaining the text part is crucial. Consequently, the simple and lightweight Prompt Tuning based adaptation strategy that we employ outperforms the previous SOTA approach by 5.01% mAP and 6.61% accuracy while utilizing less than one third of the training data (200k images as compared to 720k). To assess the real-world applicability of our proposed models, we conduct a comprehensive evaluation across various scenarios. This involves rigorous testing on images sourced from 21 distinct datasets, including those generated by GANs-based, Diffusion-based and Commercial tools.},

keywords = {WP3: Media Content Production and Analysis},

pubstate = {published},

tppubtype = {conference}

}

@conference{AIComment,

title = {AiCommentator: A Multimodal Conversational Agent for Embedded Visualization in Football Viewing},

author = {Peter Andrews and Oda Elise Nordberg and Frode Guribye and Kazuyuki Fujita and Morten Fjeld and Njål Borch},

url = {https://mediafutures.no/acm_iui_24_aicommentator_peterandrews-1/},

year  = {2024},

date = {2024-03-18},

urldate = {2024-03-18},

booktitle = {Intelligent User Interfaces (IUI)},

journal = {Intelligent User Interfaces (IUI)},

abstract = {Traditionally, sports commentators provide viewers with diverse information, encompassing in-game developments and player performances. Yet young adult football viewers increasingly use mobile devices for deeper insights during football matches. Such insights into players on the pitch and performance statistics support viewers’ understanding of game stakes, creating a more engaging viewing experience. Inspired by commentators’ traditional roles and to incorporate information into a single platform, we developed AiCommentator, a Multimodal Conversational Agent (MCA) for embedded visualization and conversational interactions in football broadcast video. AiCommentator integrates embedded visualization, either with an automated non-interactive or with a responsive interactive commentary mode. Our system builds upon multimodal techniques, integrating computer vision and large language models, to demonstrate ways for designing tailored, interactive sports-viewing content. AiCommentator’s event system infers game states based on a multi-object tracking algorithm and computer vision backend, facilitating automated responsive commentary. We address three key topics: evaluating young adults’ satisfaction and immersion across the two viewing modes, enhancing viewer understanding of in-game events and players on the pitch, and devising methods to present this information in a usable manner. In a mixed-method evaluation (n=16) of AiCommentator, we found that the participants appreciated aspects of both system modes but preferred the interactive mode, expressing a higher degree of engagement and satisfaction. Our paper reports on our development of AiCommentator and presents the results from our user study, demonstrating the promise of interactive MCA for a more engaging sports viewing experience. Systems like AiCommentator could be pivotal in transforming the interactivity and accessibility of sports content, revolutionizing how sports viewers engage with video content.},

keywords = {WP4: Media Content Interaction and Accessibility},

pubstate = {published},

tppubtype = {conference}

}

@conference{deepfakede,

title = {Deepfake Detection: Analysing Model Generalisation Across Architectures, Datasets and Pre-Training Paradigms},

author = {Sohail Ahmed Khan and Duc-Tien Dang-Nguyen},

url = {https://mediafutures.no/ieee_access_2023___deepfake_comparative_analysis/},

year  = {2023},

date = {2023-12-15},

booktitle = {IEEE Access},

abstract = {As deepfake technology gains traction, the need for reliable detection systems is crucial. Recent research has introduced various deep learning-based detection systems, yet they exhibit limitations in generalizing effectively across diverse data distributions that differ from the training data. Our study focuses on understanding the generalization challenges by exploring specific aspects such as deep learning model architecture, pre-training strategy and datasets. Through a comprehensive comparative analysis, we evaluate multiple supervised and self-supervised deep learning models for deepfake detection.



Specifically, we evaluate eight supervised deep learning architectures and two transformer-based models pre-trained using self-supervised strategies (DINO, CLIP) on four different deepfake detection benchmarks (FakeAVCeleb, CelebDF-V2, DFDC and FaceForensics++). Our analysis includes intra-dataset and inter-dataset evaluations, examining the best performing models, generalisation capabilities and impact of augmentations. We also investigate the trade-off between model size, efficiency and performance. Our main goal is to provide insights into the effectiveness of different deep learning architectures (transformers, CNNs), training strategies (supervised, self-supervised) and deepfake detection benchmarks. Through our extensive analysis, we established that Transformer models outperform CNN models in deepfake detection. Also, we show that FaceForensics++ and DFDC datasets equip models with comparably better generalisation capabilities, as compared to FakeAVCeleb and CelebDF-V2 datasets. Our analysis also show that image augmentations can be helpful in achieving better performance, at least for the Transformer models.},

keywords = {WP3: Media Content Production and Analysis},

pubstate = {published},

tppubtype = {conference}

}

@conference{CENTERIS,

title = {The future technologies of journalism},

author = {Bjørnar Tessem and Are Tverberg and Njål Borch },

url = {https://mediafutures.no/centeris/},

year  = {2023},

date = {2023-11-10},

booktitle = {Centeris},

abstract = {The practice of journalism has undergone many changes in the last few years, with changes in technology being the 

main driver of these changes. We present a future study where we aim to get an understanding of what technologies 

will become important for the journalist and further change the journalist’s workplace. The new technological 

solutions will have to be implemented in the media houses’ information systems, and knowledge about what 

technologies will have the greatest impact will influence IS strategies in the media house. In the study we 

interviewed 16 experts on how they envision the future technologies of the journalist. We analyzed the interviews 

with a qualitative research approach. Our analysis shows that technologies for multi-platform news production, 

automated news content generation, cloud services for flexible production, content search, and content verification 

are the most important in terms of needs and competitiveness.},

keywords = {WP3: Media Content Production and Analysis},

pubstate = {published},

tppubtype = {conference}

}

@conference{intrs23_session3,

title = {Using Visual and Linguistic Framing to Support Sustainable Decisions in an Online Store},

author = {Alain Starke and Kimia Emami and Andrea Makarová and Bruce Ferwerda },

url = {https://mediafutures.no/intrs23_session3_paper_2/},

year  = {2023},

date = {2023-09-18},

urldate = {2023-09-18},

booktitle = {Association for Computing Machinery (ACM) RecSys ’23,},

keywords = {WP2: User Modeling Personalization and Engagement},

pubstate = {published},

tppubtype = {conference}

}

@conference{inra2023,

title = {The Interplay between Food Knowledge, Nudges, and Preference Elicitation Methods Determines the Evaluation of a Recipe Recommender System},

author = {Ayoub El Majjodi and Alain D. Starke and Christoph Trattner },

url = {https://mediafutures.no/intrs2023-2/},

year  = {2023},

date = {2023-09-18},

urldate = {2023-09-18},

booktitle = {Association for Computing Machinery (ACM) RecSys ’23},

keywords = {WP2: User Modeling Personalization and Engagement},

pubstate = {published},

tppubtype = {conference}

}

@conference{RecSys_2023_LBR,

title = {Evaluating The Effects of Calibrated Popularity Bias Mitigation: A Field Study},

author = {Anastasiia Klimashevskaia and Mehdi Elahi and Dietmar Jannach and Lars Skjærven and Astrid Tessem and Christoph Trattner},

url = {https://mediafutures.no/recsys_2023_lbr/},

year  = {2023},

date = {2023-09-18},

booktitle = {Association for Computing Machinery (ACM) RecSys ’23},

keywords = {WP2: User Modeling Personalization and Engagement},

pubstate = {published},

tppubtype = {conference}

}

@conference{inra2023-1,

title = {Topical Preference Trumps Other Features in News Recommendation: A Conjoint Analysis on a Representative Sample from Norway},

author = {Erik Knudsen and Alain D. Starke and Christoph Trattner },

url = {https://mediafutures.no/inra2023-1/},

year  = {2023},

date = {2023-09-18},

urldate = {2023-09-18},

booktitle = {Association for Computing Machinery (ACM) RecSys ’23},

keywords = {WP1: Understanding Media Experiences, WP2: User Modeling Personalization and Engagement},

pubstate = {published},

tppubtype = {conference}

}

@conference{behavrec2023,

title = {Understanding How News Recommender Systems Influence Selective Exposure},

author = {Khadiga Seddik and Erik Knudsen and Damian Trilling and Christoph Trattner },

url = {https://mediafutures.no/behavrec2023/},

year  = {2023},

date = {2023-09-18},

urldate = {2023-09-18},

booktitle = {Association for Computing Machinery (ACM) RecSys ’23},

keywords = {WP1: Understanding Media Experiences, WP2: User Modeling Personalization and Engagement},

pubstate = {published},

tppubtype = {conference}

}

@conference{Sheikhi2023,

title = {Automated Claim Detection for Fact-checking: A Case Study using Norwegian Pre-trained Language Models},

author = {Ghazaal Sheikhi and Samia Touileb and Sohail Ahmed Khan },

url = {https://mediafutures.no/2023_nodalida-1_1/},

year  = {2023},

date = {2023-05-24},

urldate = {2023-05-24},

abstract = {We investigate to what extent pre-trained language models can be used for automated claim detection for fact-checking in a low resource setting. We explore this idea by fine-tuning four Norwegian pre-trained language models to perform the binary classification task of determining if a claim should be discarded or upheld to be further processed by human fact-checkers. We conduct a set of experiments to compare the performance of the language models, and provide a simple baseline model using SVM with tf-idf features. Since we are focusing on claim detection, the recall score for the upheld class is to be emphasized over other performance measures. Our experiments indicate that the language models are superior to the baseline system in terms of F1, while the baseline model results in the highest precision. However, the two Norwegian models, NorBERT2 and NB-BERT_large, give respectively superior F1 and recall values. We argue that large language models could be successfully employed to solve the automated claim detection problem. The choice of the model depends on the desired end-goal. Moreover, our error analysis shows that language models are generally less sensitive to the changes in claim length and source than the SVM model.},

keywords = {WP5: Norwegian Language Technologies},

pubstate = {published},

tppubtype = {conference}

}

@conference{Barnes2023,

title = {Identifying Token-Level Dialectal Features in Social Media},

author = {Jeremy Barnes, Samia Touileb, Petter Mæhlum and Pierre Lison },

url = {https://mediafutures.no/2023_nodalida-1_16/},

year  = {2023},

date = {2023-05-24},

abstract = {Dialectal variation is present in many human languages and is attracting a growing interest in NLP. Most previous work concentrated on either (1) classifying dialectal varieties at the document or sentence level or (2) performing standard NLP tasks on dialectal data. In this paper, we propose the novel task of token-level dialectal feature prediction. We present a set of fine-grained annotation guidelines for Norwegian dialects, expand a corpus of dialectal tweets, and manually annotate them using the introduced guidelines. Furthermore, to evaluate the learnability of our task, we conduct labeling experiments using a collection of baselines, weakly supervised and supervised sequence labeling models. The obtained results show that, despite the difficulty of the task and the scarcity of training data, many dialectal features can be predicted with reasonably high accuracy.},

keywords = {WP5: Norwegian Language Technologies},

pubstate = {published},

tppubtype = {conference}

}

@conference{Samuel2023,

title = {NorBench – A Benchmark for Norwegian Language Models},

author = {David Samuel and Andrey Kutuzov and Samia Touileb and Erik Velldal and Lilja Øvrelid and Egil Rønningstad and Elina Sigdel and Anna Palatkina},

url = {https://mediafutures.no/2023_nodalida-1_61/},

year  = {2023},

date = {2023-05-24},

urldate = {2023-05-24},

abstract = {We present NorBench: a streamlined suite of NLP tasks and probes for evaluating Norwegian language models (LMs) on standardized data splits and evaluation metrics. We also introduce a range of new Norwegian language models (both encoder and encoder-decoder based). Finally, we compare and analyze their performance, along with other existing LMs, across the different benchmark tests of NorBench.},

keywords = {WP5: Norwegian Language Technologies},

pubstate = {published},

tppubtype = {conference}

}

@conference{Touileb2023,

title = {Measuring Normative and Descriptive Biases in Language Models Using Census Data},

author = {Samia Touileb and Lilja Øvrelid and Erik Velldal},

url = {https://mediafutures.no/2023_eacl-main_164/},

year  = {2023},

date = {2023-05-02},

abstract = {We investigate in this paper how distributions of occupations with respect to gender is reflected

in pre-trained language models. Such distributions are not always aligned to normative ideals, nor do they necessarily reflect a descriptive assessment of reality. In this paper, we introduce an approach for measuring to what degree pre-trained language models are aligned to normative and descriptive occupational distributions. To this end, we use official demographic information about gender–occupation distributions provided by the national statistics agencies of France, Norway, United Kingdom, and the United States. We manually generate template-based sentences combining gendered pronouns and nouns with occupations,

and subsequently probe a selection of ten language models covering the English, French, and Norwegian languages. The scoring system we introduce in this work is language independent, and can be used on any combination of

template-based sentences, occupations, and languages. The approach could also be extended to other dimensions of national census data and other demographic variables.},

keywords = {WP5: Norwegian Language Technologies},

pubstate = {published},

tppubtype = {conference}

}

@conference{Touileb2022b,

title = {Measuring Harmful Representations in Scandinavian Language Models},

author = {Samia Touileb and Debora Nozza},

url = {https://mediafutures.no/2211-11678/},

year  = {2022},

date = {2022-11-21},

urldate = {2022-11-21},

abstract = {Scandinavian countries are perceived as rolemodels when it comes to gender equality. With the advent of pre-trained language models and their widespread usage, we investigate to what extent gender-based harmful and toxic content exist in selected Scandinavian language models. We examine nine models, covering Danish, Swedish, and Norwegian, by manually creating template-based sentences and probing

the models for completion. We evaluate the completions using two methods for measuring harmful and toxic completions and provide a thorough analysis of the results. We show that Scandinavian pre-trained language models contain harmful and gender-based stereotypes with similar values across all languages.

This finding goes against the general expectations related to gender equality in Scandinavian countries and shows the possible problematic outcomes of using such models in real world settings.},

keywords = {WP5: Norwegian Language Technologies},

pubstate = {published},

tppubtype = {conference}

}