@conference{AIMod24,

title = {AiModerator: A Co-Pilot for Hyper-Contextualization in Political Debate Video},

author = {Peter Andrews and Njål Borch and Morten Fjeld},

url = {https://mediafutures.no/iui25-78/},

year  = {2025},

date = {2025-03-01},

booktitle = {Intelligent User Interfaces (IUI) 2025},

abstract = {Political debates are essential in political discourse for democratic societies. Advancements in technology have significantly transformed the structure of political debates, the ways in which politicians communicate, and the platforms through which audiences engage with them. Originally a forum for improving understanding, political debates have increasingly favored theatrics over substance, risking young adult disengagement. To bring substance back to this medium we developed AiModerator, a political debate co-pilot acting as a Multimodal Conversational Agent (MCA). AiModerator aims to promote engagement while improving understanding by analyzing video content to provide contextually relevant information. This consolidated information facilitates understanding while keeping users synchronized with the debate viewing experience. Our system builds upon multimodal techniques, integrating computer vision and large language models to demonstrate ways of improving content delivery and engagement. AiModerator's backend system extracts events from identified speech data, allowing the user to interact with these events through a touch interface on an iPad application. We address three key topics: evaluating young adults' engagement, satisfaction, and preference compared to traditional second screening, and determining whether AiModerator can improve subjective understanding. To evaluate these measures we conducted a mixed-method evaluation (n=20) within-group design A-B study. Our analysis found AiModerator excelled in promoting engagement and satisfaction while delivering clear, contextually relevant information to the user which improved their understanding of debate topics more than the second screening mode. Our qualitative analysis offers broader insights, particularly in terms of a trade-off between automation and information consolidation versus autonomy and control.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{designing_for_automated24,

title = {Designing for Automated Sports Commentary Systems},

author = {Peter Andrews and Oda Elise Nordberg and Frode Guribye and Morten Fjeld and Njål Borch },

url = {https://mediafutures.no/designing_for_automated_sports_commentary_systems-2/},

year  = {2024},

date = {2024-06-12},

booktitle = {IMX'24},

abstract = {Advancements in Natural Language Processing (NLP) and Computer Vision (CV) are revolutionizing how we experience sports broadcasting. Traditionally, sports commentary has played a crucial role in enhancing viewer understanding and engagement with live games. Yet, the prospects of automated commentary, especially in light of these technological advancements and their impact on viewers’ experience, remain largely unexplored. This paper elaborates upon an innovative automated commentary system that integrates NLP and CV to provide a multimodal experience, combining auditory feedback through text-to-speech and visual cues, known as italicizing, for real-time in-game commentary. The system supports color commentary, which aims to inform the viewer of information surrounding the game by pulling additional content from a database. Moreover, it also supports play-by-play commentary covering in-game developments derived from an event system based on CV. As the system reinvents the role of commentary in sports video, we must consider the design and implications of multimodal artificial commentators. A focused user study with eight participants aimed at understanding the design implications of such multimodal artificial commentators reveals critical insights. Key findings emphasize the importance of language precision, content relevance, and delivery style in automated commentary, underscoring the necessity for personalization to meet diverse viewer preferences. Our results validate the potential value and effectiveness of multimodal feedback and derive design considerations, particularly in personalizing content to revolutionize the role of commentary in sports broadcasts.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Footyvision1,

title = {FootyVision: Multi-Object Tracking, Localisation, and Augmentation of Players and Ball in Football Video},

author = {Peter Andrews and Njål Borch and Morten Fjeld},

url = {https://mediafutures.no/peterandrews-footyvision-icmip24-final/},

year  = {2024},

date = {2024-04-20},

booktitle = {ACM ICMIP},

abstract = {Football video content analysis is a rapidly evolving field aiming to enrich the viewing experience of football matches. Current research often focuses on specific tasks like player and/or ball detection, tracking, and localisation in top-down views. Our study strives to integrate these efforts into a comprehensive Multi-Object Tracking (MOT) model capable of handling perspective transformations. Our framework, FootyVision, employs a YOLOv7 backbone trained on an extended player and ball dataset. The MOT module builds a gallery and assigns identities via the Hungarian algorithm based on feature embeddings, bounding box intersection over union, distance, and velocity. A novel component of our model is the perspective transformation module that leverages activation maps from the YOLOv7 backbone to compute homographies using lines, intersection points, and ellipses. This method effectively adapts to dynamic and uncalibrated video data, even in viewpoints with limited visual information. In terms of performance, FootyVision sets new benchmarks. The model achieves a mean average precision (mAP) of 95.7% and an F1-score of 95.5% in object detection. For MOT, it demonstrates robust capabilities, with an IDF1 score of approximately 93% on both ISSIA and SoccerNet datasets. For SoccerNet, it reaches a MOTA of 94.04% and shows competitive results for ISSIA. Additionally, FootyVision scores a HOTA(0) of 93.1% and an overall HOTA of 72.16% for the SoccerNet dataset. Our ablation study confirms the effectiveness of the selected tracking features and identifies key attributes for further improvement. While the model excels in maintaining track accuracy throughout the testing dataset, we recognise the potential to enhance spatial-location accuracy.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{AIComment,

title = {AiCommentator: A Multimodal Conversational Agent for Embedded Visualization in Football Viewing},

author = {Peter Andrews and Oda Elise Nordberg and Frode Guribye and Kazuyuki Fujita and Morten Fjeld and Njål Borch},

url = {https://mediafutures.no/acm_iui_24_aicommentator_peterandrews-1/},

year  = {2024},

date = {2024-03-18},

urldate = {2024-03-18},

booktitle = {Intelligent User Interfaces (IUI)},

journal = {Intelligent User Interfaces (IUI)},

abstract = {Traditionally, sports commentators provide viewers with diverse information, encompassing in-game developments and player performances. Yet young adult football viewers increasingly use mobile devices for deeper insights during football matches. Such insights into players on the pitch and performance statistics support viewers’ understanding of game stakes, creating a more engaging viewing experience. Inspired by commentators’ traditional roles and to incorporate information into a single platform, we developed AiCommentator, a Multimodal Conversational Agent (MCA) for embedded visualization and conversational interactions in football broadcast video. AiCommentator integrates embedded visualization, either with an automated non-interactive or with a responsive interactive commentary mode. Our system builds upon multimodal techniques, integrating computer vision and large language models, to demonstrate ways for designing tailored, interactive sports-viewing content. AiCommentator’s event system infers game states based on a multi-object tracking algorithm and computer vision backend, facilitating automated responsive commentary. We address three key topics: evaluating young adults’ satisfaction and immersion across the two viewing modes, enhancing viewer understanding of in-game events and players on the pitch, and devising methods to present this information in a usable manner. In a mixed-method evaluation (n=16) of AiCommentator, we found that the participants appreciated aspects of both system modes but preferred the interactive mode, expressing a higher degree of engagement and satisfaction. Our paper reports on our development of AiCommentator and presents the results from our user study, demonstrating the promise of interactive MCA for a more engaging sports viewing experience. Systems like AiCommentator could be pivotal in transforming the interactivity and accessibility of sports content, revolutionizing how sports viewers engage with video content.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Fjeld2022,

title = {RedirectedDoors: Redirection While Opening Doors in Virtual Reality},

author = {Morten Fjeld and Yukai Hoshikawa and Kazuyuki Fujita and Kazuki Takashima and Yoshifumi Kitamura },

year  = {2022},

date = {2022-03-12},

urldate = {2022-03-12},

booktitle = {RedirectedDoors: Redirection While Opening Doors in Virtual Reality.},

abstract = {We propose RedirectedDoors, a novel technique for redirection in VR focused on door-opening behavior. This technique manipulates the user's walking direction by rotating the entire virtual environment at a certain angular ratio of the door being opened, while the virtual door's position is kept unmanipulated to ensure door-opening realism. Results of a user study using two types of door-opening interfaces (with and without a passive haptic prop) revealed that the estimated detection thresholds generally showed a higher space efficiency of redirection. Following the results, we derived usage guidelines for our technique that provide lower noticeability and higher acceptability.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Le2021b,

title = {VXSlate: Combining Head Movement and Mobile Touch for Large Virtual Display Interaction},

author = {Khanh-Duy Le and Tanh Quang Tran and Karol Chlasta and Krzysztof Krejtz and Morten Fjeld and Andreas Kunz},

url = {https://conferences.computer.org/vrpub/pdfs/VRW2021-2ANNoldm4A10Ml9f63uYC9/136700a528/136700a528.pdf

https://www.youtube.com/watch?v=N8ZJlKWj4mk&ab_channel=DuyL%C3%AAKh%C3%A1nh},

doi = { 10.1109/VRW52623.2021.00146},

year  = {2021},

date = {2021-02-12},

pages = {528-529},

publisher = {IEEE The Institute of Electrical and Electronics Engineers, Inc.},

organization = {2021 IEEE Conference on Virtual Reality and 3D User Interfaces Abstracts and Workshops (VRW).},

abstract = {Virtual Reality (VR) headsets can open opportunities for users to accomplish complex tasks on large virtual displays, using compact setups. However, interacting with large virtual displays using existing interaction techniques might cause fatigue, especially for precise manipulations, due to the lack of physical surfaces. We designed VXSlate, an interaction technique that uses a large virtual display, as an expansion of a tablet. VXSlate combines a user’s head movements, as tracked by the VR headset, and touch interaction on the tablet. The user’s head movements position both a virtual representation of the tablet and of the user’s hand on the large virtual display. The user’s multi-touch interactions perform finely-tuned content manipulations.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Okopnyi2020,

title = {Unpacking Editorial Agreements in Collaborative Video Production},

author = {Pavel Okopnyi and Oskar Juhlin and Frode Guribye},

url = {https://www.researchgate.net/publication/342251635_Unpacking_Editorial_Agreements_in_Collaborative_Video_Production},

doi = {https://doi.org/10.1145/3391614.3393652},

year  = {2020},

date = {2020-06-01},

booktitle = {IMX '20: ACM International Conference on Interactive Media Experiences},

pages = {117–126},

address = {New York},

abstract = {Video production is a collaborative process involving creative, artistic and technical elements that require a multitude of specialised skill sets. This open-ended work is often marked by uncertainty and interpretive flexibility in terms of what the product is and should be. At the same time, most current video production tools are designed for single users. There is a growing interest, both in industry and academia, to design features that support key collaborative processes in editing, such as commenting on videos. We add to current research by unpacking specific forms of collaboration, in particular the social mechanisms and strategies employed to reduce interpretive flexibility and uncertainty in achieving agreements between editors and other collaborators. The findings contribute to the emerging design interest by identifying general design paths for how to support collaboration in video editing through scaffolding, iconic referencing, and suggestive editing.},

note = {Pre SFI},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Flobak2019,

title = {Participatory Design of VR Scenarios for Exposure Therapy},

author = {Eivind Flobak and Jo Dugstad Wake and Joakim Vindenes and Smiti Kahlon and T. Nordgreen and Frode Guribye},

url = {https://www.researchgate.net/publication/330205387_Participatory_Design_of_VR_Scenarios_for_Exposure_Therapy},

doi = {10.1145/3290605.3300799 },

year  = {2019},

date = {2019-05-01},

booktitle = {Proceedings of the 2019 CHI Conference on Human Factors in Computing Systems (CHI '19)},

number = {Paper 569},

address = {New York},

abstract = {Virtual reality (VR) applications for exposure therapy predominantly use computer-generated imagery to create controlled environments in which users can be exposed to their fears. Creating 3D animations, however, is demanding and time-consuming. This paper presents a participatory approach for prototyping VR scenarios that are enabled by 360° video and grounded in lived experiences. We organized a participa-tory workshop with adolescents to prototype such scenarios, consisting of iterative phases of ideation, storyboarding, live-action plays recorded by a 360° camera, and group evaluation. Through an analysis of the participants' interactions, we outline how they worked to design prototypes that depict situations relevant to those with a fear of public speaking. Our analysis also explores how participants used their experiences and refections as resources for design. Six clinical psychologists evaluated the prototypes from the workshop and concluded they were viable therapeutic tools, emphasizing the immer-sive, realistic experience they presented. We argue that our approach makes the design of VR scenarios more accessible.},

note = {Pre SFI},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Arntzen2018b,

title = {Multi-device Linear Composition on the Web, Enabling Multi-device Linear Media with HTMLTimingObject and Shared Motion},

author = {Ingar Mæhlum Arntzen and Njål Borch and François Daoust and Dominique Hazael-Massieux

},

url = {https://www.researchgate.net/publication/324991987_Multi-device_Linear_Composition_on_the_Web_Enabling_Multi-device_Linear_Media_with_HTMLTimingObject_and_Shared_Motion},

year  = {2018},

date = {2018-01-01},

address = {Brussels},

organization = {Media Synchronization Workshop},

abstract = {Composition is a hallmark of the Web, yet it does not fully extend to linear media. This paper defines linear composition as the ability to form linear media by coordinated playback of independent linear components. We argue that native Web support for linear composition is a key enabler for Web-based multi-device linear media, and that precise multi-device timing is the main technical challenge. This paper proposes the introduction of an HTMLTimingObject as basis for linear composition in the single-device scenario. Linear composition in the multi-device scenario is ensured as HTMLTimingObjects may integrate with Shared Motion, a generic timing mechanism for the Web. By connecting HTMLMediaElements and HTMLTrackElements with a multi-device timing mechanism, a powerful programming model for multi-device linear media is unlocked.},

note = {Pre SFI},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{Borch2016,

title = {Timing - small step for developers, giant leap for the media industry, IBC 2016},

author = {Njål Borch and François Daoust and Ingar Mæhlum Arntzen},

url = {https://www.w3.org/community/webtiming/files/2016/09/Borch_IBC2016-final.pdf},

year  = {2017},

date = {2017-02-11},

note = {Pre SFI},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}