Demo: A Latency-Optimized LLM-based Multimodal Dialogue System for Embodied Conversational Agents in VR

@inproceedings{Kuehlem2025,
author = {W. K\"{u}hlem, Konstantin and Ehret, Jonathan and W. Kuhlen, Torsten and B\"{o}nsch, Andrea},
title = {A Latency-Optimized LLM-based Multimodal Dialogue System for Embodied Conversational Agents in VR},
year = {2025},
isbn = {9798400715082},
publisher = {Association for Computing Machinery},
doi = {10.1145/3717511.3749287},
abstract = {Interactions with Embodied Conversational Agents (ECAs) are essential in many social Virtual Reality (VR) applications, highlighting the growing demand for free-flowing, context-aware conversations supported by low-latency, multimodal ECA responses. We introduce a modular, extensible framework powered by an Large Language Model (LLM), featuring streaming-based optimization techniques specially crafted for multimodal responses. Our system is capable of controlling self-behavior and task execution, in the form of moving through the Immersive Virtual Environment (IVE) directly controlled by the LLM, and is also capable of reacting to events in the IVE. In our study, our applied optimizations achieved a latency improvement of about (66\%) on average compared to having no optimizations.},
booktitle = {Proceedings of the 25th ACM International Conference on Intelligent Virtual Agents},
articleno = {49},
numpages = {3},
series = {IVA '25}
}