dream-publications.bib

@inproceedings{Bavaresco-etal-2026,
	author = 	 {Anna Bavaresco and Marianne de Heer Kloots and Sandro Pezzelle and Raquel Fern\'{a}ndez},
	title = 	 {Vision-Language Models Align with Human Neural Representations in Concept Processing},
	booktitle = 	 {Proceedings of the The 19th Conference of the European Chapter of the Association for Computational Linguistics (EACL)},
	year = 	 {2026},
	note= {To appear.},
	abstract = {Recent studies suggest that transformer-based vision-language models (VLMs) capture the multimodality of concept processing in the human brain. However, a systematic evaluation exploring different types of VLM architectures and the role played by visual and textual context is still lacking. Here, we analyse multiple VLMs employing different strategies to integrate visual and textual modalities, along with language-only counterparts. We measure the alignment between concept representations by models and existing (fMRI) brain responses to concept words presented in two experimental conditions, where either visual (pictures) or textual (sentences) context is provided. Our results reveal that VLMs outperform the language-only counterparts in both experimental conditions. However, controlled ablation studies show that only for some VLMs, such as LXMERT and IDEFICS2, brain alignment stems from genuinely learning more human-like concepts during pretraining, while others are highly sensitive to the context provided at inference. Additionally, we find that vision-language encoders are more brain-aligned than more recent, generative VLMs. Altogether, our study shows that VLMs align with human neural representations in concept processing, while highlighting differences among architectures.}
}

@inproceedings{Testoni-etal-emnlp-2024,
  author = 	 {Alberto Testoni and Barbara Plank and Raquel Fern\'{a}ndez},
  title = 	 {RACQUET: Unveiling the Dangers of Overlooked Referential Ambiguity in Visual LLMs},
  booktitle = {Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
  year = 	 {2025},
  url={https://aclanthology.org/2025.emnlp-main.1206/},
  abstract={Ambiguity resolution is key to effective communication. While humans effortlessly address ambiguity through conversational grounding strategies, the extent to which current language models can emulate these strategies remains unclear. In this work, we examine referential ambiguity in image-based question answering by introducing RACQUET, a carefully curated dataset targeting distinct aspects of ambiguity. Through a series of evaluations, we reveal significant limitations and problems of overconfidence of state-of-the-art large multimodal language models in addressing ambiguity in their responses. The overconfidence issue becomes particularly relevant for RACQUET-BIAS, a subset designed to analyze a critical yet underexplored problem: failing to address ambiguity leads to stereotypical, socially biased responses. Our results underscore the urgency of equipping models with robust strategies to deal with uncertainty without resorting to undesirable stereotypes.}
}

@inproceedings{Horst-etal-acl-2025,
  author = 	 {Nicola Horst and Davide Mazzaccara and Antonia Schmidt and Michael Sullivan and Filippo Momentè and Luca Franceschetti and Philipp Sadler and Sherzod Hakimov and Alberto Testoni and Raffaella Bernardi and Raquel Fern\'{a}ndez and Alexander Koller and Oliver Lemon and David Schlangen and Mario Giulianelli and Alessandro Suglia},
  title = 	 {Playpen: An Environment for Exploring Learning Through Conversational Interaction},
  booktitle = 	 {Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
  year = 	 {2025},
  url = 	 {https://aclanthology.org/2025.emnlp-main.1517/},
  url_github = {https://github.com/lm-playpen/playpen},
  abstract = 	 {Are we running out of learning signal? Predicting the next word in an existing text has turned out to be a powerful signal, at least at scale. But there are signs that we are running out of this resource. In recent months, interaction between learner and feedback-giver has come into focus, both for "alignment" (with a reward model judging the quality of instruction following attempts) and for improving "reasoning" (process- and outcome-based verifiers judging reasoning steps). In this paper, we explore to what extent synthetic interaction in what we call Dialogue Games -- goal-directed and rule-governed activities driven predominantly by verbal actions -- can provide a learning signal, and how this signal can be used. We introduce an environment for producing such interaction data (with the help of a Large Language Model as counterpart to the learner model), both offline and online. We investigate the effects of supervised fine-tuning on this data, as well as reinforcement learning setups such as DPO, and GRPO; showing that all of these approaches achieve some improvements in in-domain games, but only GRPO demonstrates the ability to generalise to out-of-domain games as well as retain competitive performance in reference-based tasks. We release the framework and the baseline training setups in the hope that this can foster research in this promising new direction.}
}

@inproceedings{Momente-etal-2025,
  title={Triangulating LLM Progress through Benchmarks, Games, and Cognitive Tests}, 
  author={Filippo Moment\`{e} and Alessandro Suglia and Mario Giulianelli and Ambra Ferrari and Alexander Koller and Oliver Lemon and David Schlangen and Raquel Fern\'{a}ndez and Raffaella Bernardi},
  year={2025},
  booktitle = {Findings of the Association for Computational Linguistics: EMNLP 2025},
  url = {https://aclanthology.org/2025.findings-emnlp.1092/},
  abstract = {We examine three evaluation paradigms: large question-answering benchmarks (e.g., MMLU and BBH), interactive games (e.g., Signalling Games or Taboo), and cognitive tests (e.g., for working memory or theory of mind). First, we investigate which of the former two-benchmarks or games-is most effective at discriminating LLMs of varying quality. Then, inspired by human cognitive assessments, we compile a suite of targeted tests that measure cognitive abilities deemed essential for effective language use, and we investigate their correlation with model performance in benchmarks and games. Our analyses reveal that interactive games are superior to standard benchmarks in discriminating models. Causal and logical reasoning correlate with both static and interactive tests, while differences emerge regarding core executive functions and social/emotional skills, which correlate more with games. We advocate the development of new interactive benchmarks and targeted cognitive tasks inspired by assessing human abilities but designed specifically for LLMs.}
}
@article{Bavaresco-deHeerKloots-etal-arxiv-2025,
title={Modelling Multimodal Integration in Human Concept Processing with Vision-Language Models},
author={Anna Bavaresco and Marianne de Heer Kloots and Sandro Pezzelle and Raquel Fern\'{a}ndez},
journal = {ArXiv},
year={Preprints},
url={https://arxiv.org/abs/2407.17914},
note={2025.},
abstract={Text representations from language models have proven remarkably predictive of human neural activity involved in language processing, 
with the recent transformer-based models outperforming previous architectures in downstream tasks and prediction of brain responses. 
However, the word representations learnt by language-only models may be limited in that they lack sensory information from other modalities, 
which several cognitive and neuroscience studies showed to be reflected in human meaning representations. Here, we leverage current pre-trained 
vision-language models (VLMs) to investigate whether the integration of visuo-linguistic information they operate leads to representations 
that are more aligned with human brain activity than those obtained by models trained with language-only input. We focus on fMRI responses 
recorded while participants read concept words in the context of either a full sentence or a picture. Our results reveal that VLM representations 
correlate more strongly than those by language-only models with activations in brain areas functionally related to language processing. 
Additionally, we find that transformer-based vision-language encoders -- e.g., LXMERT and VisualBERT -- yield more brain-aligned representations 
than generative VLMs, whose autoregressive abilities do not seem to provide an advantage when modelling single words. Finally, our ablation 
analyses suggest that the high brain alignment achieved by some of the VLMs we evaluate results from semantic information acquired specifically 
during multimodal pretraining as opposed to being already encoded in their unimodal modules. Altogether, our findings indicate an advantage of 
multimodal models in predicting human brain activations, which reveals that modelling language and vision integration has the potential to 
capture the multimodal nature of human concept representations.}
}


@inproceedings{Bavaresco-Fernandez-conll-2025,
  author = 	 {Anna Bavaresco and Raquel Fern\'{a}ndez},
  title = 	 {Experiential Semantic Information and Brain Alignment: Are Multimodal Models Better than Language Models?},
  booktitle = 	 {Proceedings of the 29th Conference on Computational Natural Language Learning (CoNLL)},
  year = 	 {2025},
  url = {https://aclanthology.org/2025.conll-1.10/},
  abstract = {A common assumption in Computational Linguistics is that text representations learnt by multimodal models are richer and more human-like than those by language-only models, as they are grounded in images or audio -- similar to how human language is grounded in real-world experiences. However, empirical studies checking whether this is true are largely lacking. We address this gap by comparing word representations from contrastive multimodal models vs. language-only ones in the extent to which they capture experiential information -- as defined by an existing norm-based 'experiential model' -- and align with human fMRI responses. Our results indicate that, surprisingly, language-only models are superior to multimodal ones in both respects. Additionally, they learn more unique brain-relevant semantic information beyond that shared with the experiential model. Overall, our study highlights the need to develop computational models that better integrate the complementary semantic information provided by multimodal data sources.}
}

@inproceedings{ghaleb-etal-acl-2025, 
  author={Ghaleb, Esam and Khaertdinov, Bulat and {\"O}zy{\"u}rek, Asl{\i} and Fern{\'a}ndez, Raquel},
  title={I see what you mean: Co-Speech Gestures for Reference Resolution in Multimodal Dialogue},
  booktitle={Proceedings of the of the 63rd Conference of the Association for Computational Linguistics (ACL Findings)},
  year={2025},
  url={https://aclanthology.org/2025.findings-acl.682/},
  url_github={https://github.com/EsamGhaleb/MultimodalReferenceResolution},
  abstract={In face-to-face interaction, we use multiple modalities, including speech and gestures, to communicate information and resolve references to objects. However, how representational co-speech gestures refer to objects remains understudied from a computational perspective. In this work, we address this gap by introducing a multimodal reference resolution task centred on representational gestures, while simultaneously tackling the challenge of learning robust gesture embeddings. We propose a self-supervised pre-training approach to gesture representation learning that grounds body movements in spoken language. Our experiments show that the learned embeddings align with expert annotations and have significant predictive power. Moreover, reference resolution accuracy further improves when (1) using multimodal gesture representations, even when speech is unavailable at inference time, and (2) leveraging dialogue history. Overall, our findings highlight the complementary roles of gesture and speech in reference resolution, offering a step towards more naturalistic models of human-machine interaction.}
}

@inproceedings{judgebench-acl-2025,
  title={LLMs instead of Human Judges? A Large Scale Empirical Study across 20 NLP Evaluation Tasks},
  author={Anna Bavaresco and Raffaella Bernardi and Leonardo Bertolazzi and Desmond Elliott and Raquel Fern\'{a}ndez and Albert Gatt and Esam Ghaleb and Mario Giulianelli and Michael Hanna and Alexander Koller and Andr\'{e} F. T. Martins and Philipp Mondorf and Vera Neplenbroek and Sandro Pezzelle and Barbara Plank and David Schlangen and Alessandro Suglia and Aditya K. Surikuchi and Ece Takmaz and Alberto Testoni},
  year={2025},
  booktitle={Proceedings of the of the 63rd Conference of the Association for Computational Linguistics (ACL 2025)},     
  url={https://aclanthology.org/2025.acl-short.20/},
  url_github = {https://github.com/dmg-illc/JUDGE-BENCH},
  abstract={There is an increasing trend towards evaluating NLP models with LLM-generated judgments instead of human judgments. 
In the absence of a comparison against human data, this raises concerns about the validity of these evaluations; 
in case they are conducted with proprietary models, this also raises concerns over reproducibility. We provide JUDGE-BENCH, 
a collection of 20 NLP datasets with human annotations, and comprehensively evaluate 11 current LLMs, covering both open-weight 
and proprietary models, for their ability to replicate the annotations. Our evaluations show that each LLM exhibits a large 
variance across datasets in its correlation to human judgments. We conclude that LLMs are not yet ready to systematically 
replace human judges in NLP.}
}

@inproceedings{surikuchi-etal-2024,
title={Not (yet) the whole story: Evaluating Visual Storytelling Requires More than Measuring Coherence, Grounding, and Repetition},
author={Aditya K. Surikuchi and Raquel Fern\'{a}ndez and Sandro Pezzelle},
year={2024},
booktitle={Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing (EMNLP Findings)},
url={https://aclanthology.org/2024.findings-emnlp.679/},
url_github = {https://github.com/akskuchi/dHM-visual-storytelling},
abstract={Visual storytelling consists in generating a natural language story given a temporally ordered sequence of images. This task is not only challenging for models, but also very difficult to evaluate with automatic metrics since there is no consensus about what makes a story 'good'. In this paper, we introduce a novel method that measures story quality in terms of human likeness regarding three key aspects highlighted in previous work: visual grounding, coherence, and repetitiveness. We then use this method to evaluate the stories generated by several models, showing that the foundation model LLaVA obtains the best result, but only slightly so compared to TAPM, a 50-times smaller visual storytelling model. Upgrading the visual and language components of TAPM results in a model that yields competitive performance with a relatively low number of parameters. Finally, we carry out a human evaluation study, whose results suggest that a 'good' story may require more than a human-like level of visual grounding, coherence, and repetition.}
}

@inproceedings{ghaleb-etal-icmi-2024,
  title={Learning Co-Speech Gesture Representations in Dialogue through Contrastive Learning: An Intrinsic Evaluation},
  author={Ghaleb, Esam and Khaertdinov, Bulat and Rasenberg, Marlou and Pouw, Wim and Holler, Judith and {\"O}zy{\"u}rek, Asl{\i} and Fern{\'a}ndez, Raquel},
  booktitle={Proceedings of the 26th ACM International Conference on Multimodal Interaction (ICMI)},  
  year={2024},
  url ={https://dl.acm.org/doi/10.1145/3678957.3685707},
  url_github = {https://github.com/EsamGhaleb/Learning-Co-Speech-Gesture-Representations},
  abstract={In face-to-face dialogues, the form-meaning relationship of co-speech gestures varies depending on contextual factors such as what the gestures refer to and the individual characteristics of speakers. These factors make co-speech gesture representation learning challenging. How can we learn meaningful gestures representations considering gestures' variability and relationship with speech? This paper tackles this challenge by employing self-supervised contrastive learning techniques to learn gesture representations from skeletal and speech information. We propose an approach that includes both unimodal and multimodal pre-training to ground gesture representations in co-occurring speech. For training, we utilize a face-to-face dialogue dataset rich with representational iconic gestures. We conduct thorough intrinsic evaluations of the learned representations through comparison with human-annotated pairwise gesture similarity. Moreover, we perform a diagnostic probing analysis to assess the possibility of recovering interpretable gesture features from the learned representations. Our results show a significant positive correlation with human-annotated gesture similarity and reveal that the similarity between the learned representations is consistent with well-motivated patterns related to the dynamics of dialogue interaction. Moreover, our findings demonstrate that several features concerning the form of gestures can be recovered from the latent representations. Overall, this study shows that multimodal contrastive learning is a promising approach for learning gesture representations, which opens the door to using such representations in larger-scale gesture analysis studies.}
}

@inproceedings{bavaresco-etal-acl-2024,
title={Don't Buy it! Reassessing the Ad Understanding Abilities of Contrastive Multimodal Models},
author={Anna Bavaresco and Alberto Testoni and Raquel Fern{\'a}ndez},
booktitle={Proceedings of the 62nd Conference of the Association for Computational Linguistics (ACL 2024)},
year={2024},
url={https://aclanthology.org/2024.acl-short.77/},
url_data={https://zenodo.org/doi/10.5281/zenodo.11355891},
url_github = {https://github.com/dmg-illc/trade},
abstract={Image-based advertisements are complex multimodal stimuli that often contain unusual visual elements and figurative language. Previous research on automatic ad understanding has reported impressive zero-shot accuracy of contrastive vision-and-language models (VLMs) on an ad-explanation retrieval task. Here, we examine the original task setup and show that contrastive VLMs can solve it by exploiting grounding heuristics. To control for this confound, we introduce TRADE, a new evaluation test set with adversarial grounded explanations. While these explanations look implausible to humans, we show that they "fool" four different contrastive VLMs. Our findings highlight the need for an improved operationalisation of automatic ad understanding that truly evaluates VLMs' multimodal reasoning abilities.}
}

@inproceedings{testoni-etal-acl-2024,
title={Naming, Describing, and Quantifying Visual Objects in Humans and LLMs},
author={Alberto Testoni and Juell Sprott and Sandro Pezzelle},
booktitle={Proceedings of the 62nd Conference of the Association for Computational Linguistics (ACL 2024)},
year={2024},
url={https://aclanthology.org/2024.acl-short.50/},
abstract={While human speakers use a variety of different expressions when describing the same object in an image, giving rise to a distribution of plausible labels driven by pragmatic constraints, the extent to which current Vision and Language Large Language Models (VLLMs) can mimic this crucial feature of language use is an open question. This applies to common, everyday objects, but it is particularly interesting for uncommon or novel objects for which a category label may be lacking or fuzzy. Furthermore, humans show clear production preferences for highly context-sensitive expressions, such as the quantifiers `few' or `most'. In our work, we evaluate VLLMs (FROMAGe, BLIP-2, LLaVA) on three categories (nouns, attributes, and quantifiers) where humans show great subjective variability concerning the distribution over plausible labels, using datasets and resources mostly under-explored in previous work. Our results reveal mixed evidence on the ability of VLLMs to capture human naming preferences, with all models failing in tasks that require high-level reasoning such as assigning quantifiers.}
}

@article{luden-etat-clin-2024,
 title = "Beyond Perplexity: Examining Temporal Generalization in Large Language Models via Definition Generation",
 author = "Iris Luden and Mario Giulianelli and Raquel Fern{\'a}ndez",
 journal={Computational Linguistics in the Netherlands Journal}, 
 volume={13}, 
 year = "2024",
 pages = "205--232",
 url= {https://clinjournal.org/clinj/article/view/181}, 
 url_github = {https://github.com/IrisLuden/Beyond_Perplexity-TemporalGeneralization-DefinitionGeneration},
 abstract = "The advent of large language models (LLMs) has significantly improved performance across various Natural Language Processing tasks. However, the performance of LLMs has been shown to deteriorate over time, indicating a lack of temporal generalization. To date, performance deterioration of LLMs is primarily attributed to the factual changes in the real world over time. However, not only the facts of the world, but also the language we use to describe it constantly changes. Recent studies have indicated a relationship between performance deterioration and semantic change. This is typically measured using perplexity scores and relative performance on downstream tasks. Yet, perplexity and accuracy do not explain the effects of temporally shifted data on LLMs in practice. In this work, we propose to assess lexico-semantic temporal generalization of a language model by exploiting the task of contextualized word definition generation. This in-depth semantic assessment enables interpretable insights into the possible mistakes a model may perpetrate due to meaning shift, and can be used to complement more coarse-grained measures like perplexity scores. To assess how semantic change impacts performance, we design the task by differentiating between semantically stable, changing, and emerging target words, and experiment with T5-base, fine-tuned for contextualized definition generation. Our results indicate that (i) the model’s performance deteriorates for the task of contextualized word definition generation, (ii) the performance deteriorates more for semantically changing words compared to semantically stable words, (iii) the model exhibits significantly lower performance and potential bias for emerging words, and (iv) the performance does not correlate with cross-entropy or (pseudo)-perplexity scores. Overall, our results show that definition generation can be a promising task to assess a model’s capacity for temporal generalization with respect to semantic change."
}

@inproceedings{testoni-fernandez-eacl-2024,
    title = "Asking the Right Question at the Right Time: Human and Model Uncertainty Guidance To Ask Clarification Questions",
    author = "Alberto Testoni and Raquel Fern{\'a}ndez",
    booktitle = "Proceedings of the European Chapter of the Association for Computational Linguistics (EACL)",
    year = "2024",
    url = "https://aclanthology.org/2024.eacl-long.16/",
    abstract = "Clarification questions are an essential dialogue tool to signal misunderstanding, ambiguities, and under-specification in language use. While humans are able to resolve uncertainty by asking questions since childhood, modern dialogue systems struggle to generate effective questions. To make progress in this direction, in this work we take a collaborative dialogue task as a testbed and study how model uncertainty relates to human uncertainty -- an as yet under-explored problem. We show that model uncertainty does not mirror human clarification-seeking behavior, which suggests that using human clarification questions as supervision for deciding when to ask may not be the most effective way to resolve model uncertainty. To address this issue, we propose an approach to generating clarification questions based on model uncertainty estimation, compare it to several alternatives, and show that it leads to significant improvements in terms of task success. Our findings highlight the importance of equipping dialogue systems with the ability to assess their own uncertainty and exploit in interaction."
}

@inproceedings{testoni-etal-acl-2024,
title={Naming, Describing, and Quantifying Visual Objects in Humans and LLMs},
author={Alberto Testoni and Juell Sprott and Sandro Pezzelle},
booktitle={Proceedings of the 62nd Conference of the Association for Computational Linguistics (ACL 2024)},
year={2024},
url={https://aclanthology.org/2024.acl-short.50/},
abstract={While human speakers use a variety of different expressions when describing the same object in an image, giving rise to a distribution of plausible labels driven by pragmatic constraints, the extent to which current Vision and Language Large Language Models (VLLMs) can mimic this crucial feature of language use is an open question. This applies to common, everyday objects, but it is particularly interesting for uncommon or novel objects for which a category label may be lacking or fuzzy. Furthermore, humans show clear production preferences for highly context-sensitive expressions, such as the quantifiers `few' or `most'. In our work, we evaluate VLLMs (FROMAGe, BLIP-2, LLaVA) on three categories (nouns, attributes, and quantifiers) where humans show great subjective variability concerning the distribution over plausible labels, using datasets and resources mostly under-explored in previous work. Our results reveal mixed evidence on the ability of VLLMs to capture human naming preferences, with all models failing in tasks that require high-level reasoning such as assigning quantifiers.}
}

@inproceedings{ghaleb-etal-cogsci-2024,
  title={Analysing Cross-Speaker Convergence through the Lens of Automatically Detected Shared Linguistic Constructions},
  author={Ghaleb, Esam and Rasenberg, Marlou and Pouw, Wim and Toni, Ivan and Holler, Judith and {\"O}zy{\"u}rek, Asl{\i} and Fern{\'a}ndez, Raquel},
  booktitle={Proceedings of the Annual Meeting of the Cognitive Science Society},
  volume={46},
  year={2024},
  url={https://escholarship.org/uc/item/43h970fc},
  abstract={Conversation requires a substantial amount of coordination between dialogue participants, from managing turn taking to negotiating mutual understanding. Part of this coordination effort surfaces as the reuse of linguistic behaviour across speakers, a process often referred to as alignment. While the presence of linguistic alignment is well documented in the literature, several questions remain open, including the extent to which patterns of reuse across speakers have an impact on the emergence of labelling conventions for novel referents. In this study, we put forward a methodology for automatically detecting shared lemmatised constructions—expressions with a common lexical core used by both speakers within a dialogue—and apply it to a referential communication corpus where participants aim to identify novel objects for which no established labels exist. Our analyses uncover the usage patterns of shared constructions in interaction and reveal that features such as their frequency and the amount of different constructions used for a referent are associated with the degree of object labelling convergence the participants exhibit after social interaction. More generally, the present study shows that automatically detected shared constructions offer a useful level of analysis to investigate the dynamics of reference negotiation in dialogue.}
}

@inproceedings{takmaz-etal-eacl-2024,
    title = "Describing Images \textit{Fast and Slow}: Quantifying and Predicting the Variation in Human Signals during Visuo-Linguistic Processes",
    author = "Ece Takmaz and Sandro Pezzelle and Raquel Fern{\'a}ndez",
    booktitle = "Proceedings of the European Chapter of the Association for Computational Linguistics (EACL)",
    year = "2024",
    url = "https://aclanthology.org/2024.eacl-long.126/",
    abstract = "There is an intricate relation between the properties of an image and how humans behave while describing the image. This behavior shows ample variation, as manifested in human signals such as eye movements and when humans start to describe the image. Despite the value of such signals of visuo-linguistic variation, they are virtually disregarded in the training of current pretrained models, which motivates further investigation. Using a corpus of Dutch image descriptions with concurrently collected eye-tracking data, we explore the nature of the variation in visuo-linguistic signals, and find that they correlate with each other. Given this result, we hypothesize that variation stems partly from the properties of the images, and explore whether image representations encoded by pretrained vision encoders can capture such variation. Our results indicate that pretrained models do so to a weak-to-moderate degree, suggesting that the models lack biases about what makes a stimulus complex for humans and what leads to variations in human outputs.",
    url_github = "https://github.com/ecekt/visuolinguistic_signal_variation"
}

@inproceedings{giulianelli-etal-2023-information,
    title = "Information Value: Measuring Utterance Predictability as Distance from Plausible Alternatives",
    author = "Mario Giulianelli and Sarenne Wallbridge and Raquel Fern{\'a}ndez",
    booktitle = "Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
    year = "2023",
    url ="https://aclanthology.org/2023.emnlp-main.343/",
    abstract = "We present information value, a measure which quantifies the predictability of an utterance relative to a set of plausible alternatives. We introduce a method to obtain interpretable estimates of information value using neural text generators, and exploit their psychometric predictive power to investigate the dimensions of predictability that drive human comprehension behaviour. Information value is a stronger predictor of utterance acceptability in written and spoken dialogue than aggregates of token-level surprisal and it is complementary to surprisal for predicting eye-tracked reading times."
}

@inproceedings{giulianelli-etal-2023-arxiv,
    title = "What Comes Next? Evaluating Uncertainty in Neural Text Generators Against Human Production Variability",
    author = " Giulianelli, Mario and Baan, Joris and Aziz, Wilker and Fern{\'a}ndez, Raquel and Plank, Barbara",
    booktitle = "Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
    year = "2023",
    url = "https://aclanthology.org/2023.emnlp-main.887/",
    abstract = "In Natural Language Generation (NLG) tasks, for any input, multiple communicative goals are plausible, and any goal can be put into words, or produced, in multiple ways. We characterise the extent to which human production varies lexically, syntactically, and semantically across four NLG tasks, connecting human production variability to aleatoric or data uncertainty. We then inspect the space of output strings shaped by a generation system{'}s predicted probability distribution and decoding algorithm to probe its uncertainty. For each test input, we measure the generator{'}s calibration to human production variability. Following this instance-level approach, we analyse NLG models and decoding strategies, demonstrating that probing a generator with multiple samples and, when possible, multiple references, provides the level of detail necessary to gain understanding of a model{'}s representation of uncertainty."
}


@inproceedings{giulianelli-etal-2023,
    title = "Interpretable Word Sense Representations via Definition Generation: The Case of Semantic Change Analysis",
    author = "Giulianelli, Mario  and
      Luden, Iris  and
      Fern{\'a}ndez, Raquel  and
      Kutuzov, Andrey",
    booktitle = "Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
    year = "2023",
    address = "Toronto, Canada",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.acl-long.176",
    pages = "3130--3148",
    abstract = "We propose using automatically generated natural language definitions of contextualised word usages as interpretable word and word sense representations.Given a collection of usage examples for a target word, and the corresponding data-driven usage clusters (i.e., word senses), a definition is generated for each usage with a specialised Flan-T5 language model, and the most prototypical definition in a usage cluster is chosen as the sense label. We demonstrate how the resulting sense labels can make existing approaches to semantic change analysis more interpretable, and how they can allow users {---} historical linguists, lexicographers, or social scientists {---} to explore and intuitively explain diachronic trajectories of word meaning. Semantic change analysis is only one of many possible applications of the {`}definitions as representations{'} paradigm. Beyond being human-readable, contextualised definitions also outperform token or usage sentence embeddings in word-in-context semantic similarity judgements, making them a new promising type of lexical representation for NLP.",
}

@inproceedings{takmaz-etal-2023-speaking,
    title = "Speaking the Language of Your Listener: Audience-Aware Adaptation via Plug-and-Play Theory of Mind",
    author = "Takmaz, Ece  and
      Brandizzi, Nicolo{'}  and
      Giulianelli, Mario  and
      Pezzelle, Sandro  and
      Fern{\'a}ndez, Raquel",
    booktitle = "Findings of the Association for Computational Linguistics: ACL 2023",
    year = "2023",
    address = "Toronto, Canada",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.findings-acl.258",
    url_github = "https://github.com/nicofirst1/speaker-adaptation",
    pages = "4198--4217",
    abstract = "Dialogue participants may have varying levels of knowledge about the topic under discussion. In such cases, it is essential for speakers to adapt their utterances by taking their audience into account. Yet, it is an open question how such adaptation can be modelled in computational agents. In this paper, we model a visually grounded referential game between a knowledgeable speaker and a listener with more limited visual and linguistic experience. Inspired by psycholinguistic theories, we endow our speaker with the ability to adapt its referring expressions via a simulation module that monitors the effectiveness of planned utterances from the listener{'}s perspective. We propose an adaptation mechanism building on plug-and-play approaches to controlled language generation, where utterance generation is steered on the fly by the simulator without finetuning the speaker{'}s underlying language model. Our results and analyses show that our approach is effective: the speaker{'}s utterances become closer to the listener{'}s domain of expertise, which leads to higher communicative success.",
}

@article{Pezzelle-Fernandez-2023-cogsci,
author = {Pezzelle, Sandro and Fern{\'a}ndez, Raquel},
title = {Semantic Adaptation to the Interpretation of Gradable Adjectives via Active Linguistic Interaction},
journal = {Cognitive Science},
volume = {47},
number = {2},
pages = {e13248},
keywords = {Gradable adjectives, Semantic adaptation, Vagueness, Questions, Linguistic interaction, Visual grounding},
doi = {https://doi.org/10.1111/cogs.13248},
url = {https://onlinelibrary.wiley.com/doi/abs/10.1111/cogs.13248},
eprint = {https://onlinelibrary.wiley.com/doi/pdf/10.1111/cogs.13248},
abstract = {Abstract When communicating, people adapt their linguistic representations to those of their interlocutors. Previous studies have shown that this also occurs at the semantic level for vague and context-dependent terms such as quantifiers and uncertainty expressions. However, work to date has mostly focused on passive exposure to a given speaker's interpretation, without considering the possible role of active linguistic interaction. In this study, we focus on gradable adjectives big and small and develop a novel experimental paradigm that allows participants to ask clarification questions to figure out their interlocutor's interpretation. We find that, when in doubt, speakers do resort to this strategy, despite its inherent cognitive cost, and that doing so results in higher semantic alignment measured in terms of communicative success. While not all question–answer pairs are equally informative, we show that speakers become better questioners as the interaction progresses. Yet, the higher semantic alignment observed when speakers are able to ask questions does not increase over time. This suggests that conversational interaction's key advantage may be to boost coordination without committing to long-term semantic updates. Our findings shed new light on the mechanisms used by speakers to achieve semantic alignment and on how language is shaped by communication.},
year = {2023}
}

@inproceedings{surikuchi-etal-2023-emnlp,
    title = "GROOViST: A Metric for Grounding Objects in Visual Storytelling",
    author = "Aditya Surikuchi and Sandro Pezzelle and Raquel Fern{\'a}ndez",
    booktitle = "Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
    year = "2023",
    url ="https://aclanthology.org/2023.emnlp-main.202/",
    abstract = "A proper evaluation of stories generated for a sequence of images{---}the task commonly referred to as visual storytelling{---}must consider multiple aspects, such as coherence, grammatical correctness, and visual grounding. In this work, we focus on evaluating the degree of grounding, that is, the extent to which a story is about the entities shown in the images. We analyze current metrics, both designed for this purpose and for general vision-text alignment. Given their observed shortcomings, we propose a novel evaluation tool, GROOViST, that accounts for cross-modal dependencies, \textit{temporal misalignments} (the fact that the order in which entities appear in the story and the image sequence may not match), and human intuitions on visual grounding. An additional advantage of GROOViST is its modular design, where the contribution of each component can be assessed and interpreted individually."
}

@inproceedings{jansen-etal-2022-controllable,
    title = "Controllable Text Generation for All Ages: Evaluating a Plug-and-Play Approach to Age-Adapted Dialogue",
    author = "Jansen, Lennert  and
      Laichter, {\v{S}}t{\v{e}}p{\'a}n Lars  and
      Sinclair, Arabella  and
      van der Goot, Margot  and
      Fern{\'a}ndez, Raquel  and
      Pezzelle, Sandro",
    booktitle = "Proceedings of the 2nd Workshop on Natural Language Generation, Evaluation, and Metrics (GEM)",
    year = "2022",
    address = "Abu Dhabi, United Arab Emirates (Hybrid)",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.gem-1.14",
    pages = "172--188",
    abstract = "To be trusted and perceived as natural and coherent, conversational systems must adapt to the language of their users. While personalized dialogue is a promising direction, controlling generation for fine-grained language features remains a challenge in this approach. A recent line of research showed the effectiveness of leveraging pre-trained language models toward adapting to a text{'}s topic or sentiment. In this study, we build on these approaches and focus on a higher-level dimension of language variation: speakers{'} age. We frame the task as a dialogue response generation, and test methods based on bag-of-words (BoW) and neural discriminators (Disc) to condition the output of GPT-2 and DialoGPT without altering the parameters of the language models. We show that Disc models achieve a higher degree of detectable control than BoW models based on automatic evaluation. In contrast, humans can partially detect age differences in BoW but not Disc responses. Since BoW responses are deemed better than Disc ones by humans, simple controllable methods thus appear to be a better tradeoff between adaptation and language quality. Our work confirms the challenges of adapting to higher-level dimensions of language variation. Moreover, it highlights the need to evaluate natural language generation thoroughly.",
}

@inproceedings{baan-etal-2022-emnlp,
    title = "Stop Measuring Calibration When Humans Disagree",
    author = "Baan, Joris  and
      Aziz, Wilker  and
      Plank, Barbara  and
      Fern{\'a}ndez, Raquel",
    booktitle = "Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing",
    year = "2022",
    address = "Abu Dhabi, United Arab Emirates",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.emnlp-main.124",
    pages = "1892--1915",
    abstract = "Calibration is a popular framework to evaluate whether a classifier knows when it does not know - i.e., its predictive probabilities are a good indication of how likely a prediction is to be correct. Correctness is commonly estimated against the human majority class. Recently, calibration to human majority has been measured on tasks where humans inherently disagree about which class applies. We show that measuring calibration to human majority given inherent disagreements is theoretically problematic, demonstrate this empirically on the ChaosNLI dataset, and derive several instance-level measures of calibration that capture key statistical properties of human judgements - including class frequency, ranking and entropy.",
}

@inproceedings{giulianelli-2022-emnlp,
    title = "Towards Pragmatic Production Strategies for Natural Language Generation Tasks",
    author = "Giulianelli, Mario",
    booktitle = "Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing",
    year = "2022",
    address = "Abu Dhabi, United Arab Emirates",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.emnlp-main.544",
    pages = "7978--7984",
    abstract = "This position paper proposes a conceptual framework for the design of Natural Language Generation (NLG) systems that follow efficient and effective production strategies in order to achieve complex communicative goals. In this general framework, efficiency is characterised as the parsimonious regulation of production and comprehension costs while effectiveness is measured with respect to task-oriented and contextually grounded communicative goals. We provide concrete suggestions for the estimation of goals, costs, and utility via modern statistical methods, demonstrating applications of our framework to the classic pragmatic task of visually grounded referential games and to abstractive text summarisation, two popular generation tasks with real-world applications. In sum, we advocate for the development of NLG systems that learn to make pragmatic production decisions from experience, by reasoning about goals, costs, and utility in a human-like way.",
}


@inproceedings{giulianelli-etal-2022-construction,
    title = "Construction Repetition Reduces Information Rate in Dialogue",
    author = "Giulianelli, Mario  and
      Sinclair, Arabella  and
      Fern{\'a}ndez, Raquel",
    booktitle = "Proceedings of the 2nd Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics and the 12th International Joint Conference on Natural Language Processing",
    month = nov,
    year = "2022",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.aacl-main.51",
    pages = "665--682",
    abstract = "Speakers repeat constructions frequently in dialogue. Due to their peculiar information-theoretic properties, repetitions can be thought of as a strategy for cost-effective communication. In this study, we focus on the repetition of lexicalised constructions{---}i.e., recurring multi-word units{---}in English open-domain spoken dialogues. We hypothesise that speakers use construction repetition to mitigate information rate, leading to an overall decrease in utterance information content over the course of a dialogue. We conduct a quantitative analysis, measuring the information content of constructions and that of their containing utterances, estimating information content with an adaptive neural language model. We observe that construction usage lowers the information content of utterances. This facilitating effect (i) increases throughout dialogues, (ii) is boosted by repetition, (iii) grows as a function of repetition frequency and density, and (iv) is stronger for repetitions of referential constructions.",
}

@article{sinclair-fernandez-system-2022,
title = {Alignment of code switching varies with proficiency in second language learning dialogue},
journal = {System. Special Issue on Linguistic alignment in Second Language Acquisition: occurrences, learning effects, and beyond},
year = {2022},
issn = {0346-251X},
publisher = {Elsevier},
doi = {https://doi.org/10.1016/j.system.2022.102952},
url = {https://www.sciencedirect.com/science/article/pii/S0346251X22002342},
author = {Arabella J. Sinclair and Raquel Fern\'andez},
abstract = {Speakers in dialogue tend to adopt the language patterns of the other, aligning their language to their interlocutor. This can happen at many levels of communication, including the tendency to code switch (CS), or change to another language. Alignment has often been considered the result of an unconscious automatic process that facilitates speakers' mutual understanding. In dialogues with a second language (L2) learner, alignment is constrained by the proficiency of the learner, and additional non-automatic processes will be at play, namely the individual pedagogical goals of learner and tutor. In this study, we investigate alignment in dialogues between Spanish/Catalan learners of English and their tutors. We analyse CS incidence, whether code switching can be explained as automatic alignment between speakers, and whether this is independent of other, non-automatic factors related to speakers’ goals. We find that alignment of code switching is present, varies with learner proficiency, and that code switching can additionally be triggered by lexical overlap and turn taking asymmetry, which we attribute to conscious pedagogical choices on the part of both tutor, at lower levels, and learner, at higher levels of student proficiency.}
}

@Article{sinclair-etal-2022-tacl,
  author = 	 "Arabella Sinclair and Jaap Jumelet and Willem Zuidema and Raquel Fern\'andez",
  title = 	 "Structural Persistence in Language Models: Priming as a Window into Abstract Language Representations",
  journal = 	 "Transactions of the Association for Computational Linguistics (TACL)",
  year = "2022",
  url = "https://direct.mit.edu/tacl/article/doi/10.1162/tacl_a_00504/113019/Structural-Persistence-in-Language-Models-Priming",
  abstract = "We investigate the extent to which modern, neural language models are susceptible to structural priming,
  the phenomenon whereby the structure of a sentence makes the same structure more probable in a follow-up  sentence.
  We explore how priming can be used to study the potential of these models to learn abstract structural information,
  which is a prerequisite for good performance on tasks that require natural language understanding skills. We introduce
  a novel metric and release PRIME-LM, a large corpus where we control for various linguistic factors which interact with priming strength.
  We find that Transformer models indeed show evidence of structural priming, but also that the generalisations they learned are to
  some extent modulated by semantic information. Our experiments also show that the representations acquired by the models
  may not only encode abstract sequential structure but involve certain level of hierarchical syntactic information.
  More generally, our study shows that the priming paradigm is a useful, additional tool for gaining insights into the capacities
  of language models and opens the door to future priming-based investigations that probe the model's internal states."
}

@inproceedings{ryb-etal-2022-analog,
    title = "{A}na{L}og: Testing Analytical and Deductive Logic Learnability in Language Models",
    author = "Ryb, Samuel  and
      Giulianelli, Mario  and
      Sinclair, Arabella  and
      Fern{\'a}ndez, Raquel",
    booktitle = "Proceedings of the 11th Joint Conference on Lexical and Computational Semantics",
    month = jul,
    year = "2022",
    address = "Seattle, Washington",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.starsem-1.5",
    pages = "55--68",
    abstract = "We investigate the extent to which pre-trained language models acquire analytical and deductive logical reasoning
    capabilities as a side effect of learning word prediction. We present AnaLog, a natural language inference task designed to
    probe models for these capabilities, controlling for different invalid heuristics the models may adopt instead of learning the
    desired generalisations. We test four languagemodels on AnaLog, finding that they have all learned, to a different extent,
    to encode information that is predictive of entailment beyond shallow heuristics such as lexical overlap and grammaticality.
    We closely analyse the best performing language model and show that while it performs more consistently than other language
    models across logical connectives and reasoning domains, it still is sensitive to lexical and syntactic variations in the
    realisation of logical statements."
}

@inproceedings{takmaz-etal-2022-cmcl,
    title = "Less Descriptive yet Discriminative: Quantifying the Properties of Multimodal Referring Utterances via {CLIP}",
    author = "Takmaz, Ece  and Pezzelle, Sandro  and Fern{\'a}ndez, Raquel",
    booktitle = "Proceedings of the Workshop on Cognitive Modeling and Computational Linguistics",
    month = may,
    year = "2022",
    address = "Dublin, Ireland",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.cmcl-1.4",
    pages = "36--42",
    abstract = "In this work, we use a transformer-based pre-trained multimodal model, CLIP,
    to shed light on the mechanisms employed by human speakers when referring to visual entities.
    In particular, we use CLIP to quantify the degree of descriptiveness (how well an utterance
    describes an image in isolation) and discriminativeness (to what extent an utterance is effective
    in picking out a single image among similar images) of human referring utterances within multimodal
    dialogues. Overall, our results show that utterances become less descriptive over time while their
    discriminativeness remains unchanged. Through analysis, we propose that this trend could be due to
    participants relying on the previous mentions in the dialogue history, as well as being able to
    distill the most discriminative information from the visual context. In general, our study opens
    up the possibility of using this and similar models to quantify patterns in human data and shed
    light on the underlying cognitive mechanisms."
}

@inproceedings{takmaz-2022-team,
    title = "Team {DMG} at {CMCL} 2022 Shared Task: Transformer Adapters for the Multi- and Cross-Lingual Prediction of Human Reading Behavior",
    author = "Takmaz, Ece",
    booktitle = "Proceedings of the Workshop on Cognitive Modeling and Computational Linguistics",
    month = may,
    year = "2022",
    bibbase_note = {Best Shared Task Paper Award},
    address = "Dublin, Ireland",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.cmcl-1.16",
    pages = "136--144",
    abstract = "In this paper, we present the details of our approaches that attained the second place
    in the shared task of the ACL 2022 Cognitive Modeling and Computational Linguistics Workshop.
    The shared task is focused on multi- and cross-lingual prediction of eye movement features in
    human reading behavior, which could provide valuable information regarding language processing.
    To this end, we train {`}adapters{'} inserted into the layers of frozen transformer-based pretrained
    language models. We find that multilingual models equipped with adapters perform well in predicting
    eye-tracking features. Our results suggest that utilizing language- and task-specific adapters is
    beneficial and translating test sets into similar languages that exist in the training set could
    help with zero-shot transferability in the prediction of human reading behavior."
}

@inproceedings{giulianelli-etal-2022-fire,
    title = "Do Not Fire the Linguist: Grammatical Profiles Help Language Models Detect Semantic Change",
    author = "Giulianelli, Mario  and Kutuzov, Andrey  and Pivovarova, Lidia",
    booktitle = "Proceedings of the 3rd Workshop on Computational Approaches to Historical Language Change",
    month = may,
    year = "2022",
    address = "Dublin, Ireland",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.lchange-1.6",
    pages = "54--67",
    abstract = "Morphological and syntactic changes in word usage {---} as captured, e.g., by
    grammatical profiles {---} have been shown to be good predictors of a word{'}s meaning change.
    In this work, we explore whether large pre-trained contextualised language models, a common tool
    for lexical semantic change detection, are sensitive to such morphosyntactic changes. To this end,
    we first compare the performance of grammatical profiles against that of a multilingual neural
    language model (XLM-R) on 10 datasets, covering 7 languages, and then combine the two approaches
    in ensembles to assess their complementarity. Our results show that ensembling grammatical
    profiles with XLM-R improves semantic change detection performance for most datasets and languages.
    This indicates that language models do not fully cover the fine-grained morphological and syntactic
    signals that are explicitly represented in grammatical profiles. An interesting exception are the
    test sets where the time spans under analysis are much longer than the time gap between them
    (for example, century-long spans with a one-year gap between them). Morphosyntactic change is
    slow so grammatical profiles do not detect in such cases. In contrast, language models, thanks
    to their access to lexical information, are able to detect fast topical changes."
}


@Article{pezzelle-etal-2021-tacl,
  author = 	 "Sandro Pezzelle and Ece Takmaz and Raquel Fern\'andez",
  title = 	 "Word Representation Learning in Multimodal Pre-Trained Transformers: An Intrinsic Evaluation",
  journal = 	 "Transactions of the Association for Computational Linguistics (TACL)",
  year = "2021",
  url = "https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00443/1979754/tacl_a_00443.pdf",
  url_github = "https://github.com/sandropezzelle/multimodal-evaluation",
  abstract = "This study carries out a systematic /intrinsic/ evaluation of the semantic representations
	learned by state-of-the-art pre-trained multimodal Transformers.
	These representations are claimed to be task-agnostic and shown to help on many downstream language-and-vision tasks.
	However, the extent to which they align with human semantic intuitions remains unclear.
	We experiment with various models and obtain /static/ word representations from the /contextualized/ ones they learn.
	We then evaluate them against the semantic judgements provided by human speakers. In line with previous evidence,
	we observe a generalized advantage of multimodal representations over language-only ones on concrete word pairs,
  but not on abstract ones. On the one hand, this confirms the effectiveness of these models to
	align language and vision, which results in better semantic representations for concepts that are /grounded/ in images.
	On the other hand, models are shown to follow different representation learning patterns, which
	sheds some light on /how/ and /when/ they perform multimodal integration."
}


@inproceedings{jansen-etal-2021-detecting,
    title = "Detecting Age-Related Linguistic Patterns in Dialogue: Toward Adaptive Conversational Systems",
    author = "Jansen, Lennert and Sinclair, Arabella and van der Goot, Margot J. and Fern{\'a}ndez, Raquel and Pezzelle, Sandro",
    booktitle = "Proceedings of the Eighth Italian Conference on Computational Linguistics (CLiC-it)",
    year = "2021",
    url = "http://ceur-ws.org/Vol-3033/paper47.pdf",
    url_github = "https://github.com/ lennertjansen/detecting-age-in-dialogue",
    abstract = "This work explores an important dimension of variation in the language used by dialogue
    participants: their age. While previous work showed differences at various linguistic levels between
    age groups when experimenting with written discourse data (e.g., blog posts), previous work on dialogue
    has largely been limited to acoustic information related to voice and prosody. Detecting fine-grained
    linguistic properties of human dialogues is of crucial importance for developing AI- based conversational
    systems which are able to adapt to their human interlocutors. We therefore investigate whether, and to
    what extent, current text-based NLP models can detect such linguistic differences, and what the features
    driving their predictions are. We show that models achieve a fairly good performance on age- group prediction,
    though the task appears to be more challenging compared to discourse. Through in-depth analysis of the
    best models’ errors and the most predictive cues, we show that, in dialogue, differences among age groups
    mostly concern stylistic and lexical choices. We believe these findings can inform future work on developing
    controlled generation models for adaptive conversational systems."
}

@inproceedings{giulianelli-etal-2021-information,
    title = "Is Information Density Uniform in Task-Oriented Dialogues?",
    author = "Giulianelli, Mario and Sinclair, Arabella and Fern{\'a}ndez, Raquel",
    booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
    year = "2021",
    publisher = "Association for Computational Linguistics.",
    url = "https://aclanthology.org/2021.emnlp-main.652/",
    url_github = "https://github.com/dmg-illc/uid-dialogue",
    abstract = "The Uniform Information Density principle states that speakers plan their utterances to reduce fluctuations
    in the density of the information transmitted. In this paper, we test whether, and within which contextual units this
    principle holds in task-oriented dialogues. We show that there is evidence supporting the principle in written dialogues
    where participants play a cooperative reference game as well as in spoken dialogues involving instruction giving and following.
    Our study underlines the importance of identifying the relevant contextual components, showing that information content
    increases particularly within topically and referentially related contextual units."
}

@inproceedings{giulianelli-fernandez-2021-analysing,
    title = "Analysing Human Strategies of Information Transmission as a Function of Discourse Context",
    author = "Giulianelli, Mario and Fern{\'a}ndez, Raquel",
    booktitle = "Proceedings of the 25th Conference on Computational Natural Language Learning (CoNLL)",
    year = "2021",
    publisher = "Association for Computational Linguistics.",
    url = "https://aclanthology.org/2021.conll-1.50/",
    url_github = "https://github.com/dmg-illc/uid-dialogue",
    abstract = "Speakers are thought to use rational information transmission strategies for efficient communication.
    Previous work analysing these strategies in sentence production has failed to take into account how the information
    content of sentences varies as a function of the available discourse context.
    In this study, we estimate sentence information content within discourse context.
    We find that speakers transmit information at a stable rate---i.e., rationally---in English newspaper articles
    but that this rate decreases in spoken open domain and written task-oriented dialogues.
    We also observe that speakers' choices are not oriented towards local uniformity of information,
    which is another hypothesised rational strategy. We suggest that a more faithful model of communication
    should explicitly include production costs and goal-oriented rewards."
}

@inproceedings{sinclair-fernandez-2021-construction,
    title = "Construction coordination in first and second language acquisition",
    author = "Sinclair, Arabella  and Fern{\'a}ndez, Raquel",
    booktitle = "Proceedings of the 25th Workshop on the Semantics and Pragmatics of Dialogue",
    year = "2021",
    address = "Potsdam, Germany",
    publisher = "SEMDIAL",
    url = "http://semdial.org/anthology/Z21-Sinclair_semdial_0018.pdf",
    abstract = "Repetition of linguistic forms is a pervasive coordination mechanism in interactive language use.
    In this paper, we investigate patterns of cross-participant repetition in dialogues where participants have
    different levels of linguistic ability. Achieving a better understanding of these patterns can not only shed
    light on how humans coordinate in conversation, but may also contribute to developing more natural and effective
    dialogue agents in education contexts related to language learning. Our approach is novel in several respects:
    We focus on multi-word constructions at the lexical and morphosyntactic level, consider both first and second
    acquisition dialogue, and contrast these setups with adult native conversation. The results of our study show
    that language acquisition scenarios are characterised by richer inventories of shared constructions but lower
    usage rates than fluent adult dialogues, and that shared construction use evolves as the linguistic ability
    of the learners increases, arguably leading to a process of routinisation."
}

@inproceedings{giulianelli-etal-2021-profiling,
    title = "Grammatical Profiling for Semantic Change Detection",
    author = "Giulianelli, Mario and Kutuzov, Andrey and Pivovarova, Lidia",
    booktitle = "Proceedings of the 25th Conference on Computational Natural Language Learning (CoNLL)",
    year = "2021",
    publisher = "Association for Computational Linguistics.",
    url = "https://aclanthology.org/2021.conll-1.33/",
    url_GitHub = "https://github.com/glnmario/semchange-profiling",
    abstract = "Semantics, morphology and syntax are strongly interdependent. However, the majority of computational methods for semantic change detection use distributional word representations which encode mostly semantics. We investigate an alternative method, grammatical profiling, based entirely on changes in the morphosyntactic behaviour of words. We demonstrate that it can be used for semantic change detection and even outperforms some distributional semantic methods. We present an in-depth qualitative and quantitative analysis of the predictions made by our grammatical profiling system, showing that they are plausible and interpretable."
}

@inproceedings{parfenova-etal-2021-probing,
    title = "Probing Cross-Modal Representations in Multi-Step Relational Reasoning",
    author = "Parfenova, Iuliia  and
      Elliott, Desmond  and
      Fern{\'a}ndez, Raquel  and
      Pezzelle, Sandro",
    booktitle = "Proceedings of the 6th Workshop on Representation Learning for NLP (RepL4NLP-2021)",
    month = aug,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.repl4nlp-1.16.pdf",
    doi = "10.18653/v1/2021.repl4nlp-1.16",
    url_GitHub = "https://github.com/jig-san/multi-step-size-reasoning",
    pages = "152--162",
    abstract = "We investigate the representations learned by vision and language models in tasks that require relational reasoning. Focusing on the problem of assessing the relative size of objects in abstract visual contexts, we analyse both one-step and two-step reasoning. For the latter, we construct a new dataset of three-image scenes and define a task that requires reasoning at the level of the individual images and across images in a scene. We probe the learned model representations using diagnostic classifiers. Our experiments show that pretrained multimodal transformer-based architectures can perform higher-level relational reasoning, and are able to learn representations for novel tasks and data that are very different from what was seen in pretraining.",
}

@inproceedings{noble-etal-2021-semantic,
    title = "Semantic shift in social networks",
    author = "Noble, Bill  and
      Sayeed, Asad  and
      Fern{\'a}ndez, Raquel  and
      Larsson, Staffan",
    booktitle = "Proceedings of *SEM 2021: The Tenth Joint Conference on Lexical and Computational Semantics",
    month = aug,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.starsem-1.3.pdf",
    doi = "10.18653/v1/2021.starsem-1.3",
    url_GitHub = "https://github.com/GU-CLASP/semantic-shift-in-social-networks",
    pages = "26--37",
    abstract = "Just as the meaning of words is tied to the communities in which they are used, so too is semantic change. But how does lexical semantic change manifest differently across different communities? In this work, we investigate the relationship between community structure and semantic change in 45 communities from the social media website Reddit. We use distributional methods to quantify lexical semantic change and induce a social network on communities, based on interactions between members. We explore the relationship between semantic change and the clustering coefficient of a community{'}s social network graph, as well as community size and stability. While none of these factors are found to be significant on their own, we report a significant effect of their three-way interaction. We also report on significant word-level effects of frequency and change in frequency, which replicate previous findings.",
}

@inproceedings{takmaz-etal-2020-refer,
    title = "{R}efer, {R}euse, {R}educe: {G}enerating {S}ubsequent {R}eferences in {V}isual and {C}onversational {C}ontexts",
    author = "Takmaz, Ece  and
      Giulianelli, Mario  and
      Pezzelle, Sandro  and
      Sinclair, Arabella  and
      Fern{\'a}ndez, Raquel",
    booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
    month = nov,
    year = "2020",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2020.emnlp-main.353.pdf",
    url_dataset = "https://dmg-photobook.github.io",
    doi = "10.18653/v1/2020.emnlp-main.353",
    pages = "4350--4368",
    abstract = "Dialogue participants often refer to entities or situations repeatedly within a conversation, which contributes to its cohesiveness. Subsequent references exploit the common ground accumulated by the interlocutors and hence have several interesting properties, namely, they tend to be shorter and reuse expressions that were effective in previous mentions. In this paper, we tackle the generation of first and subsequent references in visually grounded dialogue. We propose a generation model that produces referring utterances grounded in both the visual and the conversational context. To assess the referring effectiveness of its output, we also implement a reference resolution system. Our experiments and analyses show that the model produces better, more effective referring utterances than a model not grounded in the dialogue context, and generates subsequent references that exhibit linguistic patterns akin to humans.",
}

@inproceedings{takmaz-etal-2020-generating,
    title = "{G}enerating {I}mage {D}escriptions via {S}equential {C}ross-{M}odal {A}lignment {G}uided by {H}uman {G}aze",
    author = "Takmaz, Ece  and
      Pezzelle, Sandro  and
      Beinborn, Lisa  and
      Fern{\'a}ndez, Raquel",
    booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
    month = nov,
    year = "2020",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2020.emnlp-main.377.pdf",
    url_GitHub = "https://github.com/dmg-illc/didec-seq-gen",
    doi = "10.18653/v1/2020.emnlp-main.353",
    pages = "4664--4677",
    abstract = "When speakers describe an image, they tend to look at objects before mentioning them. In this paper, we investigate such sequential cross-modal alignment by modelling the image description generation process computationally. We take as our starting point a state-of-the-art image captioning system and develop several model variants that exploit information from human gaze patterns recorded during language production. In particular, we propose the first approach to image description generation where visual processing is modelled sequentially. Our experiments and analyses confirm that better descriptions can be obtained by exploiting gaze-driven attention and shed light on human cognitive processes by comparing different ways of aligning the gaze modality with language production. We find that processing gaze data sequentially leads to descriptions that are better aligned to those produced by speakers, more diverse, and more natural{---}particularly when gaze is encoded with a dedicated recurrent component.",
}


@inproceedings{giulianelli-etal-2020-analysing,
    title = "Analysing Lexical Semantic Change with Contextualised Word Representations",
    author = "Giulianelli, Mario  and
      Del Tredici, Marco  and
      Fern{\'a}ndez, Raquel",
    booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics (ACL)",
    month = jul,
    year = "2020",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2020.acl-main.365.pdf",
    url_Dataset = {https://doi.org/10.5281/zenodo.3773250},
    doi = "10.18653/v1/2020.acl-main.365",
    pages = "3960--3973",
    abstract = "This paper presents the first unsupervised approach to lexical semantic change that makes use of contextualised word representations. We propose a novel method that exploits the BERT neural language model to obtain representations of word usages, clusters these representations into usage types, and measures change along time with three proposed metrics. We create a new evaluation dataset and show that the model representations and the detected semantic shifts are positively correlated with human judgements. Our extensive qualitative analysis demonstrates that our method captures a variety of synchronic and diachronic linguistic phenomena. We expect our work to inspire further research in this direction.",
}

@inproceedings{kutuzov-giulianelli-2020-change,
    title = "{U}i{O}-{U}v{A} at {S}em{E}val-2020 Task 1: Contextualised Embeddings for Lexical Semantic Change Detection",
    author = "Kutuzov, Andrey  and
      Giulianelli, Mario",
    booktitle = "Proceedings of the Fourteenth Workshop on Semantic Evaluation",
    month = dec,
    year = "2020",
    address = "Barcelona (online)",
    publisher = "International Committee for Computational Linguistics",
    url = "https://aclanthology.org/2020.semeval-1.14",
    url_GitHub = "https://github.com/akutuzov/semeval2020",
    pages = "126--134",
    abstract = "We apply contextualised word embeddings to lexical semantic change detection in the SemEval-2020 Shared Task 1. This paper focuses on Subtask 2, ranking words by the degree of their semantic drift over time. We analyse the performance of two contextualising architectures (BERT and ELMo) and three change detection algorithms. We find that the most effective algorithms rely on the cosine similarity between averaged token embeddings and the pairwise distances between token embeddings. They outperform strong baselines by a large margin (in the post-evaluation phase, we have the best Subtask 2 submission for SemEval-2020 Task 1), but interestingly, the choice of a particular algorithm depends on the distribution of gold scores in the test set.",
}


@inproceedings{gualdoni-etal:2020:clicit,
  author={Eleonora Gualdoni and Raffaella Bernardi and Raquel Fern\'andez and Sandro Pezzelle},
  title={Grounded and Ungrounded Referring Expressions in Human Dialogues: Language Mirrors Different Grounding Conditions},
  booktitle={Proceedings of the 7th Italian Conference on Computational Linguistics (CLiC-it)},
  year={2020},
  url={http://ceur-ws.org/Vol-2769/paper_38.pdf}
}