{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/understanding-and-detecting-hallucinations-in","title":"Understanding and Detecting Hallucinations in Neural Machine Translation via Model Introspection","arxiv_id":"2301.07779","date":"2023-01-18","proceeding":null,"authors":["Weijia Xu","Sweta Agrawal","Eleftheria Briakou","Marianna J. Martindale","Marine Carpuat"],"abstract":"Neural sequence generation models are known to \"hallucinate\", by producing outputs that are unrelated to the source text. These hallucinations are potentially harmful, yet it remains unclear in what conditions they arise and how to mitigate their impact. In this work, we first identify internal model symptoms of hallucinations by analyzing the relative token contributions to the generation in contrastive hallucinated vs. non-hallucinated outputs generated via source perturbations. We then show that these symptoms are reliable indicators of natural hallucinations, by using them to design a lightweight hallucination detector which outperforms both model-free baselines and strong classifiers based on quality estimation or large pre-trained models on manually annotated English-Chinese and German-English translation test beds.","url_abs":"https://arxiv.org/abs/2301.07779v2","url_pdf":"https://arxiv.org/pdf/2301.07779v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"understanding-and-detecting-hallucinations-in","repo_url":"https://github.com/weijia-xu/hallucinations-in-nmt","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"tf","reach":null}],"tasks":[{"task_slug":"hallucination","task_name":"Hallucination"},{"task_slug":"machine-translation","task_name":"Machine Translation"},{"task_slug":"translation","task_name":"Translation"}],"methods":[{"method_slug":"test","method_name":"Test"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2301.07779","atlas_url":"https://app.syntology.ai/?focus=2301.07779","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2301.07779"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/weijia-xu/hallucinations-in-nmt","reach":null}],"summary":{"ran_violates":1,"ran_draft_wrong":1,"ran_fixture":1},"by_repo_kind":{"official":{"samples":3,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":3,"samples":[{"code_sha256_prefix":"0b01d4b617050453","entry":"compute_similarity","repo":"weijia-xu/hallucinations-in-nmt","repo_kind":"official","path":"code/classifier.py","file_url":"https://github.com/weijia-xu/hallucinations-in-nmt/blob/HEAD/code/classifier.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0b01d4b617050453"}},{"code_sha256_prefix":"b1d7ded7fe1a46f9","entry":"deduplicate","repo":"weijia-xu/hallucinations-in-nmt","repo_kind":"official","path":"code/classifier.py","file_url":"https://github.com/weijia-xu/hallucinations-in-nmt/blob/HEAD/code/classifier.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b1d7ded7fe1a46f9"}},{"code_sha256_prefix":"51f757dfe483d3ae","entry":"sliding_window_similarity","repo":"weijia-xu/hallucinations-in-nmt","repo_kind":"official","path":"code/classifier.py","file_url":"https://github.com/weijia-xu/hallucinations-in-nmt/blob/HEAD/code/classifier.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"51f757dfe483d3ae"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}