{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/valhalla-visual-hallucination-for-machine","title":"VALHALLA: Visual Hallucination for Machine Translation","arxiv_id":"2206.00100","date":"2022-05-31","proceeding":"CVPR 2022 1","authors":["Yi Li","Rameswar Panda","Yoon Kim","Chun-Fu","Chen","Rogerio Feris","David Cox","Nuno Vasconcelos"],"abstract":"Designing better machine translation systems by considering auxiliary inputs such as images has attracted much attention in recent years. While existing methods show promising performance over the conventional text-only translation systems, they typically require paired text and image as input during inference, which limits their applicability to real-world scenarios. In this paper, we introduce a visual hallucination framework, called VALHALLA, which requires only source sentences at inference time and instead uses hallucinated visual representations for multimodal machine translation. In particular, given a source sentence an autoregressive hallucination transformer is used to predict a discrete visual representation from the input text, and the combined text and hallucinated representations are utilized to obtain the target translation. We train the hallucination transformer jointly with the translation transformer using standard backpropagation with cross-entropy losses while being guided by an additional loss that encourages consistency between predictions using either ground-truth or hallucinated visual representations. Extensive experiments on three standard translation datasets with a diverse set of language pairs demonstrate the effectiveness of our approach over both text-only baselines and state-of-the-art methods. Project page: http://www.svcl.ucsd.edu/projects/valhalla.","url_abs":"https://arxiv.org/abs/2206.00100v1","url_pdf":"https://arxiv.org/pdf/2206.00100v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"valhalla-visual-hallucination-for-machine","repo_url":"https://github.com/jerryyli/valhalla-nmt","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"hallucination","task_name":"Hallucination"},{"task_slug":"machine-translation","task_name":"Machine Translation"},{"task_slug":"multimodal-machine-translation","task_name":"Multimodal Machine Translation"},{"task_slug":"sentence","task_name":"Sentence"},{"task_slug":"translation","task_name":"Translation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2206.00100","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.00100"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jerryyli/valhalla-nmt","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":4,"ran_violates":2,"unverified":4},"by_repo_kind":{"listed":{"samples":10,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"fe5dd5258046898c","entry":"always","repo":"jerryyli/valhalla-nmt","repo_kind":"listed","path":"models/hallucinate/dalle.py","file_url":"https://github.com/jerryyli/valhalla-nmt/blob/HEAD/models/hallucinate/dalle.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fe5dd5258046898c"}},{"code_sha256_prefix":"7ae941c6130274cb","entry":"cp_path_to_dir","repo":"jerryyli/valhalla-nmt","repo_kind":"listed","path":"DALLE-pytorch/train_dalle.py","file_url":"https://github.com/jerryyli/valhalla-nmt/blob/HEAD/DALLE-pytorch/train_dalle.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7ae941c6130274cb"}},{"code_sha256_prefix":"60fff7c3c400d7ff","entry":"default","repo":"jerryyli/valhalla-nmt","repo_kind":"listed","path":"models/hallucinate/dalle.py","file_url":"https://github.com/jerryyli/valhalla-nmt/blob/HEAD/models/hallucinate/dalle.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"60fff7c3c400d7ff"}},{"code_sha256_prefix":"aa5486a3650902d8","entry":"exists","repo":"jerryyli/valhalla-nmt","repo_kind":"listed","path":"DALLE-pytorch/train_dalle.py","file_url":"https://github.com/jerryyli/valhalla-nmt/blob/HEAD/DALLE-pytorch/train_dalle.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"aa5486a3650902d8"}},{"code_sha256_prefix":"88ce019380f10478","entry":"get_symbols_to_strip_from_output","repo":"jerryyli/valhalla-nmt","repo_kind":"listed","path":"generate.py","file_url":"https://github.com/jerryyli/valhalla-nmt/blob/HEAD/generate.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"88ce019380f10478"}},{"code_sha256_prefix":"863c2d358b6285ad","entry":"get_trainable_params","repo":"jerryyli/valhalla-nmt","repo_kind":"listed","path":"DALLE-pytorch/train_dalle.py","file_url":"https://github.com/jerryyli/valhalla-nmt/blob/HEAD/DALLE-pytorch/train_dalle.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"863c2d358b6285ad"}},{"code_sha256_prefix":"87478659023a0eda","entry":"cast_tuple","repo":"jerryyli/valhalla-nmt","repo_kind":"listed","path":"models/hallucinate/transformer.py","file_url":"https://github.com/jerryyli/valhalla-nmt/blob/HEAD/models/hallucinate/transformer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"87478659023a0eda"}},{"code_sha256_prefix":"ad7205079f7814cf","entry":"eval_str_list","repo":"jerryyli/valhalla-nmt","repo_kind":"listed","path":"dataclass/utils.py","file_url":"https://github.com/jerryyli/valhalla-nmt/blob/HEAD/dataclass/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ad7205079f7814cf"}},{"code_sha256_prefix":"cd971280708e837c","entry":"interpret_dc_type","repo":"jerryyli/valhalla-nmt","repo_kind":"listed","path":"dataclass/utils.py","file_url":"https://github.com/jerryyli/valhalla-nmt/blob/HEAD/dataclass/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cd971280708e837c"}},{"code_sha256_prefix":"006572a6fdddbaed","entry":"module_name_fordropout","repo":"jerryyli/valhalla-nmt","repo_kind":"listed","path":"models/transformer_discrete_encoder.py","file_url":"https://github.com/jerryyli/valhalla-nmt/blob/HEAD/models/transformer_discrete_encoder.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"006572a6fdddbaed"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}