{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/variance-aware-machine-translation-test-sets","title":"Variance-Aware Machine Translation Test Sets","arxiv_id":"2111.04079","date":"2021-11-07","proceeding":null,"authors":["Runzhe Zhan","Xuebo Liu","Derek F. Wong","Lidia S. Chao"],"abstract":"We release 70 small and discriminative test sets for machine translation (MT) evaluation called variance-aware test sets (VAT), covering 35 translation directions from WMT16 to WMT20 competitions. VAT is automatically created by a novel variance-aware filtering method that filters the indiscriminative test instances of the current MT test sets without any human labor. Experimental results show that VAT outperforms the original WMT test sets in terms of the correlation with human judgement across mainstream language pairs and test sets. Further analysis on the properties of VAT reveals the challenging linguistic features (e.g., translation of low-frequency words and proper nouns) for competitive MT systems, providing guidance for constructing future MT test sets. The test sets and the code for preparing variance-aware MT test sets are freely available at https://github.com/NLP2CT/Variance-Aware-MT-Test-Sets .","url_abs":"https://arxiv.org/abs/2111.04079v1","url_pdf":"https://arxiv.org/pdf/2111.04079v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"variance-aware-machine-translation-test-sets","repo_url":"https://github.com/nlp2ct/variance-aware-mt-test-sets","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[{"task_slug":"machine-translation","task_name":"Machine Translation"},{"task_slug":"translation","task_name":"Translation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2111.04079","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2111.04079"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/NLP2CT/Variance-Aware-MT-Test-Sets","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/nlp2ct/variance-aware-mt-test-sets","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran_fixture":1,"ran_draft_wrong":2,"ran_honours":1},"by_repo_kind":{"official":{"samples":4,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"949889d4e3b475fc","entry":"cal_diff_measure","repo":"NLP2CT/Variance-Aware-MT-Test-Sets","repo_kind":"official","path":"cal_filtering.py","file_url":"https://github.com/NLP2CT/Variance-Aware-MT-Test-Sets/blob/HEAD/cal_filtering.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"949889d4e3b475fc"}},{"code_sha256_prefix":"a45f37443a759856","entry":"gen_score_table","repo":"NLP2CT/Variance-Aware-MT-Test-Sets","repo_kind":"official","path":"score_bert.py","file_url":"https://github.com/NLP2CT/Variance-Aware-MT-Test-Sets/blob/HEAD/score_bert.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"a45f37443a759856"}},{"code_sha256_prefix":"3efd010c538874f3","entry":"read_dump_csv","repo":"NLP2CT/Variance-Aware-MT-Test-Sets","repo_kind":"official","path":"cal_filtering.py","file_url":"https://github.com/NLP2CT/Variance-Aware-MT-Test-Sets/blob/HEAD/cal_filtering.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"3efd010c538874f3"}},{"code_sha256_prefix":"a09938cc9f07abc4","entry":"read_text_score","repo":"NLP2CT/Variance-Aware-MT-Test-Sets","repo_kind":"official","path":"cal_filtering.py","file_url":"https://github.com/NLP2CT/Variance-Aware-MT-Test-Sets/blob/HEAD/cal_filtering.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"a09938cc9f07abc4"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}