{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/right-for-the-wrong-reasons-diagnosing","title":"Right for the Wrong Reasons: Diagnosing Syntactic Heuristics in Natural Language Inference","arxiv_id":"1902.01007","date":"2019-02-04","proceeding":"ACL 2019 7","authors":["R. Thomas McCoy","Ellie Pavlick","Tal Linzen"],"abstract":"A machine learning system can score well on a given test set by relying on heuristics that are effective for frequent example types but break down in more challenging cases. We study this issue within natural language inference (NLI), the task of determining whether one sentence entails another. We hypothesize that statistical NLI models may adopt three fallible syntactic heuristics: the lexical overlap heuristic, the subsequence heuristic, and the constituent heuristic. To determine whether models have adopted these heuristics, we introduce a controlled evaluation set called HANS (Heuristic Analysis for NLI Systems), which contains many examples where the heuristics fail. We find that models trained on MNLI, including BERT, a state-of-the-art model, perform very poorly on HANS, suggesting that they have indeed adopted these heuristics. We conclude that there is substantial room for improvement in NLI systems, and that the HANS dataset can motivate and measure progress in this area","url_abs":"https://arxiv.org/abs/1902.01007v4","url_pdf":"https://arxiv.org/pdf/1902.01007v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"right-for-the-wrong-reasons-diagnosing","repo_url":"https://github.com/tommccoy1/hans","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":null},{"paper_slug":"right-for-the-wrong-reasons-diagnosing","repo_url":"https://github.com/UKPLab/acl2020-confidence-regularization","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"right-for-the-wrong-reasons-diagnosing","repo_url":"https://github.com/UKPLab/emnlp2020-debiasing-unknown","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"right-for-the-wrong-reasons-diagnosing","repo_url":"https://github.com/chrisc36/debias","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"right-for-the-wrong-reasons-diagnosing","repo_url":"https://github.com/yg211/bert_nli","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"right-for-the-wrong-reasons-diagnosing","repo_url":"https://github.com/yyhappier/shortcutsuite","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}}],"tasks":[{"task_slug":"natural-language-inference","task_name":"Natural Language Inference"},{"task_slug":"sentence","task_name":"Sentence"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1902.01007","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1902.01007"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yg211/bert_nli","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/tommccoy1/hans","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/UKPLab/acl2020-confidence-regularization","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/UKPLab/emnlp2020-debiasing-unknown","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yyhappier/shortcutsuite","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/chrisc36/debias","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_draft_wrong":1},"by_repo_kind":{"official":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"36f026ae6824d388","entry":"format_label","repo":"tommccoy1/hans","repo_kind":"official","path":"evaluate_heur_output.py","file_url":"https://github.com/tommccoy1/hans/blob/HEAD/evaluate_heur_output.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"36f026ae6824d388"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}