{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/lexical-generalization-improves-with-larger","title":"Lexical Generalization Improves with Larger Models and Longer Training","arxiv_id":"2210.12673","date":"2022-10-23","proceeding":null,"authors":["Elron Bandel","Yoav Goldberg","Yanai Elazar"],"abstract":"While fine-tuned language models perform well on many tasks, they were also shown to rely on superficial surface features such as lexical overlap. Excessive utilization of such heuristics can lead to failure on challenging inputs. We analyze the use of lexical overlap heuristics in natural language inference, paraphrase detection, and reading comprehension (using a novel contrastive dataset), and find that larger models are much less susceptible to adopting lexical overlap heuristics. We also find that longer training leads models to abandon lexical overlap heuristics. Finally, we provide evidence that the disparity between models size has its source in the pre-trained model","url_abs":"https://arxiv.org/abs/2210.12673v2","url_pdf":"https://arxiv.org/pdf/2210.12673v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"lexical-generalization-improves-with-larger","repo_url":"https://github.com/elronbandel/lexical-generalization","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"natural-language-inference","task_name":"Natural Language Inference"},{"task_slug":"reading-comprehension","task_name":"Reading Comprehension"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2210.12673","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.12673"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/elronbandel/lexical-generalization","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":3},"by_repo_kind":{"official":{"samples":3,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"d99b197c170d6224","entry":"evaluate_answerability_classification","repo":"elronbandel/lexical-generalization","repo_kind":"official","path":"evaluate.py","file_url":"https://github.com/elronbandel/lexical-generalization/blob/HEAD/evaluate.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d99b197c170d6224"}},{"code_sha256_prefix":"54843112ca346919","entry":"evaluate_answerability_squad","repo":"elronbandel/lexical-generalization","repo_kind":"official","path":"evaluate.py","file_url":"https://github.com/elronbandel/lexical-generalization/blob/HEAD/evaluate.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"54843112ca346919"}},{"code_sha256_prefix":"e04f4588491e3ec1","entry":"evaluate_entailment_classification","repo":"elronbandel/lexical-generalization","repo_kind":"official","path":"evaluate.py","file_url":"https://github.com/elronbandel/lexical-generalization/blob/HEAD/evaluate.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e04f4588491e3ec1"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}