{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/on-provable-length-and-compositional","title":"On Provable Length and Compositional Generalization","arxiv_id":"2402.04875","date":"2024-02-07","proceeding":null,"authors":["Kartik Ahuja","Amin Mansouri"],"abstract":"Out-of-distribution generalization capabilities of sequence-to-sequence models can be studied from the lens of two crucial forms of generalization: length generalization -- the ability to generalize to longer sequences than ones seen during training, and compositional generalization: the ability to generalize to token combinations not seen during training. In this work, we provide first provable guarantees on length and compositional generalization for common sequence-to-sequence models -- deep sets, transformers, state space models, and recurrent neural nets -- trained to minimize the prediction error. We show that limited capacity versions of these different architectures achieve both length and compositional generalization provided the training distribution is sufficiently diverse. In the first part, we study structured limited capacity variants of different architectures and arrive at the generalization guarantees with limited diversity requirements on the training distribution. In the second part, we study limited capacity variants with less structural assumptions and arrive at generalization guarantees but with more diversity requirements on the training distribution.","url_abs":"https://arxiv.org/abs/2402.04875v5","url_pdf":"https://arxiv.org/pdf/2402.04875v5.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"on-provable-length-and-compositional","repo_url":"https://github.com/facebookresearch/length-and-compositional-generalization","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[{"task_slug":"diversity","task_name":"Diversity"},{"task_slug":"out-of-distribution-generalization","task_name":"Out-of-Distribution Generalization"},{"task_slug":"state-space-models","task_name":"State Space Models"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2402.04875","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.04875"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/length-and-compositional-generalization","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran":4,"ran_draft_wrong":1,"unverified":1},"by_repo_kind":{"official":{"samples":6,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":6,"samples":[{"code_sha256_prefix":"367de505399f49a3","entry":"float_division","repo":"facebookresearch/length-and-compositional-generalization","repo_kind":"official","path":"src/utils/hydra_custom_resolvers.py","file_url":"https://github.com/facebookresearch/length-and-compositional-generalization/blob/HEAD/src/utils/hydra_custom_resolvers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"367de505399f49a3"}},{"code_sha256_prefix":"ca0c9668ad6f95d2","entry":"floor_division","repo":"facebookresearch/length-and-compositional-generalization","repo_kind":"official","path":"src/utils/hydra_custom_resolvers.py","file_url":"https://github.com/facebookresearch/length-and-compositional-generalization/blob/HEAD/src/utils/hydra_custom_resolvers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"ca0c9668ad6f95d2"}},{"code_sha256_prefix":"61139ec62260b411","entry":"get_logger","repo":"facebookresearch/length-and-compositional-generalization","repo_kind":"official","path":"src/utils/general.py","file_url":"https://github.com/facebookresearch/length-and-compositional-generalization/blob/HEAD/src/utils/general.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"61139ec62260b411"}},{"code_sha256_prefix":"2515c24df76ce408","entry":"get_pylogger","repo":"facebookresearch/length-and-compositional-generalization","repo_kind":"official","path":"src/utils/general.py","file_url":"https://github.com/facebookresearch/length-and-compositional-generalization/blob/HEAD/src/utils/general.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"2515c24df76ce408"}},{"code_sha256_prefix":"539cdc39227789a2","entry":"num_files_in_directory","repo":"facebookresearch/length-and-compositional-generalization","repo_kind":"official","path":"src/utils/hydra_custom_resolvers.py","file_url":"https://github.com/facebookresearch/length-and-compositional-generalization/blob/HEAD/src/utils/hydra_custom_resolvers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"539cdc39227789a2"}},{"code_sha256_prefix":"1f8a21421ae215d9","entry":"select_distribution","repo":"facebookresearch/length-and-compositional-generalization","repo_kind":"official","path":"src/utils/datamodule.py","file_url":"https://github.com/facebookresearch/length-and-compositional-generalization/blob/HEAD/src/utils/datamodule.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"1f8a21421ae215d9"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}