{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/attribute-diversity-determines-the","title":"Attribute Diversity Determines the Systematicity Gap in VQA","arxiv_id":"2311.08695","date":"2023-11-15","proceeding":null,"authors":["Ian Berlot-Attwell","Kumar Krishna Agrawal","A. Michael Carrell","Yash Sharma","Naomi Saphra"],"abstract":"Although modern neural networks often generalize to new combinations of familiar concepts, the conditions that enable such compositionality have long been an open question. In this work, we study the systematicity gap in visual question answering: the performance difference between reasoning on previously seen and unseen combinations of object attributes. To test, we introduce a novel diagnostic dataset, CLEVR-HOPE. We find that the systematicity gap is not reduced by increasing the quantity of training data, but is reduced by increasing the diversity of training data. In particular, our experiments suggest that the more distinct attribute type combinations are seen during training, the more systematic we can expect the resulting model to be.","url_abs":"https://arxiv.org/abs/2311.08695v3","url_pdf":"https://arxiv.org/pdf/2311.08695v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"attribute-diversity-determines-the","repo_url":"https://github.com/ikb-a/systematicity-gap-in-vqa","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok"}}],"tasks":[{"task_slug":"attribute","task_name":"Attribute"},{"task_slug":"diagnostic","task_name":"Diagnostic"},{"task_slug":"diversity","task_name":"Diversity"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2311.08695","atlas_url":"https://app.syntology.ai/?focus=2311.08695","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.08695"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ikb-a/systematicity-gap-in-vqa","reach":{"status":"ok"}}],"summary":{"ran":6},"by_repo_kind":{"official":{"samples":6,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":6,"samples":[{"code_sha256_prefix":"f2e53fda32893cde","entry":"add_ee_args","repo":"ikb-a/systematicity-gap-in-vqa","repo_kind":"official","path":"vqa-framework/src/vqa_framework/models/symbolic_exec.py","file_url":"https://github.com/ikb-a/systematicity-gap-in-vqa/blob/HEAD/vqa-framework/src/vqa_framework/models/symbolic_exec.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f2e53fda32893cde"}},{"code_sha256_prefix":"8dcc2074cc1eac64","entry":"calculate_acc_from_logits","repo":"ikb-a/systematicity-gap-in-vqa","repo_kind":"official","path":"vqa-framework/src/vqa_framework/utils/misc.py","file_url":"https://github.com/ikb-a/systematicity-gap-in-vqa/blob/HEAD/vqa-framework/src/vqa_framework/utils/misc.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8dcc2074cc1eac64"}},{"code_sha256_prefix":"463a7f698850cba0","entry":"clevr_collate","repo":"ikb-a/systematicity-gap-in-vqa","repo_kind":"official","path":"vqa-framework/src/vqa_framework/vr/data.py","file_url":"https://github.com/ikb-a/systematicity-gap-in-vqa/blob/HEAD/vqa-framework/src/vqa_framework/vr/data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"463a7f698850cba0"}},{"code_sha256_prefix":"e9407d5b3ae1763a","entry":"load_scenes","repo":"ikb-a/systematicity-gap-in-vqa","repo_kind":"official","path":"vqa-framework/src/vqa_framework/vr/data.py","file_url":"https://github.com/ikb-a/systematicity-gap-in-vqa/blob/HEAD/vqa-framework/src/vqa_framework/vr/data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e9407d5b3ae1763a"}},{"code_sha256_prefix":"1d697b119d4b1e3f","entry":"path_to_h5_filename","repo":"ikb-a/systematicity-gap-in-vqa","repo_kind":"official","path":"vqa-framework/src/vqa_framework/data_modules/generic_clevr_loader.py","file_url":"https://github.com/ikb-a/systematicity-gap-in-vqa/blob/HEAD/vqa-framework/src/vqa_framework/data_modules/generic_clevr_loader.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1d697b119d4b1e3f"}},{"code_sha256_prefix":"039a47d6cbd99391","entry":"strtobool","repo":"ikb-a/systematicity-gap-in-vqa","repo_kind":"official","path":"vqa-framework/src/vqa_framework/utils/misc.py","file_url":"https://github.com/ikb-a/systematicity-gap-in-vqa/blob/HEAD/vqa-framework/src/vqa_framework/utils/misc.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"039a47d6cbd99391"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}