{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/systematic-generalization-on-gscan-what-is","title":"Systematic Generalization on gSCAN: What is Nearly Solved and What is Next?","arxiv_id":"2109.12243","date":"2021-09-25","proceeding":"EMNLP 2021 11","authors":["Linlu Qiu","Hexiang Hu","BoWen Zhang","Peter Shaw","Fei Sha"],"abstract":"We analyze the grounded SCAN (gSCAN) benchmark, which was recently proposed to study systematic generalization for grounded language understanding. First, we study which aspects of the original benchmark can be solved by commonly used methods in multi-modal research. We find that a general-purpose Transformer-based model with cross-modal attention achieves strong performance on a majority of the gSCAN splits, surprisingly outperforming more specialized approaches from prior work. Furthermore, our analysis suggests that many of the remaining errors reveal the same fundamental challenge in systematic generalization of linguistic constructs regardless of visual context. Second, inspired by this finding, we propose challenging new tasks for gSCAN by generating data to incorporate relations between objects in the visual environment. Finally, we find that current models are surprisingly data inefficient given the narrow scope of commands in gSCAN, suggesting another challenge for future work.","url_abs":"https://arxiv.org/abs/2109.12243v1","url_pdf":"https://arxiv.org/pdf/2109.12243v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"systematic-generalization-on-gscan-what-is","repo_url":"https://github.com/LauraRuis/groundedSCAN","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"systematic-generalization-on-gscan-what-is","repo_url":"https://github.com/ankursikarwar/grounded-compositional-generalization","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"compositional-generalization-avg","task_name":"Compositional Generalization (AVG)"},{"task_slug":"systematic-generalization","task_name":"Systematic Generalization"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2109.12243","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.12243"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ankursikarwar/grounded-compositional-generalization","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/LauraRuis/groundedSCAN","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/google-research/language","reach":null}],"summary":{"ran":1,"unverified":6},"by_repo_kind":{"official":{"samples":7,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"beb7d468d5185af5","entry":"reject_next_to","repo":"LauraRuis/groundedSCAN","repo_kind":"official","path":"GroundedScan/gym_minigrid/roomgrid.py","file_url":"https://github.com/LauraRuis/groundedSCAN/blob/HEAD/GroundedScan/gym_minigrid/roomgrid.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"beb7d468d5185af5"}},{"code_sha256_prefix":"a3376b1d09612ac6","entry":"accept_weights","repo":"LauraRuis/groundedSCAN","repo_kind":"official","path":"GroundedScan/helpers.py","file_url":"https://github.com/LauraRuis/groundedSCAN/blob/HEAD/GroundedScan/helpers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a3376b1d09612ac6"}},{"code_sha256_prefix":"9b7e9319720f8075","entry":"object_to_repr","repo":"LauraRuis/groundedSCAN","repo_kind":"official","path":"GroundedScan/world.py","file_url":"https://github.com/LauraRuis/groundedSCAN/blob/HEAD/GroundedScan/world.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9b7e9319720f8075"}},{"code_sha256_prefix":"d4958fde313aabbd","entry":"position_to_repr","repo":"LauraRuis/groundedSCAN","repo_kind":"official","path":"GroundedScan/world.py","file_url":"https://github.com/LauraRuis/groundedSCAN/blob/HEAD/GroundedScan/world.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d4958fde313aabbd"}},{"code_sha256_prefix":"81990c62bb44b1ad","entry":"positioned_object_to_repr","repo":"LauraRuis/groundedSCAN","repo_kind":"official","path":"GroundedScan/world.py","file_url":"https://github.com/LauraRuis/groundedSCAN/blob/HEAD/GroundedScan/world.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"81990c62bb44b1ad"}},{"code_sha256_prefix":"664487d895a0b6f8","entry":"random_weights","repo":"LauraRuis/groundedSCAN","repo_kind":"official","path":"GroundedScan/helpers.py","file_url":"https://github.com/LauraRuis/groundedSCAN/blob/HEAD/GroundedScan/helpers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"664487d895a0b6f8"}},{"code_sha256_prefix":"b5738fb945e99170","entry":"topo_sort","repo":"LauraRuis/groundedSCAN","repo_kind":"official","path":"GroundedScan/helpers.py","file_url":"https://github.com/LauraRuis/groundedSCAN/blob/HEAD/GroundedScan/helpers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b5738fb945e99170"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}