{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/abstractors-transformer-modules-for-symbolic","title":"Abstractors and relational cross-attention: An inductive bias for explicit relational reasoning in Transformers","arxiv_id":"2304.00195","date":"2023-04-01","proceeding":null,"authors":["Awni Altabaa","Taylor Webb","Jonathan Cohen","John Lafferty"],"abstract":"An extension of Transformers is proposed that enables explicit relational reasoning through a novel module called the Abstractor. At the core of the Abstractor is a variant of attention called relational cross-attention. The approach is motivated by an architectural inductive bias for relational learning that disentangles relational information from object-level features. This enables explicit relational reasoning, supporting abstraction and generalization from limited data. The Abstractor is first evaluated on simple discriminative relational tasks and compared to existing relational architectures. Next, the Abstractor is evaluated on purely relational sequence-to-sequence tasks, where dramatic improvements are seen in sample efficiency compared to standard Transformers. Finally, Abstractors are evaluated on a collection of tasks based on mathematical problem solving, where consistent improvements in performance and sample efficiency are observed.","url_abs":"https://arxiv.org/abs/2304.00195v4","url_pdf":"https://arxiv.org/pdf/2304.00195v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"abstractors-transformer-modules-for-symbolic","repo_url":"https://github.com/awni00/abstractor","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"tf","reach":null}],"tasks":[{"task_slug":"inductive-bias","task_name":"Inductive Bias"},{"task_slug":"mathematical-problem-solving","task_name":"Mathematical Problem-Solving"},{"task_slug":"relational-reasoning","task_name":"Relational Reasoning"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2304.00195","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.00195"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/awni00/abstractor","reach":null}],"summary":{"ran":5,"ran_draft_wrong":3,"ran_honours":1,"unverified":3},"by_repo_kind":{"official":{"samples":12,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":12,"samples":[{"code_sha256_prefix":"902fac1c90d0a3a9","entry":"AddPositionalEmbedding","repo":"awni00/abstractor","repo_kind":"official","path":"abstracters.py","file_url":"https://github.com/awni00/abstractor/blob/HEAD/abstracters.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"902fac1c90d0a3a9"}},{"code_sha256_prefix":"b6ee3cfab9e23fe6","entry":"BaseAttention","repo":"awni00/abstractor","repo_kind":"official","path":"abstracters.py","file_url":"https://github.com/awni00/abstractor/blob/HEAD/abstracters.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"b6ee3cfab9e23fe6"}},{"code_sha256_prefix":"afc5f405a2e74f64","entry":"FeedForward","repo":"awni00/abstractor","repo_kind":"official","path":"abstracters.py","file_url":"https://github.com/awni00/abstractor/blob/HEAD/abstracters.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"afc5f405a2e74f64"}},{"code_sha256_prefix":"cd3c52764d10eb2a","entry":"GlobalSelfAttention","repo":"awni00/abstractor","repo_kind":"official","path":"abstracters.py","file_url":"https://github.com/awni00/abstractor/blob/HEAD/abstracters.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"cd3c52764d10eb2a"}},{"code_sha256_prefix":"2159f6374bdb0366","entry":"MultiHeadAttention","repo":"awni00/abstractor","repo_kind":"official","path":"abstracters.py","file_url":"https://github.com/awni00/abstractor/blob/HEAD/abstracters.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"2159f6374bdb0366"}},{"code_sha256_prefix":"ac594e861d76d88b","entry":"_build_attention_equation","repo":"awni00/abstractor","repo_kind":"official","path":"abstracters.py","file_url":"https://github.com/awni00/abstractor/blob/HEAD/abstracters.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"ac594e861d76d88b"}},{"code_sha256_prefix":"a19c79a5dd4448d8","entry":"_build_proj_equation","repo":"awni00/abstractor","repo_kind":"official","path":"abstracters.py","file_url":"https://github.com/awni00/abstractor/blob/HEAD/abstracters.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"a19c79a5dd4448d8"}},{"code_sha256_prefix":"8dfc0cd5c0042e8f","entry":"_get_output_shape","repo":"awni00/abstractor","repo_kind":"official","path":"abstracters.py","file_url":"https://github.com/awni00/abstractor/blob/HEAD/abstracters.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"8dfc0cd5c0042e8f"}},{"code_sha256_prefix":"e4da176e568a85ee","entry":"create_positional_encoding","repo":"awni00/abstractor","repo_kind":"official","path":"abstracters.py","file_url":"https://github.com/awni00/abstractor/blob/HEAD/abstracters.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"e4da176e568a85ee"}},{"code_sha256_prefix":"dc606b9fae039cdc","entry":"RelationalAbstracter","repo":"awni00/abstractor","repo_kind":"official","path":"abstracters.py","file_url":"https://github.com/awni00/abstractor/blob/HEAD/abstracters.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"dc606b9fae039cdc"}},{"code_sha256_prefix":"bcb5267bdfd0bada","entry":"RelationalAbstracterLayer","repo":"awni00/abstractor","repo_kind":"official","path":"abstracters.py","file_url":"https://github.com/awni00/abstractor/blob/HEAD/abstracters.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"bcb5267bdfd0bada"}},{"code_sha256_prefix":"ab85963b77d97a76","entry":"RelationalAttention","repo":"awni00/abstractor","repo_kind":"official","path":"abstracters.py","file_url":"https://github.com/awni00/abstractor/blob/HEAD/abstracters.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"ab85963b77d97a76"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}