{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/if-attention-serves-as-a-cognitive-model-of","title":"If Attention Serves as a Cognitive Model of Human Memory Retrieval, What is the Plausible Memory Representation?","arxiv_id":"2502.11469","date":"2025-02-17","proceeding":null,"authors":["Ryo Yoshida","Shinnosuke Isono","Kohei Kajikawa","Taiga Someya","Yushi Sugimito","Yohei Oseki"],"abstract":"Recent work in computational psycholinguistics has revealed intriguing parallels between attention mechanisms and human memory retrieval, focusing primarily on Transformer architectures that operate on token-level representations. However, computational psycholinguistic research has also established that syntactic structures provide compelling explanations for human sentence processing that word-level factors alone cannot fully account for. In this study, we investigate whether the attention mechanism of Transformer Grammar (TG), which uniquely operates on syntactic structures as representational units, can serve as a cognitive model of human memory retrieval, using Normalized Attention Entropy (NAE) as a linking hypothesis between model behavior and human processing difficulty. Our experiments demonstrate that TG's attention achieves superior predictive power for self-paced reading times compared to vanilla Transformer's, with further analyses revealing independent contributions from both models. These findings suggest that human sentence processing involves dual memory representations -- one based on syntactic structures and another on token sequences -- with attention serving as the general retrieval algorithm, while highlighting the importance of incorporating syntactic structures as representational units.","url_abs":"https://arxiv.org/abs/2502.11469v1","url_pdf":"https://arxiv.org/pdf/2502.11469v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[],"tasks":[{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"sentence","task_name":"Sentence"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2502.11469","atlas_url":"https://app.syntology.ai/?focus=2502.11469","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.11469"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/aistairc/rnng-pytorch","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/osekilab/TG-NAE","reach":{"status":"ok"}}],"summary":{"ran_draft_wrong":1,"ran":4,"ran_violates":1,"unverified":2},"by_repo_kind":{"found_in_text":{"samples":8,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"2347da1339a984a0","entry":"get_between_brackets","repo":"aistairc/rnng-pytorch","repo_kind":"found_in_text","path":"preprocess.py","file_url":"https://github.com/aistairc/rnng-pytorch/blob/HEAD/preprocess.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2347da1339a984a0"}},{"code_sha256_prefix":"d5f368ce59e66adf","entry":"get_in_order_actions","repo":"aistairc/rnng-pytorch","repo_kind":"found_in_text","path":"utils.py","file_url":"https://github.com/aistairc/rnng-pytorch/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d5f368ce59e66adf"}},{"code_sha256_prefix":"150d1c5aa4363bf4","entry":"get_len_from_padded","repo":"aistairc/rnng-pytorch","repo_kind":"found_in_text","path":"non_incremental_lm.py","file_url":"https://github.com/aistairc/rnng-pytorch/blob/HEAD/non_incremental_lm.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"150d1c5aa4363bf4"}},{"code_sha256_prefix":"1198e73fcec21ef0","entry":"get_next_bracket_index","repo":"aistairc/rnng-pytorch","repo_kind":"found_in_text","path":"preprocess.py","file_url":"https://github.com/aistairc/rnng-pytorch/blob/HEAD/preprocess.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1198e73fcec21ef0"}},{"code_sha256_prefix":"625f98c10e41a690","entry":"is_next_open_bracket","repo":"aistairc/rnng-pytorch","repo_kind":"found_in_text","path":"preprocess.py","file_url":"https://github.com/aistairc/rnng-pytorch/blob/HEAD/preprocess.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"625f98c10e41a690"}},{"code_sha256_prefix":"1b066f480d7472db","entry":"load_model","repo":"aistairc/rnng-pytorch","repo_kind":"found_in_text","path":"beam_search.py","file_url":"https://github.com/aistairc/rnng-pytorch/blob/HEAD/beam_search.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1b066f480d7472db"}},{"code_sha256_prefix":"eaef2bc09a44696e","entry":"get_in_order_max_stack_size","repo":"aistairc/rnng-pytorch","repo_kind":"found_in_text","path":"utils.py","file_url":"https://github.com/aistairc/rnng-pytorch/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"eaef2bc09a44696e"}},{"code_sha256_prefix":"32207d9ca8369ad2","entry":"get_top_down_max_stack_size","repo":"aistairc/rnng-pytorch","repo_kind":"found_in_text","path":"utils.py","file_url":"https://github.com/aistairc/rnng-pytorch/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"32207d9ca8369ad2"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}