{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/a-mechanistic-analysis-of-a-transformer","title":"A Mechanistic Analysis of a Transformer Trained on a Symbolic Multi-Step Reasoning Task","arxiv_id":"2402.11917","date":"2024-02-19","proceeding":null,"authors":["Jannik Brinkmann","Abhay Sheshadri","Victor Levoso","Paul Swoboda","Christian Bartelt"],"abstract":"Transformers demonstrate impressive performance on a range of reasoning benchmarks. To evaluate the degree to which these abilities are a result of actual reasoning, existing work has focused on developing sophisticated benchmarks for behavioral studies. However, these studies do not provide insights into the internal mechanisms driving the observed capabilities. To improve our understanding of the internal mechanisms of transformers, we present a comprehensive mechanistic analysis of a transformer trained on a synthetic reasoning task. We identify a set of interpretable mechanisms the model uses to solve the task, and validate our findings using correlational and causal evidence. Our results suggest that it implements a depth-bounded recurrent mechanisms that operates in parallel and stores intermediate results in selected token positions. We anticipate that the motifs we identified in our synthetic setting can provide valuable insights into the broader operating principles of transformers and thus provide a basis for understanding more complex models.","url_abs":"https://arxiv.org/abs/2402.11917v3","url_pdf":"https://arxiv.org/pdf/2402.11917v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"a-mechanistic-analysis-of-a-transformer","repo_url":"https://github.com/abhay-sheshadri/backward-chaining-circuits","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[],"methods":[{"method_slug":"set","method_name":"SET"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2402.11917","atlas_url":"https://app.syntology.ai/?focus=2402.11917","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.11917"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/abhay-sheshadri/backward-chaining-circuits","reach":{"status":"ok"}}],"summary":{"ran":8,"unverified":3},"by_repo_kind":{"official":{"samples":11,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":11,"samples":[{"code_sha256_prefix":"1ed19ad2693bee3e","entry":"extract_adj_matrix","repo":"abhay-sheshadri/backward-chaining-circuits","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/abhay-sheshadri/backward-chaining-circuits/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1ed19ad2693bee3e"}},{"code_sha256_prefix":"1dcef83bda5714f3","entry":"get_example_cache","repo":"abhay-sheshadri/backward-chaining-circuits","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/abhay-sheshadri/backward-chaining-circuits/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1dcef83bda5714f3"}},{"code_sha256_prefix":"acd6a741f6a7a07f","entry":"get_loaders","repo":"abhay-sheshadri/backward-chaining-circuits","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/abhay-sheshadri/backward-chaining-circuits/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"acd6a741f6a7a07f"}},{"code_sha256_prefix":"cd57abb736d9cb6d","entry":"hierarchy_pos","repo":"abhay-sheshadri/backward-chaining-circuits","repo_kind":"official","path":"src/tree_generation/viz.py","file_url":"https://github.com/abhay-sheshadri/backward-chaining-circuits/blob/HEAD/src/tree_generation/viz.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"cd57abb736d9cb6d"}},{"code_sha256_prefix":"0b3394b38c5b85c6","entry":"kl_divergence","repo":"abhay-sheshadri/backward-chaining-circuits","repo_kind":"official","path":"src/attention_knockout.py","file_url":"https://github.com/abhay-sheshadri/backward-chaining-circuits/blob/HEAD/src/attention_knockout.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0b3394b38c5b85c6"}},{"code_sha256_prefix":"dff7b2d24930b654","entry":"sample_tree_graph","repo":"abhay-sheshadri/backward-chaining-circuits","repo_kind":"official","path":"src/tree_generation/gen.py","file_url":"https://github.com/abhay-sheshadri/backward-chaining-circuits/blob/HEAD/src/tree_generation/gen.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"dff7b2d24930b654"}},{"code_sha256_prefix":"896fdf4ea0b858b0","entry":"shortest_path","repo":"abhay-sheshadri/backward-chaining-circuits","repo_kind":"official","path":"src/tree_generation/gen.py","file_url":"https://github.com/abhay-sheshadri/backward-chaining-circuits/blob/HEAD/src/tree_generation/gen.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"896fdf4ea0b858b0"}},{"code_sha256_prefix":"4f1daa704a312a04","entry":"topological_sort_edges","repo":"abhay-sheshadri/backward-chaining-circuits","repo_kind":"official","path":"src/tree_generation/gen.py","file_url":"https://github.com/abhay-sheshadri/backward-chaining-circuits/blob/HEAD/src/tree_generation/gen.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4f1daa704a312a04"}},{"code_sha256_prefix":"867c6b6984a3ebc7","entry":"add_attention_blockout","repo":"abhay-sheshadri/backward-chaining-circuits","repo_kind":"official","path":"src/attention_knockout.py","file_url":"https://github.com/abhay-sheshadri/backward-chaining-circuits/blob/HEAD/src/attention_knockout.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"867c6b6984a3ebc7"}},{"code_sha256_prefix":"b1c6a36e5eed0d39","entry":"add_attention_blockout_parallel","repo":"abhay-sheshadri/backward-chaining-circuits","repo_kind":"official","path":"src/attention_knockout.py","file_url":"https://github.com/abhay-sheshadri/backward-chaining-circuits/blob/HEAD/src/attention_knockout.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b1c6a36e5eed0d39"}},{"code_sha256_prefix":"e6652ed7bc481af1","entry":"logits_to_logit_diff","repo":"abhay-sheshadri/backward-chaining-circuits","repo_kind":"official","path":"src/interp_utils.py","file_url":"https://github.com/abhay-sheshadri/backward-chaining-circuits/blob/HEAD/src/interp_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e6652ed7bc481af1"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}