{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/deciphering-the-interplay-of-parametric-and","title":"Deciphering the Interplay of Parametric and Non-parametric Memory in Retrieval-augmented Language Models","arxiv_id":"2410.05162","date":"2024-10-07","proceeding":null,"authors":["Mehrdad Farahani","Richard Johansson"],"abstract":"Generative language models often struggle with specialized or less-discussed knowledge. A potential solution is found in Retrieval-Augmented Generation (RAG) models which act like retrieving information before generating responses. In this study, we explore how the \\textsc{Atlas} approach, a RAG model, decides between what it already knows (parametric) and what it retrieves (non-parametric). We use causal mediation analysis and controlled experiments to examine how internal representations influence information processing. Our findings disentangle the effects of parametric knowledge and the retrieved context. They indicate that in cases where the model can choose between both types of information (parametric and non-parametric), it relies more on the context than the parametric knowledge. Furthermore, the analysis investigates the computations involved in \\emph{how} the model uses the information from the context. We find that multiple mechanisms are active within the model and can be detected with mediation analysis: first, the decision of \\emph{whether the context is relevant}, and second, how the encoder computes output representations to support copying when relevant.","url_abs":"https://arxiv.org/abs/2410.05162v1","url_pdf":"https://arxiv.org/pdf/2410.05162v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"deciphering-the-interplay-of-parametric-and","repo_url":"https://github.com/m3hrdadfi/rag-memory-interplay","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"rag","task_name":"RAG"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"retrieval-augmented-generation","task_name":"Retrieval-augmented Generation"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bart","method_name":"BART"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"rag","method_name":"RAG"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2410.05162","atlas_url":"https://app.syntology.ai/?focus=2410.05162","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.05162"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/m3hrdadfi/rag-memory-interplay","reach":{"status":"ok"}}],"summary":{"ran":12,"ran_draft_wrong":1,"ran_honours":1,"unverified":3},"by_repo_kind":{"official":{"samples":17,"ran":14,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":17,"samples":[{"code_sha256_prefix":"59ea1420911ddd97","entry":"all_gather","repo":"m3hrdadfi/rag-memory-interplay","repo_kind":"official","path":"src/atlas/dist_utils.py","file_url":"https://github.com/m3hrdadfi/rag-memory-interplay/blob/HEAD/src/atlas/dist_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"59ea1420911ddd97"}},{"code_sha256_prefix":"a47d716bd6cdafab","entry":"deserialize_listdocs","repo":"m3hrdadfi/rag-memory-interplay","repo_kind":"official","path":"src/atlas/index.py","file_url":"https://github.com/m3hrdadfi/rag-memory-interplay/blob/HEAD/src/atlas/index.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a47d716bd6cdafab"}},{"code_sha256_prefix":"fbd05734c3267af1","entry":"dict_to_args_list","repo":"m3hrdadfi/rag-memory-interplay","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/m3hrdadfi/rag-memory-interplay/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fbd05734c3267af1"}},{"code_sha256_prefix":"38c3b1ee62b65838","entry":"em","repo":"m3hrdadfi/rag-memory-interplay","repo_kind":"official","path":"src/atlas/evaluation.py","file_url":"https://github.com/m3hrdadfi/rag-memory-interplay/blob/HEAD/src/atlas/evaluation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"38c3b1ee62b65838"}},{"code_sha256_prefix":"e4a21bc7a716330c","entry":"encode_passages","repo":"m3hrdadfi/rag-memory-interplay","repo_kind":"official","path":"src/atlas/atlas.py","file_url":"https://github.com/m3hrdadfi/rag-memory-interplay/blob/HEAD/src/atlas/atlas.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e4a21bc7a716330c"}},{"code_sha256_prefix":"5e2f0b798a3d70d2","entry":"f1","repo":"m3hrdadfi/rag-memory-interplay","repo_kind":"official","path":"src/atlas/evaluation.py","file_url":"https://github.com/m3hrdadfi/rag-memory-interplay/blob/HEAD/src/atlas/evaluation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5e2f0b798a3d70d2"}},{"code_sha256_prefix":"094781db771538f0","entry":"gather_wgrad","repo":"m3hrdadfi/rag-memory-interplay","repo_kind":"official","path":"src/atlas/dist_utils.py","file_url":"https://github.com/m3hrdadfi/rag-memory-interplay/blob/HEAD/src/atlas/dist_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"094781db771538f0"}},{"code_sha256_prefix":"bb1041c0bde2033c","entry":"isfloat","repo":"m3hrdadfi/rag-memory-interplay","repo_kind":"official","path":"src/causal_trace.py","file_url":"https://github.com/m3hrdadfi/rag-memory-interplay/blob/HEAD/src/causal_trace.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bb1041c0bde2033c"}},{"code_sha256_prefix":"6205dd62c2627dde","entry":"normalize_answer","repo":"m3hrdadfi/rag-memory-interplay","repo_kind":"official","path":"src/atlas/evaluation.py","file_url":"https://github.com/m3hrdadfi/rag-memory-interplay/blob/HEAD/src/atlas/evaluation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6205dd62c2627dde"}},{"code_sha256_prefix":"c268b2c0da209fa9","entry":"parse_attributes","repo":"m3hrdadfi/rag-memory-interplay","repo_kind":"official","path":"src/causal_trace.py","file_url":"https://github.com/m3hrdadfi/rag-memory-interplay/blob/HEAD/src/causal_trace.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c268b2c0da209fa9"}},{"code_sha256_prefix":"07609cc6c26d3a72","entry":"read_json_file","repo":"m3hrdadfi/rag-memory-interplay","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/m3hrdadfi/rag-memory-interplay/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"07609cc6c26d3a72"}},{"code_sha256_prefix":"c4bca944bda65eb2","entry":"select_crossattention_scores","repo":"m3hrdadfi/rag-memory-interplay","repo_kind":"official","path":"src/atlas/atlas.py","file_url":"https://github.com/m3hrdadfi/rag-memory-interplay/blob/HEAD/src/atlas/atlas.py","link_basis":"plan_row","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c4bca944bda65eb2"}},{"code_sha256_prefix":"55955fc3f98f2a29","entry":"serialize_listdocs","repo":"m3hrdadfi/rag-memory-interplay","repo_kind":"official","path":"src/atlas/index.py","file_url":"https://github.com/m3hrdadfi/rag-memory-interplay/blob/HEAD/src/atlas/index.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"55955fc3f98f2a29"}},{"code_sha256_prefix":"fc996bc980b83cab","entry":"varsize_all_gather","repo":"m3hrdadfi/rag-memory-interplay","repo_kind":"official","path":"src/atlas/dist_utils.py","file_url":"https://github.com/m3hrdadfi/rag-memory-interplay/blob/HEAD/src/atlas/dist_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fc996bc980b83cab"}},{"code_sha256_prefix":"a94b66d049f10dc8","entry":"cross_attention_forward","repo":"m3hrdadfi/rag-memory-interplay","repo_kind":"official","path":"src/atlas/fid.py","file_url":"https://github.com/m3hrdadfi/rag-memory-interplay/blob/HEAD/src/atlas/fid.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a94b66d049f10dc8"}},{"code_sha256_prefix":"4f77c9ee1641e1ec","entry":"load_or_initialize_index","repo":"m3hrdadfi/rag-memory-interplay","repo_kind":"official","path":"src/atlas/index_io.py","file_url":"https://github.com/m3hrdadfi/rag-memory-interplay/blob/HEAD/src/atlas/index_io.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4f77c9ee1641e1ec"}},{"code_sha256_prefix":"af330ecdce90bbd8","entry":"load_passages","repo":"m3hrdadfi/rag-memory-interplay","repo_kind":"official","path":"src/atlas/index_io.py","file_url":"https://github.com/m3hrdadfi/rag-memory-interplay/blob/HEAD/src/atlas/index_io.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"af330ecdce90bbd8"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}