{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/bayesian-attention-mechanism-a-probabilistic","title":"Bayesian Attention Mechanism: A Probabilistic Framework for Positional Encoding and Context Length Extrapolation","arxiv_id":"2505.22842","date":"2025-05-28","proceeding":null,"authors":["Arthur S. Bianchessi","Rodrigo C. Barros","Lucas S. Kupssinskü"],"abstract":"Transformer-based language models rely on positional encoding (PE) to handle token order and support context length extrapolation. However, existing PE methods lack theoretical clarity and rely on limited evaluation metrics to substantiate their extrapolation claims. We propose the Bayesian Attention Mechanism (BAM), a theoretical framework that formulates positional encoding as a prior within a probabilistic model. BAM unifies existing methods (e.g., NoPE and ALiBi) and motivates a new Generalized Gaussian positional prior that substantially improves long-context generalization. Empirically, BAM enables accurate information retrieval at $500\\times$ the training context length, outperforming previous state-of-the-art context length generalization in long context retrieval accuracy while maintaining comparable perplexity and introducing minimal additional parameters.","url_abs":"https://arxiv.org/abs/2505.22842v1","url_pdf":"https://arxiv.org/pdf/2505.22842v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"bayesian-attention-mechanism-a-probabilistic","repo_url":"https://github.com/ArthurSBianchessi/BAM","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"information-retrieval","task_name":"Information Retrieval"},{"task_slug":"retrieval","task_name":"Retrieval"}],"methods":[{"method_slug":"attention","method_name":"Attention"},{"method_slug":"softmax","method_name":"Softmax"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2505.22842","atlas_url":"https://app.syntology.ai/?focus=2505.22842","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.22842"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ArthurSBianchessi/BAM","reach":{"status":"ok"}}],"summary":{"ran_fixture":1,"ran_honours":1},"by_repo_kind":{"official":{"samples":2,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":2,"samples":[{"code_sha256_prefix":"6d2a08dcf3466514","entry":"repeat_kv","repo":"arthursbianchessi/bam","repo_kind":"official","path":"models/bam.py","file_url":"https://github.com/arthursbianchessi/bam/blob/HEAD/models/bam.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":2,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6d2a08dcf3466514"}},{"code_sha256_prefix":"43e6934a82b49af9","entry":"get_slopes","repo":"arthursbianchessi/bam","repo_kind":"official","path":"models/bam.py","file_url":"https://github.com/arthursbianchessi/bam/blob/HEAD/models/bam.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"43e6934a82b49af9"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}