{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/liere-generalizing-rotary-position-encodings","title":"LieRE: Generalizing Rotary Position Encodings","arxiv_id":"2406.10322","date":"2024-06-14","proceeding":null,"authors":["Sophie Ostmeier","Brian Axelrod","Michael E. Moseley","Akshay Chaudhari","Curtis Langlotz"],"abstract":"While Rotary Position Embeddings (RoPE) for large language models have become widely adopted, their application for other modalities has been slower. Here, we introduce Lie group Relative position Encodings (LieRE) that goes beyond RoPE in supporting n-dimensional inputs. We evaluate the performance of LieRE on 2D and 3D image classification tasks and observe that LieRE leads to marked relative improvements in performance (up to 9.7% for 2D and up to 25.5% for 3D), training efficiency (3.5x reduction), data efficiency (30%) compared to the baselines of DeiT III, RoPE-Mixed and Vision-Llama. https://github.com/Stanford-AIMI/LieRE","url_abs":"https://arxiv.org/abs/2406.10322v2","url_pdf":"https://arxiv.org/pdf/2406.10322v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"liere-generalizing-rotary-position-encodings","repo_url":"https://github.com/stanford-aimi/liere","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":null,"task_name":"Position"},{"task_slug":"image-classification","task_name":"image-classification"}],"methods":[{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"deit","method_name":"DeiT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"feedforward-network","method_name":"Feedforward Network"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"relative-position-encodings","method_name":"Relative Position Encodings"},{"method_slug":"softmax","method_name":"Softmax"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2406.10322","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.10322"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/stanford-aimi/liere","reach":null}],"summary":{"ran":1,"unverified":2},"by_repo_kind":{"official":{"samples":3,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"ff5d8d9cd945f2e0","entry":"PositionEncoderBase","repo":"stanford-aimi/liere","repo_kind":"official","path":"models/rope_vit.py","file_url":"https://github.com/stanford-aimi/liere/blob/HEAD/models/rope_vit.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ff5d8d9cd945f2e0"}},{"code_sha256_prefix":"cf79539a466e67e0","entry":"ExponentiationMethod","repo":"stanford-aimi/liere","repo_kind":"official","path":"models/rope_vit.py","file_url":"https://github.com/stanford-aimi/liere/blob/HEAD/models/rope_vit.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cf79539a466e67e0"}},{"code_sha256_prefix":"ab297ea8908d95e1","entry":"LierePositionEncoder","repo":"stanford-aimi/liere","repo_kind":"official","path":"models/rope_vit.py","file_url":"https://github.com/stanford-aimi/liere/blob/HEAD/models/rope_vit.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ab297ea8908d95e1"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}