{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/chordmixer-a-scalable-neural-attention-model","title":"ChordMixer: A Scalable Neural Attention Model for Sequences with Different Lengths","arxiv_id":"2206.05852","date":"2022-06-12","proceeding":null,"authors":["Ruslan Khalitov","Tong Yu","Lei Cheng","Zhirong Yang"],"abstract":"Sequential data naturally have different lengths in many domains, with some very long sequences. As an important modeling tool, neural attention should capture long-range interaction in such sequences. However, most existing neural attention models admit only short sequences, or they have to employ chunking or padding to enforce a constant input length. Here we propose a simple neural network building block called ChordMixer which can model the attention for long sequences with variable lengths. Each ChordMixer block consists of a position-wise rotation layer without learnable parameters and an element-wise MLP layer. Repeatedly applying such blocks forms an effective network backbone that mixes the input signals towards the learning targets. We have tested ChordMixer on the synthetic adding problem, long document classification, and DNA sequence-based taxonomy classification. The experiment results show that our method substantially outperforms other neural attention models.","url_abs":"https://arxiv.org/abs/2206.05852v2","url_pdf":"https://arxiv.org/pdf/2206.05852v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"chordmixer-a-scalable-neural-attention-model","repo_url":"https://github.com/ruslankhalitov/chordmixer","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"chunking","task_name":"Chunking"},{"task_slug":"document-classification","task_name":"Document Classification"},{"task_slug":"long-range-modeling","task_name":"Long-range modeling"},{"task_slug":null,"task_name":"Position"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2206.05852","atlas_url":"https://app.syntology.ai/?focus=2206.05852","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.05852"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ruslankhalitov/chordmixer","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":2,"unverified":1},"by_repo_kind":{"official":{"samples":3,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"f885ff8606a5cb13","entry":"LoadGrayscale","repo":"ruslankhalitov/chordmixer","repo_kind":"official","path":"dataloaders/lra_pathfinder.py","file_url":"https://github.com/ruslankhalitov/chordmixer/blob/HEAD/dataloaders/lra_pathfinder.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f885ff8606a5cb13"}},{"code_sha256_prefix":"6b6635adc313504f","entry":"map_norm","repo":"ruslankhalitov/chordmixer","repo_kind":"official","path":"models/chordmixer.py","file_url":"https://github.com/ruslankhalitov/chordmixer/blob/HEAD/models/chordmixer.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6b6635adc313504f"}},{"code_sha256_prefix":"72d464924b5dee96","entry":"listops_tokenizer","repo":"ruslankhalitov/chordmixer","repo_kind":"official","path":"dataloaders/lra_listops.py","file_url":"https://github.com/ruslankhalitov/chordmixer/blob/HEAD/dataloaders/lra_listops.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"72d464924b5dee96"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}