{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/colt5-faster-long-range-transformers-with","title":"CoLT5: Faster Long-Range Transformers with Conditional Computation","arxiv_id":"2303.09752","date":"2023-03-17","proceeding":null,"authors":["Joshua Ainslie","Tao Lei","Michiel de Jong","Santiago Ontañón","Siddhartha Brahma","Yury Zemlyanskiy","David Uthus","Mandy Guo","James Lee-Thorp","Yi Tay","Yun-Hsuan Sung","Sumit Sanghai"],"abstract":"Many natural language processing tasks benefit from long inputs, but processing long documents with Transformers is expensive -- not only due to quadratic attention complexity but also from applying feedforward and projection layers to every token. However, not all tokens are equally important, especially for longer documents. We propose CoLT5, a long-input Transformer model that builds on this intuition by employing conditional computation, devoting more resources to important tokens in both feedforward and attention layers. We show that CoLT5 achieves stronger performance than LongT5 with much faster training and inference, achieving SOTA on the long-input SCROLLS benchmark. Moreover, CoLT5 can effectively and tractably make use of extremely long inputs, showing strong gains up to 64k input length.","url_abs":"https://arxiv.org/abs/2303.09752v3","url_pdf":"https://arxiv.org/pdf/2303.09752v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[],"tasks":[{"task_slug":"long-range-modeling","task_name":"Long-range modeling"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/long-range-modeling-on-scrolls","task":"Long-range modeling","dataset":"SCROLLS","model":"CoLT5 XL","rank_in_archive_order":1,"of":13,"metrics":{"Avg.":"43.51","CNLI":"88.4","GovRep":"61.3/32.2/33.8","Nrtv":"31.1","QALT EM-T/H":"48.1/43.8","QMSum":"36.2/12.9/24.3","Qspr":"53.9","SumScr":"36.4/10.2/21.7"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2303.09752","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.09752"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/google/flaxformer","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":9},"by_repo_kind":{"found_in_text":{"samples":9,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"57e16ed2adc5f653","entry":"check_params_and_axis_names_match","repo":"google/flaxformer","repo_kind":"found_in_text","path":"flaxformer/sharding.py","file_url":"https://github.com/google/flaxformer/blob/HEAD/flaxformer/sharding.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"57e16ed2adc5f653"}},{"code_sha256_prefix":"b23204e3d76a4881","entry":"convert_tf_params","repo":"google/flaxformer","repo_kind":"found_in_text","path":"flaxformer/param_conversion_util.py","file_url":"https://github.com/google/flaxformer/blob/HEAD/flaxformer/param_conversion_util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b23204e3d76a4881"}},{"code_sha256_prefix":"b9e1282bcb6f5fed","entry":"crop_train_batch","repo":"google/flaxformer","repo_kind":"found_in_text","path":"flaxformer/architectures/perceiver_ar/t5_models.py","file_url":"https://github.com/google/flaxformer/blob/HEAD/flaxformer/architectures/perceiver_ar/t5_models.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b9e1282bcb6f5fed"}},{"code_sha256_prefix":"171e53e587dfe99f","entry":"get_axis_names","repo":"google/flaxformer","repo_kind":"found_in_text","path":"flaxformer/sharding.py","file_url":"https://github.com/google/flaxformer/blob/HEAD/flaxformer/sharding.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"171e53e587dfe99f"}},{"code_sha256_prefix":"1f9f1e7ebe408df5","entry":"get_int_regex_matches","repo":"google/flaxformer","repo_kind":"found_in_text","path":"flaxformer/param_conversion_util.py","file_url":"https://github.com/google/flaxformer/blob/HEAD/flaxformer/param_conversion_util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1f9f1e7ebe408df5"}},{"code_sha256_prefix":"692429c64001e4a5","entry":"load_tf_params","repo":"google/flaxformer","repo_kind":"found_in_text","path":"flaxformer/param_conversion_util.py","file_url":"https://github.com/google/flaxformer/blob/HEAD/flaxformer/param_conversion_util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"692429c64001e4a5"}},{"code_sha256_prefix":"9cc93ab2811f82f5","entry":"reduce_fn","repo":"google/flaxformer","repo_kind":"found_in_text","path":"flaxformer/sharding.py","file_url":"https://github.com/google/flaxformer/blob/HEAD/flaxformer/sharding.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9cc93ab2811f82f5"}},{"code_sha256_prefix":"6219a1b59a2276fd","entry":"with_sharding","repo":"google/flaxformer","repo_kind":"found_in_text","path":"flaxformer/activation_partitioning.py","file_url":"https://github.com/google/flaxformer/blob/HEAD/flaxformer/activation_partitioning.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6219a1b59a2276fd"}},{"code_sha256_prefix":"a896ada3b6e0ee35","entry":"with_sharding_migration","repo":"google/flaxformer","repo_kind":"found_in_text","path":"flaxformer/activation_partitioning.py","file_url":"https://github.com/google/flaxformer/blob/HEAD/flaxformer/activation_partitioning.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a896ada3b6e0ee35"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}