{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/byte-latent-transformer-patches-scale-better","title":"Byte Latent Transformer: Patches Scale Better Than Tokens","arxiv_id":"2412.09871","date":"2024-12-13","proceeding":null,"authors":["Artidoro Pagnoni","Ram Pasunuru","Pedro Rodriguez","John Nguyen","Benjamin Muller","Margaret Li","Chunting Zhou","Lili Yu","Jason Weston","Luke Zettlemoyer","Gargi Ghosh","Mike Lewis","Ari Holtzman","Srinivasan Iyer"],"abstract":"We introduce the Byte Latent Transformer (BLT), a new byte-level LLM architecture that, for the first time, matches tokenization-based LLM performance at scale with significant improvements in inference efficiency and robustness. BLT encodes bytes into dynamically sized patches, which serve as the primary units of computation. Patches are segmented based on the entropy of the next byte, allocating more compute and model capacity where increased data complexity demands it. We present the first FLOP controlled scaling study of byte-level models up to 8B parameters and 4T training bytes. Our results demonstrate the feasibility of scaling models trained on raw bytes without a fixed vocabulary. Both training and inference efficiency improve due to dynamically selecting long patches when data is predictable, along with qualitative improvements on reasoning and long tail generalization. Overall, for fixed inference costs, BLT shows significantly better scaling than tokenization-based models, by simultaneously growing both patch and model size.","url_abs":"https://arxiv.org/abs/2412.09871v1","url_pdf":"https://arxiv.org/pdf/2412.09871v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"byte-latent-transformer-patches-scale-better","repo_url":"https://github.com/facebookresearch/blt","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2412.09871","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2412.09871"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/blt","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran":2,"unverified":20},"by_repo_kind":{"official":{"samples":22,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":22,"samples":[{"code_sha256_prefix":"26ff74d6538ab161","entry":"attention_flops_per_token","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/model/blt.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/model/blt.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"26ff74d6538ab161"}},{"code_sha256_prefix":"5e36da6fe12b04f3","entry":"get_num_flop_per_token","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/model/blt.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/model/blt.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5e36da6fe12b04f3"}},{"code_sha256_prefix":"09761695b9f13821","entry":"causal_mask","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/model/blt.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/model/blt.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"09761695b9f13821"}},{"code_sha256_prefix":"f20a2b559a7b8775","entry":"consolidate_checkpoints","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/checkpoint.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/checkpoint.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f20a2b559a7b8775"}},{"code_sha256_prefix":"5a9d087d5195cea8","entry":"convert_to_bytes","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/tokenizers/blt_tokenizer.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/tokenizers/blt_tokenizer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"5a9d087d5195cea8"}},{"code_sha256_prefix":"1d34daf1ab8748db","entry":"cross_entropy","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/base_transformer.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/base_transformer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1d34daf1ab8748db"}},{"code_sha256_prefix":"75d20eeaeb19679a","entry":"dist_max","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/distributed.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/distributed.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"75d20eeaeb19679a"}},{"code_sha256_prefix":"f766bed8cc55159b","entry":"dist_min","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/distributed.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/distributed.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f766bed8cc55159b"}},{"code_sha256_prefix":"ebf689175ca08206","entry":"get_device_mesh","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/distributed.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/distributed.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ebf689175ca08206"}},{"code_sha256_prefix":"c5734202cfd3e300","entry":"get_pydantic_default_args","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/config_parser.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/config_parser.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"c5734202cfd3e300"}},{"code_sha256_prefix":"1d8b97adda18ea42","entry":"get_rng_state","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/args.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/args.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1d8b97adda18ea42"}},{"code_sha256_prefix":"30e68c9cadfddd32","entry":"matmul","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/float8.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/float8.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"30e68c9cadfddd32"}},{"code_sha256_prefix":"aaa4b37e564944fe","entry":"named_replace","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/float8.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/float8.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"aaa4b37e564944fe"}},{"code_sha256_prefix":"8751ebfdda7f9c83","entry":"parse_file_config","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/config_parser.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/config_parser.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8751ebfdda7f9c83"}},{"code_sha256_prefix":"1775c1f6cd5bd79c","entry":"precompute_freqs_cis","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/base_transformer.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/base_transformer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1775c1f6cd5bd79c"}},{"code_sha256_prefix":"ff8e64af8f92c325","entry":"recursively_parse_config","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/config_parser.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/config_parser.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ff8e64af8f92c325"}},{"code_sha256_prefix":"705c70c44ddc8080","entry":"repeat_kv","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/base_transformer.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/base_transformer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"705c70c44ddc8080"}},{"code_sha256_prefix":"294621d07f3c12f0","entry":"sample_tokens","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/generate.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/generate.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"294621d07f3c12f0"}},{"code_sha256_prefix":"5b1313e8dd2cbcfa","entry":"sample_top_k","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/generate.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/generate.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5b1313e8dd2cbcfa"}},{"code_sha256_prefix":"2f2141ad0c4f3dbd","entry":"sample_top_p","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/generate.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/generate.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2f2141ad0c4f3dbd"}},{"code_sha256_prefix":"2848341791f9a450","entry":"scale","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/float8.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/float8.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2848341791f9a450"}},{"code_sha256_prefix":"0febe46155d9d826","entry":"text2bytes_bpe_delims","repo":"facebookresearch/blt","repo_kind":"official","path":"bytelatent/tokenizers/blt_tokenizer.py","file_url":"https://github.com/facebookresearch/blt/blob/HEAD/bytelatent/tokenizers/blt_tokenizer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"0febe46155d9d826"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}