{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/caduceus-bi-directional-equivariant-long","title":"Caduceus: Bi-Directional Equivariant Long-Range DNA Sequence Modeling","arxiv_id":"2403.03234","date":"2024-03-05","proceeding":null,"authors":["Yair Schiff","Chia-Hsiang Kao","Aaron Gokaslan","Tri Dao","Albert Gu","Volodymyr Kuleshov"],"abstract":"Large-scale sequence modeling has sparked rapid advances that now extend into biology and genomics. However, modeling genomic sequences introduces challenges such as the need to model long-range token interactions, the effects of upstream and downstream regions of the genome, and the reverse complementarity (RC) of DNA. Here, we propose an architecture motivated by these challenges that builds off the long-range Mamba block, and extends it to a BiMamba component that supports bi-directionality, and to a MambaDNA block that additionally supports RC equivariance. We use MambaDNA as the basis of Caduceus, the first family of RC equivariant bi-directional long-range DNA language models, and we introduce pre-training and fine-tuning strategies that yield Caduceus DNA foundation models. Caduceus outperforms previous long-range models on downstream benchmarks; on a challenging long-range variant effect prediction task, Caduceus exceeds the performance of 10x larger models that do not leverage bi-directionality or equivariance.","url_abs":"https://arxiv.org/abs/2403.03234v2","url_pdf":"https://arxiv.org/pdf/2403.03234v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"caduceus-bi-directional-equivariant-long","repo_url":"https://github.com/kuleshov-group/caduceus","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"caduceus-bi-directional-equivariant-long","repo_url":"https://github.com/frederikkemarin/bend","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"BSD-3-Clause"}},{"paper_slug":"caduceus-bi-directional-equivariant-long","repo_url":"https://github.com/jimmylihui/OpenGenome","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"caduceus-bi-directional-equivariant-long","repo_url":"https://github.com/jimmylihui/genbench","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"mamba","task_name":"Mamba"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2403.03234","atlas_url":"https://app.syntology.ai/?focus=2403.03234","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.03234"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jimmylihui/genbench","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kuleshov-group/caduceus","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/frederikkemarin/bend","reach":{"status":"ok","spdx":"BSD-3-Clause"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jimmylihui/OpenGenome","reach":null}],"summary":{"ran":4,"ran_fixture":2,"ran_draft_wrong":1,"unverified":9},"by_repo_kind":{"official":{"samples":7,"ran":5,"repositories":1},"listed":{"samples":9,"ran":2,"repositories":3}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"1b4e43606258990e","entry":"CaduceusEmbedder","repo":"frederikkemarin/bend","repo_kind":"listed","path":"bend/utils/embedders.py","file_url":"https://github.com/frederikkemarin/bend/blob/HEAD/bend/utils/embedders.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"1b4e43606258990e"}},{"code_sha256_prefix":"83de732cfd3e479f","entry":"cross_entropy","repo":"kuleshov-group/caduceus","repo_kind":"official","path":"caduceus/modeling_caduceus.py","file_url":"https://github.com/kuleshov-group/caduceus/blob/HEAD/caduceus/modeling_caduceus.py","link_basis":"plan_row","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"83de732cfd3e479f"}},{"code_sha256_prefix":"741eb38fcf150c88","entry":"fftconv_ref","repo":"kuleshov-group/caduceus","repo_kind":"official","path":"src/ops/fftconv.py","file_url":"https://github.com/kuleshov-group/caduceus/blob/HEAD/src/ops/fftconv.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"741eb38fcf150c88"}},{"code_sha256_prefix":"31a8ada19e8c38e8","entry":"fsspec_exists","repo":"kuleshov-group/caduceus","repo_kind":"official","path":"vep_embeddings.py","file_url":"https://github.com/kuleshov-group/caduceus/blob/HEAD/vep_embeddings.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"31a8ada19e8c38e8"}},{"code_sha256_prefix":"16c2a70f52e70a43","entry":"inject_substring","repo":"jimmylihui/genbench","repo_kind":"listed","path":"huggingface.py","file_url":"https://github.com/jimmylihui/genbench/blob/HEAD/huggingface.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"16c2a70f52e70a43"}},{"code_sha256_prefix":"c42e2917c6490693","entry":"recast_chromosome_tissue_dist2TSS","repo":"kuleshov-group/caduceus","repo_kind":"official","path":"vep_embeddings.py","file_url":"https://github.com/kuleshov-group/caduceus/blob/HEAD/vep_embeddings.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c42e2917c6490693"}},{"code_sha256_prefix":"2ef47d00e83953ae","entry":"weighted_cross_entropy","repo":"kuleshov-group/caduceus","repo_kind":"official","path":"caduceus/modeling_caduceus.py","file_url":"https://github.com/kuleshov-group/caduceus/blob/HEAD/caduceus/modeling_caduceus.py","link_basis":"plan_row","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2ef47d00e83953ae"}},{"code_sha256_prefix":"7d9802ebb95082ad","entry":"BaseEmbedder","repo":"frederikkemarin/bend","repo_kind":"listed","path":"bend/utils/embedders.py","file_url":"https://github.com/frederikkemarin/bend/blob/HEAD/bend/utils/embedders.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"7d9802ebb95082ad"}},{"code_sha256_prefix":"0bd488da131151fe","entry":"HyenaOperator","repo":"jimmylihui/OpenGenome","repo_kind":"listed","path":"src/models/sequence/hyena.py","file_url":"https://github.com/jimmylihui/OpenGenome/blob/HEAD/src/models/sequence/hyena.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0bd488da131151fe"}},{"code_sha256_prefix":"d8cff84d50ba79b0","entry":"adaptive_coarsegrain_gpu","repo":"jimmylihui/genbench","repo_kind":"listed","path":"selene_utils2.py","file_url":"https://github.com/jimmylihui/genbench/blob/HEAD/selene_utils2.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d8cff84d50ba79b0"}},{"code_sha256_prefix":"25edc240223b76c0","entry":"deprecated","repo":"jimmylihui/genbench","repo_kind":"listed","path":"src/dataloaders/base.py","file_url":"https://github.com/jimmylihui/genbench/blob/HEAD/src/dataloaders/base.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"25edc240223b76c0"}},{"code_sha256_prefix":"5d11b94982686ba5","entry":"fftconv_h3_ref","repo":"kuleshov-group/caduceus","repo_kind":"official","path":"src/ops/fftconv.py","file_url":"https://github.com/kuleshov-group/caduceus/blob/HEAD/src/ops/fftconv.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5d11b94982686ba5"}},{"code_sha256_prefix":"db0136eb80a50383","entry":"fsspec_listdir","repo":"kuleshov-group/caduceus","repo_kind":"official","path":"vep_embeddings.py","file_url":"https://github.com/kuleshov-group/caduceus/blob/HEAD/vep_embeddings.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"db0136eb80a50383"}},{"code_sha256_prefix":"36671e7e4ba9ec81","entry":"load_weights","repo":"jimmylihui/genbench","repo_kind":"listed","path":"huggingface.py","file_url":"https://github.com/jimmylihui/genbench/blob/HEAD/huggingface.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"36671e7e4ba9ec81"}},{"code_sha256_prefix":"9d0777602eda354e","entry":"time_features","repo":"jimmylihui/genbench","repo_kind":"listed","path":"src/dataloaders/et.py","file_url":"https://github.com/jimmylihui/genbench/blob/HEAD/src/dataloaders/et.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9d0777602eda354e"}},{"code_sha256_prefix":"f8544563682146e5","entry":"time_features_from_frequency_str","repo":"jimmylihui/genbench","repo_kind":"listed","path":"src/dataloaders/et.py","file_url":"https://github.com/jimmylihui/genbench/blob/HEAD/src/dataloaders/et.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f8544563682146e5"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}