{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/progen2-exploring-the-boundaries-of-protein","title":"ProGen2: Exploring the Boundaries of Protein Language Models","arxiv_id":"2206.13517","date":"2022-06-27","proceeding":null,"authors":["Erik Nijkamp","Jeffrey Ruffolo","Eli N. Weinstein","Nikhil Naik","Ali Madani"],"abstract":"Attention-based models trained on protein sequences have demonstrated incredible success at classification and generation tasks relevant for artificial intelligence-driven protein design. However, we lack a sufficient understanding of how very large-scale models and data play a role in effective protein model development. We introduce a suite of protein language models, named ProGen2, that are scaled up to 6.4B parameters and trained on different sequence datasets drawn from over a billion proteins from genomic, metagenomic, and immune repertoire databases. ProGen2 models show state-of-the-art performance in capturing the distribution of observed evolutionary sequences, generating novel viable sequences, and predicting protein fitness without additional finetuning. As large model sizes and raw numbers of protein sequences continue to become more widely accessible, our results suggest that a growing emphasis needs to be placed on the data distribution provided to a protein sequence model. We release the ProGen2 models and code at https://github.com/salesforce/progen.","url_abs":"https://arxiv.org/abs/2206.13517v1","url_pdf":"https://arxiv.org/pdf/2206.13517v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"progen2-exploring-the-boundaries-of-protein","repo_url":"https://github.com/salesforce/progen","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null},{"paper_slug":"progen2-exploring-the-boundaries-of-protein","repo_url":"https://github.com/oatml-markslab/proteingym","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"progen2-exploring-the-boundaries-of-protein","repo_url":"https://github.com/oatml-markslab/tranception","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"progen2-exploring-the-boundaries-of-protein","repo_url":"https://github.com/salesforce/jaxformer","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok","spdx":"BSD-3-Clause"}}],"tasks":[{"task_slug":"protein-design","task_name":"Protein Design"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2206.13517","atlas_url":"https://app.syntology.ai/?focus=2206.13517","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.13517"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/salesforce/progen","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/oatml-markslab/proteingym","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/oatml-markslab/tranception","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/salesforce/jaxformer","reach":{"status":"ok","spdx":"BSD-3-Clause"}}],"summary":{"ran_fixture":2,"ran_draft_wrong":3,"ran":8,"unverified":5},"by_repo_kind":{"official":{"samples":2,"ran":2,"repositories":1},"listed":{"samples":14,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":2,"samples":[{"code_sha256_prefix":"2f6618868885e7ca","entry":"apply_rotary_pos_emb","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"2f6618868885e7ca"}},{"code_sha256_prefix":"58a861ed3065cdb3","entry":"cast","repo":"salesforce/jaxformer","repo_kind":"listed","path":"jaxformer/hf/sample.py","file_url":"https://github.com/salesforce/jaxformer/blob/HEAD/jaxformer/hf/sample.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"58a861ed3065cdb3"}},{"code_sha256_prefix":"03e99c761c545617","entry":"duplicate_interleave","repo":"salesforce/jaxformer","repo_kind":"listed","path":"jaxformer/hf/codegen/modeling_codegen.py","file_url":"https://github.com/salesforce/jaxformer/blob/HEAD/jaxformer/hf/codegen/modeling_codegen.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"03e99c761c545617"}},{"code_sha256_prefix":"72fa614b434d062b","entry":"fixed_pos_embedding","repo":"salesforce/progen","repo_kind":"official","path":"progen2/models/progen/modeling_progen.py","file_url":"https://github.com/salesforce/progen/blob/HEAD/progen2/models/progen/modeling_progen.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"72fa614b434d062b"}},{"code_sha256_prefix":"7ea21b9da7ef5be6","entry":"fixed_pos_embedding","repo":"salesforce/jaxformer","repo_kind":"listed","path":"jaxformer/hf/codegen/modeling_codegen.py","file_url":"https://github.com/salesforce/jaxformer/blob/HEAD/jaxformer/hf/codegen/modeling_codegen.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"7ea21b9da7ef5be6"}},{"code_sha256_prefix":"943b81269283516d","entry":"loss","repo":"salesforce/jaxformer","repo_kind":"listed","path":"jaxformer/models/decoder/inter/model.py","file_url":"https://github.com/salesforce/jaxformer/blob/HEAD/jaxformer/models/decoder/inter/model.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"943b81269283516d"}},{"code_sha256_prefix":"c66149010337c505","entry":"rotate_every_two","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"c66149010337c505"}},{"code_sha256_prefix":"621a8a98538cd46d","entry":"rotate_every_two","repo":"salesforce/jaxformer","repo_kind":"listed","path":"jaxformer/hf/codegen/modeling_codegen.py","file_url":"https://github.com/salesforce/jaxformer/blob/HEAD/jaxformer/hf/codegen/modeling_codegen.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"621a8a98538cd46d"}},{"code_sha256_prefix":"127d82bb96d01476","entry":"run_return","repo":"salesforce/jaxformer","repo_kind":"listed","path":"jaxformer/utils.py","file_url":"https://github.com/salesforce/jaxformer/blob/HEAD/jaxformer/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"127d82bb96d01476"}},{"code_sha256_prefix":"5f6ee6a61b771e9f","entry":"sample","repo":"salesforce/progen","repo_kind":"official","path":"progen2/likelihood.py","file_url":"https://github.com/salesforce/progen/blob/HEAD/progen2/likelihood.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"5f6ee6a61b771e9f"}},{"code_sha256_prefix":"28250166becef619","entry":"set_default_config","repo":"salesforce/jaxformer","repo_kind":"listed","path":"jaxformer/run/trainer.py","file_url":"https://github.com/salesforce/jaxformer/blob/HEAD/jaxformer/run/trainer.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"28250166becef619"}},{"code_sha256_prefix":"36f7d178ef808204","entry":"sh_ret","repo":"salesforce/jaxformer","repo_kind":"listed","path":"jaxformer/utils.py","file_url":"https://github.com/salesforce/jaxformer/blob/HEAD/jaxformer/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"36f7d178ef808204"}},{"code_sha256_prefix":"a0fd9e51c4dc05fd","entry":"tree_flatten_with_names","repo":"salesforce/jaxformer","repo_kind":"listed","path":"jaxformer/hf/convert.py","file_url":"https://github.com/salesforce/jaxformer/blob/HEAD/jaxformer/hf/convert.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"a0fd9e51c4dc05fd"}},{"code_sha256_prefix":"5679796dda9bd742","entry":"apply_rotary_pos_emb","repo":"salesforce/jaxformer","repo_kind":"listed","path":"jaxformer/models/decoder/inter/positional.py","file_url":"https://github.com/salesforce/jaxformer/blob/HEAD/jaxformer/models/decoder/inter/positional.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"5679796dda9bd742"}},{"code_sha256_prefix":"f1a5638f90df6384","entry":"create_master","repo":"salesforce/jaxformer","repo_kind":"listed","path":"jaxformer/run/trainer.py","file_url":"https://github.com/salesforce/jaxformer/blob/HEAD/jaxformer/run/trainer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"f1a5638f90df6384"}},{"code_sha256_prefix":"f5761899e7fefe36","entry":"include_whitespace","repo":"salesforce/jaxformer","repo_kind":"listed","path":"jaxformer/hf/sample.py","file_url":"https://github.com/salesforce/jaxformer/blob/HEAD/jaxformer/hf/sample.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"f5761899e7fefe36"}},{"code_sha256_prefix":"365cda55aef11b01","entry":"run_loop","repo":"salesforce/jaxformer","repo_kind":"listed","path":"jaxformer/utils.py","file_url":"https://github.com/salesforce/jaxformer/blob/HEAD/jaxformer/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"365cda55aef11b01"}},{"code_sha256_prefix":"2c193137483f61dd","entry":"tree_leaves_with_names","repo":"salesforce/jaxformer","repo_kind":"listed","path":"jaxformer/hf/convert.py","file_url":"https://github.com/salesforce/jaxformer/blob/HEAD/jaxformer/hf/convert.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"2c193137483f61dd"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}