{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/scirepeval-a-multi-format-benchmark-for","title":"SciRepEval: A Multi-Format Benchmark for Scientific Document Representations","arxiv_id":"2211.13308","date":"2022-11-23","proceeding":null,"authors":["Amanpreet Singh","Mike D'Arcy","Arman Cohan","Doug Downey","Sergey Feldman"],"abstract":"Learned representations of scientific documents can serve as valuable input features for downstream tasks without further fine-tuning. However, existing benchmarks for evaluating these representations fail to capture the diversity of relevant tasks. In response, we introduce SciRepEval, the first comprehensive benchmark for training and evaluating scientific document representations. It includes 24 challenging and realistic tasks, 8 of which are new, across four formats: classification, regression, ranking and search. We then use this benchmark to study and improve the generalization ability of scientific document representation models. We show how state-of-the-art models like SPECTER and SciNCL struggle to generalize across the task formats, and that simple multi-task training fails to improve them. However, a new approach that learns multiple embeddings per document, each tailored to a different format, can improve performance. We experiment with task-format-specific control codes and adapters and find they outperform the existing single-embedding state-of-the-art by over 2 points absolute. We release the resulting family of multi-format models, called SPECTER2, for the community to use and build on.","url_abs":"https://arxiv.org/abs/2211.13308v4","url_pdf":"https://arxiv.org/pdf/2211.13308v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"scirepeval-a-multi-format-benchmark-for","repo_url":"https://github.com/allenai/scirepeval","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"scirepeval-a-multi-format-benchmark-for","repo_url":"https://github.com/allenai/specter2","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"diversity","task_name":"Diversity"}],"methods":[{"method_slug":"fail","method_name":"fail"}],"datasets_introduced":[{"slug":"scirepeval","name":"SciRepEval","full_name":""}],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2211.13308","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.13308"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/allenai/specter2","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/allenai/scirepeval","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_honours":1,"unverified":4},"by_repo_kind":{"official":{"samples":5,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"40e9fee2e0b7e278","entry":"gelu","repo":"allenai/scirepeval","repo_kind":"official","path":"bert_pals.py","file_url":"https://github.com/allenai/scirepeval/blob/HEAD/bert_pals.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"40e9fee2e0b7e278"}},{"code_sha256_prefix":"32fe7fad842196bf","entry":"create_cands_df","repo":"allenai/scirepeval","repo_kind":"official","path":"update_triplets.py","file_url":"https://github.com/allenai/scirepeval/blob/HEAD/update_triplets.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"32fe7fad842196bf"}},{"code_sha256_prefix":"9373f00665af3257","entry":"get_easy_negs","repo":"allenai/scirepeval","repo_kind":"official","path":"update_triplets.py","file_url":"https://github.com/allenai/scirepeval/blob/HEAD/update_triplets.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9373f00665af3257"}},{"code_sha256_prefix":"9177a4b6ac7237e9","entry":"multiple_files_to_df","repo":"allenai/scirepeval","repo_kind":"official","path":"update_triplets.py","file_url":"https://github.com/allenai/scirepeval/blob/HEAD/update_triplets.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9177a4b6ac7237e9"}},{"code_sha256_prefix":"1ad2f64f3e7925fd","entry":"read_data","repo":"allenai/scirepeval","repo_kind":"official","path":"s2and_embeddings.py","file_url":"https://github.com/allenai/scirepeval/blob/HEAD/s2and_embeddings.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1ad2f64f3e7925fd"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}