{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/dynasent-a-dynamic-benchmark-for-sentiment","title":"DynaSent: A Dynamic Benchmark for Sentiment Analysis","arxiv_id":"2012.15349","date":"2020-12-30","proceeding":"ACL 2021 5","authors":["Christopher Potts","Zhengxuan Wu","Atticus Geiger","Douwe Kiela"],"abstract":"We introduce DynaSent ('Dynamic Sentiment'), a new English-language benchmark task for ternary (positive/negative/neutral) sentiment analysis. DynaSent combines naturally occurring sentences with sentences created using the open-source Dynabench Platform, which facilities human-and-model-in-the-loop dataset creation. DynaSent has a total of 121,634 sentences, each validated by five crowdworkers, and its development and test splits are designed to produce chance performance for even the best models we have been able to develop; when future models solve this task, we will use them to create DynaSent version 2, continuing the dynamic evolution of this benchmark. Here, we report on the dataset creation effort, focusing on the steps we took to increase quality and reduce artifacts. We also present evidence that DynaSent's Neutral category is more coherent than the comparable category in other benchmarks, and we motivate training models from scratch for each round over successive fine-tuning.","url_abs":"https://arxiv.org/abs/2012.15349v1","url_pdf":"https://arxiv.org/pdf/2012.15349v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"dynasent-a-dynamic-benchmark-for-sentiment","repo_url":"https://github.com/cgpotts/dynasent","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"sentiment-analysis","task_name":"Sentiment Analysis"}],"methods":[],"datasets_introduced":[{"slug":"dynasent","name":"DynaSent","full_name":""}],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2012.15349","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2012.15349"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/cgpotts/dynasent","reach":null}],"summary":{"ran_honours":1,"ran_draft_wrong":1,"unverified":1},"by_repo_kind":{"official":{"samples":3,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"3b74331a5d2b400f","entry":"get_dist_of_majority_dists","repo":"cgpotts/dynasent","repo_kind":"official","path":"dynasent_utils.py","file_url":"https://github.com/cgpotts/dynasent/blob/HEAD/dynasent_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3b74331a5d2b400f"}},{"code_sha256_prefix":"6206bbdc84ac4981","entry":"sample_examples","repo":"cgpotts/dynasent","repo_kind":"official","path":"dynasent_utils.py","file_url":"https://github.com/cgpotts/dynasent/blob/HEAD/dynasent_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6206bbdc84ac4981"}},{"code_sha256_prefix":"40f680f0fd399a9b","entry":"sample_short_examples","repo":"cgpotts/dynasent","repo_kind":"official","path":"dynasent_utils.py","file_url":"https://github.com/cgpotts/dynasent/blob/HEAD/dynasent_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"40f680f0fd399a9b"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}