{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/summarizing-differences-between-text","title":"Describing Differences between Text Distributions with Natural Language","arxiv_id":"2201.12323","date":"2022-01-28","proceeding":null,"authors":["Ruiqi Zhong","Charlie Snell","Dan Klein","Jacob Steinhardt"],"abstract":"How do two distributions of texts differ? Humans are slow at answering this, since discovering patterns might require tediously reading through hundreds of samples. We propose to automatically summarize the differences by \"learning a natural language hypothesis\": given two distributions $D_{0}$ and $D_{1}$, we search for a description that is more often true for $D_{1}$, e.g., \"is military-related.\" To tackle this problem, we fine-tune GPT-3 to propose descriptions with the prompt: \"[samples of $D_{0}$] + [samples of $D_{1}$] + the difference between them is_____.\" We then re-rank the descriptions by checking how often they hold on a larger set of samples with a learned verifier. On a benchmark of 54 real-world binary classification tasks, while GPT-3 Curie (13B) only generates a description similar to human annotation 7% of the time, the performance reaches 61% with fine-tuning and re-ranking, and our best system using GPT-3 Davinci (175B) reaches 76%. We apply our system to describe distribution shifts, debug dataset shortcuts, summarize unknown tasks, and label text clusters, and present analyses based on automatically generated descriptions.","url_abs":"https://arxiv.org/abs/2201.12323v2","url_pdf":"https://arxiv.org/pdf/2201.12323v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"summarizing-differences-between-text","repo_url":"https://github.com/ruiqi-zhong/describedistributionaldifferences","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"binary-classification","task_name":"Binary Classification"},{"task_slug":"re-ranking","task_name":"Re-Ranking"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"cosine-annealing","method_name":"Cosine Annealing"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt-3","method_name":"GPT-3"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-cosine-annealing","method_name":"Linear Warmup With Cosine Annealing"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2201.12323","atlas_url":"https://app.syntology.ai/?focus=2201.12323","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2201.12323"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ruiqi-zhong/describedistributionaldifferences","reach":null}],"summary":{"ran_draft_wrong":4},"by_repo_kind":{"official":{"samples":4,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"cd06ae2ccf500a9e","entry":"normalize","repo":"ruiqi-zhong/describedistributionaldifferences","repo_kind":"official","path":"proposer_sec_3_1.py","file_url":"https://github.com/ruiqi-zhong/describedistributionaldifferences/blob/HEAD/proposer_sec_3_1.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"cd06ae2ccf500a9e"}},{"code_sha256_prefix":"1315d5465e129745","entry":"normalize","repo":"ruiqi-zhong/describedistributionaldifferences","repo_kind":"official","path":"verifier_sec_3_2.py","file_url":"https://github.com/ruiqi-zhong/describedistributionaldifferences/blob/HEAD/verifier_sec_3_2.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1315d5465e129745"}},{"code_sha256_prefix":"7aeaa243389b0be2","entry":"qc2input","repo":"ruiqi-zhong/describedistributionaldifferences","repo_kind":"official","path":"verifier_sec_3_2.py","file_url":"https://github.com/ruiqi-zhong/describedistributionaldifferences/blob/HEAD/verifier_sec_3_2.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7aeaa243389b0be2"}},{"code_sha256_prefix":"6b195bba7246492f","entry":"sample_sentences","repo":"ruiqi-zhong/describedistributionaldifferences","repo_kind":"official","path":"proposer_sec_3_1.py","file_url":"https://github.com/ruiqi-zhong/describedistributionaldifferences/blob/HEAD/proposer_sec_3_1.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6b195bba7246492f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}