{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/can-large-language-models-infer-and-disagree","title":"Can Large Language Models Capture Dissenting Human Voices?","arxiv_id":"2305.13788","date":"2023-05-23","proceeding":null,"authors":["Noah Lee","Na Min An","James Thorne"],"abstract":"Large language models (LLMs) have shown impressive achievements in solving a broad range of tasks. Augmented by instruction fine-tuning, LLMs have also been shown to generalize in zero-shot settings as well. However, whether LLMs closely align with the human disagreement distribution has not been well-studied, especially within the scope of natural language inference (NLI). In this paper, we evaluate the performance and alignment of LLM distribution with humans using two different techniques to estimate the multinomial distribution: Monte Carlo Estimation (MCE) and Log Probability Estimation (LPE). As a result, we show LLMs exhibit limited ability in solving NLI tasks and simultaneously fail to capture human disagreement distribution. The inference and human alignment performances plunge even further on data samples with high human disagreement levels, raising concerns about their natural language understanding (NLU) ability and their representativeness to a larger human population. The source code for the experiments is available at https://github.com/xfactlab/emnlp2023-LLM-Disagreement","url_abs":"https://arxiv.org/abs/2305.13788v2","url_pdf":"https://arxiv.org/pdf/2305.13788v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"can-large-language-models-infer-and-disagree","repo_url":"https://github.com/xfactlab/emnlp2023-llm-disagreement","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"natural-language-inference","task_name":"Natural Language Inference"},{"task_slug":"natural-language-understanding","task_name":"Natural Language Understanding"}],"methods":[{"method_slug":"align","method_name":"ALIGN"},{"method_slug":"fail","method_name":"fail"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2305.13788","atlas_url":"https://app.syntology.ai/?focus=2305.13788","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.13788"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/xfactlab/emnlp2023-llm-disagreement","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/xfactlab/emnlp2023-LLM-Disagreement","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/google-research/FLAN","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_violates":1,"unverified":27},"by_repo_kind":{"official":{"samples":18,"ran":1,"repositories":1},"found_in_text":{"samples":10,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"7c508037b40522af","entry":"str2bool","repo":"xfactlab/emnlp2023-LLM-Disagreement","repo_kind":"official","path":"preprocess.py","file_url":"https://github.com/xfactlab/emnlp2023-LLM-Disagreement/blob/HEAD/preprocess.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7c508037b40522af"}},{"code_sha256_prefix":"206789c8dcfa3c19","entry":"collapse_related_tasks","repo":"google-research/FLAN","repo_kind":"found_in_text","path":"flan/task_splits.py","file_url":"https://github.com/google-research/FLAN/blob/HEAD/flan/task_splits.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"206789c8dcfa3c19"}},{"code_sha256_prefix":"6c55f5bfb0365f70","entry":"dict_postprocess","repo":"xfactlab/emnlp2023-LLM-Disagreement","repo_kind":"official","path":"sample.py","file_url":"https://github.com/xfactlab/emnlp2023-LLM-Disagreement/blob/HEAD/sample.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6c55f5bfb0365f70"}},{"code_sha256_prefix":"5c81be5eb68c1209","entry":"dict_postprocess","repo":"xfactlab/emnlp2023-llm-disagreement","repo_kind":"official","path":"utils/processing.py","file_url":"https://github.com/xfactlab/emnlp2023-llm-disagreement/blob/HEAD/utils/processing.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5c81be5eb68c1209"}},{"code_sha256_prefix":"a536795ec1c148df","entry":"entropy_sample","repo":"xfactlab/emnlp2023-LLM-Disagreement","repo_kind":"official","path":"sample.py","file_url":"https://github.com/xfactlab/emnlp2023-LLM-Disagreement/blob/HEAD/sample.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a536795ec1c148df"}},{"code_sha256_prefix":"fc3cbdfde1b3cc50","entry":"enumerate_items","repo":"google-research/FLAN","repo_kind":"found_in_text","path":"flan/tasks.py","file_url":"https://github.com/google-research/FLAN/blob/HEAD/flan/tasks.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fc3cbdfde1b3cc50"}},{"code_sha256_prefix":"48f4ea4298ef92ff","entry":"format_from_feature_dictionary","repo":"google-research/FLAN","repo_kind":"found_in_text","path":"flan/preprocessors.py","file_url":"https://github.com/google-research/FLAN/blob/HEAD/flan/preprocessors.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"48f4ea4298ef92ff"}},{"code_sha256_prefix":"e5b0490b427152f2","entry":"generate_openai","repo":"xfactlab/emnlp2023-LLM-Disagreement","repo_kind":"official","path":"utils/generation.py","file_url":"https://github.com/xfactlab/emnlp2023-LLM-Disagreement/blob/HEAD/utils/generation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e5b0490b427152f2"}},{"code_sha256_prefix":"e31192c49a763711","entry":"generate_prompt_template","repo":"xfactlab/emnlp2023-LLM-Disagreement","repo_kind":"official","path":"utils/prompts.py","file_url":"https://github.com/xfactlab/emnlp2023-LLM-Disagreement/blob/HEAD/utils/prompts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e31192c49a763711"}},{"code_sha256_prefix":"f92e9e42a889c8dd","entry":"is_superglue_task","repo":"google-research/FLAN","repo_kind":"found_in_text","path":"flan/task_splits.py","file_url":"https://github.com/google-research/FLAN/blob/HEAD/flan/task_splits.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f92e9e42a889c8dd"}},{"code_sha256_prefix":"40756590747738d4","entry":"jensen_shannon","repo":"xfactlab/emnlp2023-LLM-Disagreement","repo_kind":"official","path":"utils/metrics.py","file_url":"https://github.com/xfactlab/emnlp2023-LLM-Disagreement/blob/HEAD/utils/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"40756590747738d4"}},{"code_sha256_prefix":"cfd7ee739cccf02d","entry":"kl_divergence","repo":"xfactlab/emnlp2023-LLM-Disagreement","repo_kind":"official","path":"utils/metrics.py","file_url":"https://github.com/xfactlab/emnlp2023-LLM-Disagreement/blob/HEAD/utils/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cfd7ee739cccf02d"}},{"code_sha256_prefix":"3bf33362230ed1a5","entry":"load_data","repo":"xfactlab/emnlp2023-LLM-Disagreement","repo_kind":"official","path":"preprocess.py","file_url":"https://github.com/xfactlab/emnlp2023-LLM-Disagreement/blob/HEAD/preprocess.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3bf33362230ed1a5"}},{"code_sha256_prefix":"7eb40d424f4306a5","entry":"load_json","repo":"xfactlab/emnlp2023-LLM-Disagreement","repo_kind":"official","path":"preprocess.py","file_url":"https://github.com/xfactlab/emnlp2023-LLM-Disagreement/blob/HEAD/preprocess.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7eb40d424f4306a5"}},{"code_sha256_prefix":"df8175311a751726","entry":"load_model","repo":"xfactlab/emnlp2023-LLM-Disagreement","repo_kind":"official","path":"utils/models.py","file_url":"https://github.com/xfactlab/emnlp2023-LLM-Disagreement/blob/HEAD/utils/models.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"df8175311a751726"}},{"code_sha256_prefix":"e435cf490873e109","entry":"negate","repo":"google-research/FLAN","repo_kind":"found_in_text","path":"flan/preprocessors.py","file_url":"https://github.com/google-research/FLAN/blob/HEAD/flan/preprocessors.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e435cf490873e109"}},{"code_sha256_prefix":"003e5dada7af2bae","entry":"option_postprocess","repo":"xfactlab/emnlp2023-LLM-Disagreement","repo_kind":"official","path":"utils/processing.py","file_url":"https://github.com/xfactlab/emnlp2023-LLM-Disagreement/blob/HEAD/utils/processing.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"003e5dada7af2bae"}},{"code_sha256_prefix":"486ad3a49d33036e","entry":"parse_glm_qa_answer","repo":"google-research/FLAN","repo_kind":"found_in_text","path":"flan/postprocessors.py","file_url":"https://github.com/google-research/FLAN/blob/HEAD/flan/postprocessors.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"486ad3a49d33036e"}},{"code_sha256_prefix":"c501787fde562c3c","entry":"preprocess_data","repo":"xfactlab/emnlp2023-LLM-Disagreement","repo_kind":"official","path":"utils/visualize.py","file_url":"https://github.com/xfactlab/emnlp2023-LLM-Disagreement/blob/HEAD/utils/visualize.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c501787fde562c3c"}},{"code_sha256_prefix":"444acbf63a2aed61","entry":"preprocess_data_Nie","repo":"xfactlab/emnlp2023-LLM-Disagreement","repo_kind":"official","path":"utils/visualize.py","file_url":"https://github.com/xfactlab/emnlp2023-LLM-Disagreement/blob/HEAD/utils/visualize.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"444acbf63a2aed61"}},{"code_sha256_prefix":"425c2228b8849976","entry":"prompt_fs_generator","repo":"xfactlab/emnlp2023-LLM-Disagreement","repo_kind":"official","path":"utils/prompts.py","file_url":"https://github.com/xfactlab/emnlp2023-LLM-Disagreement/blob/HEAD/utils/prompts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"425c2228b8849976"}},{"code_sha256_prefix":"6e27415d7d2bdc18","entry":"prompt_generator","repo":"xfactlab/emnlp2023-LLM-Disagreement","repo_kind":"official","path":"utils/prompts.py","file_url":"https://github.com/xfactlab/emnlp2023-LLM-Disagreement/blob/HEAD/utils/prompts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6e27415d7d2bdc18"}},{"code_sha256_prefix":"2163e5e3c4bcebcd","entry":"random_sample","repo":"xfactlab/emnlp2023-LLM-Disagreement","repo_kind":"official","path":"sample.py","file_url":"https://github.com/xfactlab/emnlp2023-LLM-Disagreement/blob/HEAD/sample.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2163e5e3c4bcebcd"}},{"code_sha256_prefix":"7e736694212e7c19","entry":"relative_entropy","repo":"xfactlab/emnlp2023-LLM-Disagreement","repo_kind":"official","path":"utils/metrics.py","file_url":"https://github.com/xfactlab/emnlp2023-LLM-Disagreement/blob/HEAD/utils/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7e736694212e7c19"}},{"code_sha256_prefix":"252eaeffae9c5998","entry":"remove_input_patterns_options","repo":"google-research/FLAN","repo_kind":"found_in_text","path":"flan/utils.py","file_url":"https://github.com/google-research/FLAN/blob/HEAD/flan/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"252eaeffae9c5998"}},{"code_sha256_prefix":"7d7424269c38356a","entry":"remove_leading_quotes_and_spaces","repo":"google-research/FLAN","repo_kind":"found_in_text","path":"flan/postprocessors.py","file_url":"https://github.com/google-research/FLAN/blob/HEAD/flan/postprocessors.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7d7424269c38356a"}},{"code_sha256_prefix":"0688ceac63ea3074","entry":"t_name_to_flan_pattern_name","repo":"google-research/FLAN","repo_kind":"found_in_text","path":"flan/utils.py","file_url":"https://github.com/google-research/FLAN/blob/HEAD/flan/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0688ceac63ea3074"}},{"code_sha256_prefix":"a0c365753489d524","entry":"tokenize","repo":"google-research/FLAN","repo_kind":"found_in_text","path":"flan/preprocessors.py","file_url":"https://github.com/google-research/FLAN/blob/HEAD/flan/preprocessors.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a0c365753489d524"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}