{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/semantically-diverse-language-generation-for","title":"Semantically Diverse Language Generation for Uncertainty Estimation in Language Models","arxiv_id":"2406.04306","date":"2024-06-06","proceeding":null,"authors":["Lukas Aichberger","Kajetan Schweighofer","Mykyta Ielanskyi","Sepp Hochreiter"],"abstract":"Large language models (LLMs) can suffer from hallucinations when generating text. These hallucinations impede various applications in society and industry by making LLMs untrustworthy. Current LLMs generate text in an autoregressive fashion by predicting and appending text tokens. When an LLM is uncertain about the semantic meaning of the next tokens to generate, it is likely to start hallucinating. Thus, it has been suggested that hallucinations stem from predictive uncertainty. We introduce Semantically Diverse Language Generation (SDLG) to quantify predictive uncertainty in LLMs. SDLG steers the LLM to generate semantically diverse yet likely alternatives for an initially generated text. This approach provides a precise measure of aleatoric semantic uncertainty, detecting whether the initial text is likely to be hallucinated. Experiments on question-answering tasks demonstrate that SDLG consistently outperforms existing methods while being the most computationally efficient, setting a new standard for uncertainty estimation in LLMs.","url_abs":"https://arxiv.org/abs/2406.04306v1","url_pdf":"https://arxiv.org/pdf/2406.04306v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"semantically-diverse-language-generation-for","repo_url":"https://github.com/ml-jku/SDLG","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"text-generation","task_name":"Text Generation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2406.04306","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.04306"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/ml-jku/SDLG","reach":null}],"summary":{"ran_draft_wrong":4,"ran_fixture":1,"unverified":6},"by_repo_kind":{"official":{"samples":11,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":11,"samples":[{"code_sha256_prefix":"c079bf5aa1e65d85","entry":"clean_generation","repo":"ml-jku/SDLG","repo_kind":"official","path":"sdlg.py","file_url":"https://github.com/ml-jku/SDLG/blob/HEAD/sdlg.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"BSD-3-Clause-Clear","inline_ok":false,"mcp_get_code":{"code_sha256":"c079bf5aa1e65d85"}},{"code_sha256_prefix":"e32e20f5d1a27c3b","entry":"compute_likelihood","repo":"ml-jku/SDLG","repo_kind":"official","path":"sdlg.py","file_url":"https://github.com/ml-jku/SDLG/blob/HEAD/sdlg.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause-Clear","inline_ok":false,"mcp_get_code":{"code_sha256":"e32e20f5d1a27c3b"}},{"code_sha256_prefix":"4fc5bd5d74bb816b","entry":"get_word_indices","repo":"ml-jku/SDLG","repo_kind":"official","path":"sdlg.py","file_url":"https://github.com/ml-jku/SDLG/blob/HEAD/sdlg.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause-Clear","inline_ok":false,"mcp_get_code":{"code_sha256":"4fc5bd5d74bb816b"}},{"code_sha256_prefix":"c8008859407c4f6a","entry":"rank_tensor","repo":"ml-jku/SDLG","repo_kind":"official","path":"sdlg.py","file_url":"https://github.com/ml-jku/SDLG/blob/HEAD/sdlg.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"BSD-3-Clause-Clear","inline_ok":false,"mcp_get_code":{"code_sha256":"c8008859407c4f6a"}},{"code_sha256_prefix":"1a6208fbe06fee8d","entry":"remove_invalid_ids","repo":"ml-jku/SDLG","repo_kind":"official","path":"sdlg.py","file_url":"https://github.com/ml-jku/SDLG/blob/HEAD/sdlg.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"BSD-3-Clause-Clear","inline_ok":false,"mcp_get_code":{"code_sha256":"1a6208fbe06fee8d"}},{"code_sha256_prefix":"95a922573781d7aa","entry":"compute_token_score_ranking","repo":"ml-jku/SDLG","repo_kind":"official","path":"sdlg.py","file_url":"https://github.com/ml-jku/SDLG/blob/HEAD/sdlg.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause-Clear","inline_ok":false,"mcp_get_code":{"code_sha256":"95a922573781d7aa"}},{"code_sha256_prefix":"8d9f3ee549cef855","entry":"encode","repo":"ml-jku/SDLG","repo_kind":"official","path":"run_experiments.py","file_url":"https://github.com/ml-jku/SDLG/blob/HEAD/run_experiments.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause-Clear","inline_ok":false,"mcp_get_code":{"code_sha256":"8d9f3ee549cef855"}},{"code_sha256_prefix":"d4e612315aff11dd","entry":"encode_and_format_dataset","repo":"ml-jku/SDLG","repo_kind":"official","path":"run_experiments.py","file_url":"https://github.com/ml-jku/SDLG/blob/HEAD/run_experiments.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause-Clear","inline_ok":false,"mcp_get_code":{"code_sha256":"d4e612315aff11dd"}},{"code_sha256_prefix":"6974d8f454f15c31","entry":"forward_hook","repo":"ml-jku/SDLG","repo_kind":"official","path":"sdlg.py","file_url":"https://github.com/ml-jku/SDLG/blob/HEAD/sdlg.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause-Clear","inline_ok":false,"mcp_get_code":{"code_sha256":"6974d8f454f15c31"}},{"code_sha256_prefix":"4cdc3aa9b371f7d6","entry":"generate_semantically_diverse_output_sequences","repo":"ml-jku/SDLG","repo_kind":"official","path":"sdlg.py","file_url":"https://github.com/ml-jku/SDLG/blob/HEAD/sdlg.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause-Clear","inline_ok":false,"mcp_get_code":{"code_sha256":"4cdc3aa9b371f7d6"}},{"code_sha256_prefix":"a701b90b916f5466","entry":"generate_text","repo":"ml-jku/SDLG","repo_kind":"official","path":"sdlg.py","file_url":"https://github.com/ml-jku/SDLG/blob/HEAD/sdlg.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause-Clear","inline_ok":false,"mcp_get_code":{"code_sha256":"a701b90b916f5466"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}