{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/the-hallucination-dilemma-factuality-aware","title":"The Hallucination Dilemma: Factuality-Aware Reinforcement Learning for Large Reasoning Models","arxiv_id":"2505.24630","date":"2025-05-30","proceeding":null,"authors":["Junyi Li","Hwee Tou Ng"],"abstract":"Large language models (LLMs) have significantly advanced in reasoning tasks through reinforcement learning (RL) optimization, achieving impressive capabilities across various challenging benchmarks. However, our empirical analysis reveals a critical drawback: reasoning-oriented RL fine-tuning significantly increases the prevalence of hallucinations. We theoretically analyze the RL training dynamics, identifying high-variance gradient, entropy-induced randomness, and susceptibility to spurious local optima as key factors leading to hallucinations. To address this drawback, we propose Factuality-aware Step-wise Policy Optimization (FSPO), an innovative RL fine-tuning algorithm incorporating explicit factuality verification at each reasoning step. FSPO leverages automated verification against given evidence to dynamically adjust token-level advantage values, incentivizing factual correctness throughout the reasoning process. Experiments across mathematical reasoning and hallucination benchmarks using Qwen2.5 and Llama models demonstrate that FSPO effectively reduces hallucinations while enhancing reasoning accuracy, substantially improving both reliability and performance.","url_abs":"https://arxiv.org/abs/2505.24630v1","url_pdf":"https://arxiv.org/pdf/2505.24630v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"the-hallucination-dilemma-factuality-aware","repo_url":"https://github.com/nusnlp/fspo","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"hallucination","task_name":"Hallucination"},{"task_slug":"mathematical-reasoning","task_name":"Mathematical Reasoning"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"}],"methods":[{"method_slug":"llama","method_name":"LLaMA"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2505.24630","atlas_url":"https://app.syntology.ai/?focus=2505.24630","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.24630"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/nusnlp/fspo","reach":null}],"summary":{"ran_draft_wrong":5,"ran_honours":1},"by_repo_kind":{"official":{"samples":5,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"f887d72c9492d9ee","entry":"convert_to_regular_types","repo":"nusnlp/fspo","repo_kind":"official","path":"verl/trainer/fsdp_sft_trainer.py","file_url":"https://github.com/nusnlp/fspo/blob/HEAD/verl/trainer/fsdp_sft_trainer.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f887d72c9492d9ee"}},{"code_sha256_prefix":"216e28c040173a61","entry":"extract_step","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"216e28c040173a61"}},{"code_sha256_prefix":"787bc6ea79eed461","entry":"load_data","repo":"nusnlp/fspo","repo_kind":"official","path":"evaluate/inference.py","file_url":"https://github.com/nusnlp/fspo/blob/HEAD/evaluate/inference.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"787bc6ea79eed461"}},{"code_sha256_prefix":"2ce3cb087c966a7e","entry":"make_prefix","repo":"nusnlp/fspo","repo_kind":"official","path":"examples/data_preprocess/math_dataset.py","file_url":"https://github.com/nusnlp/fspo/blob/HEAD/examples/data_preprocess/math_dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2ce3cb087c966a7e"}},{"code_sha256_prefix":"2f9d8e7d769c59d5","entry":"match_answer","repo":"nusnlp/fspo","repo_kind":"official","path":"evaluate/inference.py","file_url":"https://github.com/nusnlp/fspo/blob/HEAD/evaluate/inference.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2f9d8e7d769c59d5"}},{"code_sha256_prefix":"d22718d531542ab7","entry":"normalize_answer","repo":"nusnlp/fspo","repo_kind":"official","path":"evaluate/inference.py","file_url":"https://github.com/nusnlp/fspo/blob/HEAD/evaluate/inference.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d22718d531542ab7"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}