{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2606-09124","title":"A Regret Minimization Framework on Preference Learning in Large Language Models","arxiv_id":"2606.09124","date":"2026-06-08","proceeding":"ICML","authors":["Suhwan Kim","Taehyun Cho","Geon-Hyeong Kim","Yu Jin Kim","Youngsoo Jang","Moontae Lee","Jungwoo Lee"],"abstract":"Reinforcement learning with verifiable rewards (RLVR) has enabled progress on reasoning-intensive tasks by relying on task-specific verifiers that provide automated correctness signals. However, many realistic language tasks are difficult to equip with reliable verifiers, motivating a growing reliance on reinforcement learning from human feedback (RLHF). In this setting, we argue that a closer examination of how human feedback should be interpreted is essential. We introduce Regret-based Preference Optimization $(\\textbf{RePO})$, which reframes RLHF through $\\textit{regret minimization}$ rather than reward maximization. Human preferences are often shaped by $\\textit{prospective}$ anticipation of outcomes and $\\textit{counterfactual}$ comparisons to alternative behaviors, rather than by immediate, outcome-independent utility. $\\textbf{RePO}$ captures this structure by modeling preferences as behavior-conditioned assessments of relative suboptimality. Experiments on mathematical reasoning benchmarks and human preference datasets demonstrate consistent performance gains, indicating that $\\textbf{RePO}$ is an effective and human-aligned approach for training large language models.","url_abs":"https://arxiv.org/abs/2606.09124","url_pdf":"https://arxiv.org/pdf/2606.09124","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2606.09124","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2606.09124"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/OpenRLHF/OpenRLHF","reach":{"status":"ok","spdx":"Apache-2.0"}},{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/YSLIU627/Regularized-Preference-Optimization","reach":null},{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/Vance0124/Token-level-Direct-Preference-Optimization","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":12,"ran_violates":1,"ran_draft_wrong":1,"ran_fixture":1,"unverified":3},"by_repo_kind":{"found_in_text":{"samples":18,"ran":15,"repositories":3}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"0edeb138e07fbecc","entry":"aggregate_loss","repo":"OpenRLHF/OpenRLHF","repo_kind":"found_in_text","path":"openrlhf/models/loss.py","file_url":"https://github.com/OpenRLHF/OpenRLHF/blob/HEAD/openrlhf/models/loss.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0edeb138e07fbecc"}},{"code_sha256_prefix":"5a84f9e5640fe497","entry":"apply_chat_template","repo":"YSLIU627/Regularized-Preference-Optimization","repo_kind":"found_in_text","path":"alignment-handbook/src/alignment/data.py","file_url":"https://github.com/YSLIU627/Regularized-Preference-Optimization/blob/HEAD/alignment-handbook/src/alignment/data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5a84f9e5640fe497"}},{"code_sha256_prefix":"c146f3cb0e5259a0","entry":"compute_approx_kl","repo":"OpenRLHF/OpenRLHF","repo_kind":"found_in_text","path":"openrlhf/models/utils.py","file_url":"https://github.com/OpenRLHF/OpenRLHF/blob/HEAD/openrlhf/models/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c146f3cb0e5259a0"}},{"code_sha256_prefix":"60d73ddf6afc1bc3","entry":"compute_approx_kl","repo":"YSLIU627/Regularized-Preference-Optimization","repo_kind":"found_in_text","path":"OpenRLHF/openrlhf/models/utils.py","file_url":"https://github.com/YSLIU627/Regularized-Preference-Optimization/blob/HEAD/OpenRLHF/openrlhf/models/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"60d73ddf6afc1bc3"}},{"code_sha256_prefix":"812f2e3f5cda7c93","entry":"compute_reward","repo":"OpenRLHF/OpenRLHF","repo_kind":"found_in_text","path":"openrlhf/models/utils.py","file_url":"https://github.com/OpenRLHF/OpenRLHF/blob/HEAD/openrlhf/models/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"812f2e3f5cda7c93"}},{"code_sha256_prefix":"0edeacd7a9034f53","entry":"extract_docstring","repo":"YSLIU627/Regularized-Preference-Optimization","repo_kind":"found_in_text","path":"alignment-handbook/src/alignment/decontaminate.py","file_url":"https://github.com/YSLIU627/Regularized-Preference-Optimization/blob/HEAD/alignment-handbook/src/alignment/decontaminate.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0edeacd7a9034f53"}},{"code_sha256_prefix":"557aef158c28bb84","entry":"get_loss_batch_info","repo":"OpenRLHF/OpenRLHF","repo_kind":"found_in_text","path":"openrlhf/utils/loss_utils.py","file_url":"https://github.com/OpenRLHF/OpenRLHF/blob/HEAD/openrlhf/utils/loss_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"557aef158c28bb84"}},{"code_sha256_prefix":"a9fb683d7652195d","entry":"get_seqlens_in_batch","repo":"YSLIU627/Regularized-Preference-Optimization","repo_kind":"found_in_text","path":"OpenRLHF/openrlhf/models/packing_utils.py","file_url":"https://github.com/YSLIU627/Regularized-Preference-Optimization/blob/HEAD/OpenRLHF/openrlhf/models/packing_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a9fb683d7652195d"}},{"code_sha256_prefix":"08e261d2bbc45dbc","entry":"get_unpad_data","repo":"YSLIU627/Regularized-Preference-Optimization","repo_kind":"found_in_text","path":"OpenRLHF/openrlhf/models/packing_utils.py","file_url":"https://github.com/YSLIU627/Regularized-Preference-Optimization/blob/HEAD/OpenRLHF/openrlhf/models/packing_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"08e261d2bbc45dbc"}},{"code_sha256_prefix":"4253aac010c4504f","entry":"is_openai_format","repo":"YSLIU627/Regularized-Preference-Optimization","repo_kind":"found_in_text","path":"alignment-handbook/src/alignment/data.py","file_url":"https://github.com/YSLIU627/Regularized-Preference-Optimization/blob/HEAD/alignment-handbook/src/alignment/data.py","link_basis":"plan_row","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4253aac010c4504f"}},{"code_sha256_prefix":"959df7824bf09cc1","entry":"log_probs_from_logits","repo":"YSLIU627/Regularized-Preference-Optimization","repo_kind":"found_in_text","path":"OpenRLHF/openrlhf/models/utils.py","file_url":"https://github.com/YSLIU627/Regularized-Preference-Optimization/blob/HEAD/OpenRLHF/openrlhf/models/utils.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"959df7824bf09cc1"}},{"code_sha256_prefix":"75afa2affd2a8e36","entry":"log_probs_from_logits","repo":"OpenRLHF/OpenRLHF","repo_kind":"found_in_text","path":"openrlhf/models/utils.py","file_url":"https://github.com/OpenRLHF/OpenRLHF/blob/HEAD/openrlhf/models/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"75afa2affd2a8e36"}},{"code_sha256_prefix":"f43e59d239019e86","entry":"normalize_whitespace","repo":"YSLIU627/Regularized-Preference-Optimization","repo_kind":"found_in_text","path":"alignment-handbook/src/alignment/decontaminate.py","file_url":"https://github.com/YSLIU627/Regularized-Preference-Optimization/blob/HEAD/alignment-handbook/src/alignment/decontaminate.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f43e59d239019e86"}},{"code_sha256_prefix":"f4db4c93fe22b070","entry":"preprocess_data","repo":"YSLIU627/Regularized-Preference-Optimization","repo_kind":"found_in_text","path":"OpenRLHF/openrlhf/datasets/unpaired_preference_dataset.py","file_url":"https://github.com/YSLIU627/Regularized-Preference-Optimization/blob/HEAD/OpenRLHF/openrlhf/datasets/unpaired_preference_dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f4db4c93fe22b070"}},{"code_sha256_prefix":"cbd5b92bbd58d7d3","entry":"tdpo_loss","repo":"Vance0124/Token-level-Direct-Preference-Optimization","repo_kind":"found_in_text","path":"trainers.py","file_url":"https://github.com/Vance0124/Token-level-Direct-Preference-Optimization/blob/HEAD/trainers.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cbd5b92bbd58d7d3"}},{"code_sha256_prefix":"99a2d91ea5aed978","entry":"compute_reward","repo":"YSLIU627/Regularized-Preference-Optimization","repo_kind":"found_in_text","path":"OpenRLHF/openrlhf/models/utils.py","file_url":"https://github.com/YSLIU627/Regularized-Preference-Optimization/blob/HEAD/OpenRLHF/openrlhf/models/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"99a2d91ea5aed978"}},{"code_sha256_prefix":"5110d3bb195169a8","entry":"get_tensor_in_current_ring_attn_rank","repo":"OpenRLHF/OpenRLHF","repo_kind":"found_in_text","path":"openrlhf/models/ring_attn_utils.py","file_url":"https://github.com/OpenRLHF/OpenRLHF/blob/HEAD/openrlhf/models/ring_attn_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5110d3bb195169a8"}},{"code_sha256_prefix":"e93117d99aa6b210","entry":"reset_ring_attn_position_ids","repo":"OpenRLHF/OpenRLHF","repo_kind":"found_in_text","path":"openrlhf/models/ring_attn_utils.py","file_url":"https://github.com/OpenRLHF/OpenRLHF/blob/HEAD/openrlhf/models/ring_attn_utils.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e93117d99aa6b210"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.AI","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}