{"url":"/method/dpo","slug":"dpo","name":"DPO","full_name":"Direct Preference Optimization","full_name_withheld":false,"description_markdown":null,"description_state":"absent","introduced_year":null,"introduced_by":{"title":"Direct Preference Optimization: Your Language Model is Secretly a Reward Model","paper":"/paper/direct-preference-optimization-your-language","first_author":"Rafael Rafailov","n_authors":6,"url_abs":null,"archive_paper_url":"https://paperswithcode.com/paper/direct-preference-optimization-your-language"},"source":{"url":"https://arxiv.org/abs/2305.18290v3","title":"Direct Preference Optimization: Your Language Model is Secretly a Reward Model","url_on_a_paper_host":true},"code_snippet_url":null,"code_snippet_url_on_a_code_host":false,"categories":[{"area":"Reinforcement Learning","area_id":"reinforcement-learning","collection":"Offline Reinforcement Learning Methods","url":"/methods/category/offline-reinforcement-learning-methods","pwc_aliases":[]}],"n_papers_tagged":409,"archive_num_papers":409,"papers_newest_first":[{"paper":"/paper/multi-preference-lambda-weighted-listwise-dpo","title":"Multi-Preference Lambda-weighted Listwise DPO for Dynamic Preference Alignment","date":"2025-06-24","arxiv_id":"2506.19780","n_code_links":1,"syntology":null},{"paper":null,"title":"Smart-LLaMA-DPO: Reinforced Large Language Model for Explainable Smart Contract Vulnerability Detection","date":"2025-06-23","arxiv_id":"2506.18245","n_code_links":0,"syntology":null},{"paper":"/paper/video-salmonn-2-captioning-enhanced-audio","title":"video-SALMONN 2: Captioning-Enhanced Audio-Visual Large Language Models","date":"2025-06-18","arxiv_id":"2506.15220","n_code_links":1,"syntology":{"ran":3,"of":8,"unverified":5,"pointer_only":0}},{"paper":"/paper/tgdpo-harnessing-token-level-reward-guidance","title":"TGDPO: Harnessing Token-Level Reward Guidance for Enhancing Direct Preference Optimization","date":"2025-06-17","arxiv_id":"2506.14574","n_code_links":1,"syntology":{"ran":2,"of":3,"unverified":1,"pointer_only":3}},{"paper":null,"title":"Alignment Quality Index (AQI) : Beyond Refusals: AQI as an Intrinsic Alignment Diagnostic via Latent Geometry, Cluster Divergence, and Layer wise Pooled Representations","date":"2025-06-16","arxiv_id":"2506.13901","n_code_links":0,"syntology":null},{"paper":null,"title":"CAMS: A CityGPT-Powered Agentic Framework for Urban Human Mobility Simulation","date":"2025-06-16","arxiv_id":"2506.13599","n_code_links":0,"syntology":null},{"paper":null,"title":"From Judgment to Interference: Early Stopping LLM Harmful Outputs via Streaming Content Monitoring","date":"2025-06-11","arxiv_id":"2506.09996","n_code_links":0,"syntology":null},{"paper":null,"title":"Towards Bridging the Reward-Generation Gap in Direct Alignment Algorithms","date":"2025-06-11","arxiv_id":"2506.09457","n_code_links":0,"syntology":null},{"paper":"/paper/vision-matters-simple-visual-perturbations","title":"Vision Matters: Simple Visual Perturbations Can Boost Multimodal Math Reasoning","date":"2025-06-11","arxiv_id":"2506.09736","n_code_links":1,"syntology":null},{"paper":"/paper/explicit-preference-optimization-no-need-for","title":"Explicit Preference Optimization: No Need for an Implicit Reward Model","date":"2025-06-09","arxiv_id":"2506.07492","n_code_links":1,"syntology":null},{"paper":"/paper/levo-high-quality-song-generation-with-multi","title":"LeVo: High-Quality Song Generation with Multi-Preference Alignment","date":"2025-06-09","arxiv_id":"2506.07520","n_code_links":1,"syntology":{"ran":2,"of":2,"unverified":0,"pointer_only":2}},{"paper":null,"title":"QA-LIGN: Aligning LLMs through Constitutionally Decomposed QA","date":"2025-06-09","arxiv_id":"2506.08123","n_code_links":0,"syntology":null},{"paper":"/paper/leanpo-lean-preference-optimization-for","title":"LeanPO: Lean Preference Optimization for Likelihood Alignment in Video-LLMs","date":"2025-06-05","arxiv_id":"2506.05260","n_code_links":1,"syntology":null},{"paper":null,"title":"Aligning Large Language Models with Implicit Preferences from User-Generated Content","date":"2025-06-04","arxiv_id":"2506.04463","n_code_links":0,"syntology":null},{"paper":"/paper/superwriter-reflection-driven-long-form","title":"SuperWriter: Reflection-Driven Long-Form Generation with Large Language Models","date":"2025-06-04","arxiv_id":"2506.04180","n_code_links":1,"syntology":{"ran":6,"of":6,"unverified":0,"pointer_only":6}},{"paper":null,"title":"Protein Inverse Folding From Structure Feedback","date":"2025-06-03","arxiv_id":"2506.03028","n_code_links":0,"syntology":null},{"paper":null,"title":"Smoothed Preference Optimization via ReNoise Inversion for Aligning Diffusion Models with Varied Human Preferences","date":"2025-06-03","arxiv_id":"2506.02698","n_code_links":0,"syntology":null},{"paper":null,"title":"Understanding the Impact of Sampling Quality in Direct Preference Optimization","date":"2025-06-03","arxiv_id":"2506.04272","n_code_links":0,"syntology":null},{"paper":"/paper/if-guide-influence-function-guided","title":"IF-GUIDE: Influence Function-Guided Detoxification of LLMs","date":"2025-06-02","arxiv_id":"2506.01790","n_code_links":1,"syntology":{"ran":0,"of":1,"unverified":1,"pointer_only":0}},{"paper":null,"title":"Generalizable LLM Learning of Graph Synthetic Data with Reinforcement Learning","date":"2025-06-01","arxiv_id":"2506.00845","n_code_links":0,"syntology":null},{"paper":"/paper/harnessing-negative-signals-reinforcement","title":"Harnessing Negative Signals: Reinforcement Distillation from Teacher Data for LLM Reasoning","date":"2025-05-30","arxiv_id":"2505.24850","n_code_links":1,"syntology":null},{"paper":null,"title":"Afterburner: Reinforcement Learning Facilitates Self-Improving Code Efficiency Optimization","date":"2025-05-29","arxiv_id":"2505.23387","n_code_links":0,"syntology":null},{"paper":null,"title":"Differential Information: An Information-Theoretic Perspective on Preference Optimization","date":"2025-05-29","arxiv_id":"2505.23761","n_code_links":0,"syntology":null},{"paper":null,"title":"Distortion of AI Alignment: Does Preference Optimization Optimize for Preferences?","date":"2025-05-29","arxiv_id":"2505.23749","n_code_links":0,"syntology":null},{"paper":null,"title":"Improving Multilingual Social Media Insights: Aspect-based Comment Analysis","date":"2025-05-29","arxiv_id":"2505.23037","n_code_links":0,"syntology":null},{"paper":null,"title":"MCP Safety Training: Learning to Refuse Falsely Benign MCP Exploits using Improved Preference Alignment","date":"2025-05-29","arxiv_id":"2505.23634","n_code_links":0,"syntology":null},{"paper":null,"title":"Proximalized Preference Optimization for Diverse Feedback Types: A Decomposed Perspective on DPO","date":"2025-05-29","arxiv_id":"2505.23316","n_code_links":0,"syntology":null},{"paper":null,"title":"Reinforcement Learning for Better Verbalized Confidence in Long-Form Generation","date":"2025-05-29","arxiv_id":"2505.23912","n_code_links":0,"syntology":null},{"paper":null,"title":"D-Fusion: Direct Preference Optimization for Aligning Diffusion Models with Visually Consistent Samples","date":"2025-05-28","arxiv_id":"2505.22002","n_code_links":0,"syntology":null},{"paper":"/paper/enhancing-paraphrase-type-generation-the","title":"Enhancing Paraphrase Type Generation: The Impact of DPO and RLHF Evaluated with Human-Ranked Data","date":"2025-05-28","arxiv_id":"2506.02018","n_code_links":1,"syntology":null}],"papers_shown":30,"tasks":[{"task":"/task/language-modelling","name":"Language Modelling","papers":70},{"task":"/task/language-modeling","name":"Language Modeling","papers":61},{"task":"/task/instruction-following","name":"Instruction Following","papers":29},{"task":"/task/math","name":"Math","papers":26},{"task":"/task/large-language-model","name":"Large Language Model","papers":25},{"task":"/task/reinforcement-learning-1","name":"Reinforcement Learning (RL)","papers":25},{"task":"/task/reinforcement-learning-2","name":"reinforcement-learning","papers":23},{"task":"/task/reinforcement-learning","name":"Reinforcement Learning","papers":21},{"task":"/task/question-answering","name":"Question Answering","papers":19},{"task":"/task/mathematical-reasoning","name":"Mathematical Reasoning","papers":18},{"task":"/task/hallucination","name":"Hallucination","papers":16},{"task":"/task/model","name":"model","papers":15},{"task":"/task/diversity","name":"Diversity","papers":14},{"task":"/task/image-generation","name":"Image Generation","papers":13},{"task":"/task/gsm8k","name":"GSM8K","papers":12},{"task":"/task/text-generation","name":"Text Generation","papers":9},{"task":"/task/computational-efficiency","name":"Computational Efficiency","papers":8},{"task":"/task/text-to-image-generation","name":"Text-to-Image Generation","papers":8},{"task":"/task/safety-alignment","name":"Safety Alignment","papers":7},{"task":"/task/sentence","name":"Sentence","papers":7}],"tasks_shown":20,"n_tasks":181,"usage_by_year":[{"year":"2023","papers":20},{"year":"2024","papers":240},{"year":"2025","papers":149}],"row_source":"methods_table","archive":{"source":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","archive_url":"https://paperswithcode.com/method/dpo"},"syntology_read_at":"2026-09-24T18:15:14+00:00"}