{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/some-things-are-more-cringe-than-others","title":"Some things are more CRINGE than others: Iterative Preference Optimization with the Pairwise Cringe Loss","arxiv_id":"2312.16682","date":"2023-12-27","proceeding":null,"authors":["Jing Xu","Andrew Lee","Sainbayar Sukhbaatar","Jason Weston"],"abstract":"Practitioners commonly align large language models using pairwise preferences, i.e., given labels of the type response A is preferred to response B for a given input. Perhaps less commonly, methods have also been developed for binary feedback, i.e. training models given labels of type response A is good or bad. We show how an existing performant binary feedback method, the Cringe Loss (Adolphs et al., 2022), can be generalized to the pairwise preference setting using a simple soft margin extension. Pairwise Cringe Loss is straightforward to implement and efficient to train, and we find it outperforms state-of-the-art preference optimization algorithms such as PPO and DPO on the AlpacaFarm benchmark. We show that iterations of training of our model are important for improved results, and that we can generalize DPO to Iterative DPO in the same way.","url_abs":"https://arxiv.org/abs/2312.16682v2","url_pdf":"https://arxiv.org/pdf/2312.16682v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"some-things-are-more-cringe-than-others","repo_url":"https://github.com/facebookresearch/RAM","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[],"methods":[{"method_slug":"align","method_name":"ALIGN"},{"method_slug":"dpo","method_name":"DPO"},{"method_slug":"entropy-regularization","method_name":"Entropy Regularization"},{"method_slug":"ppo","method_name":"PPO"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2312.16682","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.16682"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/RAM","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":4,"unverified":4},"by_repo_kind":{"listed":{"samples":8,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"e14330ae6c40d236","entry":"LN","repo":"facebookresearch/RAM","repo_kind":"listed","path":"projects/cocomix/models/modeling_gpt2_cocomix.py","file_url":"https://github.com/facebookresearch/RAM/blob/HEAD/projects/cocomix/models/modeling_gpt2_cocomix.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e14330ae6c40d236"}},{"code_sha256_prefix":"370b85373b9b3300","entry":"add_instruct_tokens","repo":"facebookresearch/RAM","repo_kind":"listed","path":"ram/data.py","file_url":"https://github.com/facebookresearch/RAM/blob/HEAD/ram/data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"370b85373b9b3300"}},{"code_sha256_prefix":"26ff74d6538ab161","entry":"attention_flops_per_token","repo":"facebookresearch/RAM","repo_kind":"listed","path":"projects/mta/transformer.py","file_url":"https://github.com/facebookresearch/RAM/blob/HEAD/projects/mta/transformer.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"26ff74d6538ab161"}},{"code_sha256_prefix":"62134cd7798c1a00","entry":"get_content_key","repo":"facebookresearch/RAM","repo_kind":"listed","path":"projects/mta/single_json.py","file_url":"https://github.com/facebookresearch/RAM/blob/HEAD/projects/mta/single_json.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"62134cd7798c1a00"}},{"code_sha256_prefix":"1d34daf1ab8748db","entry":"cross_entropy","repo":"facebookresearch/RAM","repo_kind":"listed","path":"projects/mta/transformer.py","file_url":"https://github.com/facebookresearch/RAM/blob/HEAD/projects/mta/transformer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1d34daf1ab8748db"}},{"code_sha256_prefix":"712ef575ecebd0ef","entry":"load_prompt_template","repo":"facebookresearch/RAM","repo_kind":"listed","path":"ram/data.py","file_url":"https://github.com/facebookresearch/RAM/blob/HEAD/ram/data.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"712ef575ecebd0ef"}},{"code_sha256_prefix":"9a0a23caa628b0b9","entry":"load_tf_weights_in_gpt2","repo":"facebookresearch/RAM","repo_kind":"listed","path":"projects/cocomix/models/modeling_gpt2_cocomix.py","file_url":"https://github.com/facebookresearch/RAM/blob/HEAD/projects/cocomix/models/modeling_gpt2_cocomix.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9a0a23caa628b0b9"}},{"code_sha256_prefix":"e8baf90dada23a73","entry":"tqdm_distributed","repo":"facebookresearch/RAM","repo_kind":"listed","path":"projects/cocomix/utils.py","file_url":"https://github.com/facebookresearch/RAM/blob/HEAD/projects/cocomix/utils.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e8baf90dada23a73"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}