{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/distillation-contrastive-decoding-improving","title":"Distillation Contrastive Decoding: Improving LLMs Reasoning with Contrastive Decoding and Distillation","arxiv_id":"2402.14874","date":"2024-02-21","proceeding":null,"authors":["Phuc Phan","Hieu Tran","Long Phan"],"abstract":"We propose a straightforward approach called Distillation Contrastive Decoding (DCD) to enhance the reasoning capabilities of Large Language Models (LLMs) during inference. In contrast to previous approaches that relied on smaller amateur models or analysis of hidden state differences, DCD employs Contrastive Chain-of-thought Prompting and advanced distillation techniques, including Dropout and Quantization. This approach effectively addresses the limitations of Contrastive Decoding (CD), which typically requires both an expert and an amateur model, thus increasing computational resource demands. By integrating contrastive prompts with distillation, DCD obviates the need for an amateur model and reduces memory usage. Our evaluations demonstrate that DCD significantly enhances LLM performance across a range of reasoning benchmarks, surpassing both CD and existing methods in the GSM8K and StrategyQA datasets.","url_abs":"https://arxiv.org/abs/2402.14874v2","url_pdf":"https://arxiv.org/pdf/2402.14874v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"distillation-contrastive-decoding-improving","repo_url":"https://github.com/pphuc25/distil-cd","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"arithmetic-reasoning","task_name":"Arithmetic Reasoning"},{"task_slug":"gsm8k","task_name":"GSM8K"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"quantization","task_name":"Quantization"},{"task_slug":"strategyqa","task_name":"StrategyQA"}],"methods":[{"method_slug":"dropout","method_name":"Dropout"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2402.14874","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.14874"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pphuc25/distil-cd","reach":{"status":"ok"}}],"summary":{"ran":5,"ran_draft_wrong":1,"unverified":4},"by_repo_kind":{"official":{"samples":10,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":10,"samples":[{"code_sha256_prefix":"8b5fd8ae1f77f932","entry":"answer_cleansing","repo":"pphuc25/distil-cd","repo_kind":"official","path":"dcd_eval/src/chorse/process_data.py","file_url":"https://github.com/pphuc25/distil-cd/blob/HEAD/dcd_eval/src/chorse/process_data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8b5fd8ae1f77f932"}},{"code_sha256_prefix":"b412890ab2fb1902","entry":"create_regex_pattern","repo":"pphuc25/distil-cd","repo_kind":"official","path":"dcd_eval/src/chorse/process_data.py","file_url":"https://github.com/pphuc25/distil-cd/blob/HEAD/dcd_eval/src/chorse/process_data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b412890ab2fb1902"}},{"code_sha256_prefix":"b452cd9127b6afcd","entry":"get_data_name_based_on_condition","repo":"pphuc25/distil-cd","repo_kind":"official","path":"dcd_eval/src/chorse/process_data.py","file_url":"https://github.com/pphuc25/distil-cd/blob/HEAD/dcd_eval/src/chorse/process_data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b452cd9127b6afcd"}},{"code_sha256_prefix":"91b5a26d6a8fe4e5","entry":"is_correct","repo":"pphuc25/distil-cd","repo_kind":"official","path":"dcd_eval/src/chorse/utils.py","file_url":"https://github.com/pphuc25/distil-cd/blob/HEAD/dcd_eval/src/chorse/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"91b5a26d6a8fe4e5"}},{"code_sha256_prefix":"593e5e4d03d4d4df","entry":"load_json","repo":"pphuc25/distil-cd","repo_kind":"official","path":"dcd_eval/src/compute_accuracy.py","file_url":"https://github.com/pphuc25/distil-cd/blob/HEAD/dcd_eval/src/compute_accuracy.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"593e5e4d03d4d4df"}},{"code_sha256_prefix":"7e57dc83814a2145","entry":"set_stop_words","repo":"pphuc25/distil-cd","repo_kind":"official","path":"src/dcd/utils.py","file_url":"https://github.com/pphuc25/distil-cd/blob/HEAD/src/dcd/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7e57dc83814a2145"}},{"code_sha256_prefix":"8e4e00b93dabaf41","entry":"create_prompt","repo":"pphuc25/distil-cd","repo_kind":"official","path":"src/dcd/prompts.py","file_url":"https://github.com/pphuc25/distil-cd/blob/HEAD/src/dcd/prompts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8e4e00b93dabaf41"}},{"code_sha256_prefix":"80618aec403147a1","entry":"create_prompt_student","repo":"pphuc25/distil-cd","repo_kind":"official","path":"src/dcd/prompts.py","file_url":"https://github.com/pphuc25/distil-cd/blob/HEAD/src/dcd/prompts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"80618aec403147a1"}},{"code_sha256_prefix":"d7b3ad79f1f8bd95","entry":"get_model_master_and_amateur","repo":"pphuc25/distil-cd","repo_kind":"official","path":"dcd_eval/src/chorse/load_model.py","file_url":"https://github.com/pphuc25/distil-cd/blob/HEAD/dcd_eval/src/chorse/load_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d7b3ad79f1f8bd95"}},{"code_sha256_prefix":"ad7789fd83efa143","entry":"greedy_search","repo":"pphuc25/distil-cd","repo_kind":"official","path":"src/dcd/generate.py","file_url":"https://github.com/pphuc25/distil-cd/blob/HEAD/src/dcd/generate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ad7789fd83efa143"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}