{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/importance-weighting-can-help-large-language","title":"Importance Weighting Can Help Large Language Models Self-Improve","arxiv_id":"2408.09849","date":"2024-08-19","proceeding":null,"authors":["Chunyang Jiang","Chi-Min Chan","Wei Xue","Qifeng Liu","Yike Guo"],"abstract":"Large language models (LLMs) have shown remarkable capability in numerous tasks and applications. However, fine-tuning LLMs using high-quality datasets under external supervision remains prohibitively expensive. In response, LLM self-improvement approaches have been vibrantly developed recently. The typical paradigm of LLM self-improvement involves training LLM on self-generated data, part of which may be detrimental and should be filtered out due to the unstable data quality. While current works primarily employs filtering strategies based on answer correctness, in this paper, we demonstrate that filtering out correct but with high distribution shift extent (DSE) samples could also benefit the results of self-improvement. Given that the actual sample distribution is usually inaccessible, we propose a new metric called DS weight to approximate DSE, inspired by the Importance Weighting methods. Consequently, we integrate DS weight with self-consistency to comprehensively filter the self-generated samples and fine-tune the language model. Experiments show that with only a tiny valid set (up to 5\\% size of the training set) to compute DS weight, our approach can notably promote the reasoning ability of current LLM self-improvement methods. The resulting performance is on par with methods that rely on external supervision from pre-trained reward models.","url_abs":"https://arxiv.org/abs/2408.09849v2","url_pdf":"https://arxiv.org/pdf/2408.09849v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"importance-weighting-can-help-large-language","repo_url":"https://github.com/rubickkcibur/IWSI","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":null,"task_name":"valid"}],"methods":[{"method_slug":"set","method_name":"SET"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2408.09849","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.09849"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/rubickkcibur/IWSI","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":4,"unverified":3},"by_repo_kind":{"official":{"samples":7,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"335e25ec1842885a","entry":"derive_num_from_answer","repo":"rubickkcibur/IWSI","repo_kind":"official","path":"src/utils/utils.py","file_url":"https://github.com/rubickkcibur/IWSI/blob/HEAD/src/utils/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"335e25ec1842885a"}},{"code_sha256_prefix":"43b2f74033592160","entry":"derive_ratings_from_answer","repo":"rubickkcibur/IWSI","repo_kind":"official","path":"src/utils/utils.py","file_url":"https://github.com/rubickkcibur/IWSI/blob/HEAD/src/utils/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"43b2f74033592160"}},{"code_sha256_prefix":"8caa1a3b5af71291","entry":"format_ground_truth_answer","repo":"rubickkcibur/IWSI","repo_kind":"official","path":"src/utils/utils.py","file_url":"https://github.com/rubickkcibur/IWSI/blob/HEAD/src/utils/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8caa1a3b5af71291"}},{"code_sha256_prefix":"ed1617f12fd3d7a4","entry":"test_batch_loss","repo":"rubickkcibur/IWSI","repo_kind":"official","path":"src/utils/evaluation.py","file_url":"https://github.com/rubickkcibur/IWSI/blob/HEAD/src/utils/evaluation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ed1617f12fd3d7a4"}},{"code_sha256_prefix":"54bb152639bca9fe","entry":"build_model","repo":"rubickkcibur/IWSI","repo_kind":"official","path":"src/model/trainLM.py","file_url":"https://github.com/rubickkcibur/IWSI/blob/HEAD/src/model/trainLM.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"54bb152639bca9fe"}},{"code_sha256_prefix":"90be5ce7647bff49","entry":"qa_preprocess","repo":"rubickkcibur/IWSI","repo_kind":"official","path":"src/model/trainLM.py","file_url":"https://github.com/rubickkcibur/IWSI/blob/HEAD/src/model/trainLM.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"90be5ce7647bff49"}},{"code_sha256_prefix":"e144e5c5b9801622","entry":"test_loss","repo":"rubickkcibur/IWSI","repo_kind":"official","path":"src/utils/evaluation.py","file_url":"https://github.com/rubickkcibur/IWSI/blob/HEAD/src/utils/evaluation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e144e5c5b9801622"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}