{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/learn-from-mistakes-through-cooperative","title":"Learning from Mistakes via Cooperative Study Assistant for Large Language Models","arxiv_id":"2305.13829","date":"2023-05-23","proceeding":null,"authors":["Danqing Wang","Lei LI"],"abstract":"Large language models (LLMs) have demonstrated their potential to refine their generation based on their own feedback. However, the feedback from LLM itself is often inaccurate, thereby limiting its benefits. In this paper, we propose Study Assistant for Large LAnguage Model (SALAM), a novel framework with an auxiliary agent to assist the main LLM in learning from mistakes through interactive cooperation. In the gathering phase, the student assistant agent probes the main LLM, analyzes its errors, and collects the interaction in a mistake memory. During the examination phase, the study assistant provides guidelines by retrieving relevant cases to help the main LLM anticipate and avoid similar errors. We first investigate the effectiveness of a general study assistant and then customize it to provide LLM-specific guidance through imitation learning from successful guidance experiences. Our experiments on three LLMs using two challenging frameworks demonstrate that SALAM can significantly boost LLMs by an accuracy margin of up to 6.6 on BBH and 12.6 on BBQ.","url_abs":"https://arxiv.org/abs/2305.13829v3","url_pdf":"https://arxiv.org/pdf/2305.13829v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"learn-from-mistakes-through-cooperative","repo_url":"https://github.com/dqwang122/salam","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"imitation-learning","task_name":"Imitation Learning"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"large-language-model","task_name":"Large Language Model"}],"methods":[{"method_slug":"base","method_name":"BASE"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2305.13829","atlas_url":"https://app.syntology.ai/?focus=2305.13829","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.13829"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/dqwang122/salam","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/madaan/self-refine","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_draft_wrong":2,"ran":1,"unverified":10},"by_repo_kind":{"official":{"samples":9,"ran":3,"repositories":1},"found_in_text":{"samples":4,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"e5bacba887a721ef","entry":"reduce_mean","repo":"dqwang122/salam","repo_kind":"official","path":"src/utils/utils.py","file_url":"https://github.com/dqwang122/salam/blob/HEAD/src/utils/utils.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e5bacba887a721ef"}},{"code_sha256_prefix":"174bfbbeb1f29e12","entry":"reduce_std","repo":"dqwang122/salam","repo_kind":"official","path":"src/utils/utils.py","file_url":"https://github.com/dqwang122/salam/blob/HEAD/src/utils/utils.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"174bfbbeb1f29e12"}},{"code_sha256_prefix":"5874b0a8910897fd","entry":"reduce_sum","repo":"dqwang122/salam","repo_kind":"official","path":"src/utils/utils.py","file_url":"https://github.com/dqwang122/salam/blob/HEAD/src/utils/utils.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5874b0a8910897fd"}},{"code_sha256_prefix":"b735e29e0de47813","entry":"check_result","repo":"dqwang122/salam","repo_kind":"official","path":"src/utils/tools.py","file_url":"https://github.com/dqwang122/salam/blob/HEAD/src/utils/tools.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b735e29e0de47813"}},{"code_sha256_prefix":"a14da09015edfc5a","entry":"make_supervised_data_module","repo":"dqwang122/salam","repo_kind":"official","path":"src/finetune.py","file_url":"https://github.com/dqwang122/salam/blob/HEAD/src/finetune.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a14da09015edfc5a"}},{"code_sha256_prefix":"94f412dc2125cdd8","entry":"normalize_scores","repo":"madaan/self-refine","repo_kind":"found_in_text","path":"src/acronym/run_mcts.py","file_url":"https://github.com/madaan/self-refine/blob/HEAD/src/acronym/run_mcts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"94f412dc2125cdd8"}},{"code_sha256_prefix":"024bf912c9a8a3df","entry":"parse_scores","repo":"madaan/self-refine","repo_kind":"found_in_text","path":"src/acronym/run_mcts.py","file_url":"https://github.com/madaan/self-refine/blob/HEAD/src/acronym/run_mcts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"024bf912c9a8a3df"}},{"code_sha256_prefix":"995424bb47930b36","entry":"postprocess_output","repo":"dqwang122/salam","repo_kind":"official","path":"src/utils/model_utils.py","file_url":"https://github.com/dqwang122/salam/blob/HEAD/src/utils/model_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"995424bb47930b36"}},{"code_sha256_prefix":"980b5f9982142070","entry":"preprocess","repo":"dqwang122/salam","repo_kind":"official","path":"src/finetune.py","file_url":"https://github.com/dqwang122/salam/blob/HEAD/src/finetune.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"980b5f9982142070"}},{"code_sha256_prefix":"31f5087bd3a92aeb","entry":"retry_parse_fail_prone_cmd","repo":"madaan/self-refine","repo_kind":"found_in_text","path":"src/utils.py","file_url":"https://github.com/madaan/self-refine/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"31f5087bd3a92aeb"}},{"code_sha256_prefix":"0b3886637ce87213","entry":"simple_check_result","repo":"dqwang122/salam","repo_kind":"official","path":"src/utils/tools.py","file_url":"https://github.com/dqwang122/salam/blob/HEAD/src/utils/tools.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0b3886637ce87213"}},{"code_sha256_prefix":"19f7c8ee0a8ecca1","entry":"train_test_split","repo":"dqwang122/salam","repo_kind":"official","path":"src/utils/tools.py","file_url":"https://github.com/dqwang122/salam/blob/HEAD/src/utils/tools.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"19f7c8ee0a8ecca1"}},{"code_sha256_prefix":"2f14a94002fc598c","entry":"weighted_sum","repo":"madaan/self-refine","repo_kind":"found_in_text","path":"src/acronym/run_mcts.py","file_url":"https://github.com/madaan/self-refine/blob/HEAD/src/acronym/run_mcts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2f14a94002fc598c"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}