{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/the-mirage-of-model-editing-revisiting","title":"The Mirage of Model Editing: Revisiting Evaluation in the Wild","arxiv_id":"2502.11177","date":"2025-02-16","proceeding":null,"authors":["Wanli Yang","Fei Sun","Jiajun Tan","Xinyu Ma","Qi Cao","Dawei Yin","HuaWei Shen","Xueqi Cheng"],"abstract":"Despite near-perfect results in artificial evaluations, the effectiveness of model editing in real-world applications remains unexplored. To bridge this gap, we propose to study model editing in question answering (QA) by establishing a rigorous evaluation practice to assess the effectiveness of editing methods in correcting LLMs' errors. It consists of QAEdit, a new benchmark derived from popular QA datasets, and a standardized evaluation framework. Our single editing experiments indicate that current editing methods perform substantially worse than previously reported (38.5% vs. ~96%). Through module analysis and controlled experiments, we demonstrate that this performance decline stems from issues in evaluation practices of prior editing research. One key issue is the inappropriate use of teacher forcing in testing prevents error propagation by feeding ground truth tokens (inaccessible in real-world scenarios) as input. Furthermore, we simulate real-world deployment by sequential editing, revealing that current approaches fail drastically with only 1000 edits. Our analysis provides a fundamental reexamination of both the real-world applicability of existing model editing methods and their evaluation practices, and establishes a rigorous evaluation framework with key insights to advance reliable and practical model editing research.","url_abs":"https://arxiv.org/abs/2502.11177v4","url_pdf":"https://arxiv.org/pdf/2502.11177v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"the-mirage-of-model-editing-revisiting","repo_url":"https://github.com/wanliyoung/revisit-editing-evaluation","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"the-mirage-of-model-editing-revisiting","repo_url":"https://github.com/zjunlp/easyedit","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"model-editing","task_name":"Model Editing"},{"task_slug":"question-answering","task_name":"Question Answering"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2502.11177","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.11177"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/zjunlp/easyedit","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/wanliyoung/revisit-editing-evaluation","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_draft_wrong":2,"ran":2,"ran_fixture":1,"unverified":3},"by_repo_kind":{"official":{"samples":8,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"27659a9c234ffb11","entry":"binary_log_probs","repo":"wanliyoung/revisit-editing-evaluation","repo_kind":"official","path":"easyeditor/trainer/losses.py","file_url":"https://github.com/wanliyoung/revisit-editing-evaluation/blob/HEAD/easyeditor/trainer/losses.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"27659a9c234ffb11"}},{"code_sha256_prefix":"920b394e7ad80c53","entry":"hierarchical_subsequence","repo":"wanliyoung/revisit-editing-evaluation","repo_kind":"official","path":"easyeditor/util/nethook.py","file_url":"https://github.com/wanliyoung/revisit-editing-evaluation/blob/HEAD/easyeditor/util/nethook.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"920b394e7ad80c53"}},{"code_sha256_prefix":"e078eeed20f03838","entry":"masked_mean","repo":"wanliyoung/revisit-editing-evaluation","repo_kind":"official","path":"easyeditor/trainer/losses.py","file_url":"https://github.com/wanliyoung/revisit-editing-evaluation/blob/HEAD/easyeditor/trainer/losses.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e078eeed20f03838"}},{"code_sha256_prefix":"70f6ab8bde55420e","entry":"recursive_copy","repo":"wanliyoung/revisit-editing-evaluation","repo_kind":"official","path":"easyeditor/util/nethook.py","file_url":"https://github.com/wanliyoung/revisit-editing-evaluation/blob/HEAD/easyeditor/util/nethook.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"70f6ab8bde55420e"}},{"code_sha256_prefix":"440ff98c2b1ae1aa","entry":"subsequence","repo":"wanliyoung/revisit-editing-evaluation","repo_kind":"official","path":"easyeditor/util/nethook.py","file_url":"https://github.com/wanliyoung/revisit-editing-evaluation/blob/HEAD/easyeditor/util/nethook.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"440ff98c2b1ae1aa"}},{"code_sha256_prefix":"d2e4c1979dd70b9a","entry":"get_model","repo":"wanliyoung/revisit-editing-evaluation","repo_kind":"official","path":"easyeditor/trainer/models.py","file_url":"https://github.com/wanliyoung/revisit-editing-evaluation/blob/HEAD/easyeditor/trainer/models.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d2e4c1979dd70b9a"}},{"code_sha256_prefix":"2defe42cf1a7f301","entry":"get_tokenizer","repo":"wanliyoung/revisit-editing-evaluation","repo_kind":"official","path":"easyeditor/trainer/models.py","file_url":"https://github.com/wanliyoung/revisit-editing-evaluation/blob/HEAD/easyeditor/trainer/models.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2defe42cf1a7f301"}},{"code_sha256_prefix":"bcc592aa5c1e2bb3","entry":"kl_loc_loss","repo":"wanliyoung/revisit-editing-evaluation","repo_kind":"official","path":"easyeditor/trainer/losses.py","file_url":"https://github.com/wanliyoung/revisit-editing-evaluation/blob/HEAD/easyeditor/trainer/losses.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bcc592aa5c1e2bb3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}