{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2606-17464","title":"CheckMIABench: Firm Foundations For Membership Inference Attacks on Language Models","arxiv_id":"2606.17464","date":"2026-06-16","proceeding":null,"authors":["Jeffrey G. Wang","Jason Wang","Marvin Li","Seth Neel"],"abstract":"Membership inference attacks (MIAs) are a canonical way to assess a machine learning model's privacy properties. Although several attempts have been made to evaluate MIAs on language models, the extant literature has suffered numerous difficulties in constructing clean evaluations to test new techniques. In particular, subtle distribution shifts between member and non-member sets can undermine the statistical validity of MIAs; recent work has underscored this by showing that \"blind\" methods with no access to the underlying model can perform far better than published methods on the same benchmarks. This paper constructs a benchmark for principled evaluation of MIAs against LLMs, by leveraging the insight that training data before and after a fixed point during training are drawn from the same distribution. Therefore, all open-source models with intermediate checkpoints and public training data can be converted into MIA testbeds. We apply our framework to a half-dozen published attacks on the Pythia and OLMo family of models, from 70M to 7B parameters. To facilitate further privacy research, we open-source a modular library for designing and implementing attacks in this setting: https://github.com/safr-ai-lab/pandora_llm.","url_abs":"https://arxiv.org/abs/2606.17464","url_pdf":"https://arxiv.org/pdf/2606.17464","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2606.17464","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2606.17464"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/safr-ai-lab/pandora_llm","reach":null}],"summary":{"ran":3,"unverified":3},"by_repo_kind":{"found_in_text":{"samples":6,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"4176099e876e565b","entry":"compute_input_ids_cross_entropy_batch","repo":"safr-ai-lab/pandora_llm","repo_kind":"found_in_text","path":"src/pandora_llm/features/DetectGPT.py","file_url":"https://github.com/safr-ai-lab/pandora_llm/blob/HEAD/src/pandora_llm/features/DetectGPT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4176099e876e565b"}},{"code_sha256_prefix":"12da529ebb71005f","entry":"compute_input_ids_grad_jl","repo":"safr-ai-lab/pandora_llm","repo_kind":"found_in_text","path":"src/pandora_llm/features/JL.py","file_url":"https://github.com/safr-ai-lab/pandora_llm/blob/HEAD/src/pandora_llm/features/JL.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"12da529ebb71005f"}},{"code_sha256_prefix":"27df21e4f0c9d8e0","entry":"pad_sequences_to_length","repo":"safr-ai-lab/pandora_llm","repo_kind":"found_in_text","path":"src/pandora_llm/features/DetectGPT.py","file_url":"https://github.com/safr-ai-lab/pandora_llm/blob/HEAD/src/pandora_llm/features/DetectGPT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"27df21e4f0c9d8e0"}},{"code_sha256_prefix":"9e05d9f93e92fe2b","entry":"compute_dataloader_cross_entropy_batch","repo":"safr-ai-lab/pandora_llm","repo_kind":"found_in_text","path":"src/pandora_llm/features/DetectGPT.py","file_url":"https://github.com/safr-ai-lab/pandora_llm/blob/HEAD/src/pandora_llm/features/DetectGPT.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9e05d9f93e92fe2b"}},{"code_sha256_prefix":"bf1e3d3ab71d8909","entry":"compute_dataloader_jl_balanced","repo":"safr-ai-lab/pandora_llm","repo_kind":"found_in_text","path":"src/pandora_llm/features/JL.py","file_url":"https://github.com/safr-ai-lab/pandora_llm/blob/HEAD/src/pandora_llm/features/JL.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bf1e3d3ab71d8909"}},{"code_sha256_prefix":"20b0b75df49ac6c1","entry":"compute_input_ids_grad_jl_balanced","repo":"safr-ai-lab/pandora_llm","repo_kind":"found_in_text","path":"src/pandora_llm/features/JL.py","file_url":"https://github.com/safr-ai-lab/pandora_llm/blob/HEAD/src/pandora_llm/features/JL.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"20b0b75df49ac6c1"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.LG","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}