{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/fool-me-twice-entailment-from-wikipedia","title":"Fool Me Twice: Entailment from Wikipedia Gamification","arxiv_id":"2104.04725","date":"2021-04-10","proceeding":"NAACL 2021 4","authors":["Julian Martin Eisenschlos","Bhuwan Dhingra","Jannis Bulian","Benjamin Börschinger","Jordan Boyd-Graber"],"abstract":"We release FoolMeTwice (FM2 for short), a large dataset of challenging entailment pairs collected through a fun multi-player game. Gamification encourages adversarial examples, drastically lowering the number of examples that can be solved using \"shortcuts\" compared to other popular entailment datasets. Players are presented with two tasks. The first task asks the player to write a plausible claim based on the evidence from a Wikipedia page. The second one shows two plausible claims written by other players, one of which is false, and the goal is to identify it before the time runs out. Players \"pay\" to see clues retrieved from the evidence pool: the more evidence the player needs, the harder the claim. Game-play between motivated players leads to diverse strategies for crafting claims, such as temporal inference and diverting to unrelated evidence, and results in higher quality data for the entailment and evidence retrieval tasks. We open source the dataset and the game code.","url_abs":"https://arxiv.org/abs/2104.04725v1","url_pdf":"https://arxiv.org/pdf/2104.04725v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"fool-me-twice-entailment-from-wikipedia","repo_url":"https://github.com/google-research/fool-me-twice","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"retrieval","task_name":"Retrieval"}],"methods":[],"datasets_introduced":[{"slug":"fm2","name":"FM2","full_name":"FoolMeTwice"}],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2104.04725","atlas_url":"https://app.syntology.ai/?focus=2104.04725","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.04725"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/google-research/fool-me-twice","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":3,"unverified":1},"by_repo_kind":{"official":{"samples":4,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"223201afd2e80500","entry":"get_users","repo":"google-research/fool-me-twice","repo_kind":"official","path":"util/create_workflow.py","file_url":"https://github.com/google-research/fool-me-twice/blob/HEAD/util/create_workflow.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"223201afd2e80500"}},{"code_sha256_prefix":"6a44f386f8de7f3d","entry":"read_categories","repo":"google-research/fool-me-twice","repo_kind":"official","path":"util/download_wiki.py","file_url":"https://github.com/google-research/fool-me-twice/blob/HEAD/util/download_wiki.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6a44f386f8de7f3d"}},{"code_sha256_prefix":"4f2f94350b76b3ba","entry":"save_claims","repo":"google-research/fool-me-twice","repo_kind":"official","path":"util/create_workflow.py","file_url":"https://github.com/google-research/fool-me-twice/blob/HEAD/util/create_workflow.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4f2f94350b76b3ba"}},{"code_sha256_prefix":"0a3bf318045dcd25","entry":"get_comparisons","repo":"google-research/fool-me-twice","repo_kind":"official","path":"util/create_workflow.py","file_url":"https://github.com/google-research/fool-me-twice/blob/HEAD/util/create_workflow.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0a3bf318045dcd25"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}