{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/negated-lama-birds-cannot-fly","title":"Negated and Misprimed Probes for Pretrained Language Models: Birds Can Talk, But Cannot Fly","arxiv_id":"1911.03343","date":"2019-11-08","proceeding":"ACL 2020 6","authors":["Nora Kassner","Hinrich Schütze"],"abstract":"Building on Petroni et al. (2019), we propose two new probing tasks analyzing factual knowledge stored in Pretrained Language Models (PLMs). (1) Negation. We find that PLMs do not distinguish between negated (\"Birds cannot [MASK]\") and non-negated (\"Birds can [MASK]\") cloze questions. (2) Mispriming. Inspired by priming methods in human psychology, we add \"misprimes\" to cloze questions (\"Talk? Birds can [MASK]\"). We find that PLMs are easily distracted by misprimes. These results suggest that PLMs still have a long way to go to adequately learn human-like factual knowledge.","url_abs":"https://arxiv.org/abs/1911.03343v3","url_pdf":"https://arxiv.org/pdf/1911.03343v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"negated-lama-birds-cannot-fly","repo_url":"https://github.com/facebookresearch/LAMA","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null},{"paper_slug":"negated-lama-birds-cannot-fly","repo_url":"https://github.com/norakassner/LAMA_primed_negated","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok"}}],"tasks":[{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"negation","task_name":"Negation"},{"task_slug":"question-answering","task_name":"Question Answering"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1911.03343","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1911.03343"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/norakassner/LAMA_primed_negated","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/LAMA","reach":null}],"summary":{"ran_draft_wrong":1},"by_repo_kind":{"official":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"0184bb9d94afd92d","entry":"get_ConceptNet_parameters","repo":"facebookresearch/LAMA","repo_kind":"official","path":"scripts/run_experiments.py","file_url":"https://github.com/facebookresearch/LAMA/blob/HEAD/scripts/run_experiments.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"0184bb9d94afd92d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}