{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/clutrr-a-diagnostic-benchmark-for-inductive","title":"CLUTRR: A Diagnostic Benchmark for Inductive Reasoning from Text","arxiv_id":"1908.06177","date":"2019-08-16","proceeding":"IJCNLP 2019 11","authors":["Koustuv Sinha","Shagun Sodhani","Jin Dong","Joelle Pineau","William L. Hamilton"],"abstract":"The recent success of natural language understanding (NLU) systems has been troubled by results highlighting the failure of these models to generalize in a systematic and robust way. In this work, we introduce a diagnostic benchmark suite, named CLUTRR, to clarify some key issues related to the robustness and systematicity of NLU systems. Motivated by classic work on inductive logic programming, CLUTRR requires that an NLU system infer kinship relations between characters in short stories. Successful performance on this task requires both extracting relationships between entities, as well as inferring the logical rules governing these relationships. CLUTRR allows us to precisely measure a model's ability for systematic generalization by evaluating on held-out combinations of logical rules, and it allows us to evaluate a model's robustness by adding curated noise facts. Our empirical results highlight a substantial performance gap between state-of-the-art NLU models (e.g., BERT and MAC) and a graph neural network model that works directly with symbolic inputs---with the graph-based model exhibiting both stronger generalization and greater robustness.","url_abs":"https://arxiv.org/abs/1908.06177v2","url_pdf":"https://arxiv.org/pdf/1908.06177v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"clutrr-a-diagnostic-benchmark-for-inductive","repo_url":"https://github.com/facebookresearch/clutrr","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":null},{"paper_slug":"clutrr-a-diagnostic-benchmark-for-inductive","repo_url":"https://github.com/WeiKangda/clutrr","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"clutrr-a-diagnostic-benchmark-for-inductive","repo_url":"https://github.com/koustuvsinha/clutrr","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"clutrr-a-diagnostic-benchmark-for-inductive","repo_url":"https://github.com/koustuvsinha/clutrr-baselines","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"clutrr-a-diagnostic-benchmark-for-inductive","repo_url":"https://github.com/koustuvsinha/clutrr-workshop","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}}],"tasks":[{"task_slug":"diagnostic","task_name":"Diagnostic"},{"task_slug":"graph-neural-network","task_name":"Graph Neural Network"},{"task_slug":"inductive-logic-programming","task_name":"Inductive logic programming"},{"task_slug":"natural-language-understanding","task_name":"Natural Language Understanding"},{"task_slug":"relational-reasoning","task_name":"Relational Reasoning"},{"task_slug":"systematic-generalization","task_name":"Systematic Generalization"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"graph-neural-network","method_name":"Graph Neural Network"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1908.06177","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1908.06177"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/clutrr","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/koustuvsinha/clutrr-baselines","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/koustuvsinha/clutrr-workshop","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/WeiKangda/clutrr","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/koustuvsinha/clutrr","reach":{"status":"ok"}}],"summary":{"ran_draft_wrong":1},"by_repo_kind":{"official":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"b466b4328a313fe6","entry":"splitDataFrameIntoSmaller","repo":"facebookresearch/clutrr","repo_kind":"official","path":"clutrr/utils/data_backend.py","file_url":"https://github.com/facebookresearch/clutrr/blob/HEAD/clutrr/utils/data_backend.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"b466b4328a313fe6"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}