{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/clicr-a-dataset-of-clinical-case-reports-for","title":"CliCR: A Dataset of Clinical Case Reports for Machine Reading Comprehension","arxiv_id":"1803.09720","date":"2018-03-26","proceeding":"NAACL 2018 6","authors":["Simon Šuster","Walter Daelemans"],"abstract":"We present a new dataset for machine comprehension in the medical domain. Our\ndataset uses clinical case reports with around 100,000 gap-filling queries\nabout these cases. We apply several baselines and state-of-the-art neural\nreaders to the dataset, and observe a considerable gap in performance (20% F1)\nbetween the best human and machine readers. We analyze the skills required for\nsuccessful answering and show how reader performance varies depending on the\napplicable skills. We find that inferences using domain knowledge and object\ntracking are the most frequently required skills, and that recognizing omitted\ninformation and spatio-temporal reasoning are the most difficult for the\nmachines.","url_abs":"http://arxiv.org/abs/1803.09720v1","url_pdf":"http://arxiv.org/pdf/1803.09720v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"clicr-a-dataset-of-clinical-case-reports-for","repo_url":"https://github.com/clips/clicr","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":null}],"tasks":[{"task_slug":"machine-reading-comprehension","task_name":"Machine Reading Comprehension"},{"task_slug":"object-tracking","task_name":"Object Tracking"},{"task_slug":"reading-comprehension","task_name":"Reading Comprehension"}],"methods":[],"datasets_introduced":[{"slug":"clicr","name":"CliCR","full_name":"CliCR"}],"methods_introduced":[],"results":[{"leaderboard":"/sota/question-answering-on-clicr","task":"Question Answering","dataset":"CliCR","model":"Gated-Attention Reader","rank_in_archive_order":1,"of":2,"metrics":{"F1":"33.9"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-clicr","task":"Question Answering","dataset":"CliCR","model":"Stanford Attentive Reader","rank_in_archive_order":2,"of":2,"metrics":{"F1":"27.2"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/1803.09720","atlas_url":"https://app.syntology.ai/?focus=1803.09720","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1803.09720"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/clips/clicr","reach":null}],"summary":{"ran_draft_wrong":3,"unverified":1},"by_repo_kind":{"official":{"samples":4,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"bef345339dd308ee","entry":"dataset_instance","repo":"clips/clicr","repo_kind":"official","path":"dataset-code/describe_data.py","file_url":"https://github.com/clips/clicr/blob/HEAD/dataset-code/describe_data.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bef345339dd308ee"}},{"code_sha256_prefix":"2d16293904cd4c35","entry":"normalize_answer","repo":"clips/clicr","repo_kind":"official","path":"dataset-code/evaluate.py","file_url":"https://github.com/clips/clicr/blob/HEAD/dataset-code/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2d16293904cd4c35"}},{"code_sha256_prefix":"8d8b459bca775c7e","entry":"to_lower","repo":"clips/clicr","repo_kind":"official","path":"dataset-code/describe_data.py","file_url":"https://github.com/clips/clicr/blob/HEAD/dataset-code/describe_data.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8d8b459bca775c7e"}},{"code_sha256_prefix":"3c7e5b8270dfe12c","entry":"lemmatize","repo":"clips/clicr","repo_kind":"official","path":"dataset-code/evaluate.py","file_url":"https://github.com/clips/clicr/blob/HEAD/dataset-code/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3c7e5b8270dfe12c"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}