{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/reading-comprehension/papers/ran/1","list_of":"/task/reading-comprehension","task":"Reading Comprehension","archive":{"snapshot":"2025-07-28"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"syntology_read_at":"2026-09-28T10:30:06+00:00","order":"ran","order_definition":"only papers where Syntology ran at least one harvested sample; date (newest first), ties by arXiv id","caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","absence":"A paper missing from this list is not a recorded non-run: it may have no arXiv id, no harvested code, or only samples that have not run yet.","page":1,"pages_in_order":2,"rows_per_page":100,"rows":[1,100],"of":139,"counts":{"archive_papers_tagged":1760,"with_a_code_link":634,"where_syntology_ran_a_sample":139,"not_listed_spam_title":0,"listed":1760,"listed_where_code_ran":139,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":120,"every_run_a_failure_of_syntologys_instrument":19,"listed_with_a_run_with_no_instrument_failure":120,"listed_every_run_a_failure_of_syntologys_instrument":19,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"first_page":"/task/reading-comprehension/papers/ran/1","prev":null,"next":"/task/reading-comprehension/papers/ran/2","papers":[{"url":"/paper/dynamic-chunking-and-selection-for-reading","slug":"dynamic-chunking-and-selection-for-reading","title":"Dynamic Chunking and Selection for Reading Comprehension of Ultra-Long Context in Large Language Models","date":"2025-06-01","arxiv_id":"2506.00773","repositories_listed":1,"syntology":{"n":9,"n_ran":9,"n_constructed":0,"n_ran_checked":1,"n_instrument":8,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":9,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 8 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/dynamic-chunking-and-selection-for-reading#ran","syntology_url":"https://syntology.ai/paper/2506.00773","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2506.00773"}},"official":{"repos":["ecnu-text-computing/dcs"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/scidqa-a-deep-reading-comprehension-dataset","slug":"scidqa-a-deep-reading-comprehension-dataset","title":"SciDQA: A Deep Reading Comprehension Dataset over Scientific Papers","date":"2024-11-08","arxiv_id":"2411.05338","repositories_listed":1,"syntology":{"n":10,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":10,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/scidqa-a-deep-reading-comprehension-dataset#ran","syntology_url":"https://syntology.ai/paper/2411.05338","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.05338"}},"official":{"repos":["yale-nlp/scidqa"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/fine-grained-prediction-of-reading","slug":"fine-grained-prediction-of-reading","title":"Fine-Grained Prediction of Reading Comprehension from Eye Movements","date":"2024-10-06","arxiv_id":"2410.04484","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/fine-grained-prediction-of-reading#ran","syntology_url":"https://syntology.ai/paper/2410.04484","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.04484"}},"official":{"repos":["lacclab/Reading-Comprehension-Prediction"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/adaptive-question-answering-enhancing","slug":"adaptive-question-answering-enhancing","title":"Adaptive Question Answering: Enhancing Language Model Proficiency for Addressing Knowledge Conflicts with Source Citations","date":"2024-10-05","arxiv_id":"2410.04241","repositories_listed":0,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/adaptive-question-answering-enhancing#ran","syntology_url":"https://syntology.ai/paper/2410.04241","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.04241"}},"official":null}},{"url":"/paper/shaking-up-vlms-comparing-transformers-and","slug":"shaking-up-vlms-comparing-transformers-and","title":"Shaking Up VLMs: Comparing Transformers and Structured State Space Models for Vision & Language Modeling","date":"2024-09-09","arxiv_id":"2409.05395","repositories_listed":1,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/shaking-up-vlms-comparing-transformers-and#ran","syntology_url":"https://syntology.ai/paper/2409.05395","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.05395"}},"official":{"repos":["gpantaz/vl_mamba"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/seemingly-plausible-distractors-in-multi-hop","slug":"seemingly-plausible-distractors-in-multi-hop","title":"Seemingly Plausible Distractors in Multi-Hop Reasoning: Are Large Language Models Attentive Readers?","date":"2024-09-08","arxiv_id":"2409.05197","repositories_listed":1,"syntology":{"n":12,"n_ran":8,"n_constructed":0,"n_ran_checked":8,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":1,"n_no_contract":7,"n_pointer_only":0,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 1 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/seemingly-plausible-distractors-in-multi-hop#ran","syntology_url":"https://syntology.ai/paper/2409.05197","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.05197"}},"official":{"repos":["zawedcvg/are-large-language-models-attentive-readers"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/harmonizing-visual-text-comprehension-and","slug":"harmonizing-visual-text-comprehension-and","title":"Harmonizing Visual Text Comprehension and Generation","date":"2024-07-23","arxiv_id":"2407.16364","repositories_listed":1,"syntology":{"n":18,"n_ran":11,"n_constructed":0,"n_ran_checked":7,"n_instrument":4,"n_unverified":7,"n_honours":1,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 1 honoured, 0 violated, 6 with no contract checked; 4 where Syntology's instrument failed) · 7 unverified","sample_list":"/paper/harmonizing-visual-text-comprehension-and#ran","syntology_url":"https://syntology.ai/paper/2407.16364","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.16364"}},"official":{"repos":["bytedance/textharmony"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":7,"ran_from_kinds":["official"]}}},{"url":"/paper/how-to-engage-your-readers-generating-guiding","slug":"how-to-engage-your-readers-generating-guiding","title":"How to Engage Your Readers? Generating Guiding Questions to Promote Active Reading","date":"2024-07-19","arxiv_id":"2407.14309","repositories_listed":1,"syntology":{"n":11,"n_ran":10,"n_constructed":0,"n_ran_checked":10,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":10,"n_pointer_only":11,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 0 honoured, 0 violated, 10 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/how-to-engage-your-readers-generating-guiding#ran","syntology_url":"https://syntology.ai/paper/2407.14309","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.14309"}},"official":{"repos":["eth-lre/engage-your-readers"],"state":"official (archive's flag): 10 ran","n_ran":10,"n_constructed":0,"n_ran_no_instrument_failure":10,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/analyzing-temporal-complex-events-with-large","slug":"analyzing-temporal-complex-events-with-large","title":"Analyzing Temporal Complex Events with Large Language Models? A Benchmark towards Temporal, Long Context Understanding","date":"2024-06-04","arxiv_id":"2406.02472","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/analyzing-temporal-complex-events-with-large#ran","syntology_url":"https://syntology.ai/paper/2406.02472","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.02472"}},"official":{"repos":["Zhihan72/TCELongBench"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/mtvqa-benchmarking-multilingual-text-centric","slug":"mtvqa-benchmarking-multilingual-text-centric","title":"MTVQA: Benchmarking Multilingual Text-Centric Visual Question Answering","date":"2024-05-20","arxiv_id":"2405.11985","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/mtvqa-benchmarking-multilingual-text-centric#ran","syntology_url":"https://syntology.ai/paper/2405.11985","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.11985"}},"official":{"repos":["bytedance/MTVQA"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/pmg-personalized-multimodal-generation-with","slug":"pmg-personalized-multimodal-generation-with","title":"PMG : Personalized Multimodal Generation with Large Language Models","date":"2024-04-07","arxiv_id":"2404.08677","repositories_listed":3,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":1,"n_instrument":3,"n_unverified":1,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":5,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/pmg-personalized-multimodal-generation-with#ran","syntology_url":"https://syntology.ai/paper/2404.08677","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.08677"}},"official":{"repos":["Suikasxt/PMG"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/sailor-open-language-models-for-south-east","slug":"sailor-open-language-models-for-south-east","title":"Sailor: Open Language Models for South-East Asia","date":"2024-04-04","arxiv_id":"2404.03608","repositories_listed":3,"syntology":{"n":12,"n_ran":8,"n_constructed":0,"n_ran_checked":8,"n_instrument":0,"n_unverified":4,"n_honours":1,"n_violates":0,"n_no_contract":7,"n_pointer_only":11,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 1 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/sailor-open-language-models-for-south-east#ran","syntology_url":"https://syntology.ai/paper/2404.03608","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.03608"}},"official":{"repos":["epfllm/megatron-llm","sail-sg/sailor-llm"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/st-llm-large-language-models-are-effective-1","slug":"st-llm-large-language-models-are-effective-1","title":"ST-LLM: Large Language Models Are Effective Temporal Learners","date":"2024-03-30","arxiv_id":"2404.00308","repositories_listed":1,"syntology":{"n":11,"n_ran":7,"n_constructed":0,"n_ran_checked":3,"n_instrument":4,"n_unverified":4,"n_honours":1,"n_violates":1,"n_no_contract":1,"n_pointer_only":7,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 1 honoured, 1 violated, 1 with no contract checked; 4 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/st-llm-large-language-models-are-effective-1#ran","syntology_url":"https://syntology.ai/paper/2404.00308","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.00308"}},"official":{"repos":["TencentARC/ST-LLM"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":2,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/arabicaqa-a-comprehensive-dataset-for-arabic","slug":"arabicaqa-a-comprehensive-dataset-for-arabic","title":"ArabicaQA: A Comprehensive Dataset for Arabic Question Answering","date":"2024-03-26","arxiv_id":"2403.17848","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/arabicaqa-a-comprehensive-dataset-for-arabic#ran","syntology_url":"https://syntology.ai/paper/2403.17848","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.17848"}},"official":{"repos":["datascienceuibk/arabicaqa"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/scaling-rectified-flow-transformers-for-high","slug":"scaling-rectified-flow-transformers-for-high","title":"Scaling Rectified Flow Transformers for High-Resolution Image Synthesis","date":"2024-03-05","arxiv_id":"2403.03206","repositories_listed":2,"syntology":{"n":11,"n_ran":10,"n_constructed":0,"n_ran_checked":10,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":10,"n_pointer_only":11,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 0 honoured, 0 violated, 10 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/scaling-rectified-flow-transformers-for-high#ran","syntology_url":"https://syntology.ai/paper/2403.03206","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.03206"}},"official":null}},{"url":"/paper/do-language-models-exhibit-the-same-cognitive","slug":"do-language-models-exhibit-the-same-cognitive","title":"Do Language Models Exhibit the Same Cognitive Biases in Problem Solving as Human Learners?","date":"2024-01-31","arxiv_id":"2401.18070","repositories_listed":1,"syntology":{"n":12,"n_ran":10,"n_constructed":0,"n_ran_checked":10,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":1,"n_no_contract":9,"n_pointer_only":12,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 0 honoured, 1 violated, 9 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/do-language-models-exhibit-the-same-cognitive#ran","syntology_url":"https://syntology.ai/paper/2401.18070","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.18070"}},"official":{"repos":["eth-lre/solving-biases"],"state":"official (archive's flag): 10 ran","n_ran":10,"n_constructed":0,"n_ran_no_instrument_failure":10,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/knowledge-fusion-of-large-language-models","slug":"knowledge-fusion-of-large-language-models","title":"Knowledge Fusion of Large Language Models","date":"2024-01-19","arxiv_id":"2401.10491","repositories_listed":3,"syntology":{"n":4,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":4,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/knowledge-fusion-of-large-language-models#ran","syntology_url":"https://syntology.ai/paper/2401.10491","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.10491"}},"official":{"repos":["fanqiwan/fusellm"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/leveraging-external-knowledge-resources-to","slug":"leveraging-external-knowledge-resources-to","title":"Towards Efficient Methods in Medical Question Answering using Knowledge Graph Embeddings","date":"2024-01-15","arxiv_id":"2401.07977","repositories_listed":1,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":1,"n_instrument":3,"n_unverified":1,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":5,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/leveraging-external-knowledge-resources-to#ran","syntology_url":"https://syntology.ai/paper/2401.07977","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.07977"}},"official":{"repos":["saptarshi059/cdqa-project"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/avoiding-data-contamination-in-language-model","slug":"avoiding-data-contamination-in-language-model","title":"LatestEval: Addressing Data Contamination in Language Model Evaluation through Dynamic and Time-Sensitive Test Construction","date":"2023-12-19","arxiv_id":"2312.12343","repositories_listed":1,"syntology":{"n":6,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":6,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/avoiding-data-contamination-in-language-model#ran","syntology_url":"https://syntology.ai/paper/2312.12343","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.12343"}},"official":{"repos":["liyucheng09/latesteval"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/gpt4point-a-unified-framework-for-point","slug":"gpt4point-a-unified-framework-for-point","title":"GPT4Point: A Unified Framework for Point-Language Understanding and Generation","date":"2023-12-05","arxiv_id":"2312.02980","repositories_listed":1,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":3,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":1,"n_no_contract":2,"n_pointer_only":2,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 1 violated, 2 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/gpt4point-a-unified-framework-for-point#ran","syntology_url":"https://syntology.ai/paper/2312.02980","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.02980"}},"official":null}},{"url":"/paper/debate-helps-supervise-unreliable-experts","slug":"debate-helps-supervise-unreliable-experts","title":"Debate Helps Supervise Unreliable Experts","date":"2023-11-15","arxiv_id":"2311.08702","repositories_listed":1,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":1,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/debate-helps-supervise-unreliable-experts#ran","syntology_url":"https://syntology.ai/paper/2311.08702","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.08702"}},"official":{"repos":["julianmichael/debate"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/mirror-a-universal-framework-for-various","slug":"mirror-a-universal-framework-for-various","title":"Mirror: A Universal Framework for Various Information Extraction Tasks","date":"2023-11-09","arxiv_id":"2311.05419","repositories_listed":1,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":4,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/mirror-a-universal-framework-for-various#ran","syntology_url":"https://syntology.ai/paper/2311.05419","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.05419"}},"official":{"repos":["Spico197/Mirror"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/in-context-pretraining-language-modeling","slug":"in-context-pretraining-language-modeling","title":"In-context Pretraining: Language Modeling Beyond Document Boundaries","date":"2023-10-16","arxiv_id":"2310.10638","repositories_listed":1,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":3,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":3,"n_no_contract":0,"n_pointer_only":6,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 3 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/in-context-pretraining-language-modeling#ran","syntology_url":"https://syntology.ai/paper/2310.10638","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.10638"}},"official":{"repos":["swj0419/in-context-pretraining"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/compresso-structured-pruning-with","slug":"compresso-structured-pruning-with","title":"Compresso: Structured Pruning with Collaborative Prompting Learns Compact Large Language Models","date":"2023-10-08","arxiv_id":"2310.05015","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":1,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/compresso-structured-pruning-with#ran","syntology_url":"https://syntology.ai/paper/2310.05015","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.05015"}},"official":{"repos":["microsoft/moonlit"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/minigpt-5-interleaved-vision-and-language","slug":"minigpt-5-interleaved-vision-and-language","title":"MiniGPT-5: Interleaved Vision-and-Language Generation via Generative Vokens","date":"2023-10-03","arxiv_id":"2310.02239","repositories_listed":1,"syntology":{"n":6,"n_ran":4,"n_constructed":0,"n_ran_checked":3,"n_instrument":1,"n_unverified":2,"n_honours":0,"n_violates":1,"n_no_contract":2,"n_pointer_only":1,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 1 violated, 2 with no contract checked; 1 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/minigpt-5-interleaved-vision-and-language#ran","syntology_url":"https://syntology.ai/paper/2310.02239","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.02239"}},"official":{"repos":["eric-ai-lab/minigpt-5"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/adapting-large-language-models-via-reading","slug":"adapting-large-language-models-via-reading","title":"Adapting Large Language Models to Domains via Reading Comprehension","date":"2023-09-18","arxiv_id":"2309.09530","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/adapting-large-language-models-via-reading#ran","syntology_url":"https://syntology.ai/paper/2309.09530","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.09530"}},"official":{"repos":["microsoft/lmops"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/korc-knowledge-oriented-reading-comprehension","slug":"korc-knowledge-oriented-reading-comprehension","title":"KoRC: Knowledge oriented Reading Comprehension Benchmark for Deep Text Understanding","date":"2023-07-06","arxiv_id":"2307.03115","repositories_listed":1,"syntology":{"n":9,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":3,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/korc-knowledge-oriented-reading-comprehension#ran","syntology_url":"https://syntology.ai/paper/2307.03115","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2307.03115"}},"official":{"repos":["thu-keg/korc"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/shifting-attention-to-relevance-towards-the","slug":"shifting-attention-to-relevance-towards-the","title":"Shifting Attention to Relevance: Towards the Predictive Uncertainty Quantification of Free-Form Large Language Models","date":"2023-07-03","arxiv_id":"2307.01379","repositories_listed":3,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":2,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 2 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/shifting-attention-to-relevance-towards-the#ran","syntology_url":"https://syntology.ai/paper/2307.01379","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2307.01379"}},"official":{"repos":["jinhaoduan/sar","jinhaoduan/shifting-attention-to-relevance"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/contrasting-intra-modal-and-ranking-cross","slug":"contrasting-intra-modal-and-ranking-cross","title":"Contrasting Intra-Modal and Ranking Cross-Modal Hard Negatives to Enhance Visio-Linguistic Compositional Understanding","date":"2023-06-15","arxiv_id":"2306.08832","repositories_listed":2,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/contrasting-intra-modal-and-ranking-cross#ran","syntology_url":"https://syntology.ai/paper/2306.08832","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.08832"}},"official":{"repos":["lezhang7/Enhance-FineGrained","magiccircuit/enhance-finegrained"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/bridging-the-gap-between-decision-and-logits","slug":"bridging-the-gap-between-decision-and-logits","title":"Bridging the Gap between Decision and Logits in Decision-based Knowledge Distillation for Pre-trained Language Models","date":"2023-06-15","arxiv_id":"2306.08909","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":1,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":1,"phrase":"1 ran (of which 1 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified; the one sample that ran constructed an object rather than computing a result","sample_list":"/paper/bridging-the-gap-between-decision-and-logits#ran","syntology_url":"https://syntology.ai/paper/2306.08909","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.08909"}},"official":{"repos":["thunlp-mt/dbkd-plm"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":1,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/a-causal-view-of-entity-bias-in-large","slug":"a-causal-view-of-entity-bias-in-large","title":"A Causal View of Entity Bias in (Large) Language Models","date":"2023-05-24","arxiv_id":"2305.14695","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/a-causal-view-of-entity-bias-in-large#ran","syntology_url":"https://syntology.ai/paper/2305.14695","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.14695"}},"official":{"repos":["luka-group/causal-view-of-entity-bias"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/a-large-cross-modal-video-retrieval-dataset","slug":"a-large-cross-modal-video-retrieval-dataset","title":"A Large Cross-Modal Video Retrieval Dataset with Reading Comprehension","date":"2023-05-05","arxiv_id":"2305.03347","repositories_listed":1,"syntology":{"n":13,"n_ran":12,"n_constructed":0,"n_ran_checked":9,"n_instrument":3,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":9,"n_pointer_only":3,"phrase":"12 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 0 honoured, 0 violated, 9 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/a-large-cross-modal-video-retrieval-dataset#ran","syntology_url":"https://syntology.ai/paper/2305.03347","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.03347"}},"official":{"repos":["callsys/textvr"],"state":"official (archive's flag): 12 ran","n_ran":12,"n_constructed":0,"n_ran_no_instrument_failure":9,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/context-faithful-prompting-for-large-language","slug":"context-faithful-prompting-for-large-language","title":"Context-faithful Prompting for Large Language Models","date":"2023-03-20","arxiv_id":"2303.11315","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/context-faithful-prompting-for-large-language#ran","syntology_url":"https://syntology.ai/paper/2303.11315","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.11315"}},"official":{"repos":["wzhouad/context-faithful-llm"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/from-clozing-to-comprehending-retrofitting","slug":"from-clozing-to-comprehending-retrofitting","title":"From Cloze to Comprehension: Retrofitting Pre-trained Masked Language Model to Pre-trained Machine Reader","date":"2022-12-09","arxiv_id":"2212.04755","repositories_listed":1,"syntology":{"n":4,"n_ran":3,"n_constructed":2,"n_ran_checked":3,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":4,"phrase":"3 ran (of which 2 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/from-clozing-to-comprehending-retrofitting#ran","syntology_url":"https://syntology.ai/paper/2212.04755","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.04755"}},"official":{"repos":["damo-nlp-sg/pmr"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":2,"n_ran_no_instrument_failure":3,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/elastic-numerical-reasoning-with-adaptive","slug":"elastic-numerical-reasoning-with-adaptive","title":"ELASTIC: Numerical Reasoning with Adaptive Symbolic Compiler","date":"2022-10-18","arxiv_id":"2210.10105","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":1,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 1 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/elastic-numerical-reasoning-with-adaptive#ran","syntology_url":"https://syntology.ai/paper/2210.10105","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.10105"}},"official":{"repos":["neurasearch/neurips-2022-submission-3358"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":1,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/screenqa-large-scale-question-answer-pairs","slug":"screenqa-large-scale-question-answer-pairs","title":"ScreenQA: Large-Scale Question-Answer Pairs over Mobile App Screenshots","date":"2022-09-16","arxiv_id":"2209.08199","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":3,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 3 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/screenqa-large-scale-question-answer-pairs#ran","syntology_url":"https://syntology.ai/paper/2209.08199","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2209.08199"}},"official":{"repos":["google-research-datasets/screen_qa"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/can-large-language-models-reason-about","slug":"can-large-language-models-reason-about","title":"Can large language models reason about medical questions?","date":"2022-07-17","arxiv_id":"2207.08143","repositories_listed":1,"syntology":{"n":10,"n_ran":10,"n_constructed":0,"n_ran_checked":10,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":10,"n_pointer_only":0,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 0 honoured, 0 violated, 10 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/can-large-language-models-reason-about#ran","syntology_url":"https://syntology.ai/paper/2207.08143","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2207.08143"}},"official":{"repos":["vlievin/medical-reasoning"],"state":"official (archive's flag): 10 ran","n_ran":10,"n_constructed":0,"n_ran_no_instrument_failure":10,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/finbert-mrc-financial-named-entity","slug":"finbert-mrc-financial-named-entity","title":"FinBERT-MRC: financial named entity recognition using BERT under the machine reading comprehension paradigm","date":"2022-05-31","arxiv_id":"2205.15485","repositories_listed":1,"syntology":{"n":10,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/finbert-mrc-financial-named-entity#ran","syntology_url":"https://syntology.ai/paper/2205.15485","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2205.15485"}},"official":{"repos":["zyz0000/finbert-mrc"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/tie-topological-information-enhanced","slug":"tie-topological-information-enhanced","title":"TIE: Topological Information Enhanced Structural Reading Comprehension on Web Pages","date":"2022-05-13","arxiv_id":"2205.06435","repositories_listed":1,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":5,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":2,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/tie-topological-information-enhanced#ran","syntology_url":"https://syntology.ai/paper/2205.06435","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2205.06435"}},"official":{"repos":["x-lance/tie"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/improving-the-numerical-reasoning-skills-of","slug":"improving-the-numerical-reasoning-skills-of","title":"Arithmetic-Based Pretraining -- Improving Numeracy of Pretrained Language Models","date":"2022-05-13","arxiv_id":"2205.06733","repositories_listed":2,"syntology":{"n":16,"n_ran":15,"n_constructed":0,"n_ran_checked":14,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":14,"n_pointer_only":1,"phrase":"15 ran (of which 0 constructed an object rather than computing a result; 14 with no instrument failure: 0 honoured, 0 violated, 14 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/improving-the-numerical-reasoning-skills-of#ran","syntology_url":"https://syntology.ai/paper/2205.06733","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2205.06733"}},"official":{"repos":["ukplab/emnlp2022-reasoning-aware-pretraining","ukplab/starsem2023-arithmetic-based-pretraining"],"state":"official (archive's flag): 15 ran","n_ran":15,"n_constructed":0,"n_ran_no_instrument_failure":14,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/palm-scaling-language-modeling-with-pathways-1","slug":"palm-scaling-language-modeling-with-pathways-1","title":"PaLM: Scaling Language Modeling with Pathways","date":"2022-04-05","arxiv_id":"2204.02311","repositories_listed":7,"syntology":{"n":37,"n_ran":32,"n_constructed":16,"n_ran_checked":24,"n_instrument":8,"n_unverified":5,"n_honours":2,"n_violates":1,"n_no_contract":21,"n_pointer_only":0,"phrase":"32 ran (of which 16 constructed an object rather than computing a result; 24 with no instrument failure: 2 honoured, 1 violated, 21 with no contract checked; 8 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/palm-scaling-language-modeling-with-pathways-1#ran","syntology_url":"https://syntology.ai/paper/2204.02311","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.02311"}},"official":null}},{"url":"/paper/adalogn-adaptive-logic-graph-network-for","slug":"adalogn-adaptive-logic-graph-network-for","title":"AdaLoGN: Adaptive Logic Graph Network for Reasoning-Based Machine Reading Comprehension","date":"2022-03-16","arxiv_id":"2203.08992","repositories_listed":1,"syntology":{"n":8,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/adalogn-adaptive-logic-graph-network-for#ran","syntology_url":"https://syntology.ai/paper/2203.08992","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.08992"}},"official":{"repos":["nju-websoft/adalogn"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/merit-meta-path-guided-contrastive-learning","slug":"merit-meta-path-guided-contrastive-learning","title":"MERIt: Meta-Path Guided Contrastive Learning for Logical Reasoning","date":"2022-03-01","arxiv_id":"2203.00357","repositories_listed":1,"syntology":{"n":7,"n_ran":5,"n_constructed":2,"n_ran_checked":2,"n_instrument":3,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"5 ran (of which 2 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 3 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/merit-meta-path-guided-contrastive-learning#ran","syntology_url":"https://syntology.ai/paper/2203.00357","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.00357"}},"official":{"repos":["sparkjiao/merit"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":2,"n_ran_no_instrument_failure":2,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/relational-surrogate-loss-learning-1","slug":"relational-surrogate-loss-learning-1","title":"Relational Surrogate Loss Learning","date":"2022-02-26","arxiv_id":"2202.13197","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/relational-surrogate-loss-learning-1#ran","syntology_url":"https://syntology.ai/paper/2202.13197","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2202.13197"}},"official":{"repos":["hunto/reloss"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/introspective-distillation-for-robust","slug":"introspective-distillation-for-robust","title":"Introspective Distillation for Robust Question Answering","date":"2021-11-01","arxiv_id":"2111.01026","repositories_listed":1,"syntology":{"n":7,"n_ran":7,"n_constructed":0,"n_ran_checked":4,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":3,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/introspective-distillation-for-robust#ran","syntology_url":"https://syntology.ai/paper/2111.01026","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2111.01026"}},"official":{"repos":["yuleiniu/introd"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/conditionalqa-a-complex-reading-comprehension","slug":"conditionalqa-a-complex-reading-comprehension","title":"ConditionalQA: A Complex Reading Comprehension Dataset with Conditional Answers","date":"2021-10-13","arxiv_id":"2110.06884","repositories_listed":2,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/conditionalqa-a-complex-reading-comprehension#ran","syntology_url":"https://syntology.ai/paper/2110.06884","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.06884"}},"official":{"repos":["haitian-sun/conditionalqa"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/entqa-entity-linking-as-question-answering","slug":"entqa-entity-linking-as-question-answering","title":"EntQA: Entity Linking as Question Answering","date":"2021-10-05","arxiv_id":"2110.02369","repositories_listed":2,"syntology":{"n":9,"n_ran":9,"n_constructed":3,"n_ran_checked":3,"n_instrument":6,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":1,"phrase":"9 ran (of which 3 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 6 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/entqa-entity-linking-as-question-answering#ran","syntology_url":"https://syntology.ai/paper/2110.02369","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.02369"}},"official":{"repos":["wenzhengzhang/entqa"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":2,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/single-dataset-experts-for-multi-dataset","slug":"single-dataset-experts-for-multi-dataset","title":"Single-dataset Experts for Multi-dataset Question Answering","date":"2021-09-28","arxiv_id":"2109.13880","repositories_listed":1,"syntology":{"n":11,"n_ran":11,"n_constructed":0,"n_ran_checked":10,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":1,"n_no_contract":8,"n_pointer_only":3,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 1 honoured, 1 violated, 8 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/single-dataset-experts-for-multi-dataset#ran","syntology_url":"https://syntology.ai/paper/2109.13880","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.13880"}},"official":{"repos":["princeton-nlp/made"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":10,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/multidoc2dial-modeling-dialogues-grounded-in","slug":"multidoc2dial-modeling-dialogues-grounded-in","title":"MultiDoc2Dial: Modeling Dialogues Grounded in Multiple Documents","date":"2021-09-26","arxiv_id":"2109.12595","repositories_listed":1,"syntology":{"n":4,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/multidoc2dial-modeling-dialogues-grounded-in#ran","syntology_url":"https://syntology.ai/paper/2109.12595","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.12595"}},"official":{"repos":["IBM/multidoc2dial"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/codeqa-a-question-answering-dataset-for","slug":"codeqa-a-question-answering-dataset-for","title":"CodeQA: A Question Answering Dataset for Source Code Comprehension","date":"2021-09-17","arxiv_id":"2109.08365","repositories_listed":1,"syntology":{"n":11,"n_ran":11,"n_constructed":0,"n_ran_checked":7,"n_instrument":4,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":2,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 4 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/codeqa-a-question-answering-dataset-for#ran","syntology_url":"https://syntology.ai/paper/2109.08365","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.08365"}},"official":{"repos":["jadecxliu/codeqa"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/zero-shot-open-information-extraction-using","slug":"zero-shot-open-information-extraction-using","title":"Context-NER : Contextual Phrase Generation at Scale","date":"2021-09-16","arxiv_id":"2109.08079","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/zero-shot-open-information-extraction-using#ran","syntology_url":"https://syntology.ai/paper/2109.08079","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.08079"}},"official":{"repos":["him1411/edgar10q-dataset"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/summarize-then-answer-generating-concise","slug":"summarize-then-answer-generating-concise","title":"Summarize-then-Answer: Generating Concise Explanations for Multi-hop Reading Comprehension","date":"2021-09-14","arxiv_id":"2109.06853","repositories_listed":1,"syntology":{"n":4,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":4,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/summarize-then-answer-generating-concise#ran","syntology_url":"https://syntology.ai/paper/2109.06853","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.06853"}},"official":{"repos":["stonybrooknlp/suqa"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/kelm-knowledge-enhanced-pre-trained-language","slug":"kelm-knowledge-enhanced-pre-trained-language","title":"KELM: Knowledge Enhanced Pre-Trained Language Representations with Message Passing on Hierarchical Relational Graphs","date":"2021-09-09","arxiv_id":"2109.04223","repositories_listed":1,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":5,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":1,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/kelm-knowledge-enhanced-pre-trained-language#ran","syntology_url":"https://syntology.ai/paper/2109.04223","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.04223"}},"official":{"repos":["nlp-anonymous-happy/anonymous-kg-guided-nlp"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/interactive-machine-comprehension-with-1","slug":"interactive-machine-comprehension-with-1","title":"Interactive Machine Comprehension with Dynamic Knowledge Graphs","date":"2021-08-31","arxiv_id":"2109.00077","repositories_listed":1,"syntology":{"n":7,"n_ran":7,"n_constructed":0,"n_ran_checked":6,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":6,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/interactive-machine-comprehension-with-1#ran","syntology_url":"https://syntology.ai/paper/2109.00077","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.00077"}},"official":{"repos":["xingdi-eric-yuan/imrc_graph_public"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/how-optimal-is-greedy-decoding-for-extractive","slug":"how-optimal-is-greedy-decoding-for-extractive","title":"How Optimal is Greedy Decoding for Extractive Question Answering?","date":"2021-08-12","arxiv_id":"2108.05857","repositories_listed":1,"syntology":{"n":5,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/how-optimal-is-greedy-decoding-for-extractive#ran","syntology_url":"https://syntology.ai/paper/2108.05857","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2108.05857"}},"official":{"repos":["ocastel/exact-extract"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/from-lsat-the-progress-and-challenges-of","slug":"from-lsat-the-progress-and-challenges-of","title":"From LSAT: The Progress and Challenges of Complex Reasoning","date":"2021-08-02","arxiv_id":"2108.00648","repositories_listed":1,"syntology":{"n":11,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":1,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/from-lsat-the-progress-and-challenges-of#ran","syntology_url":"https://syntology.ai/paper/2108.00648","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2108.00648"}},"official":null}},{"url":"/paper/keep-it-simple-unsupervised-simplification-of-1","slug":"keep-it-simple-unsupervised-simplification-of-1","title":"Keep it Simple: Unsupervised Simplification of Multi-Paragraph Text","date":"2021-07-07","arxiv_id":"2107.03444","repositories_listed":1,"syntology":{"n":13,"n_ran":11,"n_constructed":0,"n_ran_checked":11,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":11,"n_pointer_only":0,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 11 with no instrument failure: 0 honoured, 0 violated, 11 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/keep-it-simple-unsupervised-simplification-of-1#ran","syntology_url":"https://syntology.ai/paper/2107.03444","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2107.03444"}},"official":{"repos":["tingofurro/keep_it_simple"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":11,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/okgit-open-knowledge-graph-link-prediction","slug":"okgit-open-knowledge-graph-link-prediction","title":"OKGIT: Open Knowledge Graph Link Prediction with Implicit Types","date":"2021-06-24","arxiv_id":"2106.12806","repositories_listed":1,"syntology":{"n":4,"n_ran":1,"n_constructed":1,"n_ran_checked":1,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 1 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified; the one sample that ran constructed an object rather than computing a result","sample_list":"/paper/okgit-open-knowledge-graph-link-prediction#ran","syntology_url":"https://syntology.ai/paper/2106.12806","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.12806"}},"official":{"repos":["Chandrahasd/OKGIT"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":1,"n_ran_no_instrument_failure":1,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/ethical-advice-taker-do-language-models","slug":"ethical-advice-taker-do-language-models","title":"Ethical-Advice Taker: Do Language Models Understand Natural Language Interventions?","date":"2021-06-02","arxiv_id":"2106.01465","repositories_listed":1,"syntology":{"n":11,"n_ran":9,"n_constructed":0,"n_ran_checked":9,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":9,"n_pointer_only":0,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 0 honoured, 0 violated, 9 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/ethical-advice-taker-do-language-models#ran","syntology_url":"https://syntology.ai/paper/2106.01465","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.01465"}},"official":{"repos":["allenai/ethical-interventions"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":9,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/excavatorcovid-extracting-events-and","slug":"excavatorcovid-extracting-events-and","title":"ExcavatorCovid: Extracting Events and Relations from Text Corpora for Temporal and Causal Analysis for COVID-19","date":"2021-05-05","arxiv_id":"2105.01819","repositories_listed":0,"syntology":{"n":4,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/excavatorcovid-extracting-events-and#ran","syntology_url":"https://syntology.ai/paper/2105.01819","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.01819"}},"official":null}},{"url":"/paper/effect-of-vision-and-language-extensions-on","slug":"effect-of-vision-and-language-extensions-on","title":"Effect of Visual Extensions on Natural Language Understanding in Vision-and-Language Models","date":"2021-04-16","arxiv_id":"2104.08066","repositories_listed":1,"syntology":{"n":9,"n_ran":8,"n_constructed":0,"n_ran_checked":7,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":4,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/effect-of-vision-and-language-extensions-on#ran","syntology_url":"https://syntology.ai/paper/2104.08066","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.08066"}},"official":{"repos":["alab-nii/eval_vl_glue"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/open-retrieval-conversational-machine-reading","slug":"open-retrieval-conversational-machine-reading","title":"Open-Retrieval Conversational Machine Reading","date":"2021-02-17","arxiv_id":"2102.08633","repositories_listed":2,"syntology":{"n":4,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":4,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/open-retrieval-conversational-machine-reading#ran","syntology_url":"https://syntology.ai/paper/2102.08633","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2102.08633"}},"official":{"repos":["yifan-gao/open_retrieval_conversational_machine_reading"],"state":"official: no sample here; runs from other or unrecorded repositories","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","unlocated"]}}},{"url":"/paper/english-machine-reading-comprehension","slug":"english-machine-reading-comprehension","title":"English Machine Reading Comprehension Datasets: A Survey","date":"2021-01-25","arxiv_id":"2101.10421","repositories_listed":1,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":2,"n_instrument":2,"n_unverified":0,"n_honours":1,"n_violates":1,"n_no_contract":0,"n_pointer_only":4,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 1 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/english-machine-reading-comprehension#ran","syntology_url":"https://syntology.ai/paper/2101.10421","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2101.10421"}},"official":{"repos":["dariad/rczoo"],"state":"official: no sample here; runs from other or unrecorded repositories","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["found_in_text"]}}},{"url":"/paper/websrc-a-dataset-for-web-based-structural","slug":"websrc-a-dataset-for-web-based-structural","title":"WebSRC: A Dataset for Web-Based Structural Reading Comprehension","date":"2021-01-23","arxiv_id":"2101.09465","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/websrc-a-dataset-for-web-based-structural#ran","syntology_url":"https://syntology.ai/paper/2101.09465","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2101.09465"}},"official":null}},{"url":"/paper/erica-improving-entity-and-relation","slug":"erica-improving-entity-and-relation","title":"ERICA: Improving Entity and Relation Understanding for Pre-trained Language Models via Contrastive Learning","date":"2020-12-30","arxiv_id":"2012.15022","repositories_listed":1,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/erica-improving-entity-and-relation#ran","syntology_url":"https://syntology.ai/paper/2012.15022","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2012.15022"}},"official":{"repos":["thunlp/ERICA"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/russiansuperglue-a-russian-language","slug":"russiansuperglue-a-russian-language","title":"RussianSuperGLUE: A Russian Language Understanding Evaluation Benchmark","date":"2020-10-29","arxiv_id":"2010.15925","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/russiansuperglue-a-russian-language#ran","syntology_url":"https://syntology.ai/paper/2010.15925","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.15925"}},"official":{"repos":["RussianNLP/RussianSuperGLUE"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/mt5-a-massively-multilingual-pre-trained-text","slug":"mt5-a-massively-multilingual-pre-trained-text","title":"mT5: A massively multilingual pre-trained text-to-text transformer","date":"2020-10-22","arxiv_id":"2010.11934","repositories_listed":8,"syntology":{"n":13,"n_ran":10,"n_constructed":0,"n_ran_checked":10,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":10,"n_pointer_only":0,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 0 honoured, 0 violated, 10 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/mt5-a-massively-multilingual-pre-trained-text#ran","syntology_url":"https://syntology.ai/paper/2010.11934","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.11934"}},"official":{"repos":["google-research/multilingual-t5"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/towards-interpreting-bert-for-reading","slug":"towards-interpreting-bert-for-reading","title":"Towards Interpreting BERT for Reading Comprehension Based QA","date":"2020-10-18","arxiv_id":"2010.08983","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/towards-interpreting-bert-for-reading#ran","syntology_url":"https://syntology.ai/paper/2010.08983","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.08983"}},"official":{"repos":["iitmnlp/BERT-Analysis-RCQA"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/open-domain-question-answering-goes","slug":"open-domain-question-answering-goes","title":"Open-Domain Question Answering Goes Conversational via Question Rewriting","date":"2020-10-10","arxiv_id":"2010.04898","repositories_listed":2,"syntology":{"n":8,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/open-domain-question-answering-goes#ran","syntology_url":"https://syntology.ai/paper/2010.04898","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.04898"}},"official":{"repos":["apple/ml-qrecc"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/counterfactually-augmented-snli-training-data","slug":"counterfactually-augmented-snli-training-data","title":"Counterfactually-Augmented SNLI Training Data Does Not Yield Better Generalization Than Unaugmented Data","date":"2020-10-09","arxiv_id":"2010.04762","repositories_listed":1,"syntology":{"n":16,"n_ran":16,"n_constructed":0,"n_ran_checked":15,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":15,"n_pointer_only":0,"phrase":"16 ran (of which 0 constructed an object rather than computing a result; 15 with no instrument failure: 0 honoured, 0 violated, 15 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/counterfactually-augmented-snli-training-data#ran","syntology_url":"https://syntology.ai/paper/2010.04762","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.04762"}},"official":{"repos":["nyu-mll/CNLI-generalization"],"state":"official (archive's flag): 16 ran","n_ran":16,"n_constructed":0,"n_ran_no_instrument_failure":15,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/what-disease-does-this-patient-have-a-large","slug":"what-disease-does-this-patient-have-a-large","title":"What Disease does this Patient Have? A Large-scale Open Domain Question Answering Dataset from Medical Exams","date":"2020-09-28","arxiv_id":"2009.13081","repositories_listed":3,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/what-disease-does-this-patient-have-a-large#ran","syntology_url":"https://syntology.ai/paper/2009.13081","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2009.13081"}},"official":{"repos":["jind11/MedQA"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/docvqa-a-dataset-for-vqa-on-document-images","slug":"docvqa-a-dataset-for-vqa-on-document-images","title":"DocVQA: A Dataset for VQA on Document Images","date":"2020-07-01","arxiv_id":"2007.00398","repositories_listed":3,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":3,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":2,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/docvqa-a-dataset-for-vqa-on-document-images#ran","syntology_url":"https://syntology.ai/paper/2007.00398","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2007.00398"}},"official":null}},{"url":"/paper/funnel-transformer-filtering-out-sequential","slug":"funnel-transformer-filtering-out-sequential","title":"Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing","date":"2020-06-05","arxiv_id":"2006.03236","repositories_listed":3,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/funnel-transformer-filtering-out-sequential#ran","syntology_url":"https://syntology.ai/paper/2006.03236","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.03236"}},"official":{"repos":["laiguokun/Funnel-Transformer"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/deberta-decoding-enhanced-bert-with","slug":"deberta-decoding-enhanced-bert-with","title":"DeBERTa: Decoding-enhanced BERT with Disentangled Attention","date":"2020-06-05","arxiv_id":"2006.03654","repositories_listed":14,"syntology":{"n":13,"n_ran":4,"n_constructed":0,"n_ran_checked":3,"n_instrument":1,"n_unverified":9,"n_honours":2,"n_violates":1,"n_no_contract":0,"n_pointer_only":3,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 2 honoured, 1 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 9 unverified","sample_list":"/paper/deberta-decoding-enhanced-bert-with#ran","syntology_url":"https://syntology.ai/paper/2006.03654","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.03654"}},"official":{"repos":["microsoft/DeBERTa"],"state":"official: no sample here; runs from other or unrecorded repositories","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","named_in_paper","unlocated"]}}},{"url":"/paper/language-models-are-few-shot-learners","slug":"language-models-are-few-shot-learners","title":"Language Models are Few-Shot Learners","date":"2020-05-28","arxiv_id":"2005.14165","repositories_listed":67,"syntology":{"n":65,"n_ran":45,"n_constructed":0,"n_ran_checked":40,"n_instrument":5,"n_unverified":20,"n_honours":2,"n_violates":1,"n_no_contract":37,"n_pointer_only":7,"phrase":"45 ran (of which 0 constructed an object rather than computing a result; 40 with no instrument failure: 2 honoured, 1 violated, 37 with no contract checked; 5 where Syntology's instrument failed) · 20 unverified","sample_list":"/paper/language-models-are-few-shot-learners#ran","syntology_url":"https://syntology.ai/paper/2005.14165","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.14165"}},"official":{"repos":["openai/gpt-3"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/document-modeling-with-graph-attention","slug":"document-modeling-with-graph-attention","title":"Document Modeling with Graph Attention Networks for Multi-grained Machine Reading Comprehension","date":"2020-05-12","arxiv_id":"2005.05806","repositories_listed":1,"syntology":{"n":16,"n_ran":8,"n_constructed":8,"n_ran_checked":8,"n_instrument":0,"n_unverified":8,"n_honours":0,"n_violates":0,"n_no_contract":8,"n_pointer_only":16,"phrase":"8 ran (of which 8 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 0 violated, 8 with no contract checked; 0 where Syntology's instrument failed) · 8 unverified; every one of the 8 samples that ran constructed an object rather than computing a result","sample_list":"/paper/document-modeling-with-graph-attention#ran","syntology_url":"https://syntology.ai/paper/2005.05806","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.05806"}},"official":{"repos":["DancingSoul/NQ_BERT-DM"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":8,"n_ran_no_instrument_failure":8,"n_unverified":8,"ran_from_kinds":["official"]}}},{"url":"/paper/teaching-machine-comprehension-with","slug":"teaching-machine-comprehension-with","title":"Teaching Machine Comprehension with Compositional Explanations","date":"2020-05-02","arxiv_id":"2005.00806","repositories_listed":2,"syntology":{"n":10,"n_ran":10,"n_constructed":1,"n_ran_checked":6,"n_instrument":4,"n_unverified":0,"n_honours":4,"n_violates":1,"n_no_contract":1,"n_pointer_only":10,"phrase":"10 ran (of which 1 constructed an object rather than computing a result; 6 with no instrument failure: 4 honoured, 1 violated, 1 with no contract checked; 4 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/teaching-machine-comprehension-with#ran","syntology_url":"https://syntology.ai/paper/2005.00806","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.00806"}},"official":{"repos":["INK-USC/mrc-explanation","INK-USC/nl-explanation"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":1,"n_ran_no_instrument_failure":6,"n_unverified":0,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/clinical-reading-comprehension-a-thorough","slug":"clinical-reading-comprehension-a-thorough","title":"Clinical Reading Comprehension: A Thorough Analysis of the emrQA Dataset","date":"2020-05-01","arxiv_id":"2005.00574","repositories_listed":1,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":0,"n_instrument":4,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/clinical-reading-comprehension-a-thorough#ran","syntology_url":"https://syntology.ai/paper/2005.00574","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.00574"}},"official":{"repos":["xiangyue9607/CliniRC"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/starc-structured-annotations-for-reading","slug":"starc-structured-annotations-for-reading","title":"STARC: Structured Annotations for Reading Comprehension","date":"2020-04-30","arxiv_id":"2004.14797","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/starc-structured-annotations-for-reading#ran","syntology_url":"https://syntology.ai/paper/2004.14797","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.14797"}},"official":{"repos":["berzak/onestop-qa"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/the-curse-of-performance-instability-in","slug":"the-curse-of-performance-instability-in","title":"The Curse of Performance Instability in Analysis Datasets: Consequences, Source, and Suggestions","date":"2020-04-28","arxiv_id":"2004.13606","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":3,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 3 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/the-curse-of-performance-instability-in#ran","syntology_url":"https://syntology.ai/paper/2004.13606","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.13606"}},"official":{"repos":["owenzx/InstabilityAnalysis"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/logic-guided-data-augmentation-and","slug":"logic-guided-data-augmentation-and","title":"Logic-Guided Data Augmentation and Regularization for Consistent Question Answering","date":"2020-04-21","arxiv_id":"2004.10157","repositories_listed":1,"syntology":{"n":18,"n_ran":8,"n_constructed":0,"n_ran_checked":0,"n_instrument":8,"n_unverified":10,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 8 where Syntology's instrument failed) · 10 unverified","sample_list":"/paper/logic-guided-data-augmentation-and#ran","syntology_url":"https://syntology.ai/paper/2004.10157","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.10157"}},"official":{"repos":["AkariAsai/logic_guided_qa"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":10,"ran_from_kinds":["official"]}}},{"url":"/paper/a-sentence-cloze-dataset-for-chinese-machine","slug":"a-sentence-cloze-dataset-for-chinese-machine","title":"A Sentence Cloze Dataset for Chinese Machine Reading Comprehension","date":"2020-04-07","arxiv_id":"2004.03116","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":2,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 2 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/a-sentence-cloze-dataset-for-chinese-machine#ran","syntology_url":"https://syntology.ai/paper/2004.03116","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.03116"}},"official":{"repos":["ymcui/cmrc2019"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/what-do-models-learn-from-question-answering","slug":"what-do-models-learn-from-question-answering","title":"What do Models Learn from Question Answering Datasets?","date":"2020-04-07","arxiv_id":"2004.03490","repositories_listed":2,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":3,"n_instrument":2,"n_unverified":1,"n_honours":1,"n_violates":1,"n_no_contract":1,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 1 honoured, 1 violated, 1 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/what-do-models-learn-from-question-answering#ran","syntology_url":"https://syntology.ai/paper/2004.03490","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.03490"}},"official":{"repos":["amazon-research/qa-dataset-converter"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/asking-questions-the-human-way-scalable","slug":"asking-questions-the-human-way-scalable","title":"Asking Questions the Human Way: Scalable Question-Answer Generation from Text Corpus","date":"2020-01-27","arxiv_id":"2002.00748","repositories_listed":2,"syntology":{"n":8,"n_ran":7,"n_constructed":0,"n_ran_checked":6,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":2,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/asking-questions-the-human-way-scalable#ran","syntology_url":"https://syntology.ai/paper/2002.00748","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2002.00748"}},"official":{"repos":["bangliu/ACS-QG"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":0,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/differentiable-reasoning-on-large-knowledge","slug":"differentiable-reasoning-on-large-knowledge","title":"Differentiable Reasoning on Large Knowledge Bases and Natural Language","date":"2019-12-17","arxiv_id":"1912.10824","repositories_listed":3,"syntology":{"n":26,"n_ran":15,"n_constructed":0,"n_ran_checked":15,"n_instrument":0,"n_unverified":11,"n_honours":0,"n_violates":0,"n_no_contract":15,"n_pointer_only":0,"phrase":"15 ran (of which 0 constructed an object rather than computing a result; 15 with no instrument failure: 0 honoured, 0 violated, 15 with no contract checked; 0 where Syntology's instrument failed) · 11 unverified","sample_list":"/paper/differentiable-reasoning-on-large-knowledge#ran","syntology_url":"https://syntology.ai/paper/1912.10824","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1912.10824"}},"official":{"repos":["uclnlp/gntp","uclnlp/ntp","uclnlp/ctp"],"state":"official (archive's flag): 15 ran","n_ran":15,"n_constructed":0,"n_ran_no_instrument_failure":15,"n_unverified":11,"ran_from_kinds":["official"]}}},{"url":"/paper/knowledge-guided-text-retrieval-and-reading","slug":"knowledge-guided-text-retrieval-and-reading","title":"Knowledge Guided Text Retrieval and Reading for Open Domain Question Answering","date":"2019-11-10","arxiv_id":"1911.03868","repositories_listed":7,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":2,"n_instrument":4,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":4,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 4 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/knowledge-guided-text-retrieval-and-reading#ran","syntology_url":"https://syntology.ai/paper/1911.03868","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1911.03868"}},"official":null}},{"url":"/paper/dice-loss-for-data-imbalanced-nlp-tasks","slug":"dice-loss-for-data-imbalanced-nlp-tasks","title":"Dice Loss for Data-imbalanced NLP Tasks","date":"2019-11-07","arxiv_id":"1911.02855","repositories_listed":4,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/dice-loss-for-data-imbalanced-nlp-tasks#ran","syntology_url":"https://syntology.ai/paper/1911.02855","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1911.02855"}},"official":{"repos":["ShannonAI/dice_loss_for_NLP"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/select-answer-and-explain-interpretable-multi","slug":"select-answer-and-explain-interpretable-multi","title":"Select, Answer and Explain: Interpretable Multi-hop Reading Comprehension over Multiple Documents","date":"2019-11-01","arxiv_id":"1911.00484","repositories_listed":1,"syntology":{"n":15,"n_ran":12,"n_constructed":0,"n_ran_checked":12,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":12,"n_pointer_only":0,"phrase":"12 ran (of which 0 constructed an object rather than computing a result; 12 with no instrument failure: 0 honoured, 0 violated, 12 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/select-answer-and-explain-interpretable-multi#ran","syntology_url":"https://syntology.ai/paper/1911.00484","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1911.00484"}},"official":null}},{"url":"/paper/a-unified-mrc-framework-for-named-entity","slug":"a-unified-mrc-framework-for-named-entity","title":"A Unified MRC Framework for Named Entity Recognition","date":"2019-10-25","arxiv_id":"1910.11476","repositories_listed":8,"syntology":{"n":5,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":2,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 2 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/a-unified-mrc-framework-for-named-entity#ran","syntology_url":"https://syntology.ai/paper/1910.11476","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1910.11476"}},"official":{"repos":["ShannonAI/mrc-for-flat-nested-ner"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/mrqa-2019-shared-task-evaluating","slug":"mrqa-2019-shared-task-evaluating","title":"MRQA 2019 Shared Task: Evaluating Generalization in Reading Comprehension","date":"2019-10-22","arxiv_id":"1910.09753","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":2,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":1,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 1 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/mrqa-2019-shared-task-evaluating#ran","syntology_url":"https://syntology.ai/paper/1910.09753","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1910.09753"}},"official":{"repos":["mrqa/MRQA-Shared-Task-2019"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/revealing-the-importance-of-semantic","slug":"revealing-the-importance-of-semantic","title":"Revealing the Importance of Semantic Retrieval for Machine Reading at Scale","date":"2019-09-17","arxiv_id":"1909.08041","repositories_listed":2,"syntology":{"n":17,"n_ran":15,"n_constructed":0,"n_ran_checked":13,"n_instrument":2,"n_unverified":2,"n_honours":4,"n_violates":0,"n_no_contract":9,"n_pointer_only":1,"phrase":"15 ran (of which 0 constructed an object rather than computing a result; 13 with no instrument failure: 4 honoured, 0 violated, 9 with no contract checked; 2 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/revealing-the-importance-of-semantic#ran","syntology_url":"https://syntology.ai/paper/1909.08041","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1909.08041"}},"official":{"repos":["easonnie/semanticRetrievalMRS"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/megatron-lm-training-multi-billion-parameter","slug":"megatron-lm-training-multi-billion-parameter","title":"Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism","date":"2019-09-17","arxiv_id":"1909.08053","repositories_listed":10,"syntology":{"n":47,"n_ran":12,"n_constructed":2,"n_ran_checked":7,"n_instrument":5,"n_unverified":35,"n_honours":4,"n_violates":0,"n_no_contract":3,"n_pointer_only":15,"phrase":"12 ran (of which 2 constructed an object rather than computing a result; 7 with no instrument failure: 4 honoured, 0 violated, 3 with no contract checked; 5 where Syntology's instrument failed) · 35 unverified","sample_list":"/paper/megatron-lm-training-multi-billion-parameter#ran","syntology_url":"https://syntology.ai/paper/1909.08053","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1909.08053"}},"official":{"repos":["NVIDIA/Megatron-LM"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/span-selection-pre-training-for-question","slug":"span-selection-pre-training-for-question","title":"Span Selection Pre-training for Question Answering","date":"2019-09-09","arxiv_id":"1909.04120","repositories_listed":1,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/span-selection-pre-training-for-question#ran","syntology_url":"https://syntology.ai/paper/1909.04120","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1909.04120"}},"official":{"repos":["IBM/span-selection-pretraining"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/question-generation-by-transformers","slug":"question-generation-by-transformers","title":"Question Generation by Transformers","date":"2019-09-09","arxiv_id":"1909.05017","repositories_listed":1,"syntology":{"n":14,"n_ran":5,"n_constructed":0,"n_ran_checked":4,"n_instrument":1,"n_unverified":9,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 1 where Syntology's instrument failed) · 9 unverified","sample_list":"/paper/question-generation-by-transformers#ran","syntology_url":"https://syntology.ai/paper/1909.05017","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1909.05017"}},"official":null}},{"url":"/paper/semantics-aware-bert-for-language","slug":"semantics-aware-bert-for-language","title":"Semantics-aware BERT for Language Understanding","date":"2019-09-05","arxiv_id":"1909.02209","repositories_listed":1,"syntology":{"n":12,"n_ran":7,"n_constructed":0,"n_ran_checked":5,"n_instrument":2,"n_unverified":5,"n_honours":2,"n_violates":0,"n_no_contract":3,"n_pointer_only":3,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 2 honoured, 0 violated, 3 with no contract checked; 2 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/semantics-aware-bert-for-language#ran","syntology_url":"https://syntology.ai/paper/1909.02209","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1909.02209"}},"official":{"repos":["cooelf/SemBERT"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":5,"ran_from_kinds":["official"]}}},{"url":"/paper/interactive-language-learning-by-question","slug":"interactive-language-learning-by-question","title":"Interactive Language Learning by Question Answering","date":"2019-08-28","arxiv_id":"1908.10909","repositories_listed":1,"syntology":{"n":11,"n_ran":10,"n_constructed":0,"n_ran_checked":8,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":8,"n_pointer_only":10,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 0 violated, 8 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/interactive-language-learning-by-question#ran","syntology_url":"https://syntology.ai/paper/1908.10909","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1908.10909"}},"official":{"repos":["xingdi-eric-yuan/qait_public"],"state":"official (archive's flag): 10 ran","n_ran":10,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/interactive-machine-comprehension-with","slug":"interactive-machine-comprehension-with","title":"Interactive Machine Comprehension with Information Seeking Agents","date":"2019-08-27","arxiv_id":"1908.10449","repositories_listed":1,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":5,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":6,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/interactive-machine-comprehension-with#ran","syntology_url":"https://syntology.ai/paper/1908.10449","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1908.10449"}},"official":{"repos":["xingdi-eric-yuan/imrc_public"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/universal-adversarial-triggers-for-nlp","slug":"universal-adversarial-triggers-for-nlp","title":"Universal Adversarial Triggers for Attacking and Analyzing NLP","date":"2019-08-20","arxiv_id":"1908.07125","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/universal-adversarial-triggers-for-nlp#ran","syntology_url":"https://syntology.ai/paper/1908.07125","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1908.07125"}},"official":{"repos":["Eric-Wallace/universal-triggers"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/graphflow-exploiting-conversation-flow-with","slug":"graphflow-exploiting-conversation-flow-with","title":"GraphFlow: Exploiting Conversation Flow with Graph Neural Networks for Conversational Machine Comprehension","date":"2019-07-31","arxiv_id":"1908.00059","repositories_listed":1,"syntology":{"n":4,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/graphflow-exploiting-conversation-flow-with#ran","syntology_url":"https://syntology.ai/paper/1908.00059","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1908.00059"}},"official":{"repos":["hugochan/GraphFlow"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":2,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/roberta-a-robustly-optimized-bert-pretraining","slug":"roberta-a-robustly-optimized-bert-pretraining","title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach","date":"2019-07-26","arxiv_id":"1907.11692","repositories_listed":67,"syntology":{"n":48,"n_ran":37,"n_constructed":11,"n_ran_checked":36,"n_instrument":1,"n_unverified":11,"n_honours":0,"n_violates":0,"n_no_contract":36,"n_pointer_only":24,"phrase":"37 ran (of which 11 constructed an object rather than computing a result; 36 with no instrument failure: 0 honoured, 0 violated, 36 with no contract checked; 1 where Syntology's instrument failed) · 11 unverified","sample_list":"/paper/roberta-a-robustly-optimized-bert-pretraining#ran","syntology_url":"https://syntology.ai/paper/1907.11692","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1907.11692"}},"official":{"repos":["pytorch/fairseq"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}}],"record_sha256":"c8a7505643e243f4ad48ab4ac32048787a194037307149cee88b8a8a6c42aa81","record_changed_at":"2026-09-28","record_changed_at_basis":"first_hashed"}