{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/question-answering/papers/ran/11","list_of":"/task/question-answering","task":"Question Answering","archive":{"snapshot":"2025-07-28"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"syntology_read_at":"2026-09-28T10:30:06+00:00","order":"ran","order_definition":"only papers where Syntology ran at least one harvested sample; date (newest first), ties by arXiv id","caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","absence":"A paper missing from this list is not a recorded non-run: it may have no arXiv id, no harvested code, or only samples that have not run yet.","page":11,"pages_in_order":13,"rows_per_page":100,"rows":[1001,1100],"of":1274,"counts":{"archive_papers_tagged":10817,"with_a_code_link":4171,"where_syntology_ran_a_sample":1274,"not_listed_spam_title":0,"listed":10817,"listed_where_code_ran":1274,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":1073,"every_run_a_failure_of_syntologys_instrument":201,"listed_with_a_run_with_no_instrument_failure":1073,"listed_every_run_a_failure_of_syntologys_instrument":201,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"first_page":"/task/question-answering/papers/ran/1","prev":"/task/question-answering/papers/ran/10","next":"/task/question-answering/papers/ran/12","papers":[{"url":"/paper/multi-step-reasoning-over-unstructured-text","slug":"multi-step-reasoning-over-unstructured-text","title":"Multi-Step Reasoning Over Unstructured Text with Beam Dense Retrieval","date":"2021-04-13","arxiv_id":"2104.05883","repositories_listed":1,"syntology":{"n":5,"n_ran":3,"n_constructed":3,"n_ran_checked":3,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":5,"phrase":"3 ran (of which 3 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified; every one of the 3 samples that ran constructed an object rather than computing a result","sample_list":"/paper/multi-step-reasoning-over-unstructured-text#ran","syntology_url":"https://syntology.ai/paper/2104.05883","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.05883"}},"official":{"repos":["henryzhao5852/BeamDR"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":3,"n_ran_no_instrument_failure":3,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/qa-gnn-reasoning-with-language-models-and","slug":"qa-gnn-reasoning-with-language-models-and","title":"QA-GNN: Reasoning with Language Models and Knowledge Graphs for Question Answering","date":"2021-04-13","arxiv_id":"2104.06378","repositories_listed":6,"syntology":{"n":25,"n_ran":8,"n_constructed":0,"n_ran_checked":8,"n_instrument":0,"n_unverified":17,"n_honours":1,"n_violates":0,"n_no_contract":7,"n_pointer_only":5,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 1 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 17 unverified","sample_list":"/paper/qa-gnn-reasoning-with-language-models-and#ran","syntology_url":"https://syntology.ai/paper/2104.06378","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.06378"}},"official":{"repos":["michiyasunaga/qagnn","worksheets.codalab.org/worksheets/0xf215deb05edf44a2ac353c711f52a25f"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":13,"ran_from_kinds":["official"]}}},{"url":"/paper/spartqa-a-textual-question-answering","slug":"spartqa-a-textual-question-answering","title":"SpartQA: : A Textual Question Answering Benchmark for Spatial Reasoning","date":"2021-04-12","arxiv_id":"2104.05832","repositories_listed":1,"syntology":{"n":15,"n_ran":9,"n_constructed":0,"n_ran_checked":9,"n_instrument":0,"n_unverified":6,"n_honours":0,"n_violates":0,"n_no_contract":9,"n_pointer_only":0,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 0 honoured, 0 violated, 9 with no contract checked; 0 where Syntology's instrument failed) · 6 unverified","sample_list":"/paper/spartqa-a-textual-question-answering#ran","syntology_url":"https://syntology.ai/paper/2104.05832","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.05832"}},"official":{"repos":["HLR/SpartQA_generation"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":9,"n_unverified":6,"ran_from_kinds":["official"]}}},{"url":"/paper/pqa-perceptual-question-answering","slug":"pqa-perceptual-question-answering","title":"PQA: Perceptual Question Answering","date":"2021-04-08","arxiv_id":"2104.03589","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/pqa-perceptual-question-answering#ran","syntology_url":"https://syntology.ai/paper/2104.03589","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.03589"}},"official":null}},{"url":"/paper/beyond-question-based-biases-assessing","slug":"beyond-question-based-biases-assessing","title":"Beyond Question-Based Biases: Assessing Multimodal Shortcut Learning in Visual Question Answering","date":"2021-04-07","arxiv_id":"2104.03149","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/beyond-question-based-biases-assessing#ran","syntology_url":"https://syntology.ai/paper/2104.03149","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.03149"}},"official":{"repos":["cdancette/detect-shortcuts"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/an-investigation-of-critical-issues-in-bias","slug":"an-investigation-of-critical-issues-in-bias","title":"Are Bias Mitigation Techniques for Deep Learning Effective?","date":"2021-04-01","arxiv_id":"2104.00170","repositories_listed":1,"syntology":{"n":11,"n_ran":6,"n_constructed":0,"n_ran_checked":4,"n_instrument":2,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":1,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 2 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/an-investigation-of-critical-issues-in-bias#ran","syntology_url":"https://syntology.ai/paper/2104.00170","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.00170"}},"official":{"repos":["erobic/bias-mitigators"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":5,"ran_from_kinds":["official"]}}},{"url":"/paper/towards-general-purpose-vision-systems","slug":"towards-general-purpose-vision-systems","title":"Towards General Purpose Vision Systems","date":"2021-04-01","arxiv_id":"2104.00743","repositories_listed":2,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/towards-general-purpose-vision-systems#ran","syntology_url":"https://syntology.ai/paper/2104.00743","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.00743"}},"official":{"repos":["allenai/gpv-1"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/trafficqa-a-question-answering-benchmark-and","slug":"trafficqa-a-question-answering-benchmark-and","title":"SUTD-TrafficQA: A Question Answering Benchmark and an Efficient Network for Video Reasoning over Traffic Events","date":"2021-03-29","arxiv_id":"2103.15538","repositories_listed":3,"syntology":{"n":9,"n_ran":6,"n_constructed":1,"n_ran_checked":6,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":2,"phrase":"6 ran (of which 1 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/trafficqa-a-question-answering-benchmark-and#ran","syntology_url":"https://syntology.ai/paper/2103.15538","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2103.15538"}},"official":{"repos":["SUTDCV/SUTD-TrafficQA"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/generic-attention-model-explainability-for","slug":"generic-attention-model-explainability-for","title":"Generic Attention-model Explainability for Interpreting Bi-Modal and Encoder-Decoder Transformers","date":"2021-03-29","arxiv_id":"2103.15679","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/generic-attention-model-explainability-for#ran","syntology_url":"https://syntology.ai/paper/2103.15679","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2103.15679"}},"official":{"repos":["hila-chefer/Transformer-MM-Explainability"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/unicorn-on-rainbow-a-universal-commonsense","slug":"unicorn-on-rainbow-a-universal-commonsense","title":"UNICORN on RAINBOW: A Universal Commonsense Reasoning Model on a New Multitask Benchmark","date":"2021-03-24","arxiv_id":"2103.13009","repositories_listed":1,"syntology":{"n":5,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/unicorn-on-rainbow-a-universal-commonsense#ran","syntology_url":"https://syntology.ai/paper/2103.13009","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2103.13009"}},"official":{"repos":["allenai/rainbow"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/mitigating-false-negative-contexts-in-multi","slug":"mitigating-false-negative-contexts-in-multi","title":"Mitigating False-Negative Contexts in Multi-document Question Answering with Retrieval Marginalization","date":"2021-03-22","arxiv_id":"2103.12235","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":2,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":1,"n_pointer_only":1,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 0 violated, 1 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/mitigating-false-negative-contexts-in-multi#ran","syntology_url":"https://syntology.ai/paper/2103.12235","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2103.12235"}},"official":{"repos":["niansong1996/retrieval_marginalization"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/automatic-generation-of-contrast-sets-from","slug":"automatic-generation-of-contrast-sets-from","title":"Automatic Generation of Contrast Sets from Scene Graphs: Probing the Compositional Consistency of GQA","date":"2021-03-17","arxiv_id":"2103.09591","repositories_listed":2,"syntology":{"n":5,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":4,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":5,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/automatic-generation-of-contrast-sets-from#ran","syntology_url":"https://syntology.ai/paper/2103.09591","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2103.09591"}},"official":{"repos":["yonatanbitton/AutoGenOfContrastSetsFromSceneGraphs"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/logic-embeddings-for-complex-query-answering","slug":"logic-embeddings-for-complex-query-answering","title":"Logic Embeddings for Complex Query Answering","date":"2021-02-28","arxiv_id":"2103.00418","repositories_listed":4,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/logic-embeddings-for-complex-query-answering#ran","syntology_url":"https://syntology.ai/paper/2103.00418","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2103.00418"}},"official":{"repos":["francoisluus/KGReasoning"],"state":"official: no sample here; runs from other or unrecorded repositories","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["unlocated"]}}},{"url":"/paper/nystromformer-a-nystrom-based-algorithm-for","slug":"nystromformer-a-nystrom-based-algorithm-for","title":"Nyströmformer: A Nyström-Based Algorithm for Approximating Self-Attention","date":"2021-02-07","arxiv_id":"2102.03902","repositories_listed":10,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/nystromformer-a-nystrom-based-algorithm-for#ran","syntology_url":"https://syntology.ai/paper/2102.03902","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2102.03902"}},"official":{"repos":["mlpen/Nystromformer"],"state":"official: no sample here; runs from other or unrecorded repositories","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["unlocated"]}}},{"url":"/paper/unifying-vision-and-language-tasks-via-text","slug":"unifying-vision-and-language-tasks-via-text","title":"Unifying Vision-and-Language Tasks via Text Generation","date":"2021-02-04","arxiv_id":"2102.02779","repositories_listed":2,"syntology":{"n":12,"n_ran":2,"n_constructed":2,"n_ran_checked":2,"n_instrument":0,"n_unverified":10,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 2 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 10 unverified; every one of the 2 samples that ran constructed an object rather than computing a result","sample_list":"/paper/unifying-vision-and-language-tasks-via-text#ran","syntology_url":"https://syntology.ai/paper/2102.02779","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2102.02779"}},"official":{"repos":["j-min/VL-T5"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":2,"n_ran_no_instrument_failure":2,"n_unverified":10,"ran_from_kinds":["official"]}}},{"url":"/paper/what-makes-good-in-context-examples-for-gpt-3","slug":"what-makes-good-in-context-examples-for-gpt-3","title":"What Makes Good In-Context Examples for GPT-$3$?","date":"2021-01-17","arxiv_id":"2101.06804","repositories_listed":3,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/what-makes-good-in-context-examples-for-gpt-3#ran","syntology_url":"https://syntology.ai/paper/2101.06804","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2101.06804"}},"official":null}},{"url":"/paper/switch-transformers-scaling-to-trillion","slug":"switch-transformers-scaling-to-trillion","title":"Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity","date":"2021-01-11","arxiv_id":"2101.03961","repositories_listed":8,"syntology":{"n":13,"n_ran":7,"n_constructed":2,"n_ran_checked":3,"n_instrument":4,"n_unverified":6,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":8,"phrase":"7 ran (of which 2 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 4 where Syntology's instrument failed) · 6 unverified","sample_list":"/paper/switch-transformers-scaling-to-trillion#ran","syntology_url":"https://syntology.ai/paper/2101.03961","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2101.03961"}},"official":{"repos":["tensorflow/mesh"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":4,"ran_from_kinds":["listed","unlocated"]}}},{"url":"/paper/few-shot-question-answering-by-pretraining","slug":"few-shot-question-answering-by-pretraining","title":"Few-Shot Question Answering by Pretraining Span Selection","date":"2021-01-02","arxiv_id":"2101.00438","repositories_listed":5,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":2,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/few-shot-question-answering-by-pretraining#ran","syntology_url":"https://syntology.ai/paper/2101.00438","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2101.00438"}},"official":{"repos":["oriram/splinter"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/reader-guided-passage-reranking-for-open","slug":"reader-guided-passage-reranking-for-open","title":"Rider: Reader-Guided Passage Reranking for Open-Domain Question Answering","date":"2021-01-01","arxiv_id":"2101.00294","repositories_listed":1,"syntology":{"n":9,"n_ran":6,"n_constructed":0,"n_ran_checked":5,"n_instrument":1,"n_unverified":3,"n_honours":2,"n_violates":1,"n_no_contract":2,"n_pointer_only":9,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 2 honoured, 1 violated, 2 with no contract checked; 1 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/reader-guided-passage-reranking-for-open#ran","syntology_url":"https://syntology.ai/paper/2101.00294","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2101.00294"}},"official":{"repos":["morningmoni/GAR"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/erica-improving-entity-and-relation","slug":"erica-improving-entity-and-relation","title":"ERICA: Improving Entity and Relation Understanding for Pre-trained Language Models via Contrastive Learning","date":"2020-12-30","arxiv_id":"2012.15022","repositories_listed":1,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/erica-improving-entity-and-relation#ran","syntology_url":"https://syntology.ai/paper/2012.15022","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2012.15022"}},"official":{"repos":["thunlp/ERICA"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/overcoming-language-priors-with-self","slug":"overcoming-language-priors-with-self","title":"Overcoming Language Priors with Self-supervised Learning for Visual Question Answering","date":"2020-12-17","arxiv_id":"2012.11528","repositories_listed":1,"syntology":{"n":8,"n_ran":7,"n_constructed":3,"n_ran_checked":5,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":8,"phrase":"7 ran (of which 3 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/overcoming-language-priors-with-self#ran","syntology_url":"https://syntology.ai/paper/2012.11528","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2012.11528"}},"official":{"repos":["CrossmodalGroup/SSL-VQA"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":3,"n_ran_no_instrument_failure":5,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/distilling-knowledge-from-reader-to-retriever-1","slug":"distilling-knowledge-from-reader-to-retriever-1","title":"Distilling Knowledge from Reader to Retriever for Question Answering","date":"2020-12-08","arxiv_id":"2012.04584","repositories_listed":4,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/distilling-knowledge-from-reader-to-retriever-1#ran","syntology_url":"https://syntology.ai/paper/2012.04584","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2012.04584"}},"official":{"repos":["facebookresearch/FiD"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/learning-associative-inference-using-fast-1","slug":"learning-associative-inference-using-fast-1","title":"Learning Associative Inference Using Fast Weight Memory","date":"2020-11-16","arxiv_id":"2011.07831","repositories_listed":1,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":1,"n_honours":2,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 2 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/learning-associative-inference-using-fast-1#ran","syntology_url":"https://syntology.ai/paper/2011.07831","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2011.07831"}},"official":{"repos":["ischlag/Fast-Weight-Memory-public"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/learning-to-model-and-ignore-dataset-bias","slug":"learning-to-model-and-ignore-dataset-bias","title":"Learning to Model and Ignore Dataset Bias with Mixed Capacity Ensembles","date":"2020-11-07","arxiv_id":"2011.03856","repositories_listed":1,"syntology":{"n":18,"n_ran":10,"n_constructed":5,"n_ran_checked":8,"n_instrument":2,"n_unverified":8,"n_honours":2,"n_violates":1,"n_no_contract":5,"n_pointer_only":0,"phrase":"10 ran (of which 5 constructed an object rather than computing a result; 8 with no instrument failure: 2 honoured, 1 violated, 5 with no contract checked; 2 where Syntology's instrument failed) · 8 unverified","sample_list":"/paper/learning-to-model-and-ignore-dataset-bias#ran","syntology_url":"https://syntology.ai/paper/2011.03856","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2011.03856"}},"official":{"repos":["chrisc36/autobias"],"state":"official (archive's flag): 10 ran","n_ran":10,"n_constructed":5,"n_ran_no_instrument_failure":8,"n_unverified":8,"ran_from_kinds":["official"]}}},{"url":"/paper/exams-a-multi-subject-high-school","slug":"exams-a-multi-subject-high-school","title":"EXAMS: A Multi-Subject High School Examinations Dataset for Cross-Lingual and Multilingual Question Answering","date":"2020-11-05","arxiv_id":"2011.03080","repositories_listed":2,"syntology":{"n":3,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/exams-a-multi-subject-high-school#ran","syntology_url":"https://syntology.ai/paper/2011.03080","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2011.03080"}},"official":{"repos":["mhardalov/exams-qa"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/charbert-character-aware-pre-trained-language","slug":"charbert-character-aware-pre-trained-language","title":"CharBERT: Character-aware Pre-trained Language Model","date":"2020-11-03","arxiv_id":"2011.01513","repositories_listed":1,"syntology":{"n":13,"n_ran":10,"n_constructed":0,"n_ran_checked":8,"n_instrument":2,"n_unverified":3,"n_honours":2,"n_violates":0,"n_no_contract":6,"n_pointer_only":3,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 2 honoured, 0 violated, 6 with no contract checked; 2 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/charbert-character-aware-pre-trained-language#ran","syntology_url":"https://syntology.ai/paper/2011.01513","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2011.01513"}},"official":{"repos":["wtma/CharBERT"],"state":"official (archive's flag): 10 ran","n_ran":10,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/constructing-a-multi-hop-qa-dataset-for","slug":"constructing-a-multi-hop-qa-dataset-for","title":"Constructing A Multi-hop QA Dataset for Comprehensive Evaluation of Reasoning Steps","date":"2020-11-02","arxiv_id":"2011.01060","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":2,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":1,"n_no_contract":0,"n_pointer_only":1,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 1 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/constructing-a-multi-hop-qa-dataset-for#ran","syntology_url":"https://syntology.ai/paper/2011.01060","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2011.01060"}},"official":{"repos":["Alab-NII/2wikimultihop"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/loss-rescaling-vqa-revisiting-language-prior","slug":"loss-rescaling-vqa-revisiting-language-prior","title":"Loss re-scaling VQA: Revisiting the LanguagePrior Problem from a Class-imbalance View","date":"2020-10-30","arxiv_id":"2010.16010","repositories_listed":1,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":4,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":1,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/loss-rescaling-vqa-revisiting-language-prior#ran","syntology_url":"https://syntology.ai/paper/2010.16010","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.16010"}},"official":{"repos":["guoyang9/class-imbalance-VQA"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/cliniqg4qa-generating-diverse-questions-for","slug":"cliniqg4qa-generating-diverse-questions-for","title":"CliniQG4QA: Generating Diverse Questions for Domain Adaptation of Clinical Question Answering","date":"2020-10-30","arxiv_id":"2010.16021","repositories_listed":2,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":2,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":1,"n_no_contract":0,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 1 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/cliniqg4qa-generating-diverse-questions-for#ran","syntology_url":"https://syntology.ai/paper/2010.16021","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.16021"}},"official":{"repos":["panushri25/emrQA","sunlab-osu/CliniQG4QA"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/russiansuperglue-a-russian-language","slug":"russiansuperglue-a-russian-language","title":"RussianSuperGLUE: A Russian Language Understanding Evaluation Benchmark","date":"2020-10-29","arxiv_id":"2010.15925","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/russiansuperglue-a-russian-language#ran","syntology_url":"https://syntology.ai/paper/2010.15925","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.15925"}},"official":{"repos":["RussianNLP/RussianSuperGLUE"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/measuring-association-between-labels-and-free","slug":"measuring-association-between-labels-and-free","title":"Measuring Association Between Labels and Free-Text Rationales","date":"2020-10-24","arxiv_id":"2010.12762","repositories_listed":1,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":1,"n_honours":2,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 2 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/measuring-association-between-labels-and-free#ran","syntology_url":"https://syntology.ai/paper/2010.12762","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.12762"}},"official":{"repos":["allenai/label_rationale_association"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/retrieve-rerank-read-then-iterate-answering","slug":"retrieve-rerank-read-then-iterate-answering","title":"Answering Open-Domain Questions of Varying Reasoning Steps from Text","date":"2020-10-23","arxiv_id":"2010.12527","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":2,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":1,"n_no_contract":0,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 1 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/retrieve-rerank-read-then-iterate-answering#ran","syntology_url":"https://syntology.ai/paper/2010.12527","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.12527"}},"official":{"repos":["beerqa/irrr"],"state":"official: no sample here; runs from other or unrecorded repositories","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["unlocated"]}}},{"url":"/paper/xor-qa-cross-lingual-open-retrieval-question","slug":"xor-qa-cross-lingual-open-retrieval-question","title":"XOR QA: Cross-lingual Open-Retrieval Question Answering","date":"2020-10-22","arxiv_id":"2010.11856","repositories_listed":3,"syntology":{"n":9,"n_ran":8,"n_constructed":0,"n_ran_checked":6,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":2,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/xor-qa-cross-lingual-open-retrieval-question#ran","syntology_url":"https://syntology.ai/paper/2010.11856","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.11856"}},"official":{"repos":["AkariAsai/XORQA"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/mt5-a-massively-multilingual-pre-trained-text","slug":"mt5-a-massively-multilingual-pre-trained-text","title":"mT5: A massively multilingual pre-trained text-to-text transformer","date":"2020-10-22","arxiv_id":"2010.11934","repositories_listed":8,"syntology":{"n":13,"n_ran":10,"n_constructed":0,"n_ran_checked":10,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":10,"n_pointer_only":0,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 0 honoured, 0 violated, 10 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/mt5-a-massively-multilingual-pre-trained-text#ran","syntology_url":"https://syntology.ai/paper/2010.11934","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.11934"}},"official":{"repos":["google-research/multilingual-t5"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/sort-ing-vqa-models-contrastive-gradient","slug":"sort-ing-vqa-models-contrastive-gradient","title":"SOrT-ing VQA Models : Contrastive Gradient Learning for Improved Consistency","date":"2020-10-20","arxiv_id":"2010.10038","repositories_listed":1,"syntology":{"n":4,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":4,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/sort-ing-vqa-models-contrastive-gradient#ran","syntology_url":"https://syntology.ai/paper/2010.10038","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.10038"}},"official":{"repos":["sameerdharur/sorting-vqa"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/open-question-answering-over-tables-and-text-1","slug":"open-question-answering-over-tables-and-text-1","title":"Open Question Answering over Tables and Text","date":"2020-10-20","arxiv_id":"2010.10439","repositories_listed":1,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":1,"n_instrument":3,"n_unverified":1,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":3,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/open-question-answering-over-tables-and-text-1#ran","syntology_url":"https://syntology.ai/paper/2010.10439","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.10439"}},"official":{"repos":["wenhuchen/OTT-QA"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/knowledge-guided-open-attribute-value","slug":"knowledge-guided-open-attribute-value","title":"Knowledge-guided Open Attribute Value Extraction with Reinforcement Learning","date":"2020-10-19","arxiv_id":"2010.09189","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/knowledge-guided-open-attribute-value#ran","syntology_url":"https://syntology.ai/paper/2010.09189","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.09189"}},"official":{"repos":["yeliu0930/Knowledge-guided-Open-Attribute-Value-Extraction-with-Reinforcement-Learning"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/towards-interpreting-bert-for-reading","slug":"towards-interpreting-bert-for-reading","title":"Towards Interpreting BERT for Reading Comprehension Based QA","date":"2020-10-18","arxiv_id":"2010.08983","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/towards-interpreting-bert-for-reading#ran","syntology_url":"https://syntology.ai/paper/2010.08983","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.08983"}},"official":{"repos":["iitmnlp/BERT-Analysis-RCQA"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/length-adaptive-transformer-train-once-with-1","slug":"length-adaptive-transformer-train-once-with-1","title":"Length-Adaptive Transformer: Train Once with Length Drop, Use Anytime with Search","date":"2020-10-14","arxiv_id":"2010.07003","repositories_listed":1,"syntology":{"n":9,"n_ran":9,"n_constructed":0,"n_ran_checked":7,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":2,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/length-adaptive-transformer-train-once-with-1#ran","syntology_url":"https://syntology.ai/paper/2010.07003","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.07003"}},"official":{"repos":["clovaai/length-adaptive-transformer"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/f1-is-not-enough-models-and-evaluation","slug":"f1-is-not-enough-models-and-evaluation","title":"F1 is Not Enough! Models and Evaluation Towards User-Centered Explainable Question Answering","date":"2020-10-13","arxiv_id":"2010.06283","repositories_listed":1,"syntology":{"n":4,"n_ran":4,"n_constructed":4,"n_ran_checked":4,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":4,"phrase":"4 ran (of which 4 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified; every one of the 4 samples that ran constructed an object rather than computing a result","sample_list":"/paper/f1-is-not-enough-models-and-evaluation#ran","syntology_url":"https://syntology.ai/paper/2010.06283","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.06283"}},"official":{"repos":["boschresearch/f1-is-not-enough"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":4,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/open-domain-question-answering-goes","slug":"open-domain-question-answering-goes","title":"Open-Domain Question Answering Goes Conversational via Question Rewriting","date":"2020-10-10","arxiv_id":"2010.04898","repositories_listed":2,"syntology":{"n":8,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/open-domain-question-answering-goes#ran","syntology_url":"https://syntology.ai/paper/2010.04898","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.04898"}},"official":{"repos":["apple/ml-qrecc"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/autoqa-from-databases-to-qa-semantic-parsers","slug":"autoqa-from-databases-to-qa-semantic-parsers","title":"AutoQA: From Databases To QA Semantic Parsers With Only Synthetic Training Data","date":"2020-10-09","arxiv_id":"2010.04806","repositories_listed":3,"syntology":{"n":26,"n_ran":13,"n_constructed":2,"n_ran_checked":10,"n_instrument":3,"n_unverified":13,"n_honours":1,"n_violates":4,"n_no_contract":5,"n_pointer_only":26,"phrase":"13 ran (of which 2 constructed an object rather than computing a result; 10 with no instrument failure: 1 honoured, 4 violated, 5 with no contract checked; 3 where Syntology's instrument failed) · 13 unverified","sample_list":"/paper/autoqa-from-databases-to-qa-semantic-parsers#ran","syntology_url":"https://syntology.ai/paper/2010.04806","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.04806"}},"official":{"repos":["stanford-oval/genienlp","stanford-oval/genie-toolkit","stanford-oval/schema2qa"],"state":"official (archive's flag): 13 ran","n_ran":13,"n_constructed":2,"n_ran_no_instrument_failure":10,"n_unverified":13,"ran_from_kinds":["official"]}}},{"url":"/paper/learning-a-cost-effective-annotation-policy","slug":"learning-a-cost-effective-annotation-policy","title":"Learning a Cost-Effective Annotation Policy for Question Answering","date":"2020-10-07","arxiv_id":"2010.03476","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/learning-a-cost-effective-annotation-policy#ran","syntology_url":"https://syntology.ai/paper/2010.03476","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.03476"}},"official":{"repos":["bernhard2202/qa-annotation"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/exposing-shallow-heuristics-of-relation","slug":"exposing-shallow-heuristics-of-relation","title":"Exposing Shallow Heuristics of Relation Extraction Models with Challenge Data","date":"2020-10-07","arxiv_id":"2010.03656","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/exposing-shallow-heuristics-of-relation#ran","syntology_url":"https://syntology.ai/paper/2010.03656","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.03656"}},"official":{"repos":["shacharosn/CRE"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/unqovering-stereotyping-biases-via","slug":"unqovering-stereotyping-biases-via","title":"UnQovering Stereotyping Biases via Underspecified Questions","date":"2020-10-06","arxiv_id":"2010.02428","repositories_listed":1,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":0,"n_instrument":4,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/unqovering-stereotyping-biases-via#ran","syntology_url":"https://syntology.ai/paper/2010.02428","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.02428"}},"official":{"repos":["allenai/unqover"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/universal-natural-language-processing-with","slug":"universal-natural-language-processing-with","title":"Universal Natural Language Processing with Limited Annotations: Try Few-shot Textual Entailment as a Start","date":"2020-10-06","arxiv_id":"2010.02584","repositories_listed":1,"syntology":{"n":9,"n_ran":8,"n_constructed":0,"n_ran_checked":8,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":8,"n_pointer_only":0,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 0 violated, 8 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/universal-natural-language-processing-with#ran","syntology_url":"https://syntology.ai/paper/2010.02584","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.02584"}},"official":{"repos":["salesforce/UniversalFewShotNLP"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/infobert-improving-robustness-of-language-1","slug":"infobert-improving-robustness-of-language-1","title":"InfoBERT: Improving Robustness of Language Models from An Information Theoretic Perspective","date":"2020-10-05","arxiv_id":"2010.02329","repositories_listed":2,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":3,"n_instrument":1,"n_unverified":0,"n_honours":2,"n_violates":0,"n_no_contract":1,"n_pointer_only":4,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 2 honoured, 0 violated, 1 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/infobert-improving-robustness-of-language-1#ran","syntology_url":"https://syntology.ai/paper/2010.02329","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.02329"}},"official":{"repos":["AI-secure/InfoBERT"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/luke-deep-contextualized-entity","slug":"luke-deep-contextualized-entity","title":"LUKE: Deep Contextualized Entity Representations with Entity-aware Self-attention","date":"2020-10-02","arxiv_id":"2010.01057","repositories_listed":9,"syntology":{"n":10,"n_ran":3,"n_constructed":2,"n_ran_checked":3,"n_instrument":0,"n_unverified":7,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 2 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 7 unverified","sample_list":"/paper/luke-deep-contextualized-entity#ran","syntology_url":"https://syntology.ai/paper/2010.01057","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.01057"}},"official":{"repos":["studio-ousia/luke"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":2,"n_ran_no_instrument_failure":3,"n_unverified":5,"ran_from_kinds":["official"]}}},{"url":"/paper/towards-question-answering-as-an-automatic","slug":"towards-question-answering-as-an-automatic","title":"Towards Question-Answering as an Automatic Metric for Evaluating the Content Quality of a Summary","date":"2020-10-01","arxiv_id":"2010.00490","repositories_listed":2,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/towards-question-answering-as-an-automatic#ran","syntology_url":"https://syntology.ai/paper/2010.00490","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.00490"}},"official":{"repos":["CogComp/qaeval-experiments"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/what-disease-does-this-patient-have-a-large","slug":"what-disease-does-this-patient-have-a-large","title":"What Disease does this Patient Have? A Large-scale Open Domain Question Answering Dataset from Medical Exams","date":"2020-09-28","arxiv_id":"2009.13081","repositories_listed":3,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/what-disease-does-this-patient-have-a-large#ran","syntology_url":"https://syntology.ai/paper/2009.13081","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2009.13081"}},"official":{"repos":["jind11/MedQA"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/x-lxmert-paint-caption-and-answer-questions","slug":"x-lxmert-paint-caption-and-answer-questions","title":"X-LXMERT: Paint, Caption and Answer Questions with Multi-Modal Transformers","date":"2020-09-23","arxiv_id":"2009.11278","repositories_listed":2,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":2,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/x-lxmert-paint-caption-and-answer-questions#ran","syntology_url":"https://syntology.ai/paper/2009.11278","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2009.11278"}},"official":{"repos":["allenai/x-lxmert"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/filter-an-enhanced-fusion-method-for-cross","slug":"filter-an-enhanced-fusion-method-for-cross","title":"FILTER: An Enhanced Fusion Method for Cross-lingual Language Understanding","date":"2020-09-10","arxiv_id":"2009.05166","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/filter-an-enhanced-fusion-method-for-cross#ran","syntology_url":"https://syntology.ai/paper/2009.05166","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2009.05166"}},"official":null}},{"url":"/paper/kilt-a-benchmark-for-knowledge-intensive","slug":"kilt-a-benchmark-for-knowledge-intensive","title":"KILT: a Benchmark for Knowledge Intensive Language Tasks","date":"2020-09-04","arxiv_id":"2009.02252","repositories_listed":3,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/kilt-a-benchmark-for-knowledge-intensive#ran","syntology_url":"https://syntology.ai/paper/2009.02252","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2009.02252"}},"official":{"repos":["facebookresearch/KILT"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/rethinking-the-objectives-of-extractive","slug":"rethinking-the-objectives-of-extractive","title":"Rethinking the Objectives of Extractive Question Answering","date":"2020-08-28","arxiv_id":"2008.12804","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/rethinking-the-objectives-of-extractive#ran","syntology_url":"https://syntology.ai/paper/2008.12804","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2008.12804"}},"official":{"repos":["KNOT-FIT-BUT/JointSpanExtraction"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/hitter-hierarchical-transformers-for","slug":"hitter-hierarchical-transformers-for","title":"HittER: Hierarchical Transformers for Knowledge Graph Embeddings","date":"2020-08-28","arxiv_id":"2008.12813","repositories_listed":3,"syntology":{"n":7,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":1,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/hitter-hierarchical-transformers-for#ran","syntology_url":"https://syntology.ai/paper/2008.12813","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2008.12813"}},"official":null}},{"url":"/paper/location-aware-graph-convolutional-networks","slug":"location-aware-graph-convolutional-networks","title":"Location-aware Graph Convolutional Networks for Video Question Answering","date":"2020-08-07","arxiv_id":"2008.09105","repositories_listed":1,"syntology":{"n":7,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":6,"n_pointer_only":1,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 1 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/location-aware-graph-convolutional-networks#ran","syntology_url":"https://syntology.ai/paper/2008.09105","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2008.09105"}},"official":{"repos":["SunDoge/L-GCN"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/question-and-answer-test-train-overlap-in","slug":"question-and-answer-test-train-overlap-in","title":"Question and Answer Test-Train Overlap in Open-Domain Question Answering Datasets","date":"2020-08-06","arxiv_id":"2008.02637","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":2,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":1,"n_no_contract":0,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 1 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/question-and-answer-test-train-overlap-in#ran","syntology_url":"https://syntology.ai/paper/2008.02637","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2008.02637"}},"official":{"repos":["facebookresearch/QA-Overlap"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/mkqa-a-linguistically-diverse-benchmark-for","slug":"mkqa-a-linguistically-diverse-benchmark-for","title":"MKQA: A Linguistically Diverse Benchmark for Multilingual Open Domain Question Answering","date":"2020-07-30","arxiv_id":"2007.15207","repositories_listed":2,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/mkqa-a-linguistically-diverse-benchmark-for#ran","syntology_url":"https://syntology.ai/paper/2007.15207","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2007.15207"}},"official":{"repos":["apple/ml-mkqa"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/big-bird-transformers-for-longer-sequences","slug":"big-bird-transformers-for-longer-sequences","title":"Big Bird: Transformers for Longer Sequences","date":"2020-07-28","arxiv_id":"2007.14062","repositories_listed":14,"syntology":{"n":15,"n_ran":10,"n_constructed":1,"n_ran_checked":2,"n_instrument":8,"n_unverified":5,"n_honours":1,"n_violates":0,"n_no_contract":1,"n_pointer_only":11,"phrase":"10 ran (of which 1 constructed an object rather than computing a result; 2 with no instrument failure: 1 honoured, 0 violated, 1 with no contract checked; 8 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/big-bird-transformers-for-longer-sequences#ran","syntology_url":"https://syntology.ai/paper/2007.14062","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2007.14062"}},"official":{"repos":["google-research/bigbird"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/distributed-memory-based-self-supervised","slug":"distributed-memory-based-self-supervised","title":"Distributed Associative Memory Network with Memory Refreshing Loss","date":"2020-07-21","arxiv_id":"2007.10637","repositories_listed":1,"syntology":{"n":9,"n_ran":5,"n_constructed":0,"n_ran_checked":4,"n_instrument":1,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 1 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/distributed-memory-based-self-supervised#ran","syntology_url":"https://syntology.ai/paper/2007.10637","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2007.10637"}},"official":{"repos":["taewonpark/DAM"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/knowledge-based-video-question-answering-with","slug":"knowledge-based-video-question-answering-with","title":"Knowledge-Based Video Question Answering with Unsupervised Scene Descriptions","date":"2020-07-17","arxiv_id":"2007.08751","repositories_listed":1,"syntology":{"n":9,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/knowledge-based-video-question-answering-with#ran","syntology_url":"https://syntology.ai/paper/2007.08751","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2007.08751"}},"official":{"repos":["noagarcia/ROLL-VideoQA"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/kqa-pro-a-large-diagnostic-dataset-for","slug":"kqa-pro-a-large-diagnostic-dataset-for","title":"KQA Pro: A Dataset with Explicit Compositional Programs for Complex Question Answering over Knowledge Base","date":"2020-07-08","arxiv_id":"2007.03875","repositories_listed":2,"syntology":{"n":10,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":9,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 9 unverified","sample_list":"/paper/kqa-pro-a-large-diagnostic-dataset-for#ran","syntology_url":"https://syntology.ai/paper/2007.03875","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2007.03875"}},"official":{"repos":["shijx12/kqapro_baselines"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":9,"ran_from_kinds":["official"]}}},{"url":"/paper/docvqa-a-dataset-for-vqa-on-document-images","slug":"docvqa-a-dataset-for-vqa-on-document-images","title":"DocVQA: A Dataset for VQA on Document Images","date":"2020-07-01","arxiv_id":"2007.00398","repositories_listed":3,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":3,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":2,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/docvqa-a-dataset-for-vqa-on-document-images#ran","syntology_url":"https://syntology.ai/paper/2007.00398","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2007.00398"}},"official":null}},{"url":"/paper/relevance-guided-supervision-for-openqa-with","slug":"relevance-guided-supervision-for-openqa-with","title":"Relevance-guided Supervision for OpenQA with ColBERT","date":"2020-07-01","arxiv_id":"2007.00814","repositories_listed":5,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/relevance-guided-supervision-for-openqa-with#ran","syntology_url":"https://syntology.ai/paper/2007.00814","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2007.00814"}},"official":{"repos":["stanford-futuredata/ColBERT","stanfordnlp/ColBERT-QA"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/graph-optimal-transport-for-cross-domain","slug":"graph-optimal-transport-for-cross-domain","title":"Graph Optimal Transport for Cross-Domain Alignment","date":"2020-06-26","arxiv_id":"2006.14744","repositories_listed":1,"syntology":{"n":10,"n_ran":9,"n_constructed":0,"n_ran_checked":9,"n_instrument":0,"n_unverified":1,"n_honours":1,"n_violates":2,"n_no_contract":6,"n_pointer_only":3,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 1 honoured, 2 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/graph-optimal-transport-for-cross-domain#ran","syntology_url":"https://syntology.ai/paper/2006.14744","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.14744"}},"official":{"repos":["LiqunChen0606/Graph-Optimal-Transport"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":9,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/neuro-symbolic-visual-reasoning-disentangling","slug":"neuro-symbolic-visual-reasoning-disentangling","title":"Neuro-Symbolic Visual Reasoning: Disentangling \"Visual\" from \"Reasoning\"","date":"2020-06-20","arxiv_id":"2006.11524","repositories_listed":0,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/neuro-symbolic-visual-reasoning-disentangling#ran","syntology_url":"https://syntology.ai/paper/2006.11524","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.11524"}},"official":null}},{"url":"/paper/squeezebert-what-can-computer-vision-teach","slug":"squeezebert-what-can-computer-vision-teach","title":"SqueezeBERT: What can computer vision teach NLP about efficient neural networks?","date":"2020-06-19","arxiv_id":"2006.11316","repositories_listed":6,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/squeezebert-what-can-computer-vision-teach#ran","syntology_url":"https://syntology.ai/paper/2006.11316","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.11316"}},"official":{"repos":["huggingface/transformers"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/sparse-and-continuous-attention-mechanisms","slug":"sparse-and-continuous-attention-mechanisms","title":"Sparse and Continuous Attention Mechanisms","date":"2020-06-12","arxiv_id":"2006.07214","repositories_listed":2,"syntology":{"n":15,"n_ran":12,"n_constructed":9,"n_ran_checked":9,"n_instrument":3,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":9,"n_pointer_only":2,"phrase":"12 ran (of which 9 constructed an object rather than computing a result; 9 with no instrument failure: 0 honoured, 0 violated, 9 with no contract checked; 3 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/sparse-and-continuous-attention-mechanisms#ran","syntology_url":"https://syntology.ai/paper/2006.07214","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.07214"}},"official":{"repos":["deep-spin/mcan-vqa-continuous-attention"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":2,"ran_from_kinds":["listed"]}}},{"url":"/paper/large-scale-adversarial-training-for-vision","slug":"large-scale-adversarial-training-for-vision","title":"Large-Scale Adversarial Training for Vision-and-Language Representation Learning","date":"2020-06-11","arxiv_id":"2006.06195","repositories_listed":2,"syntology":{"n":20,"n_ran":12,"n_constructed":5,"n_ran_checked":9,"n_instrument":3,"n_unverified":8,"n_honours":1,"n_violates":0,"n_no_contract":8,"n_pointer_only":8,"phrase":"12 ran (of which 5 constructed an object rather than computing a result; 9 with no instrument failure: 1 honoured, 0 violated, 8 with no contract checked; 3 where Syntology's instrument failed) · 8 unverified","sample_list":"/paper/large-scale-adversarial-training-for-vision#ran","syntology_url":"https://syntology.ai/paper/2006.06195","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.06195"}},"official":{"repos":["zhegan27/LXMERT-AdvTrain","zhegan27/VILLA"],"state":"official (archive's flag): 12 ran","n_ran":12,"n_constructed":5,"n_ran_no_instrument_failure":9,"n_unverified":8,"ran_from_kinds":["official"]}}},{"url":"/paper/closed-loop-neural-symbolic-learning-via","slug":"closed-loop-neural-symbolic-learning-via","title":"Closed Loop Neural-Symbolic Learning via Integrating Neural Perception, Grammar Parsing, and Symbolic Reasoning","date":"2020-06-11","arxiv_id":"2006.06649","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":2,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/closed-loop-neural-symbolic-learning-via#ran","syntology_url":"https://syntology.ai/paper/2006.06649","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.06649"}},"official":{"repos":["liqing-ustc/NGS"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/deberta-decoding-enhanced-bert-with","slug":"deberta-decoding-enhanced-bert-with","title":"DeBERTa: Decoding-enhanced BERT with Disentangled Attention","date":"2020-06-05","arxiv_id":"2006.03654","repositories_listed":14,"syntology":{"n":13,"n_ran":4,"n_constructed":0,"n_ran_checked":3,"n_instrument":1,"n_unverified":9,"n_honours":2,"n_violates":1,"n_no_contract":0,"n_pointer_only":3,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 2 honoured, 1 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 9 unverified","sample_list":"/paper/deberta-decoding-enhanced-bert-with#ran","syntology_url":"https://syntology.ai/paper/2006.03654","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.03654"}},"official":{"repos":["microsoft/DeBERTa"],"state":"official: no sample here; runs from other or unrecorded repositories","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","named_in_paper","unlocated"]}}},{"url":"/paper/generating-diverse-and-consistent-qa-pairs","slug":"generating-diverse-and-consistent-qa-pairs","title":"Generating Diverse and Consistent QA pairs from Contexts with Information-Maximizing Hierarchical Conditional VAEs","date":"2020-05-28","arxiv_id":"2005.13837","repositories_listed":1,"syntology":{"n":16,"n_ran":11,"n_constructed":0,"n_ran_checked":11,"n_instrument":0,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":11,"n_pointer_only":3,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 11 with no instrument failure: 0 honoured, 0 violated, 11 with no contract checked; 0 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/generating-diverse-and-consistent-qa-pairs#ran","syntology_url":"https://syntology.ai/paper/2005.13837","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.13837"}},"official":{"repos":["seanie12/Info-HCVAE"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":11,"n_unverified":5,"ran_from_kinds":["official"]}}},{"url":"/paper/language-models-are-few-shot-learners","slug":"language-models-are-few-shot-learners","title":"Language Models are Few-Shot Learners","date":"2020-05-28","arxiv_id":"2005.14165","repositories_listed":67,"syntology":{"n":65,"n_ran":45,"n_constructed":0,"n_ran_checked":40,"n_instrument":5,"n_unverified":20,"n_honours":2,"n_violates":1,"n_no_contract":37,"n_pointer_only":7,"phrase":"45 ran (of which 0 constructed an object rather than computing a result; 40 with no instrument failure: 2 honoured, 1 violated, 37 with no contract checked; 5 where Syntology's instrument failed) · 20 unverified","sample_list":"/paper/language-models-are-few-shot-learners#ran","syntology_url":"https://syntology.ai/paper/2005.14165","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.14165"}},"official":{"repos":["openai/gpt-3"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/open-retrieval-conversational-question","slug":"open-retrieval-conversational-question","title":"Open-Retrieval Conversational Question Answering","date":"2020-05-22","arxiv_id":"2005.11364","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/open-retrieval-conversational-question#ran","syntology_url":"https://syntology.ai/paper/2005.11364","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.11364"}},"official":{"repos":["prdwb/orconvqa-release"],"state":"official: no sample here; runs from other or unrecorded repositories","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["unlocated"]}}},{"url":"/paper/retrieval-augmented-generation-for-knowledge","slug":"retrieval-augmented-generation-for-knowledge","title":"Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks","date":"2020-05-22","arxiv_id":"2005.11401","repositories_listed":18,"syntology":{"n":6,"n_ran":6,"n_constructed":3,"n_ran_checked":5,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"6 ran (of which 3 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/retrieval-augmented-generation-for-knowledge#ran","syntology_url":"https://syntology.ai/paper/2005.11401","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.11401"}},"official":null}},{"url":"/paper/question-driven-summarization-of-answers-to","slug":"question-driven-summarization-of-answers-to","title":"Question-Driven Summarization of Answers to Consumer Health Questions","date":"2020-05-18","arxiv_id":"2005.09067","repositories_listed":1,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":6,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/question-driven-summarization-of-answers-to#ran","syntology_url":"https://syntology.ai/paper/2005.09067","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.09067"}},"official":{"repos":["saverymax/qdriven-chiqa-summarization"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/covid-twitter-bert-a-natural-language","slug":"covid-twitter-bert-a-natural-language","title":"COVID-Twitter-BERT: A Natural Language Processing Model to Analyse COVID-19 Content on Twitter","date":"2020-05-15","arxiv_id":"2005.07503","repositories_listed":1,"syntology":{"n":9,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/covid-twitter-bert-a-natural-language#ran","syntology_url":"https://syntology.ai/paper/2005.07503","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.07503"}},"official":{"repos":["digitalepidemiologylab/covid-twitter-bert"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/dense-caption-matching-and-frame-selection","slug":"dense-caption-matching-and-frame-selection","title":"Dense-Caption Matching and Frame-Selection Gating for Temporal Localization in VideoQA","date":"2020-05-13","arxiv_id":"2005.06409","repositories_listed":1,"syntology":{"n":12,"n_ran":9,"n_constructed":6,"n_ran_checked":8,"n_instrument":1,"n_unverified":3,"n_honours":2,"n_violates":0,"n_no_contract":6,"n_pointer_only":1,"phrase":"9 ran (of which 6 constructed an object rather than computing a result; 8 with no instrument failure: 2 honoured, 0 violated, 6 with no contract checked; 1 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/dense-caption-matching-and-frame-selection#ran","syntology_url":"https://syntology.ai/paper/2005.06409","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.06409"}},"official":{"repos":["hyounghk/VideoQADenseCapFrameGate-ACL2020"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":6,"n_ran_no_instrument_failure":7,"n_unverified":3,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/beyond-accuracy-behavioral-testing-of-nlp","slug":"beyond-accuracy-behavioral-testing-of-nlp","title":"Beyond Accuracy: Behavioral Testing of NLP models with CheckList","date":"2020-05-08","arxiv_id":"2005.04118","repositories_listed":4,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/beyond-accuracy-behavioral-testing-of-nlp#ran","syntology_url":"https://syntology.ai/paper/2005.04118","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.04118"}},"official":{"repos":["marcotcr/checklist"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/cobra-contrastive-bi-modal-representation","slug":"cobra-contrastive-bi-modal-representation","title":"COBRA: Contrastive Bi-Modal Representation Algorithm","date":"2020-05-07","arxiv_id":"2005.03687","repositories_listed":1,"syntology":{"n":4,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/cobra-contrastive-bi-modal-representation#ran","syntology_url":"https://syntology.ai/paper/2005.03687","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.03687"}},"official":{"repos":["ovshake/cobra"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/connecting-the-dots-a-knowledgeable-path","slug":"connecting-the-dots-a-knowledgeable-path","title":"Connecting the Dots: A Knowledgeable Path Generator for Commonsense Question Answering","date":"2020-05-02","arxiv_id":"2005.00691","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/connecting-the-dots-a-knowledgeable-path#ran","syntology_url":"https://syntology.ai/paper/2005.00691","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.00691"}},"official":{"repos":["wangpf3/Commonsense-Path-Generator"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/deformer-decomposing-pre-trained-transformers","slug":"deformer-decomposing-pre-trained-transformers","title":"DeFormer: Decomposing Pre-trained Transformers for Faster Question Answering","date":"2020-05-02","arxiv_id":"2005.00697","repositories_listed":1,"syntology":{"n":6,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":3,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/deformer-decomposing-pre-trained-transformers#ran","syntology_url":"https://syntology.ai/paper/2005.00697","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.00697"}},"official":{"repos":["StonyBrookNLP/deformer"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/unifiedqa-crossing-format-boundaries-with-a","slug":"unifiedqa-crossing-format-boundaries-with-a","title":"UnifiedQA: Crossing Format Boundaries With a Single QA System","date":"2020-05-02","arxiv_id":"2005.00700","repositories_listed":2,"syntology":{"n":7,"n_ran":4,"n_constructed":3,"n_ran_checked":4,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":1,"n_no_contract":3,"n_pointer_only":1,"phrase":"4 ran (of which 3 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 1 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/unifiedqa-crossing-format-boundaries-with-a#ran","syntology_url":"https://syntology.ai/paper/2005.00700","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.00700"}},"official":{"repos":["allenai/unifiedqa"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":3,"n_ran_no_instrument_failure":4,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/exploring-and-predicting-transferability","slug":"exploring-and-predicting-transferability","title":"Exploring and Predicting Transferability across NLP Tasks","date":"2020-05-02","arxiv_id":"2005.00770","repositories_listed":1,"syntology":{"n":10,"n_ran":6,"n_constructed":0,"n_ran_checked":4,"n_instrument":2,"n_unverified":4,"n_honours":2,"n_violates":0,"n_no_contract":2,"n_pointer_only":4,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 2 honoured, 0 violated, 2 with no contract checked; 2 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/exploring-and-predicting-transferability#ran","syntology_url":"https://syntology.ai/paper/2005.00770","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.00770"}},"official":{"repos":["tuvuumass/task-transferability"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/hero-hierarchical-encoder-for-video-language","slug":"hero-hierarchical-encoder-for-video-language","title":"HERO: Hierarchical Encoder for Video+Language Omni-representation Pre-training","date":"2020-05-01","arxiv_id":"2005.00200","repositories_listed":3,"syntology":{"n":13,"n_ran":10,"n_constructed":0,"n_ran_checked":9,"n_instrument":1,"n_unverified":3,"n_honours":2,"n_violates":0,"n_no_contract":7,"n_pointer_only":8,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 2 honoured, 0 violated, 7 with no contract checked; 1 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/hero-hierarchical-encoder-for-video-language#ran","syntology_url":"https://syntology.ai/paper/2005.00200","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.00200"}},"official":{"repos":["linjieli222/HERO"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/clinical-reading-comprehension-a-thorough","slug":"clinical-reading-comprehension-a-thorough","title":"Clinical Reading Comprehension: A Thorough Analysis of the emrQA Dataset","date":"2020-05-01","arxiv_id":"2005.00574","repositories_listed":1,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":0,"n_instrument":4,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/clinical-reading-comprehension-a-thorough#ran","syntology_url":"https://syntology.ai/paper/2005.00574","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.00574"}},"official":{"repos":["xiangyue9607/CliniRC"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/dynamic-language-binding-in-relational-visual","slug":"dynamic-language-binding-in-relational-visual","title":"Dynamic Language Binding in Relational Visual Reasoning","date":"2020-04-30","arxiv_id":"2004.14603","repositories_listed":1,"syntology":{"n":9,"n_ran":6,"n_constructed":5,"n_ran_checked":6,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 5 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/dynamic-language-binding-in-relational-visual#ran","syntology_url":"https://syntology.ai/paper/2004.14603","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.14603"}},"official":null}},{"url":"/paper/how-do-decisions-emerge-across-layers-in","slug":"how-do-decisions-emerge-across-layers-in","title":"How do Decisions Emerge across Layers in Neural Models? Interpretation with Differentiable Masking","date":"2020-04-30","arxiv_id":"2004.14992","repositories_listed":2,"syntology":{"n":10,"n_ran":6,"n_constructed":4,"n_ran_checked":6,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 4 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/how-do-decisions-emerge-across-layers-in#ran","syntology_url":"https://syntology.ai/paper/2004.14992","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.14992"}},"official":{"repos":["nicola-decao/diffmask"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":4,"n_ran_no_instrument_failure":4,"n_unverified":1,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/subjqa-a-dataset-for-subjectivity-and-review","slug":"subjqa-a-dataset-for-subjectivity-and-review","title":"SubjQA: A Dataset for Subjectivity and Review Comprehension","date":"2020-04-29","arxiv_id":"2004.14283","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/subjqa-a-dataset-for-subjectivity-and-review#ran","syntology_url":"https://syntology.ai/paper/2004.14283","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.14283"}},"official":{"repos":["megagonlabs/SubjQA"],"state":"official: no sample here; runs from other or unrecorded repositories","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["found_in_text"]}}},{"url":"/paper/pragmatic-issue-sensitive-image-captioning","slug":"pragmatic-issue-sensitive-image-captioning","title":"Pragmatic Issue-Sensitive Image Captioning","date":"2020-04-29","arxiv_id":"2004.14451","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/pragmatic-issue-sensitive-image-captioning#ran","syntology_url":"https://syntology.ai/paper/2004.14451","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.14451"}},"official":{"repos":["windweller/Pragmatic-ISIC"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["unlocated"]}}},{"url":"/paper/event-extraction-by-answering-almost-natural","slug":"event-extraction-by-answering-almost-natural","title":"Event Extraction by Answering (Almost) Natural Questions","date":"2020-04-28","arxiv_id":"2004.13625","repositories_listed":3,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/event-extraction-by-answering-almost-natural#ran","syntology_url":"https://syntology.ai/paper/2004.13625","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.13625"}},"official":{"repos":["xinyadu/eeqa"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/se-kge-a-location-aware-knowledge-graph","slug":"se-kge-a-location-aware-knowledge-graph","title":"SE-KGE: A Location-Aware Knowledge Graph Embedding Model for Geographic Question Answering and Spatial Semantic Lifting","date":"2020-04-25","arxiv_id":"2004.14171","repositories_listed":1,"syntology":{"n":9,"n_ran":8,"n_constructed":0,"n_ran_checked":8,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":8,"n_pointer_only":0,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 0 violated, 8 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/se-kge-a-location-aware-knowledge-graph#ran","syntology_url":"https://syntology.ai/paper/2004.14171","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.14171"}},"official":null}},{"url":"/paper/g-daug-generative-data-augmentation-for","slug":"g-daug-generative-data-augmentation-for","title":"Generative Data Augmentation for Commonsense Reasoning","date":"2020-04-24","arxiv_id":"2004.11546","repositories_listed":1,"syntology":{"n":4,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":4,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/g-daug-generative-data-augmentation-for#ran","syntology_url":"https://syntology.ai/paper/2004.11546","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.11546"}},"official":{"repos":["yangyiben/G-DAUG-c-Generative-Data-Augmentation-for-Commonsense-Reasoning"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/template-based-question-generation-from","slug":"template-based-question-generation-from","title":"Template-Based Question Generation from Retrieved Sentences for Improved Unsupervised Question Answering","date":"2020-04-24","arxiv_id":"2004.11892","repositories_listed":1,"syntology":{"n":3,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/template-based-question-generation-from#ran","syntology_url":"https://syntology.ai/paper/2004.11892","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.11892"}},"official":{"repos":["awslabs/unsupervised-qa"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/logic-guided-data-augmentation-and","slug":"logic-guided-data-augmentation-and","title":"Logic-Guided Data Augmentation and Regularization for Consistent Question Answering","date":"2020-04-21","arxiv_id":"2004.10157","repositories_listed":1,"syntology":{"n":18,"n_ran":8,"n_constructed":0,"n_ran_checked":0,"n_instrument":8,"n_unverified":10,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 8 where Syntology's instrument failed) · 10 unverified","sample_list":"/paper/logic-guided-data-augmentation-and#ran","syntology_url":"https://syntology.ai/paper/2004.10157","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.10157"}},"official":{"repos":["AkariAsai/logic_guided_qa"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":10,"ran_from_kinds":["official"]}}},{"url":"/paper/ktrain-a-low-code-library-for-augmented","slug":"ktrain-a-low-code-library-for-augmented","title":"ktrain: A Low-Code Library for Augmented Machine Learning","date":"2020-04-19","arxiv_id":"2004.10703","repositories_listed":4,"syntology":{"n":16,"n_ran":4,"n_constructed":0,"n_ran_checked":4,"n_instrument":0,"n_unverified":12,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 12 unverified","sample_list":"/paper/ktrain-a-low-code-library-for-augmented#ran","syntology_url":"https://syntology.ai/paper/2004.10703","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.10703"}},"official":{"repos":["amaiya/ktrain"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":10,"ran_from_kinds":["listed"]}}},{"url":"/paper/etc-encoding-long-and-structured-data-in","slug":"etc-encoding-long-and-structured-data-in","title":"ETC: Encoding Long and Structured Inputs in Transformers","date":"2020-04-17","arxiv_id":"2004.08483","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/etc-encoding-long-and-structured-data-in#ran","syntology_url":"https://syntology.ai/paper/2004.08483","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.08483"}},"official":{"repos":["google-research/google-research"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/hybridqa-a-dataset-of-multi-hop-question","slug":"hybridqa-a-dataset-of-multi-hop-question","title":"HybridQA: A Dataset of Multi-Hop Question Answering over Tabular and Textual Data","date":"2020-04-15","arxiv_id":"2004.07347","repositories_listed":2,"syntology":{"n":15,"n_ran":12,"n_constructed":0,"n_ran_checked":9,"n_instrument":3,"n_unverified":3,"n_honours":1,"n_violates":1,"n_no_contract":7,"n_pointer_only":2,"phrase":"12 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 1 honoured, 1 violated, 7 with no contract checked; 3 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/hybridqa-a-dataset-of-multi-hop-question#ran","syntology_url":"https://syntology.ai/paper/2004.07347","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.07347"}},"official":{"repos":["wenhuchen/HybridQA"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["listed","official","unlocated"]}}},{"url":"/paper/toward-subgraph-guided-knowledge-graph","slug":"toward-subgraph-guided-knowledge-graph","title":"Toward Subgraph-Guided Knowledge Graph Question Generation with Graph Neural Networks","date":"2020-04-13","arxiv_id":"2004.06015","repositories_listed":1,"syntology":{"n":5,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/toward-subgraph-guided-knowledge-graph#ran","syntology_url":"https://syntology.ai/paper/2004.06015","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.06015"}},"official":null}},{"url":"/paper/a-negative-case-analysis-of-visual-grounding","slug":"a-negative-case-analysis-of-visual-grounding","title":"Visual Grounding Methods for VQA are Working for the Wrong Reasons!","date":"2020-04-12","arxiv_id":"2004.05704","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/a-negative-case-analysis-of-visual-grounding#ran","syntology_url":"https://syntology.ai/paper/2004.05704","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.05704"}},"official":{"repos":["erobic/negative_analysis_of_grounding"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}}],"record_sha256":"98b9a91b57abf77522c490c13955bb36cddd5cf51c2422b8fbfffbb3c46aac21","record_changed_at":"2026-09-28","record_changed_at_basis":"first_hashed"}