{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/question-answering/papers/ran/10","list_of":"/task/question-answering","task":"Question Answering","archive":{"snapshot":"2025-07-28"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"syntology_read_at":"2026-09-28T10:30:06+00:00","order":"ran","order_definition":"only papers where Syntology ran at least one harvested sample; date (newest first), ties by arXiv id","caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","absence":"A paper missing from this list is not a recorded non-run: it may have no arXiv id, no harvested code, or only samples that have not run yet.","page":10,"pages_in_order":13,"rows_per_page":100,"rows":[901,1000],"of":1274,"counts":{"archive_papers_tagged":10817,"with_a_code_link":4171,"where_syntology_ran_a_sample":1274,"not_listed_spam_title":0,"listed":10817,"listed_where_code_ran":1274,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":1073,"every_run_a_failure_of_syntologys_instrument":201,"listed_with_a_run_with_no_instrument_failure":1073,"listed_every_run_a_failure_of_syntologys_instrument":201,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"first_page":"/task/question-answering/papers/ran/1","prev":"/task/question-answering/papers/ran/9","next":"/task/question-answering/papers/ran/11","papers":[{"url":"/paper/dream-uncovering-mental-models-behind","slug":"dream-uncovering-mental-models-behind","title":"DREAM: Improving Situational QA by First Elaborating the Situation","date":"2021-12-16","arxiv_id":"2112.08656","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/dream-uncovering-mental-models-behind#ran","syntology_url":"https://syntology.ai/paper/2112.08656","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2112.08656"}},"official":{"repos":["allenai/dream"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/evidentiality-guided-generation-for-knowledge","slug":"evidentiality-guided-generation-for-knowledge","title":"Evidentiality-guided Generation for Knowledge-Intensive NLP Tasks","date":"2021-12-16","arxiv_id":"2112.08688","repositories_listed":1,"syntology":{"n":13,"n_ran":10,"n_constructed":0,"n_ran_checked":9,"n_instrument":1,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":9,"n_pointer_only":7,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 0 honoured, 0 violated, 9 with no contract checked; 1 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/evidentiality-guided-generation-for-knowledge#ran","syntology_url":"https://syntology.ai/paper/2112.08688","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2112.08688"}},"official":{"repos":["akariasai/evidentiality_qa"],"state":"official (archive's flag): 10 ran","n_ran":10,"n_constructed":0,"n_ran_no_instrument_failure":9,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/towards-unsupervised-dense-information","slug":"towards-unsupervised-dense-information","title":"Unsupervised Dense Information Retrieval with Contrastive Learning","date":"2021-12-16","arxiv_id":"2112.09118","repositories_listed":6,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":4,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":1,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/towards-unsupervised-dense-information#ran","syntology_url":"https://syntology.ai/paper/2112.09118","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2112.09118"}},"official":{"repos":["facebookresearch/contriever"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/longt5-efficient-text-to-text-transformer-for","slug":"longt5-efficient-text-to-text-transformer-for","title":"LongT5: Efficient Text-To-Text Transformer for Long Sequences","date":"2021-12-15","arxiv_id":"2112.07916","repositories_listed":4,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/longt5-efficient-text-to-text-transformer-for#ran","syntology_url":"https://syntology.ai/paper/2112.07916","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2112.07916"}},"official":{"repos":["google-research/longt5"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/video-as-conditional-graph-hierarchy-for","slug":"video-as-conditional-graph-hierarchy-for","title":"Video as Conditional Graph Hierarchy for Multi-Granular Question Answering","date":"2021-12-12","arxiv_id":"2112.06197","repositories_listed":1,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":5,"n_pointer_only":1,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 1 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/video-as-conditional-graph-hierarchy-for#ran","syntology_url":"https://syntology.ai/paper/2112.06197","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2112.06197"}},"official":{"repos":["doc-doc/hqga"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/improving-language-models-by-retrieving-from","slug":"improving-language-models-by-retrieving-from","title":"Improving language models by retrieving from trillions of tokens","date":"2021-12-08","arxiv_id":"2112.04426","repositories_listed":2,"syntology":{"n":23,"n_ran":16,"n_constructed":5,"n_ran_checked":14,"n_instrument":2,"n_unverified":7,"n_honours":0,"n_violates":3,"n_no_contract":11,"n_pointer_only":3,"phrase":"16 ran (of which 5 constructed an object rather than computing a result; 14 with no instrument failure: 0 honoured, 3 violated, 11 with no contract checked; 2 where Syntology's instrument failed) · 7 unverified","sample_list":"/paper/improving-language-models-by-retrieving-from#ran","syntology_url":"https://syntology.ai/paper/2112.04426","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2112.04426"}},"official":null}},{"url":"/paper/causal-distillation-for-language-models","slug":"causal-distillation-for-language-models","title":"Causal Distillation for Language Models","date":"2021-12-05","arxiv_id":"2112.02505","repositories_listed":1,"syntology":{"n":4,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/causal-distillation-for-language-models#ran","syntology_url":"https://syntology.ai/paper/2112.02505","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2112.02505"}},"official":{"repos":["frankaging/Causal-Distill"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/classification-regression-for-chart","slug":"classification-regression-for-chart","title":"Classification-Regression for Chart Comprehension","date":"2021-11-29","arxiv_id":"2111.14792","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/classification-regression-for-chart#ran","syntology_url":"https://syntology.ai/paper/2111.14792","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2111.14792"}},"official":{"repos":["levymsn/cqa-crct"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/swinbert-end-to-end-transformers-with-sparse","slug":"swinbert-end-to-end-transformers-with-sparse","title":"SwinBERT: End-to-End Transformers with Sparse Attention for Video Captioning","date":"2021-11-25","arxiv_id":"2111.13196","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/swinbert-end-to-end-transformers-with-sparse#ran","syntology_url":"https://syntology.ai/paper/2111.13196","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2111.13196"}},"official":{"repos":["microsoft/swinbert"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/crossing-the-format-boundary-of-text-and","slug":"crossing-the-format-boundary-of-text-and","title":"UniTAB: Unifying Text and Box Outputs for Grounded Vision-Language Modeling","date":"2021-11-23","arxiv_id":"2111.12085","repositories_listed":1,"syntology":{"n":15,"n_ran":14,"n_constructed":0,"n_ran_checked":9,"n_instrument":5,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":8,"n_pointer_only":5,"phrase":"14 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 1 honoured, 0 violated, 8 with no contract checked; 5 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/crossing-the-format-boundary-of-text-and#ran","syntology_url":"https://syntology.ai/paper/2111.12085","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2111.12085"}},"official":{"repos":["microsoft/UniTAB"],"state":"official (archive's flag): 14 ran","n_ran":14,"n_constructed":0,"n_ran_no_instrument_failure":9,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/debertav3-improving-deberta-using-electra","slug":"debertav3-improving-deberta-using-electra","title":"DeBERTaV3: Improving DeBERTa using ELECTRA-Style Pre-Training with Gradient-Disentangled Embedding Sharing","date":"2021-11-18","arxiv_id":"2111.09543","repositories_listed":3,"syntology":{"n":7,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/debertav3-improving-deberta-using-electra#ran","syntology_url":"https://syntology.ai/paper/2111.09543","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2111.09543"}},"official":{"repos":["microsoft/DeBERTa"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/introspective-distillation-for-robust","slug":"introspective-distillation-for-robust","title":"Introspective Distillation for Robust Question Answering","date":"2021-11-01","arxiv_id":"2111.01026","repositories_listed":1,"syntology":{"n":7,"n_ran":7,"n_constructed":0,"n_ran_checked":4,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":3,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/introspective-distillation-for-robust#ran","syntology_url":"https://syntology.ai/paper/2111.01026","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2111.01026"}},"official":{"repos":["yuleiniu/introd"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/metaicl-learning-to-learn-in-context","slug":"metaicl-learning-to-learn-in-context","title":"MetaICL: Learning to Learn In Context","date":"2021-10-29","arxiv_id":"2110.15943","repositories_listed":2,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/metaicl-learning-to-learn-in-context#ran","syntology_url":"https://syntology.ai/paper/2110.15943","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.15943"}},"official":{"repos":["facebookresearch/metaicl"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/sqaler-scaling-question-answering-by","slug":"sqaler-scaling-question-answering-by","title":"SQALER: Scaling Question Answering by Decoupling Multi-Hop and Logical Reasoning","date":"2021-10-27","arxiv_id":"2110.14266","repositories_listed":0,"syntology":{"n":5,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":4,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/sqaler-scaling-question-answering-by#ran","syntology_url":"https://syntology.ai/paper/2110.14266","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.14266"}},"official":null}},{"url":"/paper/perceptual-score-what-data-modalities-does","slug":"perceptual-score-what-data-modalities-does","title":"Perceptual Score: What Data Modalities Does Your Model Perceive?","date":"2021-10-27","arxiv_id":"2110.14375","repositories_listed":3,"syntology":{"n":1,"n_ran":1,"n_constructed":1,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":1,"phrase":"1 ran (of which 1 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified; the one sample that ran constructed an object rather than computing a result","sample_list":"/paper/perceptual-score-what-data-modalities-does#ran","syntology_url":"https://syntology.ai/paper/2110.14375","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.14375"}},"official":{"repos":["itaigat/perceptual-score"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":1,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/vector-valued-distance-and-gyrocalculus-on","slug":"vector-valued-distance-and-gyrocalculus-on","title":"Vector-valued Distance and Gyrocalculus on the Space of Symmetric Positive Definite Matrices","date":"2021-10-26","arxiv_id":"2110.13475","repositories_listed":1,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/vector-valued-distance-and-gyrocalculus-on#ran","syntology_url":"https://syntology.ai/paper/2110.13475","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.13475"}},"official":{"repos":["fedelopez77/gyrospd"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/alignment-attention-by-matching-key-and-query","slug":"alignment-attention-by-matching-key-and-query","title":"Alignment Attention by Matching Key and Query Distributions","date":"2021-10-25","arxiv_id":"2110.12567","repositories_listed":1,"syntology":{"n":4,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/alignment-attention-by-matching-key-and-query#ran","syntology_url":"https://syntology.ai/paper/2110.12567","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.12567"}},"official":{"repos":["szhang42/alignment_attention"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/iconqa-a-new-benchmark-for-abstract-diagram","slug":"iconqa-a-new-benchmark-for-abstract-diagram","title":"IconQA: A New Benchmark for Abstract Diagram Understanding and Visual Language Reasoning","date":"2021-10-25","arxiv_id":"2110.13214","repositories_listed":1,"syntology":{"n":10,"n_ran":8,"n_constructed":0,"n_ran_checked":8,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":8,"n_pointer_only":10,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 0 violated, 8 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/iconqa-a-new-benchmark-for-abstract-diagram#ran","syntology_url":"https://syntology.ai/paper/2110.13214","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.13214"}},"official":{"repos":["lupantech/iconqa"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/label-descriptive-patterns-and-their","slug":"label-descriptive-patterns-and-their","title":"Label-Descriptive Patterns and Their Application to Characterizing Classification Errors","date":"2021-10-18","arxiv_id":"2110.09599","repositories_listed":2,"syntology":{"n":5,"n_ran":3,"n_constructed":3,"n_ran_checked":3,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 3 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified; every one of the 3 samples that ran constructed an object rather than computing a result","sample_list":"/paper/label-descriptive-patterns-and-their#ran","syntology_url":"https://syntology.ai/paper/2110.09599","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.09599"}},"official":{"repos":["uds-lsv/premise"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/contraqa-question-answering-under-1","slug":"contraqa-question-answering-under-1","title":"Attacking Open-domain Question Answering by Injecting Misinformation","date":"2021-10-15","arxiv_id":"2110.07803","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/contraqa-question-answering-under-1#ran","syntology_url":"https://syntology.ai/paper/2110.07803","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.07803"}},"official":{"repos":["teacherpeterpan/contraqa"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/can-explanations-be-useful-for-calibrating","slug":"can-explanations-be-useful-for-calibrating","title":"Can Explanations Be Useful for Calibrating Black Box Models?","date":"2021-10-14","arxiv_id":"2110.07586","repositories_listed":2,"syntology":{"n":4,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/can-explanations-be-useful-for-calibrating#ran","syntology_url":"https://syntology.ai/paper/2110.07586","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.07586"}},"official":{"repos":["xiye17/interpcalib"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/systematic-inequalities-in-language","slug":"systematic-inequalities-in-language","title":"Systematic Inequalities in Language Technology Performance across the World's Languages","date":"2021-10-13","arxiv_id":"2110.06733","repositories_listed":2,"syntology":{"n":6,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":6,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/systematic-inequalities-in-language#ran","syntology_url":"https://syntology.ai/paper/2110.06733","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.06733"}},"official":{"repos":["neubig/globalutility"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/conditionalqa-a-complex-reading-comprehension","slug":"conditionalqa-a-complex-reading-comprehension","title":"ConditionalQA: A Complex Reading Comprehension Dataset with Conditional Answers","date":"2021-10-13","arxiv_id":"2110.06884","repositories_listed":2,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/conditionalqa-a-complex-reading-comprehension#ran","syntology_url":"https://syntology.ai/paper/2110.06884","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.06884"}},"official":{"repos":["haitian-sun/conditionalqa"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/what-makes-sentences-semantically-related-a","slug":"what-makes-sentences-semantically-related-a","title":"What Makes Sentences Semantically Related: A Textual Relatedness Dataset and Empirical Study","date":"2021-10-10","arxiv_id":"2110.04845","repositories_listed":2,"syntology":{"n":4,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/what-makes-sentences-semantically-related-a#ran","syntology_url":"https://syntology.ai/paper/2110.04845","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.04845"}},"official":{"repos":["priya22/semantic-textual-relatedness"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/distantly-supervised-evidence-retrieval","slug":"distantly-supervised-evidence-retrieval","title":"Distantly-Supervised Evidence Retrieval Enables Question Answering without Evidence Annotation","date":"2021-10-10","arxiv_id":"2110.04889","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":1,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":1,"phrase":"1 ran (of which 1 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified; the one sample that ran constructed an object rather than computing a result","sample_list":"/paper/distantly-supervised-evidence-retrieval#ran","syntology_url":"https://syntology.ai/paper/2110.04889","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.04889"}},"official":{"repos":["henryzhao5852/distdr"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":1,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/a-few-more-examples-may-be-worth-billions-of","slug":"a-few-more-examples-may-be-worth-billions-of","title":"A Few More Examples May Be Worth Billions of Parameters","date":"2021-10-08","arxiv_id":"2110.04374","repositories_listed":1,"syntology":{"n":10,"n_ran":9,"n_constructed":0,"n_ran_checked":8,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":2,"n_no_contract":6,"n_pointer_only":3,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 2 violated, 6 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/a-few-more-examples-may-be-worth-billions-of#ran","syntology_url":"https://syntology.ai/paper/2110.04374","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.04374"}},"official":{"repos":["yuvalkirstain/lm-evaluation-harness"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/coarse-to-fine-reasoning-for-visual-question","slug":"coarse-to-fine-reasoning-for-visual-question","title":"Coarse-to-Fine Reasoning for Visual Question Answering","date":"2021-10-06","arxiv_id":"2110.02526","repositories_listed":2,"syntology":{"n":9,"n_ran":7,"n_constructed":0,"n_ran_checked":6,"n_instrument":1,"n_unverified":2,"n_honours":2,"n_violates":1,"n_no_contract":3,"n_pointer_only":2,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 2 honoured, 1 violated, 3 with no contract checked; 1 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/coarse-to-fine-reasoning-for-visual-question#ran","syntology_url":"https://syntology.ai/paper/2110.02526","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.02526"}},"official":{"repos":["aioz-ai/cfr_vqa","aioz-ai/crf_vqa"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/entqa-entity-linking-as-question-answering","slug":"entqa-entity-linking-as-question-answering","title":"EntQA: Entity Linking as Question Answering","date":"2021-10-05","arxiv_id":"2110.02369","repositories_listed":2,"syntology":{"n":9,"n_ran":9,"n_constructed":3,"n_ran_checked":3,"n_instrument":6,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":1,"phrase":"9 ran (of which 3 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 6 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/entqa-entity-linking-as-question-answering#ran","syntology_url":"https://syntology.ai/paper/2110.02369","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.02369"}},"official":{"repos":["wenzhengzhang/entqa"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":2,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/single-dataset-experts-for-multi-dataset","slug":"single-dataset-experts-for-multi-dataset","title":"Single-dataset Experts for Multi-dataset Question Answering","date":"2021-09-28","arxiv_id":"2109.13880","repositories_listed":1,"syntology":{"n":11,"n_ran":11,"n_constructed":0,"n_ran_checked":10,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":1,"n_no_contract":8,"n_pointer_only":3,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 1 honoured, 1 violated, 8 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/single-dataset-experts-for-multi-dataset#ran","syntology_url":"https://syntology.ai/paper/2109.13880","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.13880"}},"official":{"repos":["princeton-nlp/made"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":10,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/improving-question-answering-performance","slug":"improving-question-answering-performance","title":"Improving Question Answering Performance Using Knowledge Distillation and Active Learning","date":"2021-09-26","arxiv_id":"2109.12662","repositories_listed":1,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/improving-question-answering-performance#ran","syntology_url":"https://syntology.ai/paper/2109.12662","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.12662"}},"official":{"repos":["mirbostani/QA-KD-AL"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/sd-qa-spoken-dialectal-question-answering-for","slug":"sd-qa-spoken-dialectal-question-answering-for","title":"SD-QA: Spoken Dialectal Question Answering for the Real World","date":"2021-09-24","arxiv_id":"2109.12072","repositories_listed":1,"syntology":{"n":22,"n_ran":20,"n_constructed":0,"n_ran_checked":20,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":20,"n_pointer_only":0,"phrase":"20 ran (of which 0 constructed an object rather than computing a result; 20 with no instrument failure: 0 honoured, 0 violated, 20 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/sd-qa-spoken-dialectal-question-answering-for#ran","syntology_url":"https://syntology.ai/paper/2109.12072","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.12072"}},"official":{"repos":["ffaisal93/sd-qa"],"state":"official (archive's flag): 20 ran","n_ran":20,"n_constructed":0,"n_ran_no_instrument_failure":20,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/codeqa-a-question-answering-dataset-for","slug":"codeqa-a-question-answering-dataset-for","title":"CodeQA: A Question Answering Dataset for Source Code Comprehension","date":"2021-09-17","arxiv_id":"2109.08365","repositories_listed":1,"syntology":{"n":11,"n_ran":11,"n_constructed":0,"n_ran_checked":7,"n_instrument":4,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":2,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 4 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/codeqa-a-question-answering-dataset-for#ran","syntology_url":"https://syntology.ai/paper/2109.08365","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.08365"}},"official":{"repos":["jadecxliu/codeqa"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/neural-unification-for-logic-reasoning-over","slug":"neural-unification-for-logic-reasoning-over","title":"Neural Unification for Logic Reasoning over Natural Language","date":"2021-09-17","arxiv_id":"2109.08460","repositories_listed":1,"syntology":{"n":7,"n_ran":4,"n_constructed":0,"n_ran_checked":4,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/neural-unification-for-logic-reasoning-over#ran","syntology_url":"https://syntology.ai/paper/2109.08460","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.08460"}},"official":{"repos":["ibm/neural_unification_for_logic_reasoning_over_language"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/simple-entity-centric-questions-challenge","slug":"simple-entity-centric-questions-challenge","title":"Simple Entity-Centric Questions Challenge Dense Retrievers","date":"2021-09-17","arxiv_id":"2109.08535","repositories_listed":1,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":5,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/simple-entity-centric-questions-challenge#ran","syntology_url":"https://syntology.ai/paper/2109.08535","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.08535"}},"official":{"repos":["princeton-nlp/entityquestions"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/rng-kbqa-generation-augmented-iterative","slug":"rng-kbqa-generation-augmented-iterative","title":"RnG-KBQA: Generation Augmented Iterative Ranking for Knowledge Base Question Answering","date":"2021-09-17","arxiv_id":"2109.08678","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/rng-kbqa-generation-augmented-iterative#ran","syntology_url":"https://syntology.ai/paper/2109.08678","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.08678"}},"official":{"repos":["salesforce/rng-kbqa"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/phrase-retrieval-learns-passage-retrieval-too","slug":"phrase-retrieval-learns-passage-retrieval-too","title":"Phrase Retrieval Learns Passage Retrieval, Too","date":"2021-09-16","arxiv_id":"2109.08133","repositories_listed":1,"syntology":{"n":19,"n_ran":11,"n_constructed":0,"n_ran_checked":6,"n_instrument":5,"n_unverified":8,"n_honours":2,"n_violates":1,"n_no_contract":3,"n_pointer_only":5,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 2 honoured, 1 violated, 3 with no contract checked; 5 where Syntology's instrument failed) · 8 unverified","sample_list":"/paper/phrase-retrieval-learns-passage-retrieval-too#ran","syntology_url":"https://syntology.ai/paper/2109.08133","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.08133"}},"official":{"repos":["princeton-nlp/DensePhrases"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":8,"ran_from_kinds":["official"]}}},{"url":"/paper/expanding-end-to-end-question-answering-on","slug":"expanding-end-to-end-question-answering-on","title":"Expanding End-to-End Question Answering on Differentiable Knowledge Graphs with Intersection","date":"2021-09-13","arxiv_id":"2109.05808","repositories_listed":1,"syntology":{"n":4,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":4,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/expanding-end-to-end-question-answering-on#ran","syntology_url":"https://syntology.ai/paper/2109.05808","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.05808"}},"official":null}},{"url":"/paper/question-answering-over-electronic-devices-a","slug":"question-answering-over-electronic-devices-a","title":"Question Answering over Electronic Devices: A New Benchmark Dataset and a Multi-Task Learning based QA Framework","date":"2021-09-13","arxiv_id":"2109.05897","repositories_listed":1,"syntology":{"n":9,"n_ran":9,"n_constructed":0,"n_ran_checked":8,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":8,"n_pointer_only":0,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 0 violated, 8 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/question-answering-over-electronic-devices-a#ran","syntology_url":"https://syntology.ai/paper/2109.05897","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.05897"}},"official":{"repos":["abhi1nandy2/emnlp-2021-findings"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/reasonbert-pre-trained-to-reason-with-distant","slug":"reasonbert-pre-trained-to-reason-with-distant","title":"ReasonBERT: Pre-trained to Reason with Distant Supervision","date":"2021-09-10","arxiv_id":"2109.04912","repositories_listed":1,"syntology":{"n":11,"n_ran":9,"n_constructed":0,"n_ran_checked":8,"n_instrument":1,"n_unverified":2,"n_honours":1,"n_violates":1,"n_no_contract":6,"n_pointer_only":0,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 1 honoured, 1 violated, 6 with no contract checked; 1 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/reasonbert-pre-trained-to-reason-with-distant#ran","syntology_url":"https://syntology.ai/paper/2109.04912","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.04912"}},"official":{"repos":["sunlab-osu/reasonbert"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/an-empirical-study-of-gpt-3-for-few-shot","slug":"an-empirical-study-of-gpt-3-for-few-shot","title":"An Empirical Study of GPT-3 for Few-Shot Knowledge-Based VQA","date":"2021-09-10","arxiv_id":"2109.05014","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/an-empirical-study-of-gpt-3-for-few-shot#ran","syntology_url":"https://syntology.ai/paper/2109.05014","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.05014"}},"official":{"repos":["microsoft/PICa"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/weakly-supervised-visual-retriever-reader-for","slug":"weakly-supervised-visual-retriever-reader-for","title":"Weakly-Supervised Visual-Retriever-Reader for Knowledge-based Question Answering","date":"2021-09-09","arxiv_id":"2109.04014","repositories_listed":2,"syntology":{"n":4,"n_ran":3,"n_constructed":1,"n_ran_checked":1,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":1,"phrase":"3 ran (of which 1 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/weakly-supervised-visual-retriever-reader-for#ran","syntology_url":"https://syntology.ai/paper/2109.04014","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.04014"}},"official":{"repos":["luomancs/retriever_reader_for_okvqa"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":1,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/kelm-knowledge-enhanced-pre-trained-language","slug":"kelm-knowledge-enhanced-pre-trained-language","title":"KELM: Knowledge Enhanced Pre-Trained Language Representations with Message Passing on Hierarchical Relational Graphs","date":"2021-09-09","arxiv_id":"2109.04223","repositories_listed":1,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":5,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":1,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/kelm-knowledge-enhanced-pre-trained-language#ran","syntology_url":"https://syntology.ai/paper/2109.04223","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.04223"}},"official":{"repos":["nlp-anonymous-happy/anonymous-kg-guided-nlp"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/truthfulqa-measuring-how-models-mimic-human","slug":"truthfulqa-measuring-how-models-mimic-human","title":"TruthfulQA: Measuring How Models Mimic Human Falsehoods","date":"2021-09-08","arxiv_id":"2109.07958","repositories_listed":3,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/truthfulqa-measuring-how-models-mimic-human#ran","syntology_url":"https://syntology.ai/paper/2109.07958","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.07958"}},"official":{"repos":["sylinrl/truthfulqa"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/finqa-a-dataset-of-numerical-reasoning-over","slug":"finqa-a-dataset-of-numerical-reasoning-over","title":"FinQA: A Dataset of Numerical Reasoning over Financial Data","date":"2021-09-01","arxiv_id":"2109.00122","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":2,"n_instrument":1,"n_unverified":0,"n_honours":2,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 2 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/finqa-a-dataset-of-numerical-reasoning-over#ran","syntology_url":"https://syntology.ai/paper/2109.00122","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.00122"}},"official":{"repos":["czyssrs/finqa"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/webqa-multihop-and-multimodal-qa","slug":"webqa-multihop-and-multimodal-qa","title":"WebQA: Multihop and Multimodal QA","date":"2021-09-01","arxiv_id":"2109.00590","repositories_listed":3,"syntology":{"n":13,"n_ran":10,"n_constructed":0,"n_ran_checked":8,"n_instrument":2,"n_unverified":3,"n_honours":2,"n_violates":0,"n_no_contract":6,"n_pointer_only":12,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 2 honoured, 0 violated, 6 with no contract checked; 2 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/webqa-multihop-and-multimodal-qa#ran","syntology_url":"https://syntology.ai/paper/2109.00590","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.00590"}},"official":null}},{"url":"/paper/simvlm-simple-visual-language-model","slug":"simvlm-simple-visual-language-model","title":"SimVLM: Simple Visual Language Model Pretraining with Weak Supervision","date":"2021-08-24","arxiv_id":"2108.10904","repositories_listed":2,"syntology":{"n":37,"n_ran":22,"n_constructed":8,"n_ran_checked":16,"n_instrument":6,"n_unverified":15,"n_honours":1,"n_violates":3,"n_no_contract":12,"n_pointer_only":32,"phrase":"22 ran (of which 8 constructed an object rather than computing a result; 16 with no instrument failure: 1 honoured, 3 violated, 12 with no contract checked; 6 where Syntology's instrument failed) · 15 unverified","sample_list":"/paper/simvlm-simple-visual-language-model#ran","syntology_url":"https://syntology.ai/paper/2108.10904","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2108.10904"}},"official":null}},{"url":"/paper/hitab-a-hierarchical-table-dataset-for","slug":"hitab-a-hierarchical-table-dataset-for","title":"HiTab: A Hierarchical Table Dataset for Question Answering and Natural Language Generation","date":"2021-08-15","arxiv_id":"2108.06712","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/hitab-a-hierarchical-table-dataset-for#ran","syntology_url":"https://syntology.ai/paper/2108.06712","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2108.06712"}},"official":{"repos":["microsoft/hitab"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/how-optimal-is-greedy-decoding-for-extractive","slug":"how-optimal-is-greedy-decoding-for-extractive","title":"How Optimal is Greedy Decoding for Extractive Question Answering?","date":"2021-08-12","arxiv_id":"2108.05857","repositories_listed":1,"syntology":{"n":5,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/how-optimal-is-greedy-decoding-for-extractive#ran","syntology_url":"https://syntology.ai/paper/2108.05857","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2108.05857"}},"official":{"repos":["ocastel/exact-extract"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/dual-reader-parser-on-hybrid-textual-and","slug":"dual-reader-parser-on-hybrid-textual-and","title":"Dual Reader-Parser on Hybrid Textual and Tabular Evidence for Open Domain Question Answering","date":"2021-08-05","arxiv_id":"2108.02866","repositories_listed":1,"syntology":{"n":7,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":1,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/dual-reader-parser-on-hybrid-textual-and#ran","syntology_url":"https://syntology.ai/paper/2108.02866","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2108.02866"}},"official":{"repos":["awslabs/durepa-hybrid-qa"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/sparse-continuous-distributions-and-fenchel","slug":"sparse-continuous-distributions-and-fenchel","title":"Sparse Continuous Distributions and Fenchel-Young Losses","date":"2021-08-04","arxiv_id":"2108.01988","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/sparse-continuous-distributions-and-fenchel#ran","syntology_url":"https://syntology.ai/paper/2108.01988","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2108.01988"}},"official":{"repos":["deep-spin/sparse_continuous_distributions"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/musique-multi-hop-questions-via-single-hop","slug":"musique-multi-hop-questions-via-single-hop","title":"MuSiQue: Multihop Questions via Single-hop Question Composition","date":"2021-08-02","arxiv_id":"2108.00573","repositories_listed":3,"syntology":{"n":16,"n_ran":12,"n_constructed":0,"n_ran_checked":11,"n_instrument":1,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":11,"n_pointer_only":1,"phrase":"12 ran (of which 0 constructed an object rather than computing a result; 11 with no instrument failure: 0 honoured, 0 violated, 11 with no contract checked; 1 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/musique-multi-hop-questions-via-single-hop#ran","syntology_url":"https://syntology.ai/paper/2108.00573","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2108.00573"}},"official":{"repos":["stonybrooknlp/musique"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/greedy-gradient-ensemble-for-robust-visual","slug":"greedy-gradient-ensemble-for-robust-visual","title":"Greedy Gradient Ensemble for Robust Visual Question Answering","date":"2021-07-27","arxiv_id":"2107.12651","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":1,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 1 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified; the one sample that ran constructed an object rather than computing a result","sample_list":"/paper/greedy-gradient-ensemble-for-robust-visual#ran","syntology_url":"https://syntology.ai/paper/2107.12651","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2107.12651"}},"official":{"repos":["GeraldHan/GGE"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":1,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/one-question-answering-model-for-many","slug":"one-question-answering-model-for-many","title":"One Question Answering Model for Many Languages with Cross-lingual Dense Passage Retrieval","date":"2021-07-26","arxiv_id":"2107.11976","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/one-question-answering-model-for-many#ran","syntology_url":"https://syntology.ai/paper/2107.11976","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2107.11976"}},"official":{"repos":["AkariAsai/CORA"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/separating-skills-and-concepts-for-novel-1","slug":"separating-skills-and-concepts-for-novel-1","title":"Separating Skills and Concepts for Novel Visual Question Answering","date":"2021-07-19","arxiv_id":"2107.09106","repositories_listed":2,"syntology":{"n":8,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/separating-skills-and-concepts-for-novel-1#ran","syntology_url":"https://syntology.ai/paper/2107.09106","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2107.09106"}},"official":{"repos":["SpencerWhitehead/novelvqa"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/how-much-can-clip-benefit-vision-and-language","slug":"how-much-can-clip-benefit-vision-and-language","title":"How Much Can CLIP Benefit Vision-and-Language Tasks?","date":"2021-07-13","arxiv_id":"2107.06383","repositories_listed":4,"syntology":{"n":10,"n_ran":7,"n_constructed":0,"n_ran_checked":2,"n_instrument":5,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":9,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 5 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/how-much-can-clip-benefit-vision-and-language#ran","syntology_url":"https://syntology.ai/paper/2107.06383","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2107.06383"}},"official":{"repos":["clip-vil/CLIP-ViL"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["listed"]}}},{"url":"/paper/pondernet-learning-to-ponder","slug":"pondernet-learning-to-ponder","title":"PonderNet: Learning to Ponder","date":"2021-07-12","arxiv_id":"2107.05407","repositories_listed":4,"syntology":{"n":10,"n_ran":10,"n_constructed":5,"n_ran_checked":5,"n_instrument":5,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":1,"phrase":"10 ran (of which 5 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 5 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/pondernet-learning-to-ponder#ran","syntology_url":"https://syntology.ai/paper/2107.05407","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2107.05407"}},"official":null}},{"url":"/paper/a-long-short-term-memory-for-ai-applications","slug":"a-long-short-term-memory-for-ai-applications","title":"A Long Short-Term Memory for AI Applications in Spike-based Neuromorphic Hardware","date":"2021-07-08","arxiv_id":"2107.03992","repositories_listed":1,"syntology":{"n":11,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":11,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/a-long-short-term-memory-for-ai-applications#ran","syntology_url":"https://syntology.ai/paper/2107.03992","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2107.03992"}},"official":{"repos":["intel-nrc-ecosystem/models"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/mind-your-outliers-investigating-the-negative","slug":"mind-your-outliers-investigating-the-negative","title":"Mind Your Outliers! Investigating the Negative Impact of Outliers on Active Learning for Visual Question Answering","date":"2021-07-06","arxiv_id":"2107.02331","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/mind-your-outliers-investigating-the-negative#ran","syntology_url":"https://syntology.ai/paper/2107.02331","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2107.02331"}},"official":{"repos":["siddk/vqa-outliers"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/okgit-open-knowledge-graph-link-prediction","slug":"okgit-open-knowledge-graph-link-prediction","title":"OKGIT: Open Knowledge Graph Link Prediction with Implicit Types","date":"2021-06-24","arxiv_id":"2106.12806","repositories_listed":1,"syntology":{"n":4,"n_ran":1,"n_constructed":1,"n_ran_checked":1,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 1 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified; the one sample that ran constructed an object rather than computing a result","sample_list":"/paper/okgit-open-knowledge-graph-link-prediction#ran","syntology_url":"https://syntology.ai/paper/2106.12806","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.12806"}},"official":{"repos":["Chandrahasd/OKGIT"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":1,"n_ran_no_instrument_failure":1,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/learn-to-resolve-conversational-dependency-a","slug":"learn-to-resolve-conversational-dependency-a","title":"Learn to Resolve Conversational Dependency: A Consistency Training Framework for Conversational Question Answering","date":"2021-06-22","arxiv_id":"2106.11575","repositories_listed":1,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/learn-to-resolve-conversational-dependency-a#ran","syntology_url":"https://syntology.ai/paper/2106.11575","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.11575"}},"official":{"repos":["dmis-lab/excord"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/attend-what-you-need-motion-appearance","slug":"attend-what-you-need-motion-appearance","title":"Attend What You Need: Motion-Appearance Synergistic Networks for Video Question Answering","date":"2021-06-19","arxiv_id":"2106.10446","repositories_listed":1,"syntology":{"n":20,"n_ran":15,"n_constructed":14,"n_ran_checked":15,"n_instrument":0,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":15,"n_pointer_only":0,"phrase":"15 ran (of which 14 constructed an object rather than computing a result; 15 with no instrument failure: 0 honoured, 0 violated, 15 with no contract checked; 0 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/attend-what-you-need-motion-appearance#ran","syntology_url":"https://syntology.ai/paper/2106.10446","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.10446"}},"official":{"repos":["ahjeongseo/MASN-pytorch"],"state":"official (archive's flag): 15 ran","n_ran":15,"n_constructed":14,"n_ran_no_instrument_failure":15,"n_unverified":5,"ran_from_kinds":["official"]}}},{"url":"/paper/docnli-a-large-scale-dataset-for-document","slug":"docnli-a-large-scale-dataset-for-document","title":"DocNLI: A Large-scale Dataset for Document-level Natural Language Inference","date":"2021-06-17","arxiv_id":"2106.09449","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/docnli-a-large-scale-dataset-for-document#ran","syntology_url":"https://syntology.ai/paper/2106.09449","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.09449"}},"official":{"repos":["salesforce/DocNLI"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/modeling-worlds-in-text","slug":"modeling-worlds-in-text","title":"Modeling Worlds in Text","date":"2021-06-17","arxiv_id":"2106.09578","repositories_listed":1,"syntology":{"n":3,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/modeling-worlds-in-text#ran","syntology_url":"https://syntology.ai/paper/2106.09578","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.09578"}},"official":{"repos":["JerichoWorld/JerichoWorld"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/question-answering-infused-pre-training-of","slug":"question-answering-infused-pre-training-of","title":"Question Answering Infused Pre-training of General-Purpose Contextualized Representations","date":"2021-06-15","arxiv_id":"2106.08190","repositories_listed":1,"syntology":{"n":5,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":5,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/question-answering-infused-pre-training-of#ran","syntology_url":"https://syntology.ai/paper/2106.08190","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.08190"}},"official":{"repos":["facebookresearch/quip"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/thinking-like-transformers-1","slug":"thinking-like-transformers-1","title":"Thinking Like Transformers","date":"2021-06-13","arxiv_id":"2106.06981","repositories_listed":5,"syntology":{"n":18,"n_ran":11,"n_constructed":3,"n_ran_checked":7,"n_instrument":4,"n_unverified":7,"n_honours":2,"n_violates":1,"n_no_contract":4,"n_pointer_only":5,"phrase":"11 ran (of which 3 constructed an object rather than computing a result; 7 with no instrument failure: 2 honoured, 1 violated, 4 with no contract checked; 4 where Syntology's instrument failed) · 7 unverified","sample_list":"/paper/thinking-like-transformers-1#ran","syntology_url":"https://syntology.ai/paper/2106.06981","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.06981"}},"official":{"repos":["tech-srl/RASP"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":2,"n_ran_no_instrument_failure":4,"n_unverified":5,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/naaqa-a-neural-architecture-for-acoustic","slug":"naaqa-a-neural-architecture-for-acoustic","title":"NAAQA: A Neural Architecture for Acoustic Question Answering","date":"2021-06-11","arxiv_id":"2106.06147","repositories_listed":1,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/naaqa-a-neural-architecture-for-acoustic#ran","syntology_url":"https://syntology.ai/paper/2106.06147","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.06147"}},"official":null}},{"url":"/paper/improving-pretrained-cross-lingual-language","slug":"improving-pretrained-cross-lingual-language","title":"Improving Pretrained Cross-Lingual Language Models via Self-Labeled Word Alignment","date":"2021-06-11","arxiv_id":"2106.06381","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/improving-pretrained-cross-lingual-language#ran","syntology_url":"https://syntology.ai/paper/2106.06381","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.06381"}},"official":{"repos":["CZWin32768/XLM-Align"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/end-to-end-training-of-multi-document-reader","slug":"end-to-end-training-of-multi-document-reader","title":"End-to-End Training of Multi-Document Reader and Retriever for Open-Domain Question Answering","date":"2021-06-09","arxiv_id":"2106.05346","repositories_listed":2,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":0,"n_instrument":4,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":4,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/end-to-end-training-of-multi-document-reader#ran","syntology_url":"https://syntology.ai/paper/2106.05346","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.05346"}},"official":{"repos":["DevSinghSachan/emdr2"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["community","official"]}}},{"url":"/paper/value-a-multi-task-benchmark-for-video-and","slug":"value-a-multi-task-benchmark-for-video-and","title":"VALUE: A Multi-Task Benchmark for Video-and-Language Understanding Evaluation","date":"2021-06-08","arxiv_id":"2106.04632","repositories_listed":1,"syntology":{"n":7,"n_ran":5,"n_constructed":0,"n_ran_checked":4,"n_instrument":1,"n_unverified":2,"n_honours":2,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 2 honoured, 0 violated, 2 with no contract checked; 1 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/value-a-multi-task-benchmark-for-video-and#ran","syntology_url":"https://syntology.ai/paper/2106.04632","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.04632"}},"official":{"repos":["VALUE-Leaderboard/StarterCode"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/can-generative-pre-trained-language-models","slug":"can-generative-pre-trained-language-models","title":"Can Generative Pre-trained Language Models Serve as Knowledge Bases for Closed-book QA?","date":"2021-06-03","arxiv_id":"2106.01561","repositories_listed":1,"syntology":{"n":7,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/can-generative-pre-trained-language-models#ran","syntology_url":"https://syntology.ai/paper/2106.01561","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.01561"}},"official":{"repos":["wangcunxiang/Can-PLM-Serve-as-KB-for-CBQA"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/ethical-advice-taker-do-language-models","slug":"ethical-advice-taker-do-language-models","title":"Ethical-Advice Taker: Do Language Models Understand Natural Language Interventions?","date":"2021-06-02","arxiv_id":"2106.01465","repositories_listed":1,"syntology":{"n":11,"n_ran":9,"n_constructed":0,"n_ran_checked":9,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":9,"n_pointer_only":0,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 0 honoured, 0 violated, 9 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/ethical-advice-taker-do-language-models#ran","syntology_url":"https://syntology.ai/paper/2106.01465","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.01465"}},"official":{"repos":["allenai/ethical-interventions"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":9,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/dot-an-efficient-double-transformer-for-nlp","slug":"dot-an-efficient-double-transformer-for-nlp","title":"DoT: An efficient Double Transformer for NLP tasks with tables","date":"2021-06-01","arxiv_id":"2106.00479","repositories_listed":1,"syntology":{"n":9,"n_ran":5,"n_constructed":1,"n_ran_checked":1,"n_instrument":4,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"5 ran (of which 1 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 4 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/dot-an-efficient-double-transformer-for-nlp#ran","syntology_url":"https://syntology.ai/paper/2106.00479","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.00479"}},"official":{"repos":["google-research/tapas"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":1,"n_ran_no_instrument_failure":1,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/a-semantic-based-method-for-unsupervised","slug":"a-semantic-based-method-for-unsupervised","title":"A Semantic-based Method for Unsupervised Commonsense Question Answering","date":"2021-05-31","arxiv_id":"2105.14781","repositories_listed":1,"syntology":{"n":5,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/a-semantic-based-method-for-unsupervised#ran","syntology_url":"https://syntology.ai/paper/2105.14781","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.14781"}},"official":null}},{"url":"/paper/byt5-towards-a-token-free-future-with-pre","slug":"byt5-towards-a-token-free-future-with-pre","title":"ByT5: Towards a token-free future with pre-trained byte-to-byte models","date":"2021-05-28","arxiv_id":"2105.13626","repositories_listed":5,"syntology":{"n":6,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/byt5-towards-a-token-free-future-with-pre#ran","syntology_url":"https://syntology.ai/paper/2105.13626","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.13626"}},"official":{"repos":["google-research/byt5"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/scifive-a-text-to-text-transformer-model-for","slug":"scifive-a-text-to-text-transformer-model-for","title":"SciFive: a text-to-text transformer model for biomedical literature","date":"2021-05-28","arxiv_id":"2106.03598","repositories_listed":1,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/scifive-a-text-to-text-transformer-model-for#ran","syntology_url":"https://syntology.ai/paper/2106.03598","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.03598"}},"official":{"repos":["justinphan3110/SciFive"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/dynamic-semantic-graph-construction-and","slug":"dynamic-semantic-graph-construction-and","title":"Dynamic Semantic Graph Construction and Reasoning for Explainable Multi-hop Science Question Answering","date":"2021-05-25","arxiv_id":"2105.11776","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":2,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":2,"phrase":"2 ran (of which 2 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified; every one of the 2 samples that ran constructed an object rather than computing a result","sample_list":"/paper/dynamic-semantic-graph-construction-and#ran","syntology_url":"https://syntology.ai/paper/2105.11776","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.11776"}},"official":{"repos":["wwxu21/AMR-SG"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":2,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/multi-modal-understanding-and-generation-for","slug":"multi-modal-understanding-and-generation-for","title":"Multi-modal Understanding and Generation for Medical Images and Text via Vision-Language Pre-Training","date":"2021-05-24","arxiv_id":"2105.11333","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/multi-modal-understanding-and-generation-for#ran","syntology_url":"https://syntology.ai/paper/2105.11333","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.11333"}},"official":{"repos":["SuperSupermoon/MedViLL"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/multiple-meta-model-quantifying-for-medical","slug":"multiple-meta-model-quantifying-for-medical","title":"Multiple Meta-model Quantifying for Medical Visual Question Answering","date":"2021-05-19","arxiv_id":"2105.08913","repositories_listed":2,"syntology":{"n":11,"n_ran":8,"n_constructed":0,"n_ran_checked":7,"n_instrument":1,"n_unverified":3,"n_honours":0,"n_violates":3,"n_no_contract":4,"n_pointer_only":3,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 3 violated, 4 with no contract checked; 1 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/multiple-meta-model-quantifying-for-medical#ran","syntology_url":"https://syntology.ai/paper/2105.08913","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.08913"}},"official":{"repos":["aioz-ai/MICCAI21_MMQ"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":3,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/next-qa-next-phase-of-question-answering-to","slug":"next-qa-next-phase-of-question-answering-to","title":"NExT-QA:Next Phase of Question-Answering to Explaining Temporal Actions","date":"2021-05-18","arxiv_id":"2105.08276","repositories_listed":2,"syntology":{"n":7,"n_ran":7,"n_constructed":0,"n_ran_checked":5,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/next-qa-next-phase-of-question-answering-to#ran","syntology_url":"https://syntology.ai/paper/2105.08276","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.08276"}},"official":{"repos":["doc-doc/NExT-QA"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/tat-qa-a-question-answering-benchmark-on-a","slug":"tat-qa-a-question-answering-benchmark-on-a","title":"TAT-QA: A Question Answering Benchmark on a Hybrid of Tabular and Textual Content in Finance","date":"2021-05-17","arxiv_id":"2105.07624","repositories_listed":1,"syntology":{"n":12,"n_ran":11,"n_constructed":0,"n_ran_checked":11,"n_instrument":0,"n_unverified":1,"n_honours":2,"n_violates":0,"n_no_contract":9,"n_pointer_only":2,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 11 with no instrument failure: 2 honoured, 0 violated, 9 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/tat-qa-a-question-answering-benchmark-on-a#ran","syntology_url":"https://syntology.ai/paper/2105.07624","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.07624"}},"official":{"repos":["NExTplusplus/TAT-QA"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":11,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/pay-attention-to-mlps","slug":"pay-attention-to-mlps","title":"Pay Attention to MLPs","date":"2021-05-17","arxiv_id":"2105.08050","repositories_listed":20,"syntology":{"n":44,"n_ran":34,"n_constructed":15,"n_ran_checked":31,"n_instrument":3,"n_unverified":10,"n_honours":1,"n_violates":4,"n_no_contract":26,"n_pointer_only":11,"phrase":"34 ran (of which 15 constructed an object rather than computing a result; 31 with no instrument failure: 1 honoured, 4 violated, 26 with no contract checked; 3 where Syntology's instrument failed) · 10 unverified","sample_list":"/paper/pay-attention-to-mlps#ran","syntology_url":"https://syntology.ai/paper/2105.08050","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.08050"}},"official":null}},{"url":"/paper/inter-gps-interpretable-geometry-problem","slug":"inter-gps-interpretable-geometry-problem","title":"Inter-GPS: Interpretable Geometry Problem Solving with Formal Language and Symbolic Reasoning","date":"2021-05-10","arxiv_id":"2105.04165","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/inter-gps-interpretable-geometry-problem#ran","syntology_url":"https://syntology.ai/paper/2105.04165","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.04165"}},"official":{"repos":["lupantech/InterGPS"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/improving-factual-consistency-of-abstractive","slug":"improving-factual-consistency-of-abstractive","title":"Improving Factual Consistency of Abstractive Summarization via Question Answering","date":"2021-05-10","arxiv_id":"2105.04623","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/improving-factual-consistency-of-abstractive#ran","syntology_url":"https://syntology.ai/paper/2105.04623","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.04623"}},"official":null}},{"url":"/paper/passage-retrieval-for-outside-knowledge","slug":"passage-retrieval-for-outside-knowledge","title":"Passage Retrieval for Outside-Knowledge Visual Question Answering","date":"2021-05-09","arxiv_id":"2105.03938","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/passage-retrieval-for-outside-knowledge#ran","syntology_url":"https://syntology.ai/paper/2105.03938","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.03938"}},"official":{"repos":["prdwb/okvqa-release"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/a-dataset-of-information-seeking-questions","slug":"a-dataset-of-information-seeking-questions","title":"A Dataset of Information-Seeking Questions and Answers Anchored in Research Papers","date":"2021-05-07","arxiv_id":"2105.03011","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/a-dataset-of-information-seeking-questions#ran","syntology_url":"https://syntology.ai/paper/2105.03011","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.03011"}},"official":{"repos":["allenai/qasper-led-baseline"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/a-generative-symbolic-model-for-more-general","slug":"a-generative-symbolic-model-for-more-general","title":"Towards General Natural Language Understanding with Probabilistic Worldbuilding","date":"2021-05-06","arxiv_id":"2105.02486","repositories_listed":2,"syntology":{"n":14,"n_ran":8,"n_constructed":0,"n_ran_checked":8,"n_instrument":0,"n_unverified":6,"n_honours":0,"n_violates":1,"n_no_contract":7,"n_pointer_only":0,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 1 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 6 unverified","sample_list":"/paper/a-generative-symbolic-model-for-more-general#ran","syntology_url":"https://syntology.ai/paper/2105.02486","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.02486"}},"official":{"repos":["asaparov/fictionalgeoqa","asaparov/pwl"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":6,"ran_from_kinds":["official"]}}},{"url":"/paper/learning-to-perturb-word-embeddings-for-out","slug":"learning-to-perturb-word-embeddings-for-out","title":"Learning to Perturb Word Embeddings for Out-of-distribution QA","date":"2021-05-06","arxiv_id":"2105.02692","repositories_listed":1,"syntology":{"n":10,"n_ran":8,"n_constructed":0,"n_ran_checked":4,"n_instrument":4,"n_unverified":2,"n_honours":2,"n_violates":1,"n_no_contract":1,"n_pointer_only":6,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 2 honoured, 1 violated, 1 with no contract checked; 4 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/learning-to-perturb-word-embeddings-for-out#ran","syntology_url":"https://syntology.ai/paper/2105.02692","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.02692"}},"official":{"repos":["seanie12/SWEP"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/hidden-backdoors-in-human-centric-language","slug":"hidden-backdoors-in-human-centric-language","title":"Hidden Backdoors in Human-Centric Language Models","date":"2021-05-01","arxiv_id":"2105.00164","repositories_listed":1,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/hidden-backdoors-in-human-centric-language#ran","syntology_url":"https://syntology.ai/paper/2105.00164","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.00164"}},"official":{"repos":["lishaofeng/NLP_Backdoor"],"state":"official: harvested, nothing ran","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["unlocated"]}}},{"url":"/paper/entailment-as-few-shot-learner","slug":"entailment-as-few-shot-learner","title":"Entailment as Few-Shot Learner","date":"2021-04-29","arxiv_id":"2104.14690","repositories_listed":3,"syntology":{"n":3,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/entailment-as-few-shot-learner#ran","syntology_url":"https://syntology.ai/paper/2104.14690","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.14690"}},"official":null}},{"url":"/paper/mdetr-modulated-detection-for-end-to-end","slug":"mdetr-modulated-detection-for-end-to-end","title":"MDETR -- Modulated Detection for End-to-End Multi-Modal Understanding","date":"2021-04-26","arxiv_id":"2104.12763","repositories_listed":5,"syntology":{"n":11,"n_ran":7,"n_constructed":4,"n_ran_checked":6,"n_instrument":1,"n_unverified":4,"n_honours":0,"n_violates":1,"n_no_contract":5,"n_pointer_only":0,"phrase":"7 ran (of which 4 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 1 violated, 5 with no contract checked; 1 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/mdetr-modulated-detection-for-end-to-end#ran","syntology_url":"https://syntology.ai/paper/2104.12763","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.12763"}},"official":{"repos":["ashkamath/mdetr"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/when-does-pretraining-help-assessing-self","slug":"when-does-pretraining-help-assessing-self","title":"When Does Pretraining Help? Assessing Self-Supervised Learning for Law and the CaseHOLD Dataset","date":"2021-04-18","arxiv_id":"2104.08671","repositories_listed":2,"syntology":{"n":9,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/when-does-pretraining-help-assessing-self#ran","syntology_url":"https://syntology.ai/paper/2104.08671","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.08671"}},"official":{"repos":["reglab/casehold"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/gooaq-open-question-answering-with-diverse","slug":"gooaq-open-question-answering-with-diverse","title":"GooAQ: Open Question Answering with Diverse Answer Types","date":"2021-04-18","arxiv_id":"2104.08727","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/gooaq-open-question-answering-with-diverse#ran","syntology_url":"https://syntology.ai/paper/2104.08727","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.08727"}},"official":{"repos":["allenai/gooaq"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/natural-instructions-benchmarking","slug":"natural-instructions-benchmarking","title":"Cross-Task Generalization via Natural Language Crowdsourcing Instructions","date":"2021-04-18","arxiv_id":"2104.08773","repositories_listed":3,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":1,"n_instrument":4,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/natural-instructions-benchmarking#ran","syntology_url":"https://syntology.ai/paper/2104.08773","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.08773"}},"official":{"repos":["allenai/natural-instructions","allenai/natural-instructions-v1"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/explaining-answers-with-entailment-trees","slug":"explaining-answers-with-entailment-trees","title":"Explaining Answers with Entailment Trees","date":"2021-04-17","arxiv_id":"2104.08661","repositories_listed":1,"syntology":{"n":17,"n_ran":16,"n_constructed":0,"n_ran_checked":15,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":15,"n_pointer_only":1,"phrase":"16 ran (of which 0 constructed an object rather than computing a result; 15 with no instrument failure: 0 honoured, 0 violated, 15 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/explaining-answers-with-entailment-trees#ran","syntology_url":"https://syntology.ai/paper/2104.08661","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.08661"}},"official":null}},{"url":"/paper/beir-a-heterogenous-benchmark-for-zero-shot","slug":"beir-a-heterogenous-benchmark-for-zero-shot","title":"BEIR: A Heterogenous Benchmark for Zero-shot Evaluation of Information Retrieval Models","date":"2021-04-17","arxiv_id":"2104.08663","repositories_listed":3,"syntology":{"n":4,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/beir-a-heterogenous-benchmark-for-zero-shot#ran","syntology_url":"https://syntology.ai/paper/2104.08663","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.08663"}},"official":{"repos":["UKPLab/beir"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/editing-factual-knowledge-in-language-models","slug":"editing-factual-knowledge-in-language-models","title":"Editing Factual Knowledge in Language Models","date":"2021-04-16","arxiv_id":"2104.08164","repositories_listed":3,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":2,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":1,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/editing-factual-knowledge-in-language-models#ran","syntology_url":"https://syntology.ai/paper/2104.08164","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.08164"}},"official":{"repos":["nicola-decao/KnowledgeEditor"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/q-2-evaluating-factual-consistency-in","slug":"q-2-evaluating-factual-consistency-in","title":"$Q^{2}$: Evaluating Factual Consistency in Knowledge-Grounded Dialogues via Question Generation and Question Answering","date":"2021-04-16","arxiv_id":"2104.08202","repositories_listed":1,"syntology":{"n":7,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/q-2-evaluating-factual-consistency-in#ran","syntology_url":"https://syntology.ai/paper/2104.08202","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.08202"}},"official":null}},{"url":"/paper/what-to-pre-train-on-efficient-intermediate","slug":"what-to-pre-train-on-efficient-intermediate","title":"What to Pre-Train on? Efficient Intermediate Task Selection","date":"2021-04-16","arxiv_id":"2104.08247","repositories_listed":1,"syntology":{"n":6,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/what-to-pre-train-on-efficient-intermediate#ran","syntology_url":"https://syntology.ai/paper/2104.08247","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.08247"}},"official":{"repos":["adapter-hub/efficient-task-transfer"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/capturing-row-and-column-semantics-in","slug":"capturing-row-and-column-semantics-in","title":"Capturing Row and Column Semantics in Transformer Based Question Answering over Tables","date":"2021-04-16","arxiv_id":"2104.08303","repositories_listed":1,"syntology":{"n":10,"n_ran":9,"n_constructed":0,"n_ran_checked":9,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":9,"n_pointer_only":1,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 0 honoured, 0 violated, 9 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/capturing-row-and-column-semantics-in#ran","syntology_url":"https://syntology.ai/paper/2104.08303","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.08303"}},"official":{"repos":["IBM/row-column-intersection"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":9,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/designing-a-minimal-retrieve-and-read-system","slug":"designing-a-minimal-retrieve-and-read-system","title":"Designing a Minimal Retrieve-and-Read System for Open-Domain Question Answering","date":"2021-04-15","arxiv_id":"2104.07242","repositories_listed":1,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":0,"n_instrument":4,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/designing-a-minimal-retrieve-and-read-system#ran","syntology_url":"https://syntology.ai/paper/2104.07242","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.07242"}},"official":{"repos":["clovaai/minimal-rnr-qa"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}}],"record_sha256":"6d99a1cd892354a9b0edf3ed4a0ff27f8f9a70d8446289c904204c66ef4bfc18","record_changed_at":"2026-09-28","record_changed_at_basis":"first_hashed"}