{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/benchmarking/papers/ran/6","list_of":"/task/benchmarking","task":"Benchmarking","archive":{"snapshot":"2025-07-28"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"syntology_read_at":"2026-09-28T10:30:06+00:00","order":"ran","order_definition":"only papers where Syntology ran at least one harvested sample; date (newest first), ties by arXiv id","caption":"We ran code from the paper's repository; we did not run it on this task or check it against the task's benchmarks.","absence":"A paper missing from this list is not a recorded non-run: it may have no arXiv id, no harvested code, or only samples that have not run yet.","page":6,"pages_in_order":8,"rows_per_page":100,"rows":[501,600],"of":749,"counts":{"archive_papers_tagged":5548,"with_a_code_link":2658,"where_syntology_ran_a_sample":749,"not_listed_spam_title":0,"listed":5548,"listed_where_code_ran":749,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":624,"every_run_a_failure_of_syntologys_instrument":125,"listed_with_a_run_with_no_instrument_failure":624,"listed_every_run_a_failure_of_syntologys_instrument":125,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"first_page":"/task/benchmarking/papers/ran/1","prev":"/task/benchmarking/papers/ran/5","next":"/task/benchmarking/papers/ran/7","papers":[{"url":"/paper/2305-14516","slug":"2305-14516","title":"Chakra: Advancing Performance Benchmarking and Co-design using Standardized Execution Traces","date":"2023-05-23","arxiv_id":"2305.14516","repositories_listed":2,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":3,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/2305-14516#ran","syntology_url":"https://syntology.ai/paper/2305.14516","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.14516"}},"official":{"repos":["chakra-et/chakra"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/towards-benchmarking-and-assessing-visual-1","slug":"towards-benchmarking-and-assessing-visual-1","title":"Towards Benchmarking and Assessing Visual Naturalness of Physical World Adversarial Attacks","date":"2023-05-22","arxiv_id":"2305.12863","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/towards-benchmarking-and-assessing-visual-1#ran","syntology_url":"https://syntology.ai/paper/2305.12863","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.12863"}},"official":{"repos":["zhangsn-19/pan"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/evaluating-task-understanding-through","slug":"evaluating-task-understanding-through","title":"Separating form and meaning: Using self-consistency to quantify task understanding across multiple senses","date":"2023-05-19","arxiv_id":"2305.11662","repositories_listed":1,"syntology":{"n":4,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/evaluating-task-understanding-through#ran","syntology_url":"https://syntology.ai/paper/2305.11662","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.11662"}},"official":{"repos":["xeniaohmer/multisense_consistency"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/pmc-vqa-visual-instruction-tuning-for-medical","slug":"pmc-vqa-visual-instruction-tuning-for-medical","title":"PMC-VQA: Visual Instruction Tuning for Medical Visual Question Answering","date":"2023-05-17","arxiv_id":"2305.10415","repositories_listed":2,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":4,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":1,"n_no_contract":3,"n_pointer_only":2,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 1 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/pmc-vqa-visual-instruction-tuning-for-medical#ran","syntology_url":"https://syntology.ai/paper/2305.10415","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.10415"}},"official":{"repos":["xiaoman-zhang/PMC-VQA"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/infometic-an-informative-metric-for-reference","slug":"infometic-an-informative-metric-for-reference","title":"InfoMetIC: An Informative Metric for Reference-free Image Caption Evaluation","date":"2023-05-10","arxiv_id":"2305.06002","repositories_listed":1,"syntology":{"n":3,"n_ran":2,"n_constructed":2,"n_ran_checked":2,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":3,"phrase":"2 ran (of which 2 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified; every one of the 2 samples that ran constructed an object rather than computing a result","sample_list":"/paper/infometic-an-informative-metric-for-reference#ran","syntology_url":"https://syntology.ai/paper/2305.06002","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.06002"}},"official":{"repos":["hawlyq/infometic"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":2,"n_ran_no_instrument_failure":2,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/dexart-benchmarking-generalizable-dexterous","slug":"dexart-benchmarking-generalizable-dexterous","title":"DexArt: Benchmarking Generalizable Dexterous Manipulation with Articulated Objects","date":"2023-05-09","arxiv_id":"2305.05706","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/dexart-benchmarking-generalizable-dexterous#ran","syntology_url":"https://syntology.ai/paper/2305.05706","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.05706"}},"official":{"repos":["Kami-code/dexart-release"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/the-ebible-corpus-data-and-model-benchmarks","slug":"the-ebible-corpus-data-and-model-benchmarks","title":"The eBible Corpus: Data and Model Benchmarks for Bible Translation for Low-Resource Languages","date":"2023-04-19","arxiv_id":"2304.09919","repositories_listed":1,"syntology":{"n":15,"n_ran":14,"n_constructed":0,"n_ran_checked":14,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":14,"n_pointer_only":0,"phrase":"14 ran (of which 0 constructed an object rather than computing a result; 14 with no instrument failure: 0 honoured, 0 violated, 14 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/the-ebible-corpus-data-and-model-benchmarks#ran","syntology_url":"https://syntology.ai/paper/2304.09919","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.09919"}},"official":{"repos":["biblenlp/ebible-experiments"],"state":"official (archive's flag): 14 ran","n_ran":14,"n_constructed":0,"n_ran_no_instrument_failure":14,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/benchmarking-actor-critic-deep-reinforcement","slug":"benchmarking-actor-critic-deep-reinforcement","title":"Benchmarking Actor-Critic Deep Reinforcement Learning Algorithms for Robotics Control with Action Constraints","date":"2023-04-18","arxiv_id":"2304.08743","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/benchmarking-actor-critic-deep-reinforcement#ran","syntology_url":"https://syntology.ai/paper/2304.08743","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.08743"}},"official":{"repos":["omron-sinicx/action-constrained-rl-benchmark"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/openagi-when-llm-meets-domain-experts","slug":"openagi-when-llm-meets-domain-experts","title":"OpenAGI: When LLM Meets Domain Experts","date":"2023-04-10","arxiv_id":"2304.04370","repositories_listed":2,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/openagi-when-llm-meets-domain-experts#ran","syntology_url":"https://syntology.ai/paper/2304.04370","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.04370"}},"official":{"repos":["agiresearch/openagi"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/espnet-st-v2-multipurpose-spoken-language","slug":"espnet-st-v2-multipurpose-spoken-language","title":"ESPnet-ST-v2: Multipurpose Spoken Language Translation Toolkit","date":"2023-04-10","arxiv_id":"2304.04596","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/espnet-st-v2-multipurpose-spoken-language#ran","syntology_url":"https://syntology.ai/paper/2304.04596","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.04596"}},"official":{"repos":["espnet/espnet"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/on-evaluation-of-bangla-word-analogies","slug":"on-evaluation-of-bangla-word-analogies","title":"On Evaluation of Bangla Word Analogies","date":"2023-04-10","arxiv_id":"2304.04613","repositories_listed":0,"syntology":{"n":11,"n_ran":7,"n_constructed":0,"n_ran_checked":6,"n_instrument":1,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":11,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 1 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/on-evaluation-of-bangla-word-analogies#ran","syntology_url":"https://syntology.ai/paper/2304.04613","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.04613"}},"official":null}},{"url":"/paper/probing-conceptual-understanding-of-large","slug":"probing-conceptual-understanding-of-large","title":"Probing Conceptual Understanding of Large Visual-Language Models","date":"2023-04-07","arxiv_id":"2304.03659","repositories_listed":1,"syntology":{"n":4,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":4,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/probing-conceptual-understanding-of-large#ran","syntology_url":"https://syntology.ai/paper/2304.03659","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.03659"}},"official":{"repos":["Maddy12/UnderstandingVisualTextModels"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/interpretable-statistical-representations-of","slug":"interpretable-statistical-representations-of","title":"Interpretable statistical representations of neural population dynamics and geometry","date":"2023-04-06","arxiv_id":"2304.03376","repositories_listed":1,"syntology":{"n":13,"n_ran":8,"n_constructed":0,"n_ran_checked":8,"n_instrument":0,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":8,"n_pointer_only":0,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 0 violated, 8 with no contract checked; 0 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/interpretable-statistical-representations-of#ran","syntology_url":"https://syntology.ai/paper/2304.03376","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.03376"}},"official":{"repos":["Dynamics-of-Neural-Systems-Lab/MARBLE"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":5,"ran_from_kinds":["official"]}}},{"url":"/paper/what-makes-for-effective-few-shot-point-cloud","slug":"what-makes-for-effective-few-shot-point-cloud","title":"What Makes for Effective Few-shot Point Cloud Classification?","date":"2023-03-31","arxiv_id":"2304.00022","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/what-makes-for-effective-few-shot-point-cloud#ran","syntology_url":"https://syntology.ai/paper/2304.00022","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.00022"}},"official":{"repos":["cgye96/a_closer_look_at_3dfsl"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official","unlocated"]}}},{"url":"/paper/codegeex-a-pre-trained-model-for-code","slug":"codegeex-a-pre-trained-model-for-code","title":"CodeGeeX: A Pre-Trained Model for Code Generation with Multilingual Benchmarking on HumanEval-X","date":"2023-03-30","arxiv_id":"2303.17568","repositories_listed":2,"syntology":{"n":8,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":1,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/codegeex-a-pre-trained-model-for-code#ran","syntology_url":"https://syntology.ai/paper/2303.17568","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.17568"}},"official":{"repos":["THUDM/CodeGeeX"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/mgtbench-benchmarking-machine-generated-text","slug":"mgtbench-benchmarking-machine-generated-text","title":"MGTBench: Benchmarking Machine-Generated Text Detection","date":"2023-03-26","arxiv_id":"2303.14822","repositories_listed":4,"syntology":{"n":26,"n_ran":19,"n_constructed":0,"n_ran_checked":18,"n_instrument":1,"n_unverified":7,"n_honours":0,"n_violates":0,"n_no_contract":18,"n_pointer_only":5,"phrase":"19 ran (of which 0 constructed an object rather than computing a result; 18 with no instrument failure: 0 honoured, 0 violated, 18 with no contract checked; 1 where Syntology's instrument failed) · 7 unverified","sample_list":"/paper/mgtbench-benchmarking-machine-generated-text#ran","syntology_url":"https://syntology.ai/paper/2303.14822","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.14822"}},"official":{"repos":["xinleihe/mgtbench"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/mega-multilingual-evaluation-of-generative-ai","slug":"mega-multilingual-evaluation-of-generative-ai","title":"MEGA: Multilingual Evaluation of Generative AI","date":"2023-03-22","arxiv_id":"2303.12528","repositories_listed":1,"syntology":{"n":10,"n_ran":8,"n_constructed":0,"n_ran_checked":7,"n_instrument":1,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":1,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 1 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/mega-multilingual-evaluation-of-generative-ai#ran","syntology_url":"https://syntology.ai/paper/2303.12528","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.12528"}},"official":null}},{"url":"/paper/a-framework-for-benchmarking-class-out-of-1","slug":"a-framework-for-benchmarking-class-out-of-1","title":"A framework for benchmarking class-out-of-distribution detection and its application to ImageNet","date":"2023-02-23","arxiv_id":"2302.11893","repositories_listed":1,"syntology":{"n":15,"n_ran":13,"n_constructed":0,"n_ran_checked":13,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":13,"n_pointer_only":0,"phrase":"13 ran (of which 0 constructed an object rather than computing a result; 13 with no instrument failure: 0 honoured, 0 violated, 13 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/a-framework-for-benchmarking-class-out-of-1#ran","syntology_url":"https://syntology.ai/paper/2302.11893","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2302.11893"}},"official":{"repos":["mdabbah/COOD_benchmarking"],"state":"official (archive's flag): 13 ran","n_ran":13,"n_constructed":0,"n_ran_no_instrument_failure":13,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/cospgd-a-unified-white-box-adversarial-attack","slug":"cospgd-a-unified-white-box-adversarial-attack","title":"CosPGD: an efficient white-box adversarial attack for pixel-wise prediction tasks","date":"2023-02-04","arxiv_id":"2302.02213","repositories_listed":2,"syntology":{"n":3,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/cospgd-a-unified-white-box-adversarial-attack#ran","syntology_url":"https://syntology.ai/paper/2302.02213","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2302.02213"}},"official":{"repos":["shashankskagnihotri/adv-corrected-ddcat-cospgd","shashankskagnihotri/cospgd"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/temporai-facilitating-machine-learning","slug":"temporai-facilitating-machine-learning","title":"TemporAI: Facilitating Machine Learning Innovation in Time Domain Tasks for Medicine","date":"2023-01-28","arxiv_id":"2301.12260","repositories_listed":1,"syntology":{"n":9,"n_ran":9,"n_constructed":0,"n_ran_checked":9,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":9,"n_pointer_only":0,"phrase":"9 ran (of which 0 constructed an object rather than computing a result; 9 with no instrument failure: 0 honoured, 0 violated, 9 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/temporai-facilitating-machine-learning#ran","syntology_url":"https://syntology.ai/paper/2301.12260","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2301.12260"}},"official":{"repos":["vanderschaarlab/temporai"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":0,"n_ran_no_instrument_failure":9,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/task-agnostic-graph-neural-network-evaluation","slug":"task-agnostic-graph-neural-network-evaluation","title":"Task-Agnostic Graph Neural Network Evaluation via Adversarial Collaboration","date":"2023-01-27","arxiv_id":"2301.11517","repositories_listed":1,"syntology":{"n":4,"n_ran":4,"n_constructed":1,"n_ran_checked":3,"n_instrument":1,"n_unverified":0,"n_honours":2,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"4 ran (of which 1 constructed an object rather than computing a result; 3 with no instrument failure: 2 honoured, 0 violated, 1 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/task-agnostic-graph-neural-network-evaluation#ran","syntology_url":"https://syntology.ai/paper/2301.11517","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2301.11517"}},"official":{"repos":["victorzxy/graphac"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":1,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/benchmarking-the-robustness-of-lidar-semantic","slug":"benchmarking-the-robustness-of-lidar-semantic","title":"Benchmarking the Robustness of LiDAR Semantic Segmentation Models","date":"2023-01-03","arxiv_id":"2301.00970","repositories_listed":1,"syntology":{"n":9,"n_ran":7,"n_constructed":0,"n_ran_checked":5,"n_instrument":2,"n_unverified":2,"n_honours":0,"n_violates":1,"n_no_contract":4,"n_pointer_only":3,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 1 violated, 4 with no contract checked; 2 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/benchmarking-the-robustness-of-lidar-semantic#ran","syntology_url":"https://syntology.ai/paper/2301.00970","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2301.00970"}},"official":{"repos":["yanx27/2dpass"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/ultra-high-definition-low-light-image","slug":"ultra-high-definition-low-light-image","title":"Ultra-High-Definition Low-Light Image Enhancement: A Benchmark and Transformer-Based Method","date":"2022-12-22","arxiv_id":"2212.11548","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/ultra-high-definition-low-light-image#ran","syntology_url":"https://syntology.ai/paper/2212.11548","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.11548"}},"official":{"repos":["taowangzj/llformer"],"state":"official: no sample here; runs from other or unrecorded repositories","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["unlocated"]}}},{"url":"/paper/benchmarking-spatial-relationships-in-text-to","slug":"benchmarking-spatial-relationships-in-text-to","title":"Benchmarking Spatial Relationships in Text-to-Image Generation","date":"2022-12-20","arxiv_id":"2212.10015","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":2,"n_instrument":1,"n_unverified":0,"n_honours":2,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 2 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/benchmarking-spatial-relationships-in-text-to#ran","syntology_url":"https://syntology.ai/paper/2212.10015","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.10015"}},"official":{"repos":["microsoft/VISOR"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/a-comprehensive-study-and-comparison-of-the","slug":"a-comprehensive-study-and-comparison-of-the","title":"A Comprehensive Study of the Robustness for LiDAR-based 3D Object Detectors against Adversarial Attacks","date":"2022-12-20","arxiv_id":"2212.10230","repositories_listed":1,"syntology":{"n":7,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/a-comprehensive-study-and-comparison-of-the#ran","syntology_url":"https://syntology.ai/paper/2212.10230","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.10230"}},"official":{"repos":["Eaphan/Robust3DOD"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/are-multimodal-models-robust-to-image-and","slug":"are-multimodal-models-robust-to-image-and","title":"Benchmarking Robustness of Multimodal Image-Text Models under Distribution Shift","date":"2022-12-15","arxiv_id":"2212.08044","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/are-multimodal-models-robust-to-image-and#ran","syntology_url":"https://syntology.ai/paper/2212.08044","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.08044"}},"official":null}},{"url":"/paper/benchmarking-large-language-models-for","slug":"benchmarking-large-language-models-for","title":"Benchmarking Large Language Models for Automated Verilog RTL Code Generation","date":"2022-12-13","arxiv_id":"2212.11140","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/benchmarking-large-language-models-for#ran","syntology_url":"https://syntology.ai/paper/2212.11140","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.11140"}},"official":{"repos":["shailja-thakur/vgen"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/ego-body-pose-estimation-via-ego-head-pose","slug":"ego-body-pose-estimation-via-ego-head-pose","title":"Ego-Body Pose Estimation via Ego-Head Pose Estimation","date":"2022-12-09","arxiv_id":"2212.04636","repositories_listed":1,"syntology":{"n":14,"n_ran":13,"n_constructed":0,"n_ran_checked":13,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":2,"n_no_contract":11,"n_pointer_only":0,"phrase":"13 ran (of which 0 constructed an object rather than computing a result; 13 with no instrument failure: 0 honoured, 2 violated, 11 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/ego-body-pose-estimation-via-ego-head-pose#ran","syntology_url":"https://syntology.ai/paper/2212.04636","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.04636"}},"official":null}},{"url":"/paper/adsorbml-accelerating-adsorption-energy","slug":"adsorbml-accelerating-adsorption-energy","title":"AdsorbML: A Leap in Efficiency for Adsorption Energy Calculations using Generalizable Machine Learning Potentials","date":"2022-11-29","arxiv_id":"2211.16486","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/adsorbml-accelerating-adsorption-energy#ran","syntology_url":"https://syntology.ai/paper/2211.16486","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.16486"}},"official":{"repos":["open-catalyst-project/adsorbml"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/this-is-the-way-designing-and-compiling","slug":"this-is-the-way-designing-and-compiling","title":"This is the way: designing and compiling LEPISZCZE, a comprehensive NLP benchmark for Polish","date":"2022-11-23","arxiv_id":"2211.13112","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/this-is-the-way-designing-and-compiling#ran","syntology_url":"https://syntology.ai/paper/2211.13112","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.13112"}},"official":{"repos":["clarin-pl/lepiszcze"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/events-realm-event-reasoning-of-entity-states","slug":"events-realm-event-reasoning-of-entity-states","title":"EvEntS ReaLM: Event Reasoning of Entity States via Language Models","date":"2022-11-10","arxiv_id":"2211.05392","repositories_listed":0,"syntology":{"n":16,"n_ran":13,"n_constructed":0,"n_ran_checked":13,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":13,"n_pointer_only":9,"phrase":"13 ran (of which 0 constructed an object rather than computing a result; 13 with no instrument failure: 0 honoured, 0 violated, 13 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/events-realm-event-reasoning-of-entity-states#ran","syntology_url":"https://syntology.ai/paper/2211.05392","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.05392"}},"official":null}},{"url":"/paper/the-legal-argument-reasoning-task-in-civil","slug":"the-legal-argument-reasoning-task-in-civil","title":"The Legal Argument Reasoning Task in Civil Procedure","date":"2022-11-05","arxiv_id":"2211.02950","repositories_listed":1,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/the-legal-argument-reasoning-task-in-civil#ran","syntology_url":"https://syntology.ai/paper/2211.02950","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.02950"}},"official":{"repos":["trusthlt/legal-argument-reasoning-task"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/benchmarking-adversarial-patch-against-aerial","slug":"benchmarking-adversarial-patch-against-aerial","title":"Benchmarking Adversarial Patch Against Aerial Detection","date":"2022-10-30","arxiv_id":"2210.16765","repositories_listed":1,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":1,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/benchmarking-adversarial-patch-against-aerial#ran","syntology_url":"https://syntology.ai/paper/2210.16765","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.16765"}},"official":{"repos":["jiaweilian/ap-pa"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/benchmarking-language-models-for-code-syntax","slug":"benchmarking-language-models-for-code-syntax","title":"Benchmarking Language Models for Code Syntax Understanding","date":"2022-10-26","arxiv_id":"2210.14473","repositories_listed":1,"syntology":{"n":6,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":1,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/benchmarking-language-models-for-code-syntax#ran","syntology_url":"https://syntology.ai/paper/2210.14473","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.14473"}},"official":{"repos":["dashends/codesyntax"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/esb-a-benchmark-for-multi-domain-end-to-end","slug":"esb-a-benchmark-for-multi-domain-end-to-end","title":"ESB: A Benchmark For Multi-Domain End-to-End Speech Recognition","date":"2022-10-24","arxiv_id":"2210.13352","repositories_listed":2,"syntology":{"n":11,"n_ran":11,"n_constructed":0,"n_ran_checked":11,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":11,"n_pointer_only":0,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 11 with no instrument failure: 0 honoured, 0 violated, 11 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/esb-a-benchmark-for-multi-domain-end-to-end#ran","syntology_url":"https://syntology.ai/paper/2210.13352","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.13352"}},"official":null}},{"url":"/paper/a-comprehensive-study-on-large-scale-graph","slug":"a-comprehensive-study-on-large-scale-graph","title":"A Comprehensive Study on Large-Scale Graph Training: Benchmarking and Rethinking","date":"2022-10-14","arxiv_id":"2210.07494","repositories_listed":2,"syntology":{"n":6,"n_ran":4,"n_constructed":4,"n_ran_checked":4,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"4 ran (of which 4 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified; every one of the 4 samples that ran constructed an object rather than computing a result","sample_list":"/paper/a-comprehensive-study-on-large-scale-graph#ran","syntology_url":"https://syntology.ai/paper/2210.07494","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.07494"}},"official":{"repos":["vita-group/large_scale_gcn_benchmarking"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":4,"n_ran_no_instrument_failure":4,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/corl-research-oriented-deep-offline-1","slug":"corl-research-oriented-deep-offline-1","title":"CORL: Research-oriented Deep Offline Reinforcement Learning Library","date":"2022-10-13","arxiv_id":"2210.07105","repositories_listed":5,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/corl-research-oriented-deep-offline-1#ran","syntology_url":"https://syntology.ai/paper/2210.07105","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.07105"}},"official":{"repos":["corl-team/CORL","hanjuku-kaso/awesome-offline-rl","tinkoff-ai/CORL"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/openood-benchmarking-generalized-out-of","slug":"openood-benchmarking-generalized-out-of","title":"OpenOOD: Benchmarking Generalized Out-of-Distribution Detection","date":"2022-10-13","arxiv_id":"2210.07242","repositories_listed":4,"syntology":{"n":17,"n_ran":15,"n_constructed":0,"n_ran_checked":10,"n_instrument":5,"n_unverified":2,"n_honours":1,"n_violates":2,"n_no_contract":7,"n_pointer_only":2,"phrase":"15 ran (of which 0 constructed an object rather than computing a result; 10 with no instrument failure: 1 honoured, 2 violated, 7 with no contract checked; 5 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/openood-benchmarking-generalized-out-of#ran","syntology_url":"https://syntology.ai/paper/2210.07242","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.07242"}},"official":{"repos":["jingkang50/openood"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/mteb-massive-text-embedding-benchmark","slug":"mteb-massive-text-embedding-benchmark","title":"MTEB: Massive Text Embedding Benchmark","date":"2022-10-13","arxiv_id":"2210.07316","repositories_listed":5,"syntology":{"n":13,"n_ran":10,"n_constructed":0,"n_ran_checked":7,"n_instrument":3,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"10 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 3 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/mteb-massive-text-embedding-benchmark#ran","syntology_url":"https://syntology.ai/paper/2210.07316","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.07316"}},"official":{"repos":["embeddings-benchmark/mteb"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/vote-n-rank-revision-of-benchmarking-with","slug":"vote-n-rank-revision-of-benchmarking-with","title":"Vote'n'Rank: Revision of Benchmarking with Social Choice Theory","date":"2022-10-11","arxiv_id":"2210.05769","repositories_listed":1,"syntology":{"n":17,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":11,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 11 unverified","sample_list":"/paper/vote-n-rank-revision-of-benchmarking-with#ran","syntology_url":"https://syntology.ai/paper/2210.05769","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.05769"}},"official":{"repos":["pragmaticslab/vote_and_rank"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":11,"ran_from_kinds":["official"]}}},{"url":"/paper/benchmarking-reinforcement-learning-1","slug":"benchmarking-reinforcement-learning-1","title":"Benchmarking Reinforcement Learning Techniques for Autonomous Navigation","date":"2022-10-10","arxiv_id":"2210.04839","repositories_listed":1,"syntology":{"n":6,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/benchmarking-reinforcement-learning-1#ran","syntology_url":"https://syntology.ai/paper/2210.04839","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.04839"}},"official":null}},{"url":"/paper/building-normalizing-flows-with-stochastic","slug":"building-normalizing-flows-with-stochastic","title":"Building Normalizing Flows with Stochastic Interpolants","date":"2022-09-30","arxiv_id":"2209.15571","repositories_listed":1,"syntology":{"n":7,"n_ran":5,"n_constructed":3,"n_ran_checked":3,"n_instrument":2,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":1,"phrase":"5 ran (of which 3 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 2 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/building-normalizing-flows-with-stochastic#ran","syntology_url":"https://syntology.ai/paper/2209.15571","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2209.15571"}},"official":null}},{"url":"/paper/neural-methods-for-logical-reasoning-over-1","slug":"neural-methods-for-logical-reasoning-over-1","title":"Neural Methods for Logical Reasoning Over Knowledge Graphs","date":"2022-09-28","arxiv_id":"2209.14464","repositories_listed":1,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":1,"n_instrument":3,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/neural-methods-for-logical-reasoning-over-1#ran","syntology_url":"https://syntology.ai/paper/2209.14464","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2209.14464"}},"official":{"repos":["amayuelas/NNKGReasoning"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/periodic-extrapolative-generalisation-in","slug":"periodic-extrapolative-generalisation-in","title":"Periodic Extrapolative Generalisation in Neural Networks","date":"2022-09-21","arxiv_id":"2209.10280","repositories_listed":1,"syntology":{"n":8,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":0,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/periodic-extrapolative-generalisation-in#ran","syntology_url":"https://syntology.ai/paper/2209.10280","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2209.10280"}},"official":{"repos":["pbelcak/perkit"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/a-comprehensive-benchmark-for-covid-19","slug":"a-comprehensive-benchmark-for-covid-19","title":"A Comprehensive Benchmark for COVID-19 Predictive Modeling Using Electronic Health Records in Intensive Care","date":"2022-09-16","arxiv_id":"2209.07805","repositories_listed":3,"syntology":{"n":8,"n_ran":7,"n_constructed":0,"n_ran_checked":5,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":3,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/a-comprehensive-benchmark-for-covid-19#ran","syntology_url":"https://syntology.ai/paper/2209.07805","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2209.07805"}},"official":{"repos":["yhzhu99/covid-ehr-benchmarks","yhzhu99/pyehr","yhzhu99/pyehr-playground"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/screenqa-large-scale-question-answer-pairs","slug":"screenqa-large-scale-question-answer-pairs","title":"ScreenQA: Large-Scale Question-Answer Pairs over Mobile App Screenshots","date":"2022-09-16","arxiv_id":"2209.08199","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":3,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 3 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/screenqa-large-scale-question-answer-pairs#ran","syntology_url":"https://syntology.ai/paper/2209.08199","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2209.08199"}},"official":{"repos":["google-research-datasets/screen_qa"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/learning-transductions-to-test-systematic","slug":"learning-transductions-to-test-systematic","title":"Benchmarking Compositionality with Formal Languages","date":"2022-08-17","arxiv_id":"2208.08195","repositories_listed":1,"syntology":{"n":9,"n_ran":9,"n_constructed":2,"n_ran_checked":6,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":1,"phrase":"9 ran (of which 2 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/learning-transductions-to-test-systematic#ran","syntology_url":"https://syntology.ai/paper/2208.08195","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2208.08195"}},"official":{"repos":["valvoda/neuraltransducer"],"state":"official (archive's flag): 9 ran","n_ran":9,"n_constructed":2,"n_ran_no_instrument_failure":6,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/a-scalable-and-extensible-approach-to","slug":"a-scalable-and-extensible-approach-to","title":"MultiPL-E: A Scalable and Extensible Approach to Benchmarking Neural Code Generation","date":"2022-08-17","arxiv_id":"2208.08227","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/a-scalable-and-extensible-approach-to#ran","syntology_url":"https://syntology.ai/paper/2208.08227","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2208.08227"}},"official":{"repos":["nuprl/multipl-e"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/a-multifaceted-benchmarking-of-synthetic","slug":"a-multifaceted-benchmarking-of-synthetic","title":"A Multifaceted Benchmarking of Synthetic Electronic Health Record Generation Models","date":"2022-08-02","arxiv_id":"2208.01230","repositories_listed":1,"syntology":{"n":19,"n_ran":8,"n_constructed":0,"n_ran_checked":8,"n_instrument":0,"n_unverified":11,"n_honours":0,"n_violates":0,"n_no_contract":8,"n_pointer_only":0,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 0 violated, 8 with no contract checked; 0 where Syntology's instrument failed) · 11 unverified","sample_list":"/paper/a-multifaceted-benchmarking-of-synthetic#ran","syntology_url":"https://syntology.ai/paper/2208.01230","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2208.01230"}},"official":{"repos":["yy6linda/synthetic-ehr-benchmarking"],"state":"official (archive's flag): 8 ran","n_ran":8,"n_constructed":0,"n_ran_no_instrument_failure":8,"n_unverified":11,"ran_from_kinds":["official"]}}},{"url":"/paper/ferret-a-framework-for-benchmarking","slug":"ferret-a-framework-for-benchmarking","title":"ferret: a Framework for Benchmarking Explainers on Transformers","date":"2022-08-02","arxiv_id":"2208.01575","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/ferret-a-framework-for-benchmarking#ran","syntology_url":"https://syntology.ai/paper/2208.01575","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2208.01575"}},"official":{"repos":["g8a9/ferret"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/artfid-quantitative-evaluation-of-neural","slug":"artfid-quantitative-evaluation-of-neural","title":"ArtFID: Quantitative Evaluation of Neural Style Transfer","date":"2022-07-25","arxiv_id":"2207.12280","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/artfid-quantitative-evaluation-of-neural#ran","syntology_url":"https://syntology.ai/paper/2207.12280","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2207.12280"}},"official":{"repos":["matthias-wright/art-fid"],"state":"official: no sample here; runs from other or unrecorded repositories","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["unlocated"]}}},{"url":"/paper/benchmarking-omni-vision-representation","slug":"benchmarking-omni-vision-representation","title":"Benchmarking Omni-Vision Representation through the Lens of Visual Realms","date":"2022-07-14","arxiv_id":"2207.07106","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_constructed":1,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":1,"phrase":"1 ran (of which 1 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified; the one sample that ran constructed an object rather than computing a result","sample_list":"/paper/benchmarking-omni-vision-representation#ran","syntology_url":"https://syntology.ai/paper/2207.07106","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2207.07106"}},"official":{"repos":["ZhangYuanhan-AI/OmniBenchmark"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":1,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/automated-detection-of-label-errors-in","slug":"automated-detection-of-label-errors-in","title":"Automated Detection of Label Errors in Semantic Segmentation Datasets via Deep Learning and Uncertainty Quantification","date":"2022-07-13","arxiv_id":"2207.06104","repositories_listed":1,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/automated-detection-of-label-errors-in#ran","syntology_url":"https://syntology.ai/paper/2207.06104","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2207.06104"}},"official":{"repos":["mrcoee/automatic-label-error-detection"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/can-language-models-make-fun-a-case-study-in","slug":"can-language-models-make-fun-a-case-study-in","title":"Can Language Models Make Fun? A Case Study in Chinese Comical Crosstalk","date":"2022-07-02","arxiv_id":"2207.00735","repositories_listed":1,"syntology":{"n":13,"n_ran":11,"n_constructed":0,"n_ran_checked":11,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":11,"n_pointer_only":0,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 11 with no instrument failure: 0 honoured, 0 violated, 11 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/can-language-models-make-fun-a-case-study-in#ran","syntology_url":"https://syntology.ai/paper/2207.00735","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2207.00735"}},"official":{"repos":["anonno2/crosstalk-generation"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":11,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/benchopt-reproducible-efficient-and","slug":"benchopt-reproducible-efficient-and","title":"Benchopt: Reproducible, efficient and collaborative optimization benchmarks","date":"2022-06-27","arxiv_id":"2206.13424","repositories_listed":3,"syntology":{"n":10,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/benchopt-reproducible-efficient-and#ran","syntology_url":"https://syntology.ai/paper/2206.13424","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.13424"}},"official":{"repos":["benchopt/benchopt"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/openxai-towards-a-transparent-evaluation-of","slug":"openxai-towards-a-transparent-evaluation-of","title":"OpenXAI: Towards a Transparent Evaluation of Model Explanations","date":"2022-06-22","arxiv_id":"2206.11104","repositories_listed":2,"syntology":{"n":7,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":4,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 4 unverified","sample_list":"/paper/openxai-towards-a-transparent-evaluation-of#ran","syntology_url":"https://syntology.ai/paper/2206.11104","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.11104"}},"official":{"repos":["ai4life-group/openxai"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/benchmarking-constraint-inference-in-inverse","slug":"benchmarking-constraint-inference-in-inverse","title":"Benchmarking Constraint Inference in Inverse Reinforcement Learning","date":"2022-06-20","arxiv_id":"2206.09670","repositories_listed":2,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":2,"n_instrument":2,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":1,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 2 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/benchmarking-constraint-inference-in-inverse#ran","syntology_url":"https://syntology.ai/paper/2206.09670","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.09670"}},"official":{"repos":["guiliang/cirl-benchmarks-public","guiliang/icrl-benchmarks-public"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/nas-bench-graph-benchmarking-graph-neural","slug":"nas-bench-graph-benchmarking-graph-neural","title":"NAS-Bench-Graph: Benchmarking Graph Neural Architecture Search","date":"2022-06-18","arxiv_id":"2206.09166","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":0,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/nas-bench-graph-benchmarking-graph-neural#ran","syntology_url":"https://syntology.ai/paper/2206.09166","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.09166"}},"official":{"repos":["thumnlab/nas-bench-graph"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/smpl-simulated-industrial-manufacturing-and","slug":"smpl-simulated-industrial-manufacturing-and","title":"SMPL: Simulated Industrial Manufacturing and Process Control Learning Environments","date":"2022-06-17","arxiv_id":"2206.08851","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/smpl-simulated-industrial-manufacturing-and#ran","syntology_url":"https://syntology.ai/paper/2206.08851","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.08851"}},"official":{"repos":["smpl-env/smpl"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/taxonomy-of-benchmarks-in-graph","slug":"taxonomy-of-benchmarks-in-graph","title":"Taxonomy of Benchmarks in Graph Representation Learning","date":"2022-06-15","arxiv_id":"2206.07729","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/taxonomy-of-benchmarks-in-graph#ran","syntology_url":"https://syntology.ai/paper/2206.07729","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.07729"}},"official":{"repos":["g-taxonomy-workgroup/gtaxogym"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/isles-2022-a-multi-center-magnetic-resonance","slug":"isles-2022-a-multi-center-magnetic-resonance","title":"ISLES 2022: A multi-center magnetic resonance imaging stroke lesion segmentation dataset","date":"2022-06-14","arxiv_id":"2206.06694","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/isles-2022-a-multi-center-magnetic-resonance#ran","syntology_url":"https://syntology.ai/paper/2206.06694","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.06694"}},"official":{"repos":["ezequieldlrosa/isles22"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/revisiting-realistic-test-time-training","slug":"revisiting-realistic-test-time-training","title":"Revisiting Realistic Test-Time Training: Sequential Inference and Adaptation by Anchored Clustering","date":"2022-06-06","arxiv_id":"2206.02721","repositories_listed":1,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/revisiting-realistic-test-time-training#ran","syntology_url":"https://syntology.ai/paper/2206.02721","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.02721"}},"official":{"repos":["gorilla-lab-scut/ttac"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/revisiting-the-video-in-video-language","slug":"revisiting-the-video-in-video-language","title":"Revisiting the \"Video\" in Video-Language Understanding","date":"2022-06-03","arxiv_id":"2206.01720","repositories_listed":1,"syntology":{"n":5,"n_ran":3,"n_constructed":2,"n_ran_checked":3,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 2 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/revisiting-the-video-in-video-language#ran","syntology_url":"https://syntology.ai/paper/2206.01720","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.01720"}},"official":null}},{"url":"/paper/benchmarking-the-robustness-of-lidar-camera","slug":"benchmarking-the-robustness-of-lidar-camera","title":"Benchmarking the Robustness of LiDAR-Camera Fusion for 3D Object Detection","date":"2022-05-30","arxiv_id":"2205.14951","repositories_listed":1,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":3,"n_instrument":1,"n_unverified":0,"n_honours":0,"n_violates":3,"n_no_contract":0,"n_pointer_only":4,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 3 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/benchmarking-the-robustness-of-lidar-camera#ran","syntology_url":"https://syntology.ai/paper/2205.14951","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2205.14951"}},"official":{"repos":["kcyu2014/lidar-camera-robust-benchmark"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/fast-vision-transformers-with-hilo-attention","slug":"fast-vision-transformers-with-hilo-attention","title":"Fast Vision Transformers with HiLo Attention","date":"2022-05-26","arxiv_id":"2205.13213","repositories_listed":5,"syntology":{"n":13,"n_ran":11,"n_constructed":1,"n_ran_checked":10,"n_instrument":1,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":10,"n_pointer_only":2,"phrase":"11 ran (of which 1 constructed an object rather than computing a result; 10 with no instrument failure: 0 honoured, 0 violated, 10 with no contract checked; 1 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/fast-vision-transformers-with-hilo-attention#ran","syntology_url":"https://syntology.ai/paper/2205.13213","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2205.13213"}},"official":{"repos":["zip-group/litv2","ziplab/litv2"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/optimizing-performance-of-federated-person-re","slug":"optimizing-performance-of-federated-person-re","title":"Optimizing Performance of Federated Person Re-identification: Benchmarking and Analysis","date":"2022-05-24","arxiv_id":"2205.12144","repositories_listed":2,"syntology":{"n":8,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/optimizing-performance-of-federated-person-re#ran","syntology_url":"https://syntology.ai/paper/2205.12144","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2205.12144"}},"official":{"repos":["cap-ntu/FedReID","EasyFL-AI/EasyFL"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/bars-towards-open-benchmarking-for","slug":"bars-towards-open-benchmarking-for","title":"BARS: Towards Open Benchmarking for Recommender Systems","date":"2022-05-19","arxiv_id":"2205.09626","repositories_listed":5,"syntology":{"n":8,"n_ran":5,"n_constructed":0,"n_ran_checked":3,"n_instrument":2,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":1,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 2 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/bars-towards-open-benchmarking-for#ran","syntology_url":"https://syntology.ai/paper/2205.09626","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2205.09626"}},"official":{"repos":["openbenchmark/BARS"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/individual-fairness-guarantees-for-neural","slug":"individual-fairness-guarantees-for-neural","title":"Individual Fairness Guarantees for Neural Networks","date":"2022-05-11","arxiv_id":"2205.05763","repositories_listed":1,"syntology":{"n":7,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/individual-fairness-guarantees-for-neural#ran","syntology_url":"https://syntology.ai/paper/2205.05763","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2205.05763"}},"official":{"repos":["eliasbenussi/nn-cert-individual-fairness"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/bico-net-regress-globally-match-locally-for","slug":"bico-net-regress-globally-match-locally-for","title":"BiCo-Net: Regress Globally, Match Locally for Robust 6D Pose Estimation","date":"2022-05-07","arxiv_id":"2205.03536","repositories_listed":1,"syntology":{"n":6,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/bico-net-regress-globally-match-locally-for#ran","syntology_url":"https://syntology.ai/paper/2205.03536","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2205.03536"}},"official":{"repos":["gorilla-lab-scut/bico-net"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/k-lite-learning-transferable-visual-models","slug":"k-lite-learning-transferable-visual-models","title":"K-LITE: Learning Transferable Visual Models with External Knowledge","date":"2022-04-20","arxiv_id":"2204.09222","repositories_listed":2,"syntology":{"n":10,"n_ran":5,"n_constructed":3,"n_ran_checked":5,"n_instrument":0,"n_unverified":5,"n_honours":0,"n_violates":1,"n_no_contract":4,"n_pointer_only":0,"phrase":"5 ran (of which 3 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 1 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/k-lite-learning-transferable-visual-models#ran","syntology_url":"https://syntology.ai/paper/2204.09222","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.09222"}},"official":null}},{"url":"/paper/benchmarking-generalization-via-in-context","slug":"benchmarking-generalization-via-in-context","title":"Super-NaturalInstructions: Generalization via Declarative Instructions on 1600+ NLP Tasks","date":"2022-04-16","arxiv_id":"2204.07705","repositories_listed":10,"syntology":{"n":28,"n_ran":17,"n_constructed":1,"n_ran_checked":16,"n_instrument":1,"n_unverified":11,"n_honours":0,"n_violates":0,"n_no_contract":16,"n_pointer_only":4,"phrase":"17 ran (of which 1 constructed an object rather than computing a result; 16 with no instrument failure: 0 honoured, 0 violated, 16 with no contract checked; 1 where Syntology's instrument failed) · 11 unverified","sample_list":"/paper/benchmarking-generalization-via-in-context#ran","syntology_url":"https://syntology.ai/paper/2204.07705","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.07705"}},"official":{"repos":["allenai/natural-instructions"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/stress-testing-lidar-registration","slug":"stress-testing-lidar-registration","title":"Stress-Testing Point Cloud Registration on Automotive LiDAR","date":"2022-04-16","arxiv_id":"2204.07719","repositories_listed":1,"syntology":{"n":7,"n_ran":5,"n_constructed":0,"n_ran_checked":0,"n_instrument":5,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":7,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 5 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/stress-testing-lidar-registration#ran","syntology_url":"https://syntology.ai/paper/2204.07719","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.07719"}},"official":{"repos":["amnondrory/lidarregistration"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/from-cnns-to-vision-transformers-a","slug":"from-cnns-to-vision-transformers-a","title":"From Modern CNNs to Vision Transformers: Assessing the Performance, Robustness, and Classification Strategies of Deep Learning Models in Histopathology","date":"2022-04-11","arxiv_id":"2204.05044","repositories_listed":1,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":0,"n_instrument":4,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":5,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/from-cnns-to-vision-transformers-a#ran","syntology_url":"https://syntology.ai/paper/2204.05044","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.05044"}},"official":{"repos":["hhi-aml/histobenchmark"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/deep-visual-geo-localization-benchmark","slug":"deep-visual-geo-localization-benchmark","title":"Deep Visual Geo-localization Benchmark","date":"2022-04-07","arxiv_id":"2204.03444","repositories_listed":1,"syntology":{"n":13,"n_ran":7,"n_constructed":0,"n_ran_checked":6,"n_instrument":1,"n_unverified":6,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":2,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 1 where Syntology's instrument failed) · 6 unverified","sample_list":"/paper/deep-visual-geo-localization-benchmark#ran","syntology_url":"https://syntology.ai/paper/2204.03444","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.03444"}},"official":{"repos":["gmberton/deep-visual-geo-localization-benchmark"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":6,"ran_from_kinds":["official"]}}},{"url":"/paper/the-moral-integrity-corpus-a-benchmark-for","slug":"the-moral-integrity-corpus-a-benchmark-for","title":"The Moral Integrity Corpus: A Benchmark for Ethical Dialogue Systems","date":"2022-04-06","arxiv_id":"2204.03021","repositories_listed":2,"syntology":{"n":4,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":4,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/the-moral-integrity-corpus-a-benchmark-for#ran","syntology_url":"https://syntology.ai/paper/2204.03021","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.03021"}},"official":{"repos":["gt-salt/mic"],"state":"official: no sample here; runs from other or unrecorded repositories","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed","unlocated"]}}},{"url":"/paper/dynatask-a-framework-for-creating-dynamic-ai","slug":"dynatask-a-framework-for-creating-dynamic-ai","title":"Dynatask: A Framework for Creating Dynamic AI Benchmark Tasks","date":"2022-04-05","arxiv_id":"2204.01906","repositories_listed":1,"syntology":{"n":10,"n_ran":7,"n_constructed":0,"n_ran_checked":7,"n_instrument":0,"n_unverified":3,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":10,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 0 where Syntology's instrument failed) · 3 unverified","sample_list":"/paper/dynatask-a-framework-for-creating-dynamic-ai#ran","syntology_url":"https://syntology.ai/paper/2204.01906","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.01906"}},"official":{"repos":["facebookresearch/dynabench"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":3,"ran_from_kinds":["official"]}}},{"url":"/paper/parameter-efficient-fine-tuning-for-vision","slug":"parameter-efficient-fine-tuning-for-vision","title":"Parameter-efficient Model Adaptation for Vision Transformers","date":"2022-03-29","arxiv_id":"2203.16329","repositories_listed":3,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":0,"n_instrument":3,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":3,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 3 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/parameter-efficient-fine-tuning-for-vision#ran","syntology_url":"https://syntology.ai/paper/2203.16329","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.16329"}},"official":{"repos":["eric-ai-lab/pevit"],"state":"community repositories only","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["listed"]}}},{"url":"/paper/visual-abductive-reasoning","slug":"visual-abductive-reasoning","title":"Visual Abductive Reasoning","date":"2022-03-26","arxiv_id":"2203.14040","repositories_listed":1,"syntology":{"n":21,"n_ran":11,"n_constructed":8,"n_ran_checked":8,"n_instrument":3,"n_unverified":10,"n_honours":0,"n_violates":0,"n_no_contract":8,"n_pointer_only":0,"phrase":"11 ran (of which 8 constructed an object rather than computing a result; 8 with no instrument failure: 0 honoured, 0 violated, 8 with no contract checked; 3 where Syntology's instrument failed) · 10 unverified","sample_list":"/paper/visual-abductive-reasoning#ran","syntology_url":"https://syntology.ai/paper/2203.14040","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.14040"}},"official":{"repos":["leonnnop/var"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":8,"n_ran_no_instrument_failure":8,"n_unverified":10,"ran_from_kinds":["official"]}}},{"url":"/paper/on-the-usefulness-of-the-fit-on-the-test-view","slug":"on-the-usefulness-of-the-fit-on-the-test-view","title":"On the Usefulness of the Fit-on-the-Test View on Evaluating Calibration of Classifiers","date":"2022-03-16","arxiv_id":"2203.08958","repositories_listed":1,"syntology":{"n":22,"n_ran":17,"n_constructed":0,"n_ran_checked":16,"n_instrument":1,"n_unverified":5,"n_honours":0,"n_violates":1,"n_no_contract":15,"n_pointer_only":1,"phrase":"17 ran (of which 0 constructed an object rather than computing a result; 16 with no instrument failure: 0 honoured, 1 violated, 15 with no contract checked; 1 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/on-the-usefulness-of-the-fit-on-the-test-view#ran","syntology_url":"https://syntology.ai/paper/2203.08958","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.08958"}},"official":{"repos":["markus93/fit-on-the-test"],"state":"official (archive's flag): 17 ran","n_ran":17,"n_constructed":0,"n_ran_no_instrument_failure":16,"n_unverified":5,"ran_from_kinds":["official"]}}},{"url":"/paper/a-unified-framework-for-rank-based-evaluation","slug":"a-unified-framework-for-rank-based-evaluation","title":"A Unified Framework for Rank-based Evaluation Metrics for Link Prediction in Knowledge Graphs","date":"2022-03-14","arxiv_id":"2203.07544","repositories_listed":2,"syntology":{"n":5,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/a-unified-framework-for-rank-based-evaluation#ran","syntology_url":"https://syntology.ai/paper/2203.07544","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.07544"}},"official":{"repos":["pykeen/pykeen","pykeen/ranking-metrics-manuscript"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/benchmarking-graphormer-on-large-scale","slug":"benchmarking-graphormer-on-large-scale","title":"Benchmarking Graphormer on Large-Scale Molecular Modeling Datasets","date":"2022-03-09","arxiv_id":"2203.04810","repositories_listed":5,"syntology":{"n":9,"n_ran":8,"n_constructed":0,"n_ran_checked":5,"n_instrument":3,"n_unverified":1,"n_honours":0,"n_violates":1,"n_no_contract":4,"n_pointer_only":2,"phrase":"8 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 1 violated, 4 with no contract checked; 3 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/benchmarking-graphormer-on-large-scale#ran","syntology_url":"https://syntology.ai/paper/2203.04810","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.04810"}},"official":{"repos":["Microsoft/Graphormer"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/quasi-balanced-self-training-on-noise-aware","slug":"quasi-balanced-self-training-on-noise-aware","title":"Quasi-Balanced Self-Training on Noise-Aware Synthesis of Object Point Clouds for Closing Domain Gap","date":"2022-03-08","arxiv_id":"2203.03833","repositories_listed":1,"syntology":{"n":15,"n_ran":13,"n_constructed":0,"n_ran_checked":12,"n_instrument":1,"n_unverified":2,"n_honours":0,"n_violates":1,"n_no_contract":11,"n_pointer_only":1,"phrase":"13 ran (of which 0 constructed an object rather than computing a result; 12 with no instrument failure: 0 honoured, 1 violated, 11 with no contract checked; 1 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/quasi-balanced-self-training-on-noise-aware#ran","syntology_url":"https://syntology.ai/paper/2203.03833","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.03833"}},"official":{"repos":["gorilla-lab-scut/qs3"],"state":"official (archive's flag): 13 ran","n_ran":13,"n_constructed":0,"n_ran_no_instrument_failure":12,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/clearpose-large-scale-transparent-object","slug":"clearpose-large-scale-transparent-object","title":"ClearPose: Large-scale Transparent Object Dataset and Benchmark","date":"2022-03-08","arxiv_id":"2203.03890","repositories_listed":1,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":4,"n_instrument":0,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":3,"n_pointer_only":1,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 1 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/clearpose-large-scale-transparent-object#ran","syntology_url":"https://syntology.ai/paper/2203.03890","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.03890"}},"official":{"repos":["opipari/clearpose"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/hoi4d-a-4d-egocentric-dataset-for-category","slug":"hoi4d-a-4d-egocentric-dataset-for-category","title":"HOI4D: A 4D Egocentric Dataset for Category-Level Human-Object Interaction","date":"2022-03-03","arxiv_id":"2203.01577","repositories_listed":1,"syntology":{"n":5,"n_ran":5,"n_constructed":0,"n_ran_checked":5,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":5,"n_pointer_only":0,"phrase":"5 ran (of which 0 constructed an object rather than computing a result; 5 with no instrument failure: 0 honoured, 0 violated, 5 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/hoi4d-a-4d-egocentric-dataset-for-category#ran","syntology_url":"https://syntology.ai/paper/2203.01577","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.01577"}},"official":{"repos":["leolyliu/HOI4D-Instructions"],"state":"official (archive's flag): 5 ran","n_ran":5,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/3d-common-corruptions-and-data-augmentation","slug":"3d-common-corruptions-and-data-augmentation","title":"3D Common Corruptions and Data Augmentation","date":"2022-03-02","arxiv_id":"2203.01441","repositories_listed":1,"syntology":{"n":12,"n_ran":7,"n_constructed":0,"n_ran_checked":1,"n_instrument":6,"n_unverified":5,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":12,"phrase":"7 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 6 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/3d-common-corruptions-and-data-augmentation#ran","syntology_url":"https://syntology.ai/paper/2203.01441","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.01441"}},"official":{"repos":["EPFL-VILAB/3DCommonCorruptions"],"state":"official (archive's flag): 7 ran","n_ran":7,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":5,"ran_from_kinds":["official"]}}},{"url":"/paper/graphworld-fake-graphs-bring-real-insights","slug":"graphworld-fake-graphs-bring-real-insights","title":"GraphWorld: Fake Graphs Bring Real Insights for GNNs","date":"2022-02-28","arxiv_id":"2203.00112","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":0,"n_honours":2,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 2 honoured, 0 violated, 0 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/graphworld-fake-graphs-bring-real-insights#ran","syntology_url":"https://syntology.ai/paper/2203.00112","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.00112"}},"official":{"repos":["google-research/graphworld"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/benchmarking-generative-latent-variable","slug":"benchmarking-generative-latent-variable","title":"Benchmarking Generative Latent Variable Models for Speech","date":"2022-02-22","arxiv_id":"2202.12707","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_constructed":0,"n_ran_checked":1,"n_instrument":2,"n_unverified":0,"n_honours":0,"n_violates":1,"n_no_contract":0,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 1 violated, 0 with no contract checked; 2 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/benchmarking-generative-latent-variable#ran","syntology_url":"https://syntology.ai/paper/2202.12707","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2202.12707"}},"official":{"repos":["jakobhavtorn/benchmarking-lvms"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/benchmarking-the-linear-algebra-awareness-of","slug":"benchmarking-the-linear-algebra-awareness-of","title":"Benchmarking the Linear Algebra Awareness of TensorFlow and PyTorch","date":"2022-02-20","arxiv_id":"2202.09888","repositories_listed":2,"syntology":{"n":19,"n_ran":14,"n_constructed":0,"n_ran_checked":14,"n_instrument":0,"n_unverified":5,"n_honours":0,"n_violates":0,"n_no_contract":14,"n_pointer_only":0,"phrase":"14 ran (of which 0 constructed an object rather than computing a result; 14 with no instrument failure: 0 honoured, 0 violated, 14 with no contract checked; 0 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/benchmarking-the-linear-algebra-awareness-of#ran","syntology_url":"https://syntology.ai/paper/2202.09888","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2202.09888"}},"official":{"repos":["as641651/linearalgebra-awareness-benchmark"],"state":"official (archive's flag): 14 ran","n_ran":14,"n_constructed":0,"n_ran_no_instrument_failure":14,"n_unverified":4,"ran_from_kinds":["official"]}}},{"url":"/paper/metashift-a-dataset-of-datasets-for-1","slug":"metashift-a-dataset-of-datasets-for-1","title":"MetaShift: A Dataset of Datasets for Evaluating Contextual Distribution Shifts and Training Conflicts","date":"2022-02-14","arxiv_id":"2202.06523","repositories_listed":1,"syntology":{"n":6,"n_ran":4,"n_constructed":0,"n_ran_checked":0,"n_instrument":4,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":1,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 4 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/metashift-a-dataset-of-datasets-for-1#ran","syntology_url":"https://syntology.ai/paper/2202.06523","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2202.06523"}},"official":{"repos":["weixin-liang/metashift"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/wukong-100-million-large-scale-chinese-cross","slug":"wukong-100-million-large-scale-chinese-cross","title":"Wukong: A 100 Million Large-scale Chinese Cross-modal Pre-training Benchmark","date":"2022-02-14","arxiv_id":"2202.06767","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/wukong-100-million-large-scale-chinese-cross#ran","syntology_url":"https://syntology.ai/paper/2202.06767","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2202.06767"}},"official":null}},{"url":"/paper/theory-inspired-parameter-control-benchmarks","slug":"theory-inspired-parameter-control-benchmarks","title":"Theory-inspired Parameter Control Benchmarks for Dynamic Algorithm Configuration","date":"2022-02-07","arxiv_id":"2202.03259","repositories_listed":1,"syntology":{"n":6,"n_ran":6,"n_constructed":0,"n_ran_checked":6,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":6,"n_pointer_only":0,"phrase":"6 ran (of which 0 constructed an object rather than computing a result; 6 with no instrument failure: 0 honoured, 0 violated, 6 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/theory-inspired-parameter-control-benchmarks#ran","syntology_url":"https://syntology.ai/paper/2202.03259","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2202.03259"}},"official":{"repos":["caroladoerr/leadingonedac"],"state":"official (archive's flag): 6 ran","n_ran":6,"n_constructed":0,"n_ran_no_instrument_failure":6,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/benchmarking-and-analyzing-point-cloud","slug":"benchmarking-and-analyzing-point-cloud","title":"Benchmarking and Analyzing Point Cloud Classification under Corruptions","date":"2022-02-07","arxiv_id":"2202.03377","repositories_listed":4,"syntology":{"n":13,"n_ran":11,"n_constructed":0,"n_ran_checked":7,"n_instrument":4,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":7,"n_pointer_only":2,"phrase":"11 ran (of which 0 constructed an object rather than computing a result; 7 with no instrument failure: 0 honoured, 0 violated, 7 with no contract checked; 4 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/benchmarking-and-analyzing-point-cloud#ran","syntology_url":"https://syntology.ai/paper/2202.03377","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2202.03377"}},"official":{"repos":["jiawei-ren/modelnetc"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":7,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/benchmarking-robustness-of-3d-point-cloud","slug":"benchmarking-robustness-of-3d-point-cloud","title":"Benchmarking Robustness of 3D Point Cloud Recognition Against Common Corruptions","date":"2022-01-28","arxiv_id":"2201.12296","repositories_listed":6,"syntology":{"n":29,"n_ran":24,"n_constructed":0,"n_ran_checked":16,"n_instrument":8,"n_unverified":5,"n_honours":2,"n_violates":0,"n_no_contract":14,"n_pointer_only":7,"phrase":"24 ran (of which 0 constructed an object rather than computing a result; 16 with no instrument failure: 2 honoured, 0 violated, 14 with no contract checked; 8 where Syntology's instrument failed) · 5 unverified","sample_list":"/paper/benchmarking-robustness-of-3d-point-cloud#ran","syntology_url":"https://syntology.ai/paper/2201.12296","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2201.12296"}},"official":{"repos":["jiachens/ModelNet40-C"],"state":"official (archive's flag): 11 ran","n_ran":11,"n_constructed":0,"n_ran_no_instrument_failure":5,"n_unverified":2,"ran_from_kinds":["listed","official"]}}},{"url":"/paper/autonomous-reinforcement-learning-formalism-1","slug":"autonomous-reinforcement-learning-formalism-1","title":"Autonomous Reinforcement Learning: Formalism and Benchmarking","date":"2021-12-17","arxiv_id":"2112.09605","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_constructed":0,"n_ran_checked":1,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":1,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 0 honoured, 0 violated, 1 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/autonomous-reinforcement-learning-formalism-1#ran","syntology_url":"https://syntology.ai/paper/2112.09605","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2112.09605"}},"official":{"repos":["architsharma97/earl_benchmark"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":1,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/boosting-neural-image-compression-for","slug":"boosting-neural-image-compression-for","title":"Boosting Neural Image Compression for Machines Using Latent Space Masking","date":"2021-12-15","arxiv_id":"2112.08168","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_constructed":0,"n_ran_checked":0,"n_instrument":1,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":0,"n_pointer_only":0,"phrase":"1 ran (of which 0 constructed an object rather than computing a result; 0 with no instrument failure: 0 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/boosting-neural-image-compression-for#ran","syntology_url":"https://syntology.ai/paper/2112.08168","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2112.08168"}},"official":{"repos":["fau-lms/ncn_for_m2m"],"state":"official (archive's flag): 1 ran","n_ran":1,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/label-verify-correct-a-simple-few-shot-object","slug":"label-verify-correct-a-simple-few-shot-object","title":"Label, Verify, Correct: A Simple Few Shot Object Detection Method","date":"2021-12-10","arxiv_id":"2112.05749","repositories_listed":1,"syntology":{"n":3,"n_ran":2,"n_constructed":0,"n_ran_checked":2,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":2,"n_pointer_only":3,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 2 with no instrument failure: 0 honoured, 0 violated, 2 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/label-verify-correct-a-simple-few-shot-object#ran","syntology_url":"https://syntology.ai/paper/2112.05749","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2112.05749"}},"official":{"repos":["prannaykaul/lvc"],"state":"official (archive's flag): 2 ran","n_ran":2,"n_constructed":0,"n_ran_no_instrument_failure":2,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/learning-to-transfer-for-traffic-forecasting","slug":"learning-to-transfer-for-traffic-forecasting","title":"Learning to Transfer for Traffic Forecasting via Multi-task Learning","date":"2021-11-27","arxiv_id":"2111.15542","repositories_listed":1,"syntology":{"n":4,"n_ran":4,"n_constructed":0,"n_ran_checked":4,"n_instrument":0,"n_unverified":0,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":4,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/learning-to-transfer-for-traffic-forecasting#ran","syntology_url":"https://syntology.ai/paper/2111.15542","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2111.15542"}},"official":{"repos":["yichaolu/traffic4cast2021"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":0,"ran_from_kinds":["official"]}}},{"url":"/paper/evaluating-adversarial-attacks-on-imagenet-a","slug":"evaluating-adversarial-attacks-on-imagenet-a","title":"Evaluating Adversarial Attacks on ImageNet: A Reality Check on Misclassification Classes","date":"2021-11-22","arxiv_id":"2111.11056","repositories_listed":1,"syntology":{"n":5,"n_ran":4,"n_constructed":0,"n_ran_checked":4,"n_instrument":0,"n_unverified":1,"n_honours":0,"n_violates":0,"n_no_contract":4,"n_pointer_only":0,"phrase":"4 ran (of which 0 constructed an object rather than computing a result; 4 with no instrument failure: 0 honoured, 0 violated, 4 with no contract checked; 0 where Syntology's instrument failed) · 1 unverified","sample_list":"/paper/evaluating-adversarial-attacks-on-imagenet-a#ran","syntology_url":"https://syntology.ai/paper/2111.11056","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2111.11056"}},"official":{"repos":["utkuozbulak/imagenet-adversarial-image-evaluation"],"state":"official (archive's flag): 4 ran","n_ran":4,"n_constructed":0,"n_ran_no_instrument_failure":4,"n_unverified":1,"ran_from_kinds":["official"]}}},{"url":"/paper/efficient-deep-learning-models-for-land-cover","slug":"efficient-deep-learning-models-for-land-cover","title":"Benchmarking and scaling of deep learning models for land cover image classification","date":"2021-11-18","arxiv_id":"2111.09451","repositories_listed":1,"syntology":{"n":5,"n_ran":3,"n_constructed":0,"n_ran_checked":3,"n_instrument":0,"n_unverified":2,"n_honours":0,"n_violates":0,"n_no_contract":3,"n_pointer_only":0,"phrase":"3 ran (of which 0 constructed an object rather than computing a result; 3 with no instrument failure: 0 honoured, 0 violated, 3 with no contract checked; 0 where Syntology's instrument failed) · 2 unverified","sample_list":"/paper/efficient-deep-learning-models-for-land-cover#ran","syntology_url":"https://syntology.ai/paper/2111.09451","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2111.09451"}},"official":{"repos":["orion-ai-lab/efficientbigearthnet"],"state":"official (archive's flag): 3 ran","n_ran":3,"n_constructed":0,"n_ran_no_instrument_failure":3,"n_unverified":2,"ran_from_kinds":["official"]}}},{"url":"/paper/cleanrl-high-quality-single-file","slug":"cleanrl-high-quality-single-file","title":"CleanRL: High-quality Single-file Implementations of Deep Reinforcement Learning Algorithms","date":"2021-11-16","arxiv_id":"2111.08819","repositories_listed":2,"syntology":{"n":2,"n_ran":2,"n_constructed":0,"n_ran_checked":1,"n_instrument":1,"n_unverified":0,"n_honours":1,"n_violates":0,"n_no_contract":0,"n_pointer_only":2,"phrase":"2 ran (of which 0 constructed an object rather than computing a result; 1 with no instrument failure: 1 honoured, 0 violated, 0 with no contract checked; 1 where Syntology's instrument failed) · 0 unverified","sample_list":"/paper/cleanrl-high-quality-single-file#ran","syntology_url":"https://syntology.ai/paper/2111.08819","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2111.08819"}},"official":{"repos":["vwxyzjn/cleanrl"],"state":"official: no sample here; runs from other or unrecorded repositories","n_ran":0,"n_constructed":0,"n_ran_no_instrument_failure":0,"n_unverified":0,"ran_from_kinds":["unlocated"]}}}],"record_sha256":"10c686b11e6cd25e12e6fb231e8e402b1c15b8841e8a30098a34d3b5c549d00d","record_changed_at":"2026-09-28","record_changed_at_basis":"first_hashed"}