{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/read-json-file","entry":"read_json_file","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":36,"n_papers_ran":25,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":33,"n_samples_ran":22,"n_samples_fingerprinted":4,"n_places":37,"n_places_pointer_only":15,"by_status":{"ran_honours":0,"ran_violates":0,"ran_draft_wrong":7,"ran_fixture":0,"ran":15,"unverified":11},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2607.12488","paper":"/paper/arxiv-2607-12488","title":"Sample Efficient Generative Optimization for Molecular Design","date":null,"month_inferred_from_arxiv_id":"2026-07","title_source":"syntology","repo":"schwallergroup/sego","path":"saturn.py","file_url":"https://github.com/schwallergroup/sego/blob/HEAD/saturn.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"fef03036490e6666","mcp_get_code":{"code_sha256":"fef03036490e6666"}},{"arxiv_id":"2607.10915","paper":"/paper/arxiv-2607-10915","title":"Normative Alignment of Recommender Systems via Internal Label Shift","date":null,"month_inferred_from_arxiv_id":"2026-07","title_source":"syntology","repo":"johanneskruse/nails","path":"utils/_ebrec/_python.py","file_url":"https://github.com/johanneskruse/nails/blob/HEAD/utils/_ebrec/_python.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"99c38c7439598156","mcp_get_code":{"code_sha256":"99c38c7439598156"}},{"arxiv_id":"2604.04914","paper":"/paper/arxiv-2604-04914","title":"Analyzing Symbolic Properties for DRL Agents in Systems and Networking","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"mhmd97z/diffRL","path":"applications/aurora/aurora_lib/utils.py","file_url":"https://github.com/mhmd97z/diffRL/blob/HEAD/applications/aurora/aurora_lib/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"cfcfa08c689dbece","mcp_get_code":{"code_sha256":"cfcfa08c689dbece"}},{"arxiv_id":"2507.04701","paper":"/paper/xiyan-sql-a-novel-multi-generator-framework","title":"XiYan-SQL: A Novel Multi-Generator Framework For Text-to-SQL","date":"2025-07-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"xgenerationlab/xiyan_mcp_server","path":"src/xiyan_mcp_server/utils/file_util.py","file_url":"https://github.com/xgenerationlab/xiyan_mcp_server/blob/HEAD/src/xiyan_mcp_server/utils/file_util.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"eb4b54f7f4ad75d8","mcp_get_code":{"code_sha256":"eb4b54f7f4ad75d8"}},{"arxiv_id":"2506.21539","paper":"/paper/worldvla-towards-autoregressive-action-world","title":"WorldVLA: Towards Autoregressive Action World Model","date":"2025-06-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"alibaba-damo-academy/worldvla","path":"rynnvla-002/data_lerobot/concate_action_world_model_data.py","file_url":"https://github.com/alibaba-damo-academy/worldvla/blob/HEAD/rynnvla-002/data_lerobot/concate_action_world_model_data.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"63ae35e41d95f4d9","mcp_get_code":{"code_sha256":"63ae35e41d95f4d9"}},{"arxiv_id":"2506.13326","paper":"/paper/vis-shepherd-constructing-critic-for-llm","title":"VIS-Shepherd: Constructing Critic for LLM-based Data Visualization Generation","date":"2025-06-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"bopan3/vis-shepherd","path":"Evaluation/run_single_autoEvaluateLikert.py","file_url":"https://github.com/bopan3/vis-shepherd/blob/HEAD/Evaluation/run_single_autoEvaluateLikert.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"72b2964a4e373380","mcp_get_code":{"code_sha256":"72b2964a4e373380"}},{"arxiv_id":"2506.06292","paper":"/paper/mutual-taught-for-co-adapting-policy-and","title":"Mutual-Taught for Co-adapting Policy and Reward Models","date":"2025-05-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Stycoo/Mutual-Taught","path":"completion_inference/build_dataset.py","file_url":"https://github.com/Stycoo/Mutual-Taught/blob/HEAD/completion_inference/build_dataset.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"1acc67cef16f4df5","mcp_get_code":{"code_sha256":"1acc67cef16f4df5"}},{"arxiv_id":"2505.14460","paper":"/paper/visualquality-r1-reasoning-induced-image","title":"VisualQuality-R1: Reasoning-Induced Image Quality Assessment via Reinforcement Learning to Rank","date":"2025-05-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"tianhewu/visualquality-r1","path":"datasets/make_data.py","file_url":"https://github.com/tianhewu/visualquality-r1/blob/HEAD/datasets/make_data.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"b6715cf1542b45e4","mcp_get_code":{"code_sha256":"b6715cf1542b45e4"}},{"arxiv_id":"2505.13237","paper":"/paper/sakura-on-the-multi-hop-reasoning-of-large","title":"SAKURA: On the Multi-hop Reasoning of Large Audio-Language Models Based on Speech and Audio Information","date":"2025-05-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ckyang1124/sakura","path":"evaluation/llm_judge.py","file_url":"https://github.com/ckyang1124/sakura/blob/HEAD/evaluation/llm_judge.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"5a7528f5f02fdae8","mcp_get_code":{"code_sha256":"5a7528f5f02fdae8"}},{"arxiv_id":"2502.11741","paper":"/paper/sql-o1-a-self-reward-heuristic-dynamic-search","title":"SQL-o1: A Self-Reward Heuristic Dynamic Search Method for Text-to-SQL","date":"2025-02-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"shuailyu0110/sql-o1","path":"preprocess_data.py","file_url":"https://github.com/shuailyu0110/sql-o1/blob/HEAD/preprocess_data.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"de3dd83d24948774","mcp_get_code":{"code_sha256":"de3dd83d24948774"}},{"arxiv_id":"2502.07411","paper":"/paper/egotextvqa-towards-egocentric-scene-text","title":"EgoTextVQA: Towards Egocentric Scene-Text Aware Video Question Answering","date":"2025-02-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zhousheng97/egotextvqa","path":"gpt_eval.py","file_url":"https://github.com/zhousheng97/egotextvqa/blob/HEAD/gpt_eval.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e3413b9fe9207632","mcp_get_code":{"code_sha256":"e3413b9fe9207632"}},{"arxiv_id":"2501.07171","paper":"/paper/biomedica-an-open-biomedical-image-caption","title":"BIOMEDICA: An Open Biomedical Image-Caption Archive, Dataset, and Vision-Language Models Derived from Scientific Literature","date":"2025-01-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"minwoosun/biomedica-etl","path":"src/pmc_oa/data_loaders/loader_utils.py","file_url":"https://github.com/minwoosun/biomedica-etl/blob/HEAD/src/pmc_oa/data_loaders/loader_utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"97c7f58efcfba433","mcp_get_code":{"code_sha256":"97c7f58efcfba433"}},{"arxiv_id":"2412.10138","paper":"/paper/route-robust-multitask-tuning-and","title":"ROUTE: Robust Multitask Tuning and Collaboration for Text-to-SQL","date":"2024-12-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"alibaba/route","path":"route.py","file_url":"https://github.com/alibaba/route/blob/HEAD/route.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"de3dd83d24948774","mcp_get_code":{"code_sha256":"de3dd83d24948774"}},{"arxiv_id":"2412.04905","paper":"/paper/demo-reframing-dialogue-interaction-with-fine","title":"DEMO: Reframing Dialogue Interaction with Fine-grained Element Modeling","date":"2024-12-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mozerwang/demo","path":"src/utils/functions.py","file_url":"https://github.com/mozerwang/demo/blob/HEAD/src/utils/functions.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"bd0d911b1d93f670","mcp_get_code":{"code_sha256":"bd0d911b1d93f670"}},{"arxiv_id":"2411.11919","paper":"/paper/vl-uncertainty-detecting-hallucination-in","title":"VL-Uncertainty: Detecting Hallucination in Large Vision-Language Model via Uncertainty Estimation","date":"2024-11-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"JT-Sun/Filtering-WoRA","path":"cuhk_fintune_filterdelet_json.py","file_url":"https://github.com/JT-Sun/Filtering-WoRA/blob/HEAD/cuhk_fintune_filterdelet_json.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f00bedbbad13c687","mcp_get_code":{"code_sha256":"f00bedbbad13c687"}},{"arxiv_id":"2410.09870","paper":"/paper/chroknowledge-unveiling-chronological","title":"ChroKnowledge: Unveiling Chronological Knowledge of Language Models in Multiple Domains","date":"2024-10-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"dmis-lab/chroknowledge","path":"sources/utils.py","file_url":"https://github.com/dmis-lab/chroknowledge/blob/HEAD/sources/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f00bedbbad13c687","mcp_get_code":{"code_sha256":"f00bedbbad13c687"}},{"arxiv_id":"2410.05162","paper":"/paper/deciphering-the-interplay-of-parametric-and","title":"Deciphering the Interplay of Parametric and Non-parametric Memory in Retrieval-augmented Language Models","date":"2024-10-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"m3hrdadfi/rag-memory-interplay","path":"src/utils.py","file_url":"https://github.com/m3hrdadfi/rag-memory-interplay/blob/HEAD/src/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"07609cc6c26d3a72","mcp_get_code":{"code_sha256":"07609cc6c26d3a72"}},{"arxiv_id":"2409.14818","paper":"/paper/mobilevlm-a-vision-language-model-for-better","title":"MobileVLM: A Vision-Language Model for Better Intra- and Inter-UI Understanding","date":"2024-09-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"xiaomi/mobilevlm","path":"corpus/xml_test_22.py","file_url":"https://github.com/xiaomi/mobilevlm/blob/HEAD/corpus/xml_test_22.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"707a0ecb9f3c6f2d","mcp_get_code":{"code_sha256":"707a0ecb9f3c6f2d"}},{"arxiv_id":"2407.12857","paper":"/paper/automated-peer-reviewing-in-paper-sea","title":"Automated Peer Reviewing in Paper SEA: Standardization, Evaluation, and Analysis","date":"2024-07-09","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ecnu-sea/SEA","path":"inference/inference.py","file_url":"https://github.com/ecnu-sea/SEA/blob/HEAD/inference/inference.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"4ef3ac6502fb4bf8","mcp_get_code":{"code_sha256":"4ef3ac6502fb4bf8"}},{"arxiv_id":"2407.12186","paper":"/paper/directly-optimizing-for-synthesizability-in","title":"Directly Optimizing for Synthesizability in Generative Molecular Design using Retrosynthesis Models","date":"2024-07-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"schwallergroup/saturn","path":"saturn.py","file_url":"https://github.com/schwallergroup/saturn/blob/HEAD/saturn.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"fef03036490e6666","mcp_get_code":{"code_sha256":"fef03036490e6666"}},{"arxiv_id":"2407.01502","paper":"/paper/ai-agents-that-matter","title":"AI Agents That Matter","date":"2024-07-01","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"benediktstroebl/agent-evals","path":"NovelQA/novelQA-gpt-4.py","file_url":"https://github.com/benediktstroebl/agent-evals/blob/HEAD/NovelQA/novelQA-gpt-4.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"3f71176782469ee7","mcp_get_code":{"code_sha256":"3f71176782469ee7"}},{"arxiv_id":"2406.18522","paper":"/paper/chronomagic-bench-a-benchmark-for-metamorphic","title":"ChronoMagic-Bench: A Benchmark for Metamorphic Evaluation of Text-to-Time-lapse Video Generation","date":"2024-06-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"pku-yuangroup/magictime","path":"data_preprocess/step3_2_preprocess_video_caption.py","file_url":"https://github.com/pku-yuangroup/magictime/blob/HEAD/data_preprocess/step3_2_preprocess_video_caption.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"36c10de5d93aa54f","mcp_get_code":{"code_sha256":"36c10de5d93aa54f"}},{"arxiv_id":"2406.10638","paper":"/paper/seeing-clearly-answering-incorrectly-a","title":"Unveiling the Ignorance of MLLMs: Seeing Clearly, Answering Incorrectly","date":"2024-06-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"baai-dcai/multimodal-robustness-benchmark","path":"dataset/data_reformat.py","file_url":"https://github.com/baai-dcai/multimodal-robustness-benchmark/blob/HEAD/dataset/data_reformat.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"7fb31d184bedfa2b","mcp_get_code":{"code_sha256":"7fb31d184bedfa2b"}},{"arxiv_id":"2405.17066","paper":"/paper/saturn-sample-efficient-generative-molecular","title":"Saturn: Sample-efficient Generative Molecular Design using Memory Manipulation","date":"2024-05-27","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"code_sha256_prefix":"fef03036490e6666","mcp_get_code":{"code_sha256":"fef03036490e6666"}},{"arxiv_id":"2405.17066","paper":"/paper/saturn-sample-efficient-generative-molecular","title":"Saturn: Sample-efficient Generative Molecular Design using Memory Manipulation","date":"2024-05-27","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"schwallergroup/augmented_memory","path":"input.py","file_url":"https://github.com/schwallergroup/augmented_memory/blob/HEAD/input.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"217efcb055ff24df","mcp_get_code":{"code_sha256":"217efcb055ff24df"}},{"arxiv_id":"2404.18824","paper":"/paper/benchmarking-benchmark-leakage-in-large","title":"Benchmarking Benchmark Leakage in Large Language Models","date":"2024-04-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"gair-nlp/benbench","path":"src/metric_utils.py","file_url":"https://github.com/gair-nlp/benbench/blob/HEAD/src/metric_utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"3dfbdf6a9d3d18d2","mcp_get_code":{"code_sha256":"3dfbdf6a9d3d18d2"}},{"arxiv_id":"2402.15059","paper":"/paper/colbert-xm-a-modular-multi-vector","title":"ColBERT-XM: A Modular Multi-Vector Representation Model for Zero-Shot Multilingual Information Retrieval","date":"2024-02-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ant-louis/xm-retrievers","path":"src/utils/common.py","file_url":"https://github.com/ant-louis/xm-retrievers/blob/HEAD/src/utils/common.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"f2f3f6102fc4989c","mcp_get_code":{"code_sha256":"f2f3f6102fc4989c"}},{"arxiv_id":"2401.12761","paper":"/paper/muses-the-multi-sensor-semantic-perception","title":"MUSES: The Multi-Sensor Semantic Perception Dataset for Driving under Uncertainty","date":"2024-01-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"timbroed/MUSES","path":"processing/utils.py","file_url":"https://github.com/timbroed/MUSES/blob/HEAD/processing/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"8cb46a3eae892f9f","mcp_get_code":{"code_sha256":"8cb46a3eae892f9f"}},{"arxiv_id":"2401.07576","paper":"/paper/tdd-without-tears-towards-test-case","title":"PyTester: Deep Reinforcement Learning for Text-to-Testcase Generation","date":null,"month_inferred_from_arxiv_id":"2024-01","title_source":"archive","repo":"tddpytester/pytester","path":"handlers/utils.py","file_url":"https://github.com/tddpytester/pytester/blob/HEAD/handlers/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"bfa04064fa4b10d8","mcp_get_code":{"code_sha256":"bfa04064fa4b10d8"}},{"arxiv_id":"2401.02335","paper":"/paper/linguistic-profiling-of-deepfakes-an-open","title":"Linguistic Profiling of Deepfakes: An Open Database for Next-Generation Deepfake Detection","date":"2024-01-04","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"dflip3k/dflip-3k","path":"utils/download_models.py","file_url":"https://github.com/dflip3k/dflip-3k/blob/HEAD/utils/download_models.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"00b94adf3a8669df","mcp_get_code":{"code_sha256":"00b94adf3a8669df"}},{"arxiv_id":"2311.18448","paper":"/paper/hold-category-agnostic-3d-reconstruction-of","title":"HOLD: Category-agnostic 3D Reconstruction of Interacting Hands and Objects from Video","date":"2023-11-30","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zc-alexfan/hold","path":"code/summarize_metrics.py","file_url":"https://github.com/zc-alexfan/hold/blob/HEAD/code/summarize_metrics.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"9a08e7d923fb1f50","mcp_get_code":{"code_sha256":"9a08e7d923fb1f50"}},{"arxiv_id":"2310.15061","paper":"/paper/the-bla-benchmark-investigating-basic","title":"The BLA Benchmark: Investigating Basic Language Abilities of Pre-Trained Multimodal Models","date":"2023-10-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"shin-ee-chen/BLA","path":"evaluation_models/blip2/bla_utils.py","file_url":"https://github.com/shin-ee-chen/BLA/blob/HEAD/evaluation_models/blip2/bla_utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"b880398cd04cf6f1","mcp_get_code":{"code_sha256":"b880398cd04cf6f1"}},{"arxiv_id":"2309.17012","paper":"/paper/benchmarking-cognitive-biases-in-large","title":"Benchmarking Cognitive Biases in Large Language Models as Evaluators","date":"2023-09-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"minnesotanlp/cobbler","path":"competitive-llms/evaluations/evaluate.py","file_url":"https://github.com/minnesotanlp/cobbler/blob/HEAD/competitive-llms/evaluations/evaluate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"04fbd0764f0418f9","mcp_get_code":{"code_sha256":"04fbd0764f0418f9"}},{"arxiv_id":"2211.04486","paper":"/paper/active-example-selection-for-in-context","title":"Active Example Selection for In-Context Learning","date":"2022-11-08","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"chicagohai/active-example-selection","path":"src/prompting/strategies/greedy_strategies.py","file_url":"https://github.com/chicagohai/active-example-selection/blob/HEAD/src/prompting/strategies/greedy_strategies.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"c7e900c72688b821","mcp_get_code":{"code_sha256":"c7e900c72688b821"}},{"arxiv_id":"2210.12374","paper":"/paper/reastap-injecting-table-reasoning-skills","title":"ReasTAP: Injecting Table Reasoning Skills During Pre-training via Synthetic Reasoning Examples","date":"2022-10-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Yale-LILY/ReasTAP","path":"synthetic_tableqa_generation/prepare_table_data.py","file_url":"https://github.com/Yale-LILY/ReasTAP/blob/HEAD/synthetic_tableqa_generation/prepare_table_data.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"7c1e19fabfcb24da","mcp_get_code":{"code_sha256":"7c1e19fabfcb24da"}},{"arxiv_id":"2205.05198","paper":"/paper/reducing-activation-recomputation-in-large","title":"Reducing Activation Recomputation in Large Transformer Models","date":"2022-05-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"calculon-ai/calculon","path":"calculon/io.py","file_url":"https://github.com/calculon-ai/calculon/blob/HEAD/calculon/io.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"5763234cb8c69a6b","mcp_get_code":{"code_sha256":"5763234cb8c69a6b"}},{"arxiv_id":"2104.08786","paper":"/paper/fantastically-ordered-prompts-and-where-to","title":"Fantastically Ordered Prompts and Where to Find Them: Overcoming Few-Shot Prompt Order Sensitivity","date":"2021-04-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"chicagohai/active-example-selection","path":"src/prompting/strategies/ordering_strategy.py","file_url":"https://github.com/chicagohai/active-example-selection/blob/HEAD/src/prompting/strategies/ordering_strategy.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"aaa2f935eb008592","mcp_get_code":{"code_sha256":"aaa2f935eb008592"}}]}