{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/execute","entry":"execute","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":12,"n_papers_ran":4,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":13,"n_samples_ran":5,"n_samples_fingerprinted":0,"n_places":13,"n_places_pointer_only":3,"by_status":{"ran_honours":0,"ran_violates":0,"ran_draft_wrong":1,"ran_fixture":0,"ran":4,"unverified":8},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2411.02284","paper":"/paper/training-on-the-test-model-contamination-in","title":"Training on the Test Model: Contamination in Ranking Distillation","date":"2024-11-04","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Parry-Parry/ContaminatedDistillation","path":"src/contaminate/__main__.py","file_url":"https://github.com/Parry-Parry/ContaminatedDistillation/blob/HEAD/src/contaminate/__main__.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"00a1cf01d60755fc","mcp_get_code":{"code_sha256":"00a1cf01d60755fc"}},{"arxiv_id":"2406.04531","paper":"/paper/testeval-benchmarking-large-language-models","title":"TESTEVAL: Benchmarking Large Language Models for Test Case Generation","date":null,"month_inferred_from_arxiv_id":"2024-06","title_source":"archive","repo":"llm4softwaretesting/testeval","path":"eval_base.py","file_url":"https://github.com/llm4softwaretesting/testeval/blob/HEAD/eval_base.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"8671b30e9a445afa","mcp_get_code":{"code_sha256":"8671b30e9a445afa"}},{"arxiv_id":"2406.04531","paper":"/paper/testeval-benchmarking-large-language-models","title":"TESTEVAL: Benchmarking Large Language Models for Test Case Generation","date":null,"month_inferred_from_arxiv_id":"2024-06","title_source":"archive","repo":"llm4softwaretesting/testeval","path":"eval_branchcov.py","file_url":"https://github.com/llm4softwaretesting/testeval/blob/HEAD/eval_branchcov.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"d95b2ac22f4237ce","mcp_get_code":{"code_sha256":"d95b2ac22f4237ce"}},{"arxiv_id":"2404.15402","paper":"/paper/kids-sbi-simulation-based-inference-analysis","title":"KiDS-SBI: Simulation-based inference analysis of KiDS-1000 cosmic shear","date":null,"month_inferred_from_arxiv_id":"2024-04","title_source":"archive","repo":"mwiet/kids_sbi","path":"modules/scale_cuts/load_cterm.py","file_url":"https://github.com/mwiet/kids_sbi/blob/HEAD/modules/scale_cuts/load_cterm.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"c0a338e9423645a0","mcp_get_code":{"code_sha256":"c0a338e9423645a0"}},{"arxiv_id":"2402.15809","paper":"/paper/empowering-large-language-model-agents","title":"Empowering Large Language Model Agents through Action Learning","date":"2024-02-24","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zhao-ht/learnact","path":"aux_func/aux_func.py","file_url":"https://github.com/zhao-ht/learnact/blob/HEAD/aux_func/aux_func.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"a9ab3708c00ae21a","mcp_get_code":{"code_sha256":"a9ab3708c00ae21a"}},{"arxiv_id":"2311.05657","paper":"/paper/lumos-learning-agents-with-unified-data","title":"Agent Lumos: Unified and Modular Training for Open-Source Language Agents","date":"2023-11-09","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"allenai/lumos","path":"eval/webshop/run_eval.py","file_url":"https://github.com/allenai/lumos/blob/HEAD/eval/webshop/run_eval.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"f7678ddb6aba27d7","mcp_get_code":{"code_sha256":"f7678ddb6aba27d7"}},{"arxiv_id":"2306.15626","paper":"/paper/leandojo-theorem-proving-with-retrieval-1","title":"LeanDojo: Theorem Proving with Retrieval-Augmented Language Models","date":"2023-06-27","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"lean-dojo/LeanDojo","path":"src/lean_dojo/utils.py","file_url":"https://github.com/lean-dojo/LeanDojo/blob/HEAD/src/lean_dojo/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"58b9a44ed03f6513","mcp_get_code":{"code_sha256":"58b9a44ed03f6513"}},{"arxiv_id":"2306.14321","paper":"/paper/robut-a-systematic-study-of-table-qa","title":"RobuT: A Systematic Study of Table QA Robustness Against Human-Annotated Adversarial Perturbations","date":"2023-06-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"yilunzhao/RobuT","path":"utils/eval_utils.py","file_url":"https://github.com/yilunzhao/RobuT/blob/HEAD/utils/eval_utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"7998bb6f998f1b8a","mcp_get_code":{"code_sha256":"7998bb6f998f1b8a"}},{"arxiv_id":"2306.08891","paper":"/paper/interleaving-pre-trained-language-models-and","title":"Interleaving Pre-Trained Language Models and Large Language Models for Zero-Shot NL2SQL Generation","date":"2023-06-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ruc-datalab/zeronl2sql","path":"LLM_text2sql.py","file_url":"https://github.com/ruc-datalab/zeronl2sql/blob/HEAD/LLM_text2sql.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"c7c413b7001d6846","mcp_get_code":{"code_sha256":"c7c413b7001d6846"}},{"arxiv_id":"2302.08468","paper":"/paper/lever-learning-to-verify-language-to-code","title":"LEVER: Learning to Verify Language-to-Code Generation with Execution","date":"2023-02-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"niansong1996/lever","path":"execution/safe_execution_util.py","file_url":"https://github.com/niansong1996/lever/blob/HEAD/execution/safe_execution_util.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"0e6bb17163d0c70b","mcp_get_code":{"code_sha256":"0e6bb17163d0c70b"}},{"arxiv_id":"2301.07695","paper":"/paper/ehrsql-a-practical-text-to-sql-benchmark-for-1","title":"EHRSQL: A Practical Text-to-SQL Benchmark for Electronic Health Records","date":"2023-01-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"glee4810/EHRSQL","path":"evaluate.py","file_url":"https://github.com/glee4810/EHRSQL/blob/HEAD/evaluate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"CC-BY-4.0","inline_ok":false,"code_sha256_prefix":"cac3be6308daaa20","mcp_get_code":{"code_sha256":"cac3be6308daaa20"}},{"arxiv_id":"2205.14318","paper":"/paper/learning-from-self-sampled-correct-and","title":"Learning Math Reasoning from Self-Sampled Correct and Partially-Correct Solutions","date":"2022-05-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"microsoft/TraceCodegen","path":"execution/safe_execution_util.py","file_url":"https://github.com/microsoft/TraceCodegen/blob/HEAD/execution/safe_execution_util.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"41b1619d697fec62","mcp_get_code":{"code_sha256":"41b1619d697fec62"}},{"arxiv_id":"2108.04763","paper":"/paper/imitation-learning-by-reinforcement-learning","title":"Imitation Learning by Reinforcement Learning","date":"2021-08-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"spotify-research/il-by-rl","path":"utils.py","file_url":"https://github.com/spotify-research/il-by-rl/blob/HEAD/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"25bd681eec65f985","mcp_get_code":{"code_sha256":"25bd681eec65f985"}}]}