{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/exact-match","entry":"exact_match","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":23,"n_papers_ran":17,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":22,"n_samples_ran":16,"n_samples_fingerprinted":10,"n_places":23,"n_places_pointer_only":8,"by_status":{"ran_honours":1,"ran_violates":3,"ran_draft_wrong":0,"ran_fixture":0,"ran":12,"unverified":6},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2608.21386","paper":"/paper/arxiv-2608-21386","title":"Model of Models: When Does Emitting a Specialist Beat Attending, Adapting, or Tuning? A Controlled Cross-Domain Study of Amortized Weight Generation","date":null,"month_inferred_from_arxiv_id":"2026-08","title_source":"syntology","repo":"johnchowell/Model-of-Models","path":"exp16_arc.py","file_url":"https://github.com/johnchowell/Model-of-Models/blob/HEAD/exp16_arc.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"8b3aae2b84b3210e","mcp_get_code":{"code_sha256":"8b3aae2b84b3210e"}},{"arxiv_id":"2607.24884","paper":"/paper/arxiv-2607-24884","title":"Beyond \"What to Retrieve\": Uncertainty in Retrieval-Augmented Code Generation","date":null,"month_inferred_from_arxiv_id":"2026-07","title_source":"syntology","repo":"Rocky5502/OpenCoder_V1","path":"opencoder/evaluation/metrics.py","file_url":"https://github.com/Rocky5502/OpenCoder_V1/blob/HEAD/opencoder/evaluation/metrics.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"095c99ea52e24d66","mcp_get_code":{"code_sha256":"095c99ea52e24d66"}},{"arxiv_id":"2607.16431","paper":"/paper/arxiv-2607-16431","title":"RIMS: Preference Optimization via Smoothed Multi-pair Aggregation for Small-Scale LLM Retrieval-Augmented Generation","date":null,"month_inferred_from_arxiv_id":"2026-07","title_source":"syntology","repo":"tptrix29/RIMS","path":"baseline/baseline_instructrag.py","file_url":"https://github.com/tptrix29/RIMS/blob/HEAD/baseline/baseline_instructrag.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"693e3eb42781f873","mcp_get_code":{"code_sha256":"693e3eb42781f873"}},{"arxiv_id":"2606.03463","paper":"/paper/arxiv-2606-03463","title":"DMF: A Deterministic Memory Framework for Conversational AI Agents","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"matstech/dmf-benchmarks","path":"dmf_bench/benchmarks/locomo/rigorous.py","file_url":"https://github.com/matstech/dmf-benchmarks/blob/HEAD/dmf_bench/benchmarks/locomo/rigorous.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6aea303b6aec6c67","mcp_get_code":{"code_sha256":"6aea303b6aec6c67"}},{"arxiv_id":"2605.10787","paper":"/paper/arxiv-2605-10787","title":"ComplexMCP: Evaluation of LLM Agents in Dynamic, Interdependent, and Large-Scale Tool Sandbox","date":null,"month_inferred_from_arxiv_id":"2026-05","title_source":"syntology","repo":"AIDC-AI/complex-mcp","path":"benchmark/judge.py","file_url":"https://github.com/AIDC-AI/complex-mcp/blob/HEAD/benchmark/judge.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"051ed74b766c82aa","mcp_get_code":{"code_sha256":"051ed74b766c82aa"}},{"arxiv_id":"2605.00199","paper":"/paper/arxiv-2605-00199","title":"RSAT: Structured Attribution Makes Small Language Models Faithful Table Reasoners","date":null,"month_inferred_from_arxiv_id":"2026-05","title_source":"syntology","repo":"JugalGajjar/RSAT","path":"src/rewards/answer_reward.py","file_url":"https://github.com/JugalGajjar/RSAT/blob/HEAD/src/rewards/answer_reward.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"8200854ddbd775fd","mcp_get_code":{"code_sha256":"8200854ddbd775fd"}},{"arxiv_id":"2603.10624","paper":"/paper/arxiv-2603-10624","title":"Reinforcement Learning with Conditional Expectation Reward","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"changyi7231/CER","path":"recipe/cer/src/reward_manager.py","file_url":"https://github.com/changyi7231/CER/blob/HEAD/recipe/cer/src/reward_manager.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"bacda3216fdce102","mcp_get_code":{"code_sha256":"bacda3216fdce102"}},{"arxiv_id":"2602.13551","paper":"/paper/arxiv-2602-13551","title":"Small Reward Models via Backward Inference","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"yikee/FLIP","path":"open-instruct/decontamination/search.py","file_url":"https://github.com/yikee/FLIP/blob/HEAD/open-instruct/decontamination/search.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"b9ce489e37ca0182","mcp_get_code":{"code_sha256":"b9ce489e37ca0182"}},{"arxiv_id":"2509.03058","paper":"/paper/arxiv-2509-03058","title":"EverTracer: Hunting Stolen Large Language Models via Stealthy and Robust Probabilistic Fingerprint","date":null,"month_inferred_from_arxiv_id":"2025-09","title_source":"syntology","repo":"Xuzhenhua55/EverTracer","path":"Utils/evaluation.py","file_url":"https://github.com/Xuzhenhua55/EverTracer/blob/HEAD/Utils/evaluation.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"ab51854fed6480f7","mcp_get_code":{"code_sha256":"ab51854fed6480f7"}},{"arxiv_id":"2505.12212","paper":"/paper/data-whisperer-efficient-data-selection-for","title":"Data Whisperer: Efficient Data Selection for Task-Specific LLM Fine-Tuning via Few-Shot In-Context Learning","date":"2025-05-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"gszfwsb/Data-Whisperer","path":"pruning/datawhisperer_gsm_pruner.py","file_url":"https://github.com/gszfwsb/Data-Whisperer/blob/HEAD/pruning/datawhisperer_gsm_pruner.py","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"7d4e720c5eec0e6f","mcp_get_code":{"code_sha256":"7d4e720c5eec0e6f"}},{"arxiv_id":"2503.19707","paper":"/paper/mind-the-gap-benchmarking-spatial-reasoning","title":"Mind the Gap: Benchmarking Spatial Reasoning in Vision-Language Models","date":"2025-03-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"stogiannidis/srbench","path":"src/eval/acc.py","file_url":"https://github.com/stogiannidis/srbench/blob/HEAD/src/eval/acc.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"512557f1eb973e22","mcp_get_code":{"code_sha256":"512557f1eb973e22"}},{"arxiv_id":"2412.12151","paper":"/paper/smartcal-an-approach-to-self-aware-tool-use","title":"SMARTCAL: An Approach to Self-Aware Tool-Use Evaluation and Calibration","date":"2024-12-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Henrysyh2000/SMARTCAL","path":"ART_SM_CAL/art_uw.py","file_url":"https://github.com/Henrysyh2000/SMARTCAL/blob/HEAD/ART_SM_CAL/art_uw.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"a442b0bd917515d1","mcp_get_code":{"code_sha256":"a442b0bd917515d1"}},{"arxiv_id":"2409.14083","paper":"/paper/surf-teaching-large-vision-language-models-to","title":"SURf: Teaching Large Vision-Language Models to Selectively Utilize Retrieved Information","date":"2024-09-21","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"GasolSun36/SURf","path":"initial/tool_evaluate.py","file_url":"https://github.com/GasolSun36/SURf/blob/HEAD/initial/tool_evaluate.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"442f044af42b5b3f","mcp_get_code":{"code_sha256":"442f044af42b5b3f"}},{"arxiv_id":"2404.14741","paper":"/paper/generate-on-graph-treat-llm-as-both-agent-and","title":"Generate-on-Graph: Treat LLM as both Agent and KG in Incomplete Knowledge Graph Question Answering","date":"2024-04-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"YaooXu/GoG","path":"src/evaluate.py","file_url":"https://github.com/YaooXu/GoG/blob/HEAD/src/evaluate.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"d37ef44866e11f1d","mcp_get_code":{"code_sha256":"d37ef44866e11f1d"}},{"arxiv_id":"2403.19021","paper":"/paper/towards-llm-recsys-alignment-with-textual-id","title":"IDGenRec: LLM-RecSys Alignment with Textual ID Learning","date":"2024-03-27","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"agiresearch/idgenrec","path":"src/utils/generation_trie.py","file_url":"https://github.com/agiresearch/idgenrec/blob/HEAD/src/utils/generation_trie.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"ddaa9fa31e2c803d","mcp_get_code":{"code_sha256":"ddaa9fa31e2c803d"}},{"arxiv_id":"2403.04317","paper":"/paper/online-adaptation-of-language-models-with-a","title":"Online Adaptation of Language Models with a Memory of Amortized Contexts","date":"2024-03-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jihoontack/mac","path":"models/amortized_encdec.py","file_url":"https://github.com/jihoontack/mac/blob/HEAD/models/amortized_encdec.py","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"64702a446a4111bd","mcp_get_code":{"code_sha256":"64702a446a4111bd"}},{"arxiv_id":"2402.12842","paper":"/paper/promptkd-distilling-student-friendly","title":"PromptKD: Distilling Student-Friendly Knowledge for Generative Language Models via Prompt Tuning","date":"2024-02-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"gmkim-ai/promptkd","path":"rouge_metric.py","file_url":"https://github.com/gmkim-ai/promptkd/blob/HEAD/rouge_metric.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"61c5ad3a7b3c5731","mcp_get_code":{"code_sha256":"61c5ad3a7b3c5731"}},{"arxiv_id":"2312.02614","paper":"/paper/prompt-optimization-via-adversarial-in","title":"Prompt Optimization via Adversarial In-Context Learning","date":"2023-12-05","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zhaoyiran924/adv-in-context-learning","path":"eval/automatic/evaluation.py","file_url":"https://github.com/zhaoyiran924/adv-in-context-learning/blob/HEAD/eval/automatic/evaluation.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"41e978babf390372","mcp_get_code":{"code_sha256":"41e978babf390372"}},{"arxiv_id":"2310.03249","paper":"/paper/can-large-language-models-be-good-path","title":"Can Large Language Models be Good Path Planners? A Benchmark and Investigation on Spatial-temporal Reasoning","date":"2023-10-05","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mohamedaghzal/llms-as-path-planners","path":"gpt-4-path-planning/src/evaluate.py","file_url":"https://github.com/mohamedaghzal/llms-as-path-planners/blob/HEAD/gpt-4-path-planning/src/evaluate.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"806fb301720451ec","mcp_get_code":{"code_sha256":"806fb301720451ec"}},{"arxiv_id":"2203.09397","paper":"/paper/coloring-the-blank-slate-pre-training-imparts-1","title":"Coloring the Blank Slate: Pre-training Imparts a Hierarchical Inductive Bias to Sequence-to-sequence Models","date":"2022-03-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"sebschu/multilingual-transformations","path":"models/metrics.py","file_url":"https://github.com/sebschu/multilingual-transformations/blob/HEAD/models/metrics.py","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"66552297bbb6cd38","mcp_get_code":{"code_sha256":"66552297bbb6cd38"}},{"arxiv_id":"2109.10686","paper":"/paper/scale-efficiently-insights-from-pre-training","title":"Scale Efficiently: Insights from Pre-training and Fine-tuning Transformers","date":"2021-09-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"aaronmueller/emergent-syntax","path":"models/metrics.py","file_url":"https://github.com/aaronmueller/emergent-syntax/blob/HEAD/models/metrics.py","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"66552297bbb6cd38","mcp_get_code":{"code_sha256":"66552297bbb6cd38"}},{"arxiv_id":"1707.03904","paper":"/paper/quasar-datasets-for-question-answering-by","title":"Quasar: Datasets for Question Answering by Search and Reading","date":"2017-07-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"bdhingra/quasar","path":"metric.py","file_url":"https://github.com/bdhingra/quasar/blob/HEAD/metric.py","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"BSD-2-Clause","inline_ok":true,"code_sha256_prefix":"8c2b5b40e8ea29f0","mcp_get_code":{"code_sha256":"8c2b5b40e8ea29f0"}},{"arxiv_id":"2023.findings-emnlp.221","paper":null,"title":"arXiv:2023.findings-emnlp.221","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"RshNk73/PistaQ-SREQA","path":"PistaQ/triplet_extraction_NN_coref_rule_block.py","file_url":"https://github.com/RshNk73/PistaQ-SREQA/blob/HEAD/PistaQ/triplet_extraction_NN_coref_rule_block.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"fec01a8869724531","mcp_get_code":{"code_sha256":"fec01a8869724531"}}]}