{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/load-predictions","entry":"load_predictions","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":22,"n_papers_ran":11,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":18,"n_samples_ran":8,"n_samples_fingerprinted":0,"n_places":22,"n_places_pointer_only":8,"by_status":{"ran_honours":0,"ran_violates":0,"ran_draft_wrong":6,"ran_fixture":0,"ran":2,"unverified":10},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2608.13711","paper":"/paper/arxiv-2608-13711","title":"TRUE-Colon: Exposing a Consistent Transfer Asymmetry in Real-Time Polyp Detection","date":null,"month_inferred_from_arxiv_id":"2026-08","title_source":"syntology","repo":"sdoerrich97/true-colon","path":"true_colon/_ingest.py","file_url":"https://github.com/sdoerrich97/true-colon/blob/HEAD/true_colon/_ingest.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"bba14b509483ca23","mcp_get_code":{"code_sha256":"bba14b509483ca23"}},{"arxiv_id":"2606.26979","paper":"/paper/arxiv-2606-26979","title":"How Much Static Structure Do Code Agents Need? A Study of Deterministic Anchoring","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"mathieu0905/Code-Anchor","path":"evaluation/eval_localization_codex.py","file_url":"https://github.com/mathieu0905/Code-Anchor/blob/HEAD/evaluation/eval_localization_codex.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"31ac00a4e3cb419d","mcp_get_code":{"code_sha256":"31ac00a4e3cb419d"}},{"arxiv_id":"2604.23099","paper":"/paper/arxiv-2604-23099","title":"ProEval: Proactive Failure Discovery and Efficient Performance Estimation for Generative AI Evaluation","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"google-deepmind/proeval","path":"proeval/sampler/bq.py","file_url":"https://github.com/google-deepmind/proeval/blob/HEAD/proeval/sampler/bq.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c1c4afe2594fdfe1","mcp_get_code":{"code_sha256":"c1c4afe2594fdfe1"}},{"arxiv_id":"2603.00634","paper":"/paper/arxiv-2603-00634","title":"BLUFF: Benchmarking the Detection of False and Synthetic Content across 58 Low-Resource Languages","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"jsl5710/BLUFF","path":"src/evaluation/decoder/evaluate.py","file_url":"https://github.com/jsl5710/BLUFF/blob/HEAD/src/evaluation/decoder/evaluate.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"cb5244a9e27b7e1f","mcp_get_code":{"code_sha256":"cb5244a9e27b7e1f"}},{"arxiv_id":"2601.14046","paper":"/paper/arxiv-2601-14046","title":"PRiSM: Benchmarking Phone Realization in Speech Models","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"changelinglab/prism","path":"src/metrics/zeroshot_eval.py","file_url":"https://github.com/changelinglab/prism/blob/HEAD/src/metrics/zeroshot_eval.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e7cd548d1f3164bd","mcp_get_code":{"code_sha256":"e7cd548d1f3164bd"}},{"arxiv_id":"2601.11443","paper":"/paper/arxiv-2601-11443","title":"Predict the Retrieval! Test time adaptation for Retrieval Augmented Generation","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"sunxin000/TTARAG","path":"parallel_evaluation.py","file_url":"https://github.com/sunxin000/TTARAG/blob/HEAD/parallel_evaluation.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f2e9cf22b372413c","mcp_get_code":{"code_sha256":"f2e9cf22b372413c"}},{"arxiv_id":"2601.00223","paper":"/paper/arxiv-2601-00223","title":"JP-TL-Bench: Anchored Pairwise LLM Evaluation for Bidirectional Japanese-English Translation","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"lhl/liquid-ai-hackathon-tokyo","path":"eval/sample_predictions.py","file_url":"https://github.com/lhl/liquid-ai-hackathon-tokyo/blob/HEAD/eval/sample_predictions.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"efcb9dd76b5e4df3","mcp_get_code":{"code_sha256":"efcb9dd76b5e4df3"}},{"arxiv_id":"2511.17078","paper":"/paper/arxiv-2511-17078","title":"Hash Collisions in Molecular Fingerprints: Effects on Property Prediction and Bayesian Optimization","date":null,"month_inferred_from_arxiv_id":"2025-11","title_source":"syntology","repo":"wvirany/molcollisions","path":"experiments/analyze_regression_results.py","file_url":"https://github.com/wvirany/molcollisions/blob/HEAD/experiments/analyze_regression_results.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"80cb06e538b47619","mcp_get_code":{"code_sha256":"80cb06e538b47619"}},{"arxiv_id":"2505.19028","paper":"/paper/infochartqa-a-benchmark-for-multimodal","title":"InfoChartQA: A Benchmark for Multimodal Question Answering on Infographic Charts","date":"2025-05-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"orionbench/orionbench","path":"grounded_CoT/evaluate_predictions.py","file_url":"https://github.com/orionbench/orionbench/blob/HEAD/grounded_CoT/evaluate_predictions.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"36667e7b107acaec","mcp_get_code":{"code_sha256":"36667e7b107acaec"}},{"arxiv_id":"2505.18668","paper":"/paper/chartgalaxy-a-dataset-for-infographic-chart","title":"ChartGalaxy: A Dataset for Infographic Chart Understanding and Generation","date":"2025-05-24","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"chartgalaxy/chartgalaxy","path":"code_understanding/scoring.py","file_url":"https://github.com/chartgalaxy/chartgalaxy/blob/HEAD/code_understanding/scoring.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"36667e7b107acaec","mcp_get_code":{"code_sha256":"36667e7b107acaec"}},{"arxiv_id":"2505.14305","paper":"/paper/jolt-sql-joint-loss-tuning-of-text-to-sql","title":"JOLT-SQL: Joint Loss Tuning of Text-to-SQL with Confusion-aware Noisy Schema Sampling","date":"2025-05-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Songjw133/JOLT-SQL","path":"test_suite_sql_eval/evaluate_classical.py","file_url":"https://github.com/Songjw133/JOLT-SQL/blob/HEAD/test_suite_sql_eval/evaluate_classical.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c64a2627f38e10a8","mcp_get_code":{"code_sha256":"c64a2627f38e10a8"}},{"arxiv_id":"2504.17454","paper":"/paper/adaptive-orchestration-of-modular-generative","title":"Adaptive Orchestration of Modular Generative Information Access Systems","date":"2025-04-24","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"informagi/AQA","path":"Adaptive-RAG/AQA_final_eval.py","file_url":"https://github.com/informagi/AQA/blob/HEAD/Adaptive-RAG/AQA_final_eval.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"fd0f6a22513811ab","mcp_get_code":{"code_sha256":"fd0f6a22513811ab"}},{"arxiv_id":"2504.05506","paper":"/paper/chartqapro-a-more-diverse-and-challenging","title":"ChartQAPro: A More Diverse and Challenging Benchmark for Chart Question Answering","date":"2025-04-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"vis-nlp/chartqapro","path":"evaluate_predictions.py","file_url":"https://github.com/vis-nlp/chartqapro/blob/HEAD/evaluate_predictions.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"36667e7b107acaec","mcp_get_code":{"code_sha256":"36667e7b107acaec"}},{"arxiv_id":"2410.14057","paper":"/paper/towards-cross-cultural-machine-translation","title":"Towards Cross-Cultural Machine Translation with Retrieval-Augmented Generation from Multilingual Knowledge Graphs","date":"2024-10-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"apple/ml-kg-mt","path":"src/evaluation/evaluate.py","file_url":"https://github.com/apple/ml-kg-mt/blob/HEAD/src/evaluation/evaluate.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"aaca1dfcefe36000","mcp_get_code":{"code_sha256":"aaca1dfcefe36000"}},{"arxiv_id":"2409.02813","paper":"/paper/mmmu-pro-a-more-robust-multi-discipline","title":"MMMU-Pro: A More Robust Multi-discipline Multimodal Understanding Benchmark","date":"2024-09-04","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"opendatalab/pm4bench","path":"src/pm4bench/io.py","file_url":"https://github.com/opendatalab/pm4bench/blob/HEAD/src/pm4bench/io.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e6b5d81f514b3ebb","mcp_get_code":{"code_sha256":"e6b5d81f514b3ebb"}},{"arxiv_id":"2401.13512","paper":"/paper/can-gpt-3-5-generate-and-code-discharge","title":"Can GPT-3.5 Generate and Code Discharge Summaries?","date":"2024-01-24","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"edinburghclinicalnlp/chatgpt_icd_coding","path":"scripts/evaluate.py","file_url":"https://github.com/edinburghclinicalnlp/chatgpt_icd_coding/blob/HEAD/scripts/evaluate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"aae667a9e9bc2760","mcp_get_code":{"code_sha256":"aae667a9e9bc2760"}},{"arxiv_id":"2211.16799","paper":"/paper/nope-sac-neural-one-plane-ransac-for-sparse","title":"NOPE-SAC: Neural One-Plane RANSAC for Sparse-View Planar 3D Reconstruction","date":"2022-11-30","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"icetttb/nopesac","path":"vis_NopeSAC.py","file_url":"https://github.com/icetttb/nopesac/blob/HEAD/vis_NopeSAC.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"53d025ffef0faf78","mcp_get_code":{"code_sha256":"53d025ffef0faf78"}},{"arxiv_id":"2209.02071","paper":"/paper/concrete-improving-cross-lingual-fact","title":"CONCRETE: Improving Cross-lingual Fact-checking with Cross-lingual Retrieval","date":"2022-09-05","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"khuangaf/concrete","path":"src/xfact_eval.py","file_url":"https://github.com/khuangaf/concrete/blob/HEAD/src/xfact_eval.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"9f7343aa8cadeb59","mcp_get_code":{"code_sha256":"9f7343aa8cadeb59"}},{"arxiv_id":"2106.09248","paper":"/paper/x-fact-a-new-benchmark-dataset-for","title":"X-FACT: A New Benchmark Dataset for Multilingual Fact Checking","date":"2021-06-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"utahnlp/x-fact","path":"transformers/calculate_fscore.py","file_url":"https://github.com/utahnlp/x-fact/blob/HEAD/transformers/calculate_fscore.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"9f7343aa8cadeb59","mcp_get_code":{"code_sha256":"9f7343aa8cadeb59"}},{"arxiv_id":"2010.02840","paper":"/paper/semantic-evaluation-for-text-to-sql-with","title":"Semantic Evaluation for Text-to-SQL with Distilled Test Suites","date":"2020-10-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"taoyds/test-suite-sql-eval","path":"evaluate_classical.py","file_url":"https://github.com/taoyds/test-suite-sql-eval/blob/HEAD/evaluate_classical.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c64a2627f38e10a8","mcp_get_code":{"code_sha256":"c64a2627f38e10a8"}},{"arxiv_id":"1909.09436","paper":"/paper/codesearchnet-challenge-evaluating-the-state","title":"CodeSearchNet Challenge: Evaluating the State of Semantic Code Search","date":"2019-09-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"github/CodeSearchNet","path":"src/relevanceeval.py","file_url":"https://github.com/github/CodeSearchNet/blob/HEAD/src/relevanceeval.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"faa62c1fbe7df43e","mcp_get_code":{"code_sha256":"faa62c1fbe7df43e"}},{"arxiv_id":"1807.03748","paper":"/paper/representation-learning-with-contrastive","title":"Representation Learning with Contrastive Predictive Coding","date":"2018-07-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ex4sperans/freesound-classification","path":"linear_blend.py","file_url":"https://github.com/ex4sperans/freesound-classification/blob/HEAD/linear_blend.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"69d65c023623a577","mcp_get_code":{"code_sha256":"69d65c023623a577"}}]}