{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/get-results","entry":"get_results","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":29,"n_papers_ran":10,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":29,"n_samples_ran":9,"n_samples_fingerprinted":3,"n_places":30,"n_places_pointer_only":9,"by_status":{"ran_honours":2,"ran_violates":0,"ran_draft_wrong":0,"ran_fixture":2,"ran":5,"unverified":20},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2606.07726","paper":"/paper/arxiv-2606-07726","title":"Cutting LLM Evaluation Costs with SySRs : A Bandit Algorithm that Provably Exploits Model Similarity","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"zifanlyu/llm-bandits-sysrs","path":"bai_algs.py","file_url":"https://github.com/zifanlyu/llm-bandits-sysrs/blob/HEAD/bai_algs.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"edc7bce93659e21c","mcp_get_code":{"code_sha256":"edc7bce93659e21c"}},{"arxiv_id":"2602.03164","paper":"/paper/arxiv-2602-03164","title":"MemCast: Memory-Driven Time Series Forecasting with Experience-Conditioned Reasoning","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"Xiaoyu-Tao/MemCast-TS","path":"main/EPF/EPF_main_few_shot_reasoning.py","file_url":"https://github.com/Xiaoyu-Tao/MemCast-TS/blob/HEAD/main/EPF/EPF_main_few_shot_reasoning.py","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"0df49cdedfd78b55","mcp_get_code":{"code_sha256":"0df49cdedfd78b55"}},{"arxiv_id":"2501.17151","paper":"/paper/scanning-trojaned-models-using-out-of","title":"Scanning Trojaned Models Using Out-of-Distribution Samples","date":"2025-01-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"rohban-lab/trodo","path":"src/evaluate.py","file_url":"https://github.com/rohban-lab/trodo/blob/HEAD/src/evaluate.py","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"6cf451e2ce941877","mcp_get_code":{"code_sha256":"6cf451e2ce941877"}},{"arxiv_id":"2411.05403","paper":"/paper/benchmarking-distributional-alignment-of","title":"Benchmarking Distributional Alignment of Large Language Models","date":"2024-11-08","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"nicolemeister/benchmarking-distributional-alignment","path":"utils.py","file_url":"https://github.com/nicolemeister/benchmarking-distributional-alignment/blob/HEAD/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"614775cfc34dfab7","mcp_get_code":{"code_sha256":"614775cfc34dfab7"}},{"arxiv_id":"2406.17385","paper":"/paper/native-design-bias-studying-the-impact-of","title":"Native Design Bias: Studying the Impact of English Nativeness on Language Model Performance","date":"2024-06-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"manon-reusens/native_en_bias","path":"get_final_results.py","file_url":"https://github.com/manon-reusens/native_en_bias/blob/HEAD/get_final_results.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"c37774c0609de83a","mcp_get_code":{"code_sha256":"c37774c0609de83a"}},{"arxiv_id":"2406.07971","paper":"/paper/it-takes-two-on-the-seamlessness-between","title":"It Takes Two: On the Seamlessness between Reward and Policy Model in RLHF","date":"2024-06-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"taiminglu/seamless","path":"code/retrival/adv/retrival.py","file_url":"https://github.com/taiminglu/seamless/blob/HEAD/code/retrival/adv/retrival.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"48fc27d8ef2d0595","mcp_get_code":{"code_sha256":"48fc27d8ef2d0595"}},{"arxiv_id":"2403.13870","paper":"/paper/exmap-leveraging-explainability-heatmaps-for","title":"ExMap: Leveraging Explainability Heatmaps for Unsupervised Group Robustness to Spurious Correlations","date":"2024-03-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"rwchakra/exmap","path":"src/utils/common_utils.py","file_url":"https://github.com/rwchakra/exmap/blob/HEAD/src/utils/common_utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"a36a0ef6f7969176","mcp_get_code":{"code_sha256":"a36a0ef6f7969176"}},{"arxiv_id":"2401.12973","paper":"/paper/in-context-language-learning-arhitectures-and","title":"In-Context Language Learning: Architectures and Algorithms","date":"2024-01-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"berlino/seq_icl","path":"samestateprobe.py","file_url":"https://github.com/berlino/seq_icl/blob/HEAD/samestateprobe.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e4d688cc5cf62263","mcp_get_code":{"code_sha256":"e4d688cc5cf62263"}},{"arxiv_id":"2401.08550","paper":"/paper/expanding-hardware-efficiently-manipulable","title":"Expanding Hardware-Efficiently Manipulable Hilbert Space via Hamiltonian Embedding","date":null,"month_inferred_from_arxiv_id":"2024-01","title_source":"archive","repo":"jiaqileng/hamiltonian-embedding","path":"src/experiments/ionq_circuit_utils.py","file_url":"https://github.com/jiaqileng/hamiltonian-embedding/blob/HEAD/src/experiments/ionq_circuit_utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"840218e9d4dcf4ac","mcp_get_code":{"code_sha256":"840218e9d4dcf4ac"}},{"arxiv_id":"2311.07633","paper":"/paper/rethinking-and-benchmarking-predict-then","title":"Benchmarking PtO and PnO Methods in the Predictive Combinatorial Optimization Regime","date":"2023-11-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Thinklab-SJTU/PredictiveCO-Benchmark","path":"rethink_exp/collect_results.py","file_url":"https://github.com/Thinklab-SJTU/PredictiveCO-Benchmark/blob/HEAD/rethink_exp/collect_results.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"411c0a5ba07cc36a","mcp_get_code":{"code_sha256":"411c0a5ba07cc36a"}},{"arxiv_id":"2309.08825","paper":"/paper/distributionally-robust-post-hoc-classifiers","title":"Distributionally Robust Post-hoc Classifiers under Prior Shifts","date":"2023-09-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"weijiaheng/Drops","path":"utils.py","file_url":"https://github.com/weijiaheng/Drops/blob/HEAD/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"58b9298579ed4235","mcp_get_code":{"code_sha256":"58b9298579ed4235"}},{"arxiv_id":"2309.01265","paper":"/paper/soar-scene-debiasing-open-set-action-1","title":"SOAR: Scene-debiasing Open-set Action Recognition","date":"2023-09-03","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"yhZhai/SOAR","path":"experiments/analyze_recon_uncertainty.py","file_url":"https://github.com/yhZhai/SOAR/blob/HEAD/experiments/analyze_recon_uncertainty.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"8dd75826e2ff1516","mcp_get_code":{"code_sha256":"8dd75826e2ff1516"}},{"arxiv_id":"2307.02245","paper":"/paper/set-learning-for-accurate-and-calibrated","title":"Set Learning for Accurate and Calibrated Models","date":"2023-07-05","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"lukasmut/oko","path":"visualize_results.py","file_url":"https://github.com/lukasmut/oko/blob/HEAD/visualize_results.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"63f61a0824532ff2","mcp_get_code":{"code_sha256":"63f61a0824532ff2"}},{"arxiv_id":"2305.06648","paper":null,"title":"arXiv:2305.06648","date":null,"month_inferred_from_arxiv_id":"2023-05","title_source":null,"repo":"PierreMarion23/generalization-ode-resnets","path":"training.py","file_url":"https://github.com/PierreMarion23/generalization-ode-resnets/blob/HEAD/training.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"bd88f63e3839d840","mcp_get_code":{"code_sha256":"bd88f63e3839d840"}},{"arxiv_id":"2303.12528","paper":"/paper/mega-multilingual-evaluation-of-generative-ai","title":"MEGA: Multilingual Evaluation of Generative AI","date":"2023-03-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"microsoft/Multilingual-Evaluation-of-Generative-AI-MEGA","path":"mega/compile_results.py","file_url":"https://github.com/microsoft/Multilingual-Evaluation-of-Generative-AI-MEGA/blob/HEAD/mega/compile_results.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"49be2dc7b5784968","mcp_get_code":{"code_sha256":"49be2dc7b5784968"}},{"arxiv_id":"2303.12528","paper":"/paper/mega-multilingual-evaluation-of-generative-ai","title":"MEGA: Multilingual Evaluation of Generative AI","date":"2023-03-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"microsoft/Multilingual-Evaluation-of-Generative-AI-MEGA","path":"mega/compile_tag_results.py","file_url":"https://github.com/microsoft/Multilingual-Evaluation-of-Generative-AI-MEGA/blob/HEAD/mega/compile_tag_results.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"30ae16d06e2fb775","mcp_get_code":{"code_sha256":"30ae16d06e2fb775"}},{"arxiv_id":"2210.08139","paper":"/paper/partial-identification-of-treatment-effects","title":"Partial Identification of Treatment Effects with Implicit Generative Models","date":"2022-10-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"rgklab/partial_identification","path":"experiments/utils.py","file_url":"https://github.com/rgklab/partial_identification/blob/HEAD/experiments/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"fc71397370d55084","mcp_get_code":{"code_sha256":"fc71397370d55084"}},{"arxiv_id":"2206.06929","paper":"/paper/scaling-resnets-in-the-large-depth-regime","title":"Scaling ResNets in the Large-depth Regime","date":"2022-06-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"pierremarion23/scaling-resnets","path":"training.py","file_url":"https://github.com/pierremarion23/scaling-resnets/blob/HEAD/training.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"d7d3afe051d48031","mcp_get_code":{"code_sha256":"d7d3afe051d48031"}},{"arxiv_id":"2206.04968","paper":"/paper/what-information-is-necessary-and-sufficient","title":"What Information is Necessary and Sufficient to Predict Materials Properties using Machine Learning?","date":null,"month_inferred_from_arxiv_id":"2022-06","title_source":"archive","repo":"pv-lab/compstruct","path":"main_crabnet.py","file_url":"https://github.com/pv-lab/compstruct/blob/HEAD/main_crabnet.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"d135d244ca6ac10f","mcp_get_code":{"code_sha256":"d135d244ca6ac10f"}},{"arxiv_id":"2204.02937","paper":"/paper/last-layer-re-training-is-sufficient-for","title":"Last Layer Re-Training is Sufficient for Robustness to Spurious Correlations","date":"2022-04-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"PolinaKirichenko/deep_feature_reweighting","path":"utils.py","file_url":"https://github.com/PolinaKirichenko/deep_feature_reweighting/blob/HEAD/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"code_sha256_prefix":"58b9298579ed4235","mcp_get_code":{"code_sha256":"58b9298579ed4235"}},{"arxiv_id":"2203.08242","paper":"/paper/data-contamination-from-memorization-to-1","title":"Data Contamination: From Memorization to Exploitation","date":"2022-03-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"schwartz-lab-nlp/data_contamination","path":"finetune/results_parser.py","file_url":"https://github.com/schwartz-lab-nlp/data_contamination/blob/HEAD/finetune/results_parser.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"4ab877fc46921000","mcp_get_code":{"code_sha256":"4ab877fc46921000"}},{"arxiv_id":"2201.08698","paper":"/paper/natural-attack-for-pre-trained-models-of-code","title":"Natural Attack for Pre-trained Models of Code","date":null,"month_inferred_from_arxiv_id":"2022-01","title_source":"archive","repo":"soarsmu/attack-pretrain-models-of-code","path":"GraphCodeBERT/Authorship-Attribution/code/attack.py","file_url":"https://github.com/soarsmu/attack-pretrain-models-of-code/blob/HEAD/GraphCodeBERT/Authorship-Attribution/code/attack.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"f8e1aa1adadf89a0","mcp_get_code":{"code_sha256":"f8e1aa1adadf89a0"}},{"arxiv_id":"2112.08274","paper":"/paper/putting-people-in-their-place-monocular","title":"Putting People in their Place: Monocular Regression of 3D People in Depth","date":"2021-12-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Arthur151/Relative_Human","path":"RH_evaluation/evaluation.py","file_url":"https://github.com/Arthur151/Relative_Human/blob/HEAD/RH_evaluation/evaluation.py","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"bad47202c970ebbf","mcp_get_code":{"code_sha256":"bad47202c970ebbf"}},{"arxiv_id":"2110.03735","paper":"/paper/adversarial-unlearning-of-backdoors-via","title":"Adversarial Unlearning of Backdoors via Implicit Hypergradient","date":"2021-10-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"YiZeng623/I-BAU","path":"defense.py","file_url":"https://github.com/YiZeng623/I-BAU/blob/HEAD/defense.py","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"a9db05478110e6da","mcp_get_code":{"code_sha256":"a9db05478110e6da"}},{"arxiv_id":"2106.01216","paper":"/paper/evidential-turing-processes","title":"Evidential Turing Processes","date":"2021-06-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ituvisionlab/EvidentialCalibration","path":"utils.py","file_url":"https://github.com/ituvisionlab/EvidentialCalibration/blob/HEAD/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"5f5462e36c39a5aa","mcp_get_code":{"code_sha256":"5f5462e36c39a5aa"}},{"arxiv_id":"1911.11834","paper":"/paper/towards-fairness-in-visual-recognition","title":"Towards Fairness in Visual Recognition: Effective Strategies for Bias Mitigation","date":"2019-11-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"junjie1003/DL-Fairness-Study","path":"helper.py","file_url":"https://github.com/junjie1003/DL-Fairness-Study/blob/HEAD/helper.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"cc8d2d0c85a28685","mcp_get_code":{"code_sha256":"cc8d2d0c85a28685"}},{"arxiv_id":"1909.05357","paper":"/paper/out-of-domain-detection-for-low-resource-text","title":"Out-of-Domain Detection for Low-Resource Text Classification Tasks","date":"2019-08-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"SLAD-ml/few-shot-ood","path":"src/utils.py","file_url":"https://github.com/SLAD-ml/few-shot-ood/blob/HEAD/src/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"d3ebec56ea182bca","mcp_get_code":{"code_sha256":"d3ebec56ea182bca"}},{"arxiv_id":"1907.07484","paper":"/paper/benchmarking-robustness-in-object-detection","title":"Benchmarking Robustness in Object Detection: Autonomous Driving when Winter is Coming","date":"2019-07-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"bethgelab/robust-detection-benchmark","path":"data-analysis/robustness_eval.py","file_url":"https://github.com/bethgelab/robust-detection-benchmark/blob/HEAD/data-analysis/robustness_eval.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"736f6acb4c7f3097","mcp_get_code":{"code_sha256":"736f6acb4c7f3097"}},{"arxiv_id":"2024.findings-emnlp.253","paper":null,"title":"arXiv:2024.findings-emnlp.253","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"WilliamZR/ProTrix","path":"src/plan_then_reason.py","file_url":"https://github.com/WilliamZR/ProTrix/blob/HEAD/src/plan_then_reason.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"2d77048aca3e7684","mcp_get_code":{"code_sha256":"2d77048aca3e7684"}},{"arxiv_id":"2023.acl-long.452","paper":null,"title":"arXiv:2023.acl-long.452","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"terarachang/DataICL","path":"report_test_datamodels.py","file_url":"https://github.com/terarachang/DataICL/blob/HEAD/report_test_datamodels.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"451919270e7bef3f","mcp_get_code":{"code_sha256":"451919270e7bef3f"}}]}