{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/get-eval-loader","entry":"get_eval_loader","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":8,"n_papers_ran":2,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":7,"n_samples_ran":1,"n_samples_fingerprinted":0,"n_places":8,"n_places_pointer_only":0,"by_status":{"ran_honours":0,"ran_violates":0,"ran_draft_wrong":1,"ran_fixture":0,"ran":0,"unverified":6},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2303.14969","paper":"/paper/universal-few-shot-learning-of-dense","title":"Universal Few-shot Learning of Dense Prediction Tasks with Visual Token Matching","date":"2023-03-27","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"gitgyun/visual_token_matching","path":"dataset/dataloader_factory.py","file_url":"https://github.com/gitgyun/visual_token_matching/blob/HEAD/dataset/dataloader_factory.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"7ca5be9c02fa2823","mcp_get_code":{"code_sha256":"7ca5be9c02fa2823"}},{"arxiv_id":"2207.00050","paper":"/paper/semantic-image-synthesis-via-diffusion-models","title":"Semantic Image Synthesis via Diffusion Models","date":"2022-06-30","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"marinadominguez/diffusion-for-us-images","path":"semantic_diffusion_model/evaluations/default_dataset.py","file_url":"https://github.com/marinadominguez/diffusion-for-us-images/blob/HEAD/semantic_diffusion_model/evaluations/default_dataset.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"3ca9f515edac9150","mcp_get_code":{"code_sha256":"3ca9f515edac9150"}},{"arxiv_id":"2205.06262","paper":"/paper/feta-a-benchmark-for-few-sample-task-transfer","title":"FETA: A Benchmark for Few-Sample Task Transfer in Open-Domain Dialogue","date":"2022-05-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"alon-albalak/tlidb","path":"tlidb/TLiDB/data_loaders/data_loaders.py","file_url":"https://github.com/alon-albalak/tlidb/blob/HEAD/tlidb/TLiDB/data_loaders/data_loaders.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"1d86103d939c1b2c","mcp_get_code":{"code_sha256":"1d86103d939c1b2c"}},{"arxiv_id":"2204.03230","paper":"/paper/what-you-see-is-what-you-get-distributional","title":"What You See is What You Get: Principled Deep Learning via Distributional Generalization","date":"2022-04-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"yangarbiter/dp-dg","path":"wilds/common/data_loaders.py","file_url":"https://github.com/yangarbiter/dp-dg/blob/HEAD/wilds/common/data_loaders.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e32762f7bf5137a0","mcp_get_code":{"code_sha256":"e32762f7bf5137a0"}},{"arxiv_id":"2111.13733","paper":"/paper/failure-modes-of-domain-generalization","title":"Failure Modes of Domain Generalization Algorithms","date":"2021-11-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"tigrangalstyan/wilds","path":"wilds/common/data_loaders.py","file_url":"https://github.com/tigrangalstyan/wilds/blob/HEAD/wilds/common/data_loaders.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"55259103ce58bcac","mcp_get_code":{"code_sha256":"55259103ce58bcac"}},{"arxiv_id":"2111.04724","paper":"/paper/sustainbench-benchmarks-for-monitoring-the","title":"SustainBench: Benchmarks for Monitoring the Sustainable Development Goals with Machine Learning","date":"2021-11-08","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"sustainlab-group/sustainbench","path":"sustainbench/common/data_loaders.py","file_url":"https://github.com/sustainlab-group/sustainbench/blob/HEAD/sustainbench/common/data_loaders.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"7cb02ffd99728643","mcp_get_code":{"code_sha256":"7cb02ffd99728643"}},{"arxiv_id":"2012.07421","paper":"/paper/wilds-a-benchmark-of-in-the-wild-distribution","title":"WILDS: A Benchmark of in-the-Wild Distribution Shifts","date":"2020-12-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"p-lambda/wilds","path":"wilds/common/data_loaders.py","file_url":"https://github.com/p-lambda/wilds/blob/HEAD/wilds/common/data_loaders.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e32762f7bf5137a0","mcp_get_code":{"code_sha256":"e32762f7bf5137a0"}},{"arxiv_id":"Park_LANIT_Language-Driven_Image-to-Image_Translation_for_Unlabeled_Data_CVPR_2023_paper","paper":null,"title":"arXiv:Park_LANIT_Language-Driven_Image-to-Image_Translation_for_Unlabeled_Data_CVPR_2023_paper","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"KU-CVLAB/LANIT","path":"core/data_loader.py","file_url":"https://github.com/KU-CVLAB/LANIT/blob/HEAD/core/data_loader.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"0f014003d0b89418","mcp_get_code":{"code_sha256":"0f014003d0b89418"}}]}