{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/task","entry":"Task","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":17,"n_papers_ran":9,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":17,"n_samples_ran":9,"n_samples_fingerprinted":0,"n_places":17,"n_places_pointer_only":7,"by_status":{"ran_honours":0,"ran_violates":0,"ran_draft_wrong":0,"ran_fixture":0,"ran":9,"unverified":8},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2607.25765","paper":"/paper/arxiv-2607-25765","title":"WorkSurface-Bench: Benchmarking Enterprise Agents on Multi-Surface Knowledge Routing","date":null,"month_inferred_from_arxiv_id":"2026-07","title_source":"syntology","repo":"haolpku/WorkSurface-Bench","path":"worksurface/derive_tasks.py","file_url":"https://github.com/haolpku/WorkSurface-Bench/blob/HEAD/worksurface/derive_tasks.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"50555611ef7241a5","mcp_get_code":{"code_sha256":"50555611ef7241a5"}},{"arxiv_id":"2607.13411","paper":"/paper/arxiv-2607-13411","title":"Evaluating Frontier AI Agents as Autonomous Clinical Security Auditors","date":null,"month_inferred_from_arxiv_id":"2026-07","title_source":"syntology","repo":"MichaelEnny/clinical-ai-security-eval","path":"task/clinical_ai_security_eval.py","file_url":"https://github.com/MichaelEnny/clinical-ai-security-eval/blob/HEAD/task/clinical_ai_security_eval.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"2e1dd7706773f279","mcp_get_code":{"code_sha256":"2e1dd7706773f279"}},{"arxiv_id":"2606.24428","paper":"/paper/arxiv-2606-24428","title":"Escaping the Self-Confirmation Trap: An Execute-Distill-Verify Paradigm for Agentic Experience Learning","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"shidingz/EDV","path":"src/edv/pipeline.py","file_url":"https://github.com/shidingz/EDV/blob/HEAD/src/edv/pipeline.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"7e6bf386dea53a1e","mcp_get_code":{"code_sha256":"7e6bf386dea53a1e"}},{"arxiv_id":"2603.24738","paper":"/paper/arxiv-2603-24738","title":"Decentralized Task Scheduling in Distributed Systems: A Deep Reinforcement Learning Approach","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"danielbenniah/marl-distributed-scheduling","path":"marl_scheduler.py","file_url":"https://github.com/danielbenniah/marl-distributed-scheduling/blob/HEAD/marl_scheduler.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"b22560173dde05bc","mcp_get_code":{"code_sha256":"b22560173dde05bc"}},{"arxiv_id":"2407.19346","paper":"/paper/polynomial-regression-as-a-task-for","title":"Polynomial Regression as a Task for Understanding In-context Learning Through Finetuning and Alignment","date":"2024-07-27","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"dtsip/in-context-learning","path":"src/tasks.py","file_url":"https://github.com/dtsip/in-context-learning/blob/HEAD/src/tasks.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"3e016983bf585f29","mcp_get_code":{"code_sha256":"3e016983bf585f29"}},{"arxiv_id":"2402.04858","paper":"/paper/codeit-self-improving-language-models-with","title":"CodeIt: Self-Improving Language Models with Prioritized Hindsight Replay","date":"2024-02-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Qualcomm-AI-research/codeit","path":"codeit/augment/genetic.py","file_url":"https://github.com/Qualcomm-AI-research/codeit/blob/HEAD/codeit/augment/genetic.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause-Clear","inline_ok":false,"code_sha256_prefix":"3215558dec21b76e","mcp_get_code":{"code_sha256":"3215558dec21b76e"}},{"arxiv_id":"2310.15164","paper":"/paper/linc-a-neurosymbolic-approach-for-logical","title":"LINC: A Neurosymbolic Approach for Logical Reasoning by Combining Language Models with First-Order Logic Provers","date":"2023-10-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"benlipkin/linc","path":"eval/tasks/folio.py","file_url":"https://github.com/benlipkin/linc/blob/HEAD/eval/tasks/folio.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"ed34cd2d0c6826c0","mcp_get_code":{"code_sha256":"ed34cd2d0c6826c0"}},{"arxiv_id":"2310.04870","paper":"/paper/lemur-integrating-large-language-models-in","title":"Lemur: Integrating Large Language Models in Automated Program Verification","date":"2023-10-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"wu-haoze/lemur-program-verification","path":"lemur/src/verifier.py","file_url":"https://github.com/wu-haoze/lemur-program-verification/blob/HEAD/lemur/src/verifier.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"45188c4494249615","mcp_get_code":{"code_sha256":"45188c4494249615"}},{"arxiv_id":"2212.13738","paper":"/paper/tempclr-temporal-alignment-representation","title":"TempCLR: Temporal Alignment Representation with Contrastive Learning","date":"2022-12-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"yyuncong/tempclr","path":"tempclr/tasks/alignlocaltask.py","file_url":"https://github.com/yyuncong/tempclr/blob/HEAD/tempclr/tasks/alignlocaltask.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"8e1461d6cc228441","mcp_get_code":{"code_sha256":"8e1461d6cc228441"}},{"arxiv_id":"2210.03629","paper":"/paper/react-synergizing-reasoning-and-acting-in","title":"ReAct: Synergizing Reasoning and Acting in Language Models","date":"2022-10-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"sierra-research/tau-bench","path":"tau_bench/agents/chat_react_agent.py","file_url":"https://github.com/sierra-research/tau-bench/blob/HEAD/tau_bench/agents/chat_react_agent.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"c6504ae4ebf5bcab","mcp_get_code":{"code_sha256":"c6504ae4ebf5bcab"}},{"arxiv_id":"2204.07705","paper":"/paper/benchmarking-generalization-via-in-context","title":"Super-NaturalInstructions: Generalization via Declarative Instructions on 1600+ NLP Tasks","date":"2022-04-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zhaochenyang20/Prompt2Model-Self-Guide","path":"self-guide/utils/tasks.py","file_url":"https://github.com/zhaochenyang20/Prompt2Model-Self-Guide/blob/HEAD/self-guide/utils/tasks.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"588da051624b9d01","mcp_get_code":{"code_sha256":"588da051624b9d01"}},{"arxiv_id":"2203.13602","paper":"/paper/zs4ie-a-toolkit-for-zero-shot-information","title":"ZS4IE: A toolkit for Zero-Shot Information Extraction with simple Verbalizations","date":"2022-03-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"osainz59/Ask2Transformers","path":"a2t/tasks/base.py","file_url":"https://github.com/osainz59/Ask2Transformers/blob/HEAD/a2t/tasks/base.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c9033c25890f4270","mcp_get_code":{"code_sha256":"c9033c25890f4270"}},{"arxiv_id":"2102.03034","paper":"/paper/hyperparameter-optimization-is-deceiving-us","title":"Hyperparameter Optimization Is Deceiving Us, and How to Stop It","date":"2021-02-05","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"pasta41/deception","path":"src/utils/utils.py","file_url":"https://github.com/pasta41/deception/blob/HEAD/src/utils/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"1203fb727028792b","mcp_get_code":{"code_sha256":"1203fb727028792b"}},{"arxiv_id":"2003.02249","paper":"/paper/jiant-a-software-toolkit-for-research-on","title":"jiant: A Software Toolkit for Research on General-Purpose Text Understanding Models","date":"2020-03-04","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"nyu-mll/jiant","path":"jiant/proj/main/modeling/primary.py","file_url":"https://github.com/nyu-mll/jiant/blob/HEAD/jiant/proj/main/modeling/primary.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"3cc8da16c18465bd","mcp_get_code":{"code_sha256":"3cc8da16c18465bd"}},{"arxiv_id":"1909.04630","paper":"/paper/meta-learning-with-implicit-gradients","title":"Meta-Learning with Implicit Gradients","date":"2019-09-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"spiglerg/pyMeta","path":"pyMeta/metalearners/implicit_maml.py","file_url":"https://github.com/spiglerg/pyMeta/blob/HEAD/pyMeta/metalearners/implicit_maml.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"ca574f805c91d123","mcp_get_code":{"code_sha256":"ca574f805c91d123"}},{"arxiv_id":"1810.04805","paper":"/paper/bert-pre-training-of-deep-bidirectional","title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","date":"2018-10-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ABaldrati/MT-BERT","path":"model.py","file_url":"https://github.com/ABaldrati/MT-BERT/blob/HEAD/model.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"539b7f6e38fc9786","mcp_get_code":{"code_sha256":"539b7f6e38fc9786"}},{"arxiv_id":"1703.03400","paper":"/paper/model-agnostic-meta-learning-for-fast","title":"Model-Agnostic Meta-Learning for Fast Adaptation of Deep Networks","date":"2017-03-09","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"MaximeVandegar/Papers-in-100-Lines-of-Code","path":"Model_Agnostic_Meta_Learning_for_Fast_Adaptation_of_Deep_Networks/maml.py","file_url":"https://github.com/MaximeVandegar/Papers-in-100-Lines-of-Code/blob/HEAD/Model_Agnostic_Meta_Learning_for_Fast_Adaptation_of_Deep_Networks/maml.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"ec0fb012c0b639eb","mcp_get_code":{"code_sha256":"ec0fb012c0b639eb"}}]}