{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/get-tasks","entry":"get_tasks","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":21,"n_papers_ran":14,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":10,"n_samples_ran":3,"n_samples_fingerprinted":0,"n_places":21,"n_places_pointer_only":4,"by_status":{"ran_honours":0,"ran_violates":0,"ran_draft_wrong":3,"ran_fixture":0,"ran":0,"unverified":7},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2502.20766","paper":"/paper/flexprefill-a-context-aware-sparse-attention","title":"FlexPrefill: A Context-Aware Sparse Attention Mechanism for Efficient Long-Sequence Inference","date":"2025-02-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"bytedance/FlexPrefill","path":"experiments/benchmark/run_ruler.py","file_url":"https://github.com/bytedance/FlexPrefill/blob/HEAD/experiments/benchmark/run_ruler.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"0ff1542f9a950e79","mcp_get_code":{"code_sha256":"0ff1542f9a950e79"}},{"arxiv_id":"2407.02883","paper":"/paper/coir-a-comprehensive-benchmark-for-code","title":"CoIR: A Comprehensive Benchmark for Code Information Retrieval Models","date":"2024-07-03","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"coir-team/coir","path":"coir/data_loader.py","file_url":"https://github.com/coir-team/coir/blob/HEAD/coir/data_loader.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"0adc655290cb9e75","mcp_get_code":{"code_sha256":"0adc655290cb9e75"}},{"arxiv_id":"2405.01927","paper":"/paper/slotgat-slot-based-message-passing-for","title":"SlotGAT: Slot-based Message Passing for Heterogeneous Graph Neural Network","date":"2024-05-03","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"scottjiao/slotgat_icml23","path":"NC/methods/SlotGAT/pipeline_utils.py","file_url":"https://github.com/scottjiao/slotgat_icml23/blob/HEAD/NC/methods/SlotGAT/pipeline_utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"cf2857ac006179c2","mcp_get_code":{"code_sha256":"cf2857ac006179c2"}},{"arxiv_id":"2310.19152","paper":"/paper/bert-lost-patience-won-t-be-robust-to-1","title":"BERT Lost Patience Won't Be Robust to Adversarial Slowdown","date":"2023-10-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ztcoalson/waffle","path":"download_glue_data.py","file_url":"https://github.com/ztcoalson/waffle/blob/HEAD/download_glue_data.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"3caf26b311c98658","mcp_get_code":{"code_sha256":"3caf26b311c98658"}},{"arxiv_id":"2308.10873","paper":"/paper/spikingbert-distilling-bert-to-train-spiking","title":"SpikingBERT: Distilling BERT to Train Spiking Language Models Using Implicit Differentiation","date":"2023-08-21","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"neurocomplab-psu/spikingbert","path":"download_glue_data.py","file_url":"https://github.com/neurocomplab-psu/spikingbert/blob/HEAD/download_glue_data.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"3caf26b311c98658","mcp_get_code":{"code_sha256":"3caf26b311c98658"}},{"arxiv_id":"2205.06573","paper":"/paper/knowledge-graph-question-answering-datasets","title":"Knowledge Graph Question Answering Datasets and Their Generalizability: Are They Enough for Future Research?","date":"2022-05-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"KGQA/KGQA-datasets-generalization","path":"preprocess.py","file_url":"https://github.com/KGQA/KGQA-datasets-generalization/blob/HEAD/preprocess.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"ea09facb8968367c","mcp_get_code":{"code_sha256":"ea09facb8968367c"}},{"arxiv_id":"2010.10418","paper":"/paper/conjnli-natural-language-inference-over","title":"ConjNLI: Natural Language Inference Over Conjunctive Sentences","date":"2020-10-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"swarnaHub/ConjNLI","path":"download_mnli.py","file_url":"https://github.com/swarnaHub/ConjNLI/blob/HEAD/download_mnli.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"18962b43885bc12f","mcp_get_code":{"code_sha256":"18962b43885bc12f"}},{"arxiv_id":"2009.13818","paper":"/paper/a-simple-but-tough-to-beat-data-augmentation","title":"A Simple but Tough-to-Beat Data Augmentation Approach for Natural Language Understanding and Generation","date":"2020-09-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"dinghanshen/Cutoff","path":"download_glue_data.py","file_url":"https://github.com/dinghanshen/Cutoff/blob/HEAD/download_glue_data.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"18962b43885bc12f","mcp_get_code":{"code_sha256":"18962b43885bc12f"}},{"arxiv_id":"2004.06660","paper":"/paper/weight-poisoning-attacks-on-pre-trained","title":"Weight Poisoning Attacks on Pre-trained Models","date":"2020-04-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"neulab/RIPPLe","path":"download_glue_data.py","file_url":"https://github.com/neulab/RIPPLe/blob/HEAD/download_glue_data.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"code_sha256_prefix":"d8d0c155d6eee8f3","mcp_get_code":{"code_sha256":"d8d0c155d6eee8f3"}},{"arxiv_id":"2003.14058","paper":"/paper/mtl-nas-task-agnostic-neural-architecture","title":"MTL-NAS: Task-Agnostic Neural Architecture Search towards General-Purpose Multi-Task Learning","date":"2020-03-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"bhpfelix/MTLNAS","path":"core/tasks.py","file_url":"https://github.com/bhpfelix/MTLNAS/blob/HEAD/core/tasks.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"23e12e7e82ed9859","mcp_get_code":{"code_sha256":"23e12e7e82ed9859"}},{"arxiv_id":"1910.01244","paper":"/paper/linking-artificial-and-human-neural","title":"Linking artificial and human neural representations of language","date":"2019-10-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"hans/nn-decoding","path":"bin/download_glue_data.py","file_url":"https://github.com/hans/nn-decoding/blob/HEAD/bin/download_glue_data.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"18962b43885bc12f","mcp_get_code":{"code_sha256":"18962b43885bc12f"}},{"arxiv_id":"1908.08962","paper":"/paper/well-read-students-learn-better-the-impact-of","title":"Well-Read Students Learn Better: On the Importance of Pre-training Compact Models","date":"2019-08-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"the-Quert/iNLPfun","path":"code/download_glue_data.py","file_url":"https://github.com/the-Quert/iNLPfun/blob/HEAD/code/download_glue_data.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c54b935b72798660","mcp_get_code":{"code_sha256":"c54b935b72798660"}},{"arxiv_id":"1907.11692","paper":"/paper/roberta-a-robustly-optimized-bert-pretraining","title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach","date":"2019-07-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"pisalore/roberta_results","path":"glue_tasks/download_glue_data.py","file_url":"https://github.com/pisalore/roberta_results/blob/HEAD/glue_tasks/download_glue_data.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"3caf26b311c98658","mcp_get_code":{"code_sha256":"3caf26b311c98658"}},{"arxiv_id":"1906.02762","paper":"/paper/understanding-and-improving-transformer-from","title":"Understanding and Improving Transformer From a Multi-Particle Dynamic System Point of View","date":"2019-06-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zhuohan123/macaron-net","path":"bert/macaron-scripts/glue/download_glue_data.py","file_url":"https://github.com/zhuohan123/macaron-net/blob/HEAD/bert/macaron-scripts/glue/download_glue_data.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"code_sha256_prefix":"18962b43885bc12f","mcp_get_code":{"code_sha256":"18962b43885bc12f"}},{"arxiv_id":"1810.04805","paper":"/paper/bert-pre-training-of-deep-bidirectional","title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","date":"2018-10-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"coco60/bert-test","path":"download_glue_data.py","file_url":"https://github.com/coco60/bert-test/blob/HEAD/download_glue_data.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"18962b43885bc12f","mcp_get_code":{"code_sha256":"18962b43885bc12f"}},{"arxiv_id":"1804.07461","paper":"/paper/glue-a-multi-task-benchmark-and-analysis","title":"GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding","date":"2018-04-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"nyu-mll/GLUE-baselines","path":"download_glue_data.py","file_url":"https://github.com/nyu-mll/GLUE-baselines/blob/HEAD/download_glue_data.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"3caf26b311c98658","mcp_get_code":{"code_sha256":"3caf26b311c98658"}},{"arxiv_id":"1711.05225","paper":"/paper/chexnet-radiologist-level-pneumonia-detection","title":"CheXNet: Radiologist-Level Pneumonia Detection on Chest X-Rays with Deep Learning","date":"2017-11-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"SenWu/emmental-tutorials","path":"glue/download_glue_data.py","file_url":"https://github.com/SenWu/emmental-tutorials/blob/HEAD/glue/download_glue_data.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"18962b43885bc12f","mcp_get_code":{"code_sha256":"18962b43885bc12f"}},{"arxiv_id":"1611.01989","paper":"/paper/deepcoder-learning-to-write-programs","title":"DeepCoder: Learning to Write Programs","date":"2016-11-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"lcary/io-generation","path":"iogen/iogen.py","file_url":"https://github.com/lcary/io-generation/blob/HEAD/iogen/iogen.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"9a6e7bf5b92722cf","mcp_get_code":{"code_sha256":"9a6e7bf5b92722cf"}},{"arxiv_id":"aaai_5722","paper":null,"title":"arXiv:aaai_5722","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"microsoft/Distilled-Sentence-Embedding","path":"download_glue_data.py","file_url":"https://github.com/microsoft/Distilled-Sentence-Embedding/blob/HEAD/download_glue_data.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"18962b43885bc12f","mcp_get_code":{"code_sha256":"18962b43885bc12f"}},{"arxiv_id":"2023.acl-long.840","paper":null,"title":"arXiv:2023.acl-long.840","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"thunlp/DecT","path":"datasets/download_glue_data.py","file_url":"https://github.com/thunlp/DecT/blob/HEAD/datasets/download_glue_data.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"3caf26b311c98658","mcp_get_code":{"code_sha256":"3caf26b311c98658"}},{"arxiv_id":"2021.emnlp-main.154","paper":null,"title":"arXiv:2021.emnlp-main.154","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"Hazelsuko07/TextHide","path":"download_glue_data.py","file_url":"https://github.com/Hazelsuko07/TextHide/blob/HEAD/download_glue_data.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"18962b43885bc12f","mcp_get_code":{"code_sha256":"18962b43885bc12f"}}]}