{"url":"/task/code-completion","name":"Code Completion","slug":"code-completion","description_markdown":null,"categories":[{"name":"Computer Code","url":"/area/computer-code"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":212,"papers_with_code":108,"benchmarks":6,"benchmark_tables_in_archive":6,"benchmark_tables_shown":6,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":12,"subtasks":1,"parent_tasks":0},"benchmarks":[{"leaderboard":"/sota/code-completion-on-safim","slug":"code-completion-on-safim","dataset":"SAFIM","dataset_url":"/dataset/safim","rows_in_archive":15,"metrics":["Average","Algorithmic","Control","API"],"first_row_in_archive_order":{"model":"deepseek-coder-33b-base","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","paper_url":"/paper/evaluation-of-llms-on-syntax-aware-code-fill","paper_date":"2024-03-07","arxiv_id":"2403.04814","code_links":[{"title":"gonglinyuan/safim","url":"https://github.com/gonglinyuan/safim"}],"syntology":{"n":2,"n_ran":2,"n_unverified":0,"n_pointer_only":0}}},{"leaderboard":"/sota/code-completion-on-codexglue-github-java","slug":"code-completion-on-codexglue-github-java","dataset":"CodeXGLUE - Github Java Corpus","dataset_url":"/dataset/codexglue","rows_in_archive":3,"metrics":["Accuracy (token-level)","EM (line-level)","Edit Sim (line-level)"],"first_row_in_archive_order":{"model":"CodeGPT-adapted","paper_title":"CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation","paper_url":"/paper/codexglue-a-machine-learning-benchmark","paper_date":"2021-02-09","arxiv_id":"2102.04664","code_links":[{"title":"microsoft/CodeXGLUE","url":"https://github.com/microsoft/CodeXGLUE"},{"title":"facebookresearch/CodeGen","url":"https://github.com/facebookresearch/CodeGen"},{"title":"sberbank-ai/fusion_brain_aij2021","url":"https://github.com/sberbank-ai/fusion_brain_aij2021"},{"title":"kilimanj4r0/code-summarization-beyond-function-level","url":"https://github.com/kilimanj4r0/code-summarization-beyond-function-level"},{"title":"yueyuel/programgen-lms-reliability","url":"https://github.com/yueyuel/programgen-lms-reliability"},{"title":"Avmb/semantic_neq_game","url":"https://github.com/Avmb/semantic_neq_game"},{"title":"deeplearnxmu/unigencoder","url":"https://github.com/deeplearnxmu/unigencoder"}],"syntology":{"n":1,"n_ran":1,"n_unverified":0,"n_pointer_only":0}}},{"leaderboard":"/sota/code-completion-on-codexglue-py150","slug":"code-completion-on-codexglue-py150","dataset":"CodeXGLUE - PY150","dataset_url":"/dataset/codexglue","rows_in_archive":3,"metrics":["Accuracy (token-level)","EM (line-level)","Edit Sim (line-level)"],"first_row_in_archive_order":{"model":"CodeGPT-adapted","paper_title":"CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation","paper_url":"/paper/codexglue-a-machine-learning-benchmark","paper_date":"2021-02-09","arxiv_id":"2102.04664","code_links":[{"title":"microsoft/CodeXGLUE","url":"https://github.com/microsoft/CodeXGLUE"},{"title":"facebookresearch/CodeGen","url":"https://github.com/facebookresearch/CodeGen"},{"title":"sberbank-ai/fusion_brain_aij2021","url":"https://github.com/sberbank-ai/fusion_brain_aij2021"},{"title":"kilimanj4r0/code-summarization-beyond-function-level","url":"https://github.com/kilimanj4r0/code-summarization-beyond-function-level"},{"title":"yueyuel/programgen-lms-reliability","url":"https://github.com/yueyuel/programgen-lms-reliability"},{"title":"Avmb/semantic_neq_game","url":"https://github.com/Avmb/semantic_neq_game"},{"title":"deeplearnxmu/unigencoder","url":"https://github.com/deeplearnxmu/unigencoder"}],"syntology":{"n":1,"n_ran":1,"n_unverified":0,"n_pointer_only":0}}},{"leaderboard":"/sota/code-completion-on-dotprompts","slug":"code-completion-on-dotprompts","dataset":"DotPrompts","dataset_url":"/dataset/dotprompts","rows_in_archive":3,"metrics":["Compilation Rate"],"first_row_in_archive_order":{"model":"SantaCoder-MGD","paper_title":"Guiding Language Models of Code with Global Context using Monitors","paper_url":"/paper/guiding-language-models-of-code-with-global","paper_date":"2023-06-19","arxiv_id":"2306.10763","code_links":[{"title":"microsoft/monitors4codegen","url":"https://github.com/microsoft/monitors4codegen"}],"syntology":{"n":2,"n_ran":2,"n_unverified":0,"n_pointer_only":0}}},{"leaderboard":"/sota/code-completion-on-defects4j","slug":"code-completion-on-defects4j","dataset":"Defects4J","dataset_url":"/dataset/defects4j","rows_in_archive":2,"metrics":["Compilation Rate","Pass@1","BLEU"],"first_row_in_archive_order":{"model":"Rambo","paper_title":"RAMBO: Enhancing RAG-based Repository-Level Method Body Completion","paper_url":"/paper/rambo-enhancing-rag-based-repository-level","paper_date":"2024-09-23","arxiv_id":"2409.15204","code_links":[{"title":"ise-uet-vnu/rambo","url":"https://github.com/ise-uet-vnu/rambo"}],"syntology":null}},{"leaderboard":"/sota/code-completion-on-rambo-benchmark","slug":"code-completion-on-rambo-benchmark","dataset":"Rambo Benchmark","dataset_url":null,"rows_in_archive":2,"metrics":["Compilation Rate","BLEU"],"first_row_in_archive_order":{"model":"Rambo","paper_title":"RAMBO: Enhancing RAG-based Repository-Level Method Body Completion","paper_url":"/paper/rambo-enhancing-rag-based-repository-level","paper_date":"2024-09-23","arxiv_id":"2409.15204","code_links":[{"title":"ise-uet-vnu/rambo","url":"https://github.com/ise-uet-vnu/rambo"}],"syntology":null}}],"datasets":[{"url":"/dataset/codexglue","name":"CodeXGLUE","full_name":"","num_papers_in_archive":205},{"url":"/dataset/defects4j","name":"Defects4J","full_name":"Defects4J","num_papers_in_archive":9},{"url":"/dataset/pytorrent","name":"PyTorrent","full_name":"PyTorrent","num_papers_in_archive":5},{"url":"/dataset/rtl-repo","name":"RTL-Repo","full_name":"","num_papers_in_archive":5},{"url":"/dataset/safim","name":"SAFIM","full_name":"Syntax-Aware Fill-In-the-Middle","num_papers_in_archive":5},{"url":"/dataset/mmcode","name":"MMCode","full_name":"","num_papers_in_archive":3},{"url":"/dataset/slnet","name":"SLNET","full_name":"SLNET: A Redistributable Corpus of 3rd-party Simulink Models","num_papers_in_archive":3},{"url":"/dataset/spectre-v1","name":"Spectre-v1","full_name":"","num_papers_in_archive":3},{"url":"/dataset/dotprompts","name":"DotPrompts","full_name":"","num_papers_in_archive":1},{"url":"/dataset/openapi-code-completion","name":"OpenAPI completion refined","full_name":"","num_papers_in_archive":1},{"url":"/dataset/verified-smart-contract-code-comments","name":"Verified Smart Contract Code Comments","full_name":"","num_papers_in_archive":1},{"url":"/dataset/verified-smart-contracts","name":"Verified Smart Contracts","full_name":"","num_papers_in_archive":1}],"subtasks":[{"url":"/task/openapi-code-completion","name":"OpenAPI code completion"}],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (desc), then date (desc); the archive holds no stars","population":"papers tagged with this task that list at least one repository in the archive","shown":30,"of":108,"tagged_in_all":212,"items":[{"url":"/paper/codexglue-a-machine-learning-benchmark","title":"CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation","date":"2021-02-09","arxiv_id":"2102.04664","repositories_listed":7,"syntology":{"n":1,"n_ran":1,"n_unverified":0,"n_pointer_only":0}},{"url":"/paper/starcoder-2-and-the-stack-v2-the-next","title":"StarCoder 2 and The Stack v2: The Next Generation","date":"2024-02-29","arxiv_id":"2402.19173","repositories_listed":4,"syntology":null},{"url":"/paper/datasculpt-crafting-data-landscapes-for-llm","title":"DataSculpt: Crafting Data Landscapes for Long-Context LLMs through Multi-Objective Partitioning","date":"2024-09-02","arxiv_id":"2409.00997","repositories_listed":3,"syntology":null},{"url":"/paper/longllmlingua-accelerating-and-enhancing-llms","title":"LongLLMLingua: Accelerating and Enhancing LLMs in Long Context Scenarios via Prompt Compression","date":"2023-10-10","arxiv_id":"2310.06839","repositories_listed":3,"syntology":null},{"url":"/paper/longbench-a-bilingual-multitask-benchmark-for","title":"LongBench: A Bilingual, Multitask Benchmark for Long Context Understanding","date":"2023-08-28","arxiv_id":"2308.14508","repositories_listed":3,"syntology":{"n":1,"n_ran":1,"n_unverified":0,"n_pointer_only":0}},{"url":"/paper/open-vocabulary-learning-on-source-code-with","title":"Open Vocabulary Learning on Source Code with a Graph-Structured Cache","date":"2018-10-18","arxiv_id":"1810.08305","repositories_listed":3,"syntology":null},{"url":"/paper/on-the-workflows-and-smells-of-leaderboard","title":"On the Workflows and Smells of Leaderboard Operations (LBOps): An Exploratory Study of Foundation Model Leaderboards","date":"2024-07-04","arxiv_id":"2407.04065","repositories_listed":2,"syntology":null},{"url":"/paper/optimizing-large-language-models-for-openapi","title":"Optimizing Large Language Models for OpenAPI Code Completion","date":"2024-05-24","arxiv_id":"2405.15729","repositories_listed":2,"syntology":null},{"url":"/paper/exploring-safety-generalization-challenges-of","title":"CodeAttack: Revealing Safety Generalization Challenges of Large Language Models via Code Completion","date":"2024-03-12","arxiv_id":"2403.07865","repositories_listed":2,"syntology":{"n":2,"n_ran":2,"n_unverified":0,"n_pointer_only":0}},{"url":"/paper/scope-is-all-you-need-transforming-llms-for","title":"Scope is all you need: Transforming LLMs for HPC Code","date":"2023-08-18","arxiv_id":"2308.09440","repositories_listed":2,"syntology":null},{"url":"/paper/mpi-rical-data-driven-mpi-distributed","title":"MPI-rical: Data-Driven MPI Distributed Parallelism Assistance with Transformers","date":"2023-05-16","arxiv_id":"2305.09438","repositories_listed":2,"syntology":null},{"url":"/paper/codet5-open-code-large-language-models-for","title":"CodeT5+: Open Code Large Language Models for Code Understanding and Generation","date":"2023-05-13","arxiv_id":"2305.07922","repositories_listed":2,"syntology":{"n":4,"n_ran":3,"n_unverified":1,"n_pointer_only":0}},{"url":"/paper/codekgc-code-language-model-for-generative","title":"CodeKGC: Code Language Model for Generative Knowledge Graph Construction","date":"2023-04-18","arxiv_id":"2304.09048","repositories_listed":2,"syntology":{"n":16,"n_ran":1,"n_unverified":15,"n_pointer_only":0}},{"url":"/paper/more-than-you-ve-asked-for-a-comprehensive","title":"Not what you've signed up for: Compromising Real-World LLM-Integrated Applications with Indirect Prompt Injection","date":"2023-02-23","arxiv_id":"2302.12173","repositories_listed":2,"syntology":{"n":2,"n_ran":0,"n_unverified":2,"n_pointer_only":0}},{"url":"/paper/multi-lingual-evaluation-of-code-generation","title":"Multi-lingual Evaluation of Code Generation Models","date":"2022-10-26","arxiv_id":"2210.14868","repositories_listed":2,"syntology":{"n":6,"n_ran":1,"n_unverified":5,"n_pointer_only":0}},{"url":"/paper/unixcoder-unified-cross-modal-pre-training","title":"UniXcoder: Unified Cross-Modal Pre-training for Code Representation","date":"2022-03-08","arxiv_id":"2203.03850","repositories_listed":2,"syntology":{"n":4,"n_ran":2,"n_unverified":2,"n_pointer_only":0}},{"url":"/paper/energy-based-models-for-code-generation-under","title":"Energy-Based Models for Code Generation under Compilability Constraints","date":"2021-06-09","arxiv_id":"2106.04985","repositories_listed":2,"syntology":null},{"url":"/paper/neural-software-analysis","title":"Neural Software Analysis","date":"2020-11-16","arxiv_id":"2011.07986","repositories_listed":2,"syntology":null},{"url":"/paper/structural-language-models-for-any-code","title":"Structural Language Models of Code","date":"2019-09-30","arxiv_id":"1910.00577","repositories_listed":2,"syntology":{"n":8,"n_ran":0,"n_unverified":8,"n_pointer_only":0}},{"url":"/paper/seed-coder-let-the-code-model-curate-data-for","title":"Seed-Coder: Let the Code Model Curate Data for Itself","date":"2025-06-04","arxiv_id":"2506.03524","repositories_listed":1,"syntology":null},{"url":"/paper/swe-dev-evaluating-and-training-autonomous","title":"SWE-Dev: Evaluating and Training Autonomous Feature-Driven Software Development","date":"2025-05-22","arxiv_id":"2505.16975","repositories_listed":1,"syntology":{"n":5,"n_ran":5,"n_unverified":0,"n_pointer_only":5}},{"url":"/paper/can-you-really-trust-code-copilots-evaluating","title":"Can You Really Trust Code Copilots? Evaluating Large Language Models from a Code Security Perspective","date":"2025-05-15","arxiv_id":"2505.10494","repositories_listed":1,"syntology":null},{"url":"/paper/obscuracoder-powering-efficient-code-lm-pre","title":"ObscuraCoder: Powering Efficient Code LM Pre-Training Via Obfuscation Grounding","date":"2025-03-27","arxiv_id":"2504.00019","repositories_listed":1,"syntology":null},{"url":"/paper/logquant-log-distributed-2-bit-quantization","title":"LogQuant: Log-Distributed 2-Bit Quantization of KV Cache with Superior Accuracy Preservation","date":"2025-03-25","arxiv_id":"2503.19950","repositories_listed":1,"syntology":null},{"url":"/paper/castle-benchmarking-dataset-for-static-code","title":"CASTLE: Benchmarking Dataset for Static Code Analyzers and LLMs towards CWE Detection","date":"2025-03-12","arxiv_id":"2503.09433","repositories_listed":1,"syntology":null},{"url":"/paper/enhancing-high-quality-code-generation-in","title":"Enhancing High-Quality Code Generation in Large Language Models with Comparative Prefix-Tuning","date":"2025-03-12","arxiv_id":"2503.09020","repositories_listed":1,"syntology":null},{"url":"/paper/longspec-long-context-speculative-decoding","title":"LongSpec: Long-Context Speculative Decoding with Efficient Drafting and Verification","date":"2025-02-24","arxiv_id":"2502.17421","repositories_listed":1,"syntology":{"n":10,"n_ran":2,"n_unverified":8,"n_pointer_only":0}},{"url":"/paper/how-to-get-your-llm-to-generate-challenging","title":"How to Get Your LLM to Generate Challenging Problems for Evaluation","date":"2025-02-20","arxiv_id":"2502.14678","repositories_listed":1,"syntology":null},{"url":"/paper/green-code-optimizing-energy-efficiency-in","title":"GREEN-CODE: Learning to Optimize Energy Efficiency in LLM-based Code Generation","date":"2025-01-19","arxiv_id":"2501.11006","repositories_listed":1,"syntology":null},{"url":"/paper/gitchameleon-unmasking-the-version-switching","title":"GitChameleon: Unmasking the Version-Switching Capabilities of Code Generation Models","date":"2024-11-05","arxiv_id":"2411.05830","repositories_listed":1,"syntology":null}],"syntology_records":11,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-09-24T18:15:14+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"}},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"}}