{"url":"/sota/code-completion-on-safim","task":{"name":"Code Completion","url":"/task/code-completion","note":null},"dataset":{"name":"SAFIM","url":"/dataset/safim"},"category":"Computer Code","categories":["Computer Code"],"category_note":null,"description":null,"description_from":null,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","rank":"the archive's row order at snapshot; not re-ranked","rows_end_at":"2025-07-28","rows_withheld_as_spam":0,"metric_values":"the archive's strings, untouched"},"metrics":["Average","Algorithmic","Control","API"],"metric_direction":{"note":"inferred from the metric name only (the archive records no direction); null = not inferred, chart draws points only","by_metric":{"Average":null,"Algorithmic":null,"Control":null,"API":null}},"counts":{"rows":15,"rows_with_code":15,"rows_with_paper_page":15,"rows_dated":15,"rows_using_additional_data":0},"rows":[{"rank_in_archive_order":1,"model":"deepseek-coder-33b-base","metrics":{"API":"75.16","Algorithmic":"60.78","Average":"69.01","Control":"71.10"},"uses_additional_data":false,"paper_date":"2024-03-07","paper":"/paper/evaluation-of-llms-on-syntax-aware-code-fill","paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","code":"https://github.com/gonglinyuan/safim","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":0}},{"rank_in_archive_order":2,"model":"deepseek-coder-6.7b-base","metrics":{"API":"69.68","Algorithmic":"54.74","Average":"63.40","Control":"65.79"},"uses_additional_data":false,"paper_date":"2024-03-07","paper":"/paper/evaluation-of-llms-on-syntax-aware-code-fill","paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","code":"https://github.com/gonglinyuan/safim","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":0}},{"rank_in_archive_order":3,"model":"starcoderbase","metrics":{"API":"68.06","Algorithmic":"44.11","Average":"55.54","Control":"54.46"},"uses_additional_data":false,"paper_date":"2024-03-07","paper":"/paper/evaluation-of-llms-on-syntax-aware-code-fill","paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","code":"https://github.com/gonglinyuan/safim","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":0}},{"rank_in_archive_order":4,"model":"gpt-4-1106-preview","metrics":{"API":"62.58","Algorithmic":"42.11","Average":"53.28","Control":"55.15"},"uses_additional_data":false,"paper_date":"2024-03-07","paper":"/paper/evaluation-of-llms-on-syntax-aware-code-fill","paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","code":"https://github.com/gonglinyuan/safim","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":0}},{"rank_in_archive_order":5,"model":"CodeLlama-13b-hf","metrics":{"API":"59.68","Algorithmic":"41.41","Average":"52.78","Control":"57.25"},"uses_additional_data":false,"paper_date":"2024-03-07","paper":"/paper/evaluation-of-llms-on-syntax-aware-code-fill","paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","code":"https://github.com/gonglinyuan/safim","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":0}},{"rank_in_archive_order":6,"model":"deepseek-coder-1.3b-base","metrics":{"API":"62.58","Algorithmic":"41.20","Average":"52.63","Control":"54.10"},"uses_additional_data":false,"paper_date":"2024-03-07","paper":"/paper/evaluation-of-llms-on-syntax-aware-code-fill","paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","code":"https://github.com/gonglinyuan/safim","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":0}},{"rank_in_archive_order":7,"model":"CodeLlama-34b-hf","metrics":{"API":"56.45","Algorithmic":"38.55","Average":"49.66","Control":"53.98"},"uses_additional_data":false,"paper_date":"2024-03-07","paper":"/paper/evaluation-of-llms-on-syntax-aware-code-fill","paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","code":"https://github.com/gonglinyuan/safim","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":0}},{"rank_in_archive_order":8,"model":"CodeLlama-7b-hf","metrics":{"API":"46.77","Algorithmic":"34.68","Average":"45.00","Control":"53.56"},"uses_additional_data":false,"paper_date":"2024-03-07","paper":"/paper/evaluation-of-llms-on-syntax-aware-code-fill","paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","code":"https://github.com/gonglinyuan/safim","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":0}},{"rank_in_archive_order":9,"model":"gpt-3.5-turbo-0301","metrics":{"API":"53.87","Algorithmic":"31.24","Average":"40.86","Control":"37.48"},"uses_additional_data":false,"paper_date":"2024-03-07","paper":"/paper/evaluation-of-llms-on-syntax-aware-code-fill","paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","code":"https://github.com/gonglinyuan/safim","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":0}},{"rank_in_archive_order":10,"model":"incoder-6B","metrics":{"API":"48.06","Algorithmic":"25.16","Average":"33.79","Control":"28.16"},"uses_additional_data":false,"paper_date":"2024-03-07","paper":"/paper/evaluation-of-llms-on-syntax-aware-code-fill","paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","code":"https://github.com/gonglinyuan/safim","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":0}},{"rank_in_archive_order":11,"model":"codegen-16B-multi","metrics":{"API":"31.29","Algorithmic":"25.94","Average":"30.99","Control":"35.74"},"uses_additional_data":false,"paper_date":"2024-03-07","paper":"/paper/evaluation-of-llms-on-syntax-aware-code-fill","paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","code":"https://github.com/gonglinyuan/safim","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":0}},{"rank_in_archive_order":12,"model":"codegen-2B-multi","metrics":{"API":"32.26","Algorithmic":"23.49","Average":"29.55","Control":"32.89"},"uses_additional_data":false,"paper_date":"2024-03-07","paper":"/paper/evaluation-of-llms-on-syntax-aware-code-fill","paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","code":"https://github.com/gonglinyuan/safim","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":0}},{"rank_in_archive_order":13,"model":"incoder-1B","metrics":{"API":"43.87","Algorithmic":"21.06","Average":"29.27","Control":"22.89"},"uses_additional_data":false,"paper_date":"2024-03-07","paper":"/paper/evaluation-of-llms-on-syntax-aware-code-fill","paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","code":"https://github.com/gonglinyuan/safim","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":0}},{"rank_in_archive_order":14,"model":"codegen-6B-multi","metrics":{"API":"27.74","Algorithmic":"23.60","Average":"28.71","Control":"34.80"},"uses_additional_data":false,"paper_date":"2024-03-07","paper":"/paper/evaluation-of-llms-on-syntax-aware-code-fill","paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","code":"https://github.com/gonglinyuan/safim","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":0}},{"rank_in_archive_order":15,"model":"codegen-350M-multi","metrics":{"API":"26.45","Algorithmic":"16.30","Average":"22.94","Control":"26.06"},"uses_additional_data":false,"paper_date":"2024-03-07","paper":"/paper/evaluation-of-llms-on-syntax-aware-code-fill","paper_url":"https://arxiv.org/abs/2403.04814v3","paper_title":"Evaluation of LLMs on Syntax-Aware Code Fill-in-the-Middle Tasks","code":"https://github.com/gonglinyuan/safim","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":0}}],"since_archive":{"present":false,"note":"No Syntology-extracted rows are published in this build."},"syntology":{"read_at":"2026-09-24T18:15:14+00:00","claim":"Per row: N of M harvested code samples from that row's paper executed on a synthesized fixture; the other M-N are unverified. Not a reproduction of the row's number; not a correctness claim. n_pointer_only_licence counts samples the site points at rather than redistributes (a licence axis, independent of ran/unverified).","rows_with_graph_line":15,"rows_with_any_sample_ran":15,"distinct_papers_with_graph_line":1,"distinct_papers_with_any_sample_ran":1,"samples_over_distinct_papers":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":0,"note":"each paper (arXiv id) counted once, however many rows it is behind; this is the page-level figure"},"samples_row_weighted":{"n_ran":30,"n_unverified":0,"n_samples":30,"n_pointer_only_licence":0,"note":"row-weighted: a paper behind several rows is counted once per row; inflated relative to samples_over_distinct_papers by design, kept for readers summing the per-row syntology blocks"}}}