{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/api-blend-a-comprehensive-corpora-for","title":"API-BLEND: A Comprehensive Corpora for Training and Benchmarking API LLMs","arxiv_id":"2402.15491","date":"2024-02-23","proceeding":null,"authors":["Kinjal Basu","Ibrahim Abdelaziz","Subhajit Chaudhury","Soham Dan","Maxwell Crouse","Asim Munawar","Sadhana Kumaravel","Vinod Muthusamy","Pavan Kapanipathi","Luis A. Lastras"],"abstract":"There is a growing need for Large Language Models (LLMs) to effectively use tools and external Application Programming Interfaces (APIs) to plan and complete tasks. As such, there is tremendous interest in methods that can acquire sufficient quantities of train and test data that involve calls to tools / APIs. Two lines of research have emerged as the predominant strategies for addressing this challenge. The first has focused on synthetic data generation techniques, while the second has involved curating task-adjacent datasets which can be transformed into API / Tool-based tasks. In this paper, we focus on the task of identifying, curating, and transforming existing datasets and, in turn, introduce API-BLEND, a large corpora for training and systematic testing of tool-augmented LLMs. The datasets mimic real-world scenarios involving API-tasks such as API / tool detection, slot filling, and sequencing of the detected APIs. We demonstrate the utility of the API-BLEND dataset for both training and benchmarking purposes.","url_abs":"https://arxiv.org/abs/2402.15491v2","url_pdf":"https://arxiv.org/pdf/2402.15491v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"api-blend-a-comprehensive-corpora-for","repo_url":"https://github.com/ibm/api-blend","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"benchmarking","task_name":"Benchmarking"},{"task_slug":"slot-filling","task_name":"Slot Filling"},{"task_slug":"synthetic-data-generation","task_name":"Synthetic Data Generation"},{"task_slug":"slot-filling-1","task_name":"slot-filling"}],"methods":[{"method_slug":"focus","method_name":"Focus"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2402.15491","atlas_url":"https://app.syntology.ai/?focus=2402.15491","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.15491"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ibm/api-blend","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":5,"unverified":2},"by_repo_kind":{"official":{"samples":7,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"db518f6c2bc0187a","entry":"extract_nested_slots","repo":"ibm/api-blend","repo_kind":"official","path":"grammar-based-generation/SeqTopV2-data-gen.py","file_url":"https://github.com/ibm/api-blend/blob/HEAD/grammar-based-generation/SeqTopV2-data-gen.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"db518f6c2bc0187a"}},{"code_sha256_prefix":"65f427a22ae2f717","entry":"extract_raw_data","repo":"ibm/api-blend","repo_kind":"official","path":"llm-based-generation/llm-data-gen.py","file_url":"https://github.com/ibm/api-blend/blob/HEAD/llm-based-generation/llm-data-gen.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"65f427a22ae2f717"}},{"code_sha256_prefix":"27884bec3b0bbee3","entry":"get_ontologies","repo":"ibm/api-blend","repo_kind":"official","path":"grammar-based-generation/SeqTopV2-data-gen.py","file_url":"https://github.com/ibm/api-blend/blob/HEAD/grammar-based-generation/SeqTopV2-data-gen.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"27884bec3b0bbee3"}},{"code_sha256_prefix":"7b943d0c651eb0d4","entry":"parse_string_to_tree","repo":"ibm/api-blend","repo_kind":"official","path":"grammar-based-generation/SeqTopV2-data-gen.py","file_url":"https://github.com/ibm/api-blend/blob/HEAD/grammar-based-generation/SeqTopV2-data-gen.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7b943d0c651eb0d4"}},{"code_sha256_prefix":"5523c76f12760fc3","entry":"read_file","repo":"ibm/api-blend","repo_kind":"official","path":"grammar-based-generation/SeqSNIPS_SeqATIS-data-gen.py","file_url":"https://github.com/ibm/api-blend/blob/HEAD/grammar-based-generation/SeqSNIPS_SeqATIS-data-gen.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5523c76f12760fc3"}},{"code_sha256_prefix":"76682648b017f6ad","entry":"generate_llm_paraphrase","repo":"ibm/api-blend","repo_kind":"official","path":"llm-based-generation/llm-data-gen.py","file_url":"https://github.com/ibm/api-blend/blob/HEAD/llm-based-generation/llm-data-gen.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"76682648b017f6ad"}},{"code_sha256_prefix":"af99d100a0f45e95","entry":"split_string_on_delimiters","repo":"ibm/api-blend","repo_kind":"official","path":"grammar-based-generation/SeqSNIPS_SeqATIS-data-gen.py","file_url":"https://github.com/ibm/api-blend/blob/HEAD/grammar-based-generation/SeqSNIPS_SeqATIS-data-gen.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"af99d100a0f45e95"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}