{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/capo-cost-aware-prompt-optimization","title":"CAPO: Cost-Aware Prompt Optimization","arxiv_id":"2504.16005","date":"2025-04-22","proceeding":null,"authors":["Tom Zehle","Moritz Schlager","Timo Heiß","Matthias Feurer"],"abstract":"Large language models (LLMs) have revolutionized natural language processing by solving a wide range of tasks simply guided by a prompt. Yet their performance is highly sensitive to prompt formulation. While automated prompt optimization addresses this challenge by finding optimal prompts, current methods require a substantial number of LLM calls and input tokens, making prompt optimization expensive. We introduce CAPO (Cost-Aware Prompt Optimization), an algorithm that enhances prompt optimization efficiency by integrating AutoML techniques. CAPO is an evolutionary approach with LLMs as operators, incorporating racing to save evaluations and multi-objective optimization to balance performance with prompt length. It jointly optimizes instructions and few-shot examples while leveraging task descriptions for improved robustness. Our extensive experiments across diverse datasets and LLMs demonstrate that CAPO outperforms state-of-the-art discrete prompt optimization methods in 11/15 cases with improvements up to 21%p. Our algorithm achieves better performances already with smaller budgets, saves evaluations through racing, and decreases average prompt length via a length penalty, making it both cost-efficient and cost-aware. Even without few-shot examples, CAPO outperforms its competitors and generally remains robust to initial prompts. CAPO represents an important step toward making prompt optimization more powerful and accessible by improving cost-efficiency.","url_abs":"https://arxiv.org/abs/2504.16005v3","url_pdf":"https://arxiv.org/pdf/2504.16005v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"capo-cost-aware-prompt-optimization","repo_url":"https://github.com/finitearth/capo","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":null},{"paper_slug":"capo-cost-aware-prompt-optimization","repo_url":"https://github.com/finitearth/promptolution","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"arithmetic-reasoning","task_name":"Arithmetic Reasoning"},{"task_slug":"automl","task_name":"AutoML"},{"task_slug":"sentiment-analysis","task_name":"Sentiment Analysis"},{"task_slug":"subjectivity-analysis","task_name":"Subjectivity Analysis"},{"task_slug":"text-classification","task_name":"Text Classification"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/arithmetic-reasoning-on-gsm8k","task":"Arithmetic Reasoning","dataset":"GSM8K","model":"Llama-3.3-70B + CAPO","rank_in_archive_order":94,"of":164,"metrics":{"Accuracy":"73.73"},"uses_additional_data":false},{"leaderboard":"/sota/arithmetic-reasoning-on-gsm8k","task":"Arithmetic Reasoning","dataset":"GSM8K","model":"Mistral-Small-24B + CAPO","rank_in_archive_order":110,"of":164,"metrics":{"Accuracy":"65.07"},"uses_additional_data":false},{"leaderboard":"/sota/arithmetic-reasoning-on-gsm8k","task":"Arithmetic Reasoning","dataset":"GSM8K","model":"Qwen2.5-32B + CAPO","rank_in_archive_order":116,"of":164,"metrics":{"Accuracy":"60.2"},"uses_additional_data":false},{"leaderboard":"/sota/sentiment-analysis-on-sst-5-fine-grained","task":"Sentiment Analysis","dataset":"SST-5 Fine-grained classification","model":"Llama-3.3-70B + CAPO","rank_in_archive_order":1,"of":31,"metrics":{"Accuracy":"62.27"},"uses_additional_data":false},{"leaderboard":"/sota/sentiment-analysis-on-sst-5-fine-grained","task":"Sentiment Analysis","dataset":"SST-5 Fine-grained classification","model":"Mistral-Small-24B + CAPO","rank_in_archive_order":30,"of":31,"metrics":{"Accuracy ":"60.2"},"uses_additional_data":false},{"leaderboard":"/sota/sentiment-analysis-on-sst-5-fine-grained","task":"Sentiment Analysis","dataset":"SST-5 Fine-grained classification","model":"Qwen2.5-32B + CAPO","rank_in_archive_order":31,"of":31,"metrics":{"Accuracy ":"59.07"},"uses_additional_data":false},{"leaderboard":"/sota/subjectivity-analysis-on-subj","task":"Subjectivity Analysis","dataset":"SUBJ","model":"Llama-3.3-70B + CAPO","rank_in_archive_order":16,"of":19,"metrics":{"Accuracy":"91.6"},"uses_additional_data":false},{"leaderboard":"/sota/subjectivity-analysis-on-subj","task":"Subjectivity Analysis","dataset":"SUBJ","model":"Qwen2.5-32B + CAPO","rank_in_archive_order":17,"of":19,"metrics":{"Accuracy":"91"},"uses_additional_data":false},{"leaderboard":"/sota/subjectivity-analysis-on-subj","task":"Subjectivity Analysis","dataset":"SUBJ","model":"Mistral-Small-24B + CAPO","rank_in_archive_order":19,"of":19,"metrics":{"Accuracy":"81.67"},"uses_additional_data":false},{"leaderboard":"/sota/text-classification-on-ag-news","task":"Text Classification","dataset":"AG News","model":"Llama-3.3-70B + CAPO","rank_in_archive_order":21,"of":24,"metrics":{"Error":"11.2"},"uses_additional_data":false},{"leaderboard":"/sota/text-classification-on-ag-news","task":"Text Classification","dataset":"AG News","model":"Qwen2.5-32B + CAPO","rank_in_archive_order":22,"of":24,"metrics":{"Error":"12.93"},"uses_additional_data":false},{"leaderboard":"/sota/text-classification-on-ag-news","task":"Text Classification","dataset":"AG News","model":"Mistral-Small-24B + CAPO","rank_in_archive_order":24,"of":24,"metrics":{"Error":"15.7"},"uses_additional_data":false},{"leaderboard":"/sota/text-classification-on-bala-copa","task":"Text Classification","dataset":"Bala-Copa","model":"Qwen2.5-32B + CAPO","rank_in_archive_order":1,"of":3,"metrics":{"Accuracy":"98.47"},"uses_additional_data":false},{"leaderboard":"/sota/text-classification-on-bala-copa","task":"Text Classification","dataset":"Bala-Copa","model":"Llama-3.3-70B + CAPO","rank_in_archive_order":2,"of":3,"metrics":{"Accuracy":"98.27"},"uses_additional_data":false},{"leaderboard":"/sota/text-classification-on-bala-copa","task":"Text Classification","dataset":"Bala-Copa","model":"Mistral-Small-24B + CAPO","rank_in_archive_order":3,"of":3,"metrics":{"Accuracy":"95.13"},"uses_additional_data":false}],"syntology":{"atlas_url":null,"mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}