{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/omac-a-broad-optimization-framework-for-llm","title":"OMAC: A Broad Optimization Framework for LLM-Based Multi-Agent Collaboration","arxiv_id":"2505.11765","date":"2025-05-17","proceeding":null,"authors":["Shijun Li","Hilaf Hasson","Joydeep Ghosh"],"abstract":"Agents powered by advanced large language models (LLMs) have demonstrated impressive capabilities across diverse complex applications. Recently, Multi-Agent Systems (MAS), wherein multiple agents collaborate and communicate with each other, have exhibited enhanced capabilities in complex tasks, such as high-quality code generation and arithmetic reasoning. However, the development of such systems often relies on handcrafted methods, and the literature on systematic design and optimization of LLM-based MAS remains limited. In this work, we introduce OMAC, a general framework designed for holistic optimization of LLM-based MAS. Specifically, we identify five key optimization dimensions for MAS, encompassing both agent functionality and collaboration structure. Building upon these dimensions, we first propose a general algorithm, utilizing two actors termed the Semantic Initializer and the Contrastive Comparator, to optimize any single dimension. Then, we present an algorithm for joint optimization across multiple dimensions. Extensive experiments demonstrate the superior performance of OMAC on code generation, arithmetic reasoning, and general reasoning tasks against state-of-the-art approaches.","url_abs":"https://arxiv.org/abs/2505.11765v2","url_pdf":"https://arxiv.org/pdf/2505.11765v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"omac-a-broad-optimization-framework-for-llm","repo_url":"https://github.com/xiwenchao/omac-demo","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"arithmetic-reasoning","task_name":"Arithmetic Reasoning"},{"task_slug":"code-generation","task_name":"Code Generation"}],"methods":[{"method_slug":"mas","method_name":"MAS"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2505.11765","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.11765"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/xiwenchao/OMAC","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/xiwenchao/omac-demo","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":2,"ran_violates":1,"unverified":12},"by_repo_kind":{"found_in_text":{"samples":15,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"52487c571369c8c3","entry":"parse_bullets","repo":"xiwenchao/OMAC","repo_kind":"found_in_text","path":"code/MATH/evol_eval_math.py","file_url":"https://github.com/xiwenchao/OMAC/blob/HEAD/code/MATH/evol_eval_math.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"52487c571369c8c3"}},{"code_sha256_prefix":"efece58874c297b1","entry":"parse_yes_no","repo":"xiwenchao/OMAC","repo_kind":"found_in_text","path":"code/MATH/evol_eval_math.py","file_url":"https://github.com/xiwenchao/OMAC/blob/HEAD/code/MATH/evol_eval_math.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"efece58874c297b1"}},{"code_sha256_prefix":"8fce0277228f26b2","entry":"solve_math_problems","repo":"xiwenchao/OMAC","repo_kind":"found_in_text","path":"code/MATH/evol_eval_math.py","file_url":"https://github.com/xiwenchao/OMAC/blob/HEAD/code/MATH/evol_eval_math.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8fce0277228f26b2"}},{"code_sha256_prefix":"a7d74bc4eba65949","entry":"calculate_score","repo":"xiwenchao/OMAC","repo_kind":"found_in_text","path":"code/MMLU/calc_score.py","file_url":"https://github.com/xiwenchao/OMAC/blob/HEAD/code/MMLU/calc_score.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a7d74bc4eba65949"}},{"code_sha256_prefix":"23694908a81661df","entry":"calculate_score","repo":"xiwenchao/OMAC","repo_kind":"found_in_text","path":"code/HumanEval/utils_evo.py","file_url":"https://github.com/xiwenchao/OMAC/blob/HEAD/code/HumanEval/utils_evo.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"23694908a81661df"}},{"code_sha256_prefix":"97949d1f50cd1e53","entry":"construct_assistant_message","repo":"xiwenchao/OMAC","repo_kind":"found_in_text","path":"code/MATH/listwise_math.py","file_url":"https://github.com/xiwenchao/OMAC/blob/HEAD/code/MATH/listwise_math.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"97949d1f50cd1e53"}},{"code_sha256_prefix":"10cf0920c5efdb39","entry":"construct_judge_message","repo":"xiwenchao/OMAC","repo_kind":"found_in_text","path":"code/HumanEval/prompt_lib.py","file_url":"https://github.com/xiwenchao/OMAC/blob/HEAD/code/HumanEval/prompt_lib.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"10cf0920c5efdb39"}},{"code_sha256_prefix":"768aa810b639c13e","entry":"construct_message","repo":"xiwenchao/OMAC","repo_kind":"found_in_text","path":"code/HumanEval/prompt_lib.py","file_url":"https://github.com/xiwenchao/OMAC/blob/HEAD/code/HumanEval/prompt_lib.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"768aa810b639c13e"}},{"code_sha256_prefix":"ff195fa5698279c2","entry":"construct_message","repo":"xiwenchao/OMAC","repo_kind":"found_in_text","path":"code/MATH/listwise_math.py","file_url":"https://github.com/xiwenchao/OMAC/blob/HEAD/code/MATH/listwise_math.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ff195fa5698279c2"}},{"code_sha256_prefix":"19fbbb08c4288bbc","entry":"construct_prerank_message","repo":"xiwenchao/OMAC","repo_kind":"found_in_text","path":"code/MATH/prompt_lib.py","file_url":"https://github.com/xiwenchao/OMAC/blob/HEAD/code/MATH/prompt_lib.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"19fbbb08c4288bbc"}},{"code_sha256_prefix":"d8d506bb4d6eb83a","entry":"construct_ranking_message","repo":"xiwenchao/OMAC","repo_kind":"found_in_text","path":"code/HumanEval/prompt_lib.py","file_url":"https://github.com/xiwenchao/OMAC/blob/HEAD/code/HumanEval/prompt_lib.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d8d506bb4d6eb83a"}},{"code_sha256_prefix":"0280292f0bc4da2c","entry":"construct_ranking_message","repo":"xiwenchao/OMAC","repo_kind":"found_in_text","path":"code/MATH/listwise_math.py","file_url":"https://github.com/xiwenchao/OMAC/blob/HEAD/code/MATH/listwise_math.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0280292f0bc4da2c"}},{"code_sha256_prefix":"7ec82a253737d0da","entry":"construct_structure_message","repo":"xiwenchao/OMAC","repo_kind":"found_in_text","path":"code/MATH/prompt_lib.py","file_url":"https://github.com/xiwenchao/OMAC/blob/HEAD/code/MATH/prompt_lib.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7ec82a253737d0da"}},{"code_sha256_prefix":"2861b77fe609900a","entry":"parse_ranks","repo":"xiwenchao/OMAC","repo_kind":"found_in_text","path":"code/HumanEval/utils.py","file_url":"https://github.com/xiwenchao/OMAC/blob/HEAD/code/HumanEval/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2861b77fe609900a"}},{"code_sha256_prefix":"292858f6b6362eab","entry":"parse_ranks","repo":"xiwenchao/OMAC","repo_kind":"found_in_text","path":"code/MATH/prompt_lib.py","file_url":"https://github.com/xiwenchao/OMAC/blob/HEAD/code/MATH/prompt_lib.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"292858f6b6362eab"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}