{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/chain-of-ideas-revolutionizing-research-in","title":"Chain of Ideas: Revolutionizing Research Via Novel Idea Development with LLM Agents","arxiv_id":"2410.13185","date":"2024-10-17","proceeding":null,"authors":["Long Li","Weiwen Xu","Jiayan Guo","Ruochen Zhao","Xingxuan Li","Yuqian Yuan","Boqiang Zhang","Yuming Jiang","Yifei Xin","Ronghao Dang","Deli Zhao","Yu Rong","Tian Feng","Lidong Bing"],"abstract":"Effective research ideation is a critical step for scientific research. However, the exponential increase in scientific literature makes it challenging for researchers to stay current with recent advances and identify meaningful research directions. Recent developments in large language models~(LLMs) suggest a promising avenue for automating the generation of novel research ideas. However, existing methods for idea generation either trivially prompt LLMs or directly expose LLMs to extensive literature without indicating useful information. Inspired by the research process of human researchers, we propose a Chain-of-Ideas~(CoI) agent, an LLM-based agent that organizes relevant literature in a chain structure to effectively mirror the progressive development in a research domain. This organization facilitates LLMs to capture the current advancements in research, thereby enhancing their ideation capabilities. Furthermore, we propose Idea Arena, an evaluation protocol that can comprehensively evaluate idea generation methods from different perspectives, aligning closely with the preferences of human researchers. Experimental results indicate that the CoI agent consistently outperforms other methods and shows comparable quality as humans in research idea generation. Moreover, our CoI agent is budget-friendly, with a minimum cost of \\$0.50 to generate a candidate idea and its corresponding experimental design.","url_abs":"https://arxiv.org/abs/2410.13185v5","url_pdf":"https://arxiv.org/pdf/2410.13185v5.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"chain-of-ideas-revolutionizing-research-in","repo_url":"https://github.com/damo-nlp-sg/coi-agent","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"experimental-design","task_name":"Experimental Design"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2410.13185","atlas_url":"https://app.syntology.ai/?focus=2410.13185","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.13185"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/damo-nlp-sg/coi-agent","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_draft_wrong":1,"ran":13,"unverified":1},"by_repo_kind":{"official":{"samples":15,"ran":14,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"f41cb1a19b154297","entry":"encode_image","repo":"damo-nlp-sg/coi-agent","repo_kind":"official","path":"LLM.py","file_url":"https://github.com/damo-nlp-sg/coi-agent/blob/HEAD/LLM.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"code_sha256_prefix":"f650e65cc3083d84","entry":"extract","repo":"damo-nlp-sg/coi-agent","repo_kind":"official","path":"utils.py","file_url":"https://github.com/damo-nlp-sg/coi-agent/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f650e65cc3083d84"}},{"code_sha256_prefix":"4abf0ae6b3c8d8de","entry":"extract","repo":"damo-nlp-sg/coi-agent","repo_kind":"official","path":"searcher/sementic_search.py","file_url":"https://github.com/damo-nlp-sg/coi-agent/blob/HEAD/searcher/sementic_search.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4abf0ae6b3c8d8de"}},{"code_sha256_prefix":"8da3d9ba5f65d0b8","entry":"extract_json","repo":"damo-nlp-sg/coi-agent","repo_kind":"official","path":"utils.py","file_url":"https://github.com/damo-nlp-sg/coi-agent/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8da3d9ba5f65d0b8"}},{"code_sha256_prefix":"4872fb8c1582e598","entry":"fetch","repo":"damo-nlp-sg/coi-agent","repo_kind":"official","path":"searcher/sementic_search.py","file_url":"https://github.com/damo-nlp-sg/coi-agent/blob/HEAD/searcher/sementic_search.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4872fb8c1582e598"}},{"code_sha256_prefix":"a6d4e225abfd096e","entry":"get_content_between_a_b","repo":"damo-nlp-sg/coi-agent","repo_kind":"official","path":"LLM.py","file_url":"https://github.com/damo-nlp-sg/coi-agent/blob/HEAD/LLM.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a6d4e225abfd096e"}},{"code_sha256_prefix":"62498ce66167479b","entry":"get_deep_check_idea_novel_search_query_prompt","repo":"damo-nlp-sg/coi-agent","repo_kind":"official","path":"prompts/deep_research_agent_prompts.py","file_url":"https://github.com/damo-nlp-sg/coi-agent/blob/HEAD/prompts/deep_research_agent_prompts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"62498ce66167479b"}},{"code_sha256_prefix":"9f075963c52cb8a9","entry":"get_deep_rewrite_query_prompt","repo":"damo-nlp-sg/coi-agent","repo_kind":"official","path":"prompts/deep_research_agent_prompts.py","file_url":"https://github.com/damo-nlp-sg/coi-agent/blob/HEAD/prompts/deep_research_agent_prompts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9f075963c52cb8a9"}},{"code_sha256_prefix":"954ecd542ae11bc3","entry":"get_deep_search_query_prompt","repo":"damo-nlp-sg/coi-agent","repo_kind":"official","path":"prompts/deep_research_agent_prompts.py","file_url":"https://github.com/damo-nlp-sg/coi-agent/blob/HEAD/prompts/deep_research_agent_prompts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"954ecd542ae11bc3"}},{"code_sha256_prefix":"4ddd0eb717fa5b8e","entry":"get_judge_experiment_all_prompt","repo":"damo-nlp-sg/coi-agent","repo_kind":"official","path":"prompts/juder_prompts.py","file_url":"https://github.com/damo-nlp-sg/coi-agent/blob/HEAD/prompts/juder_prompts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4ddd0eb717fa5b8e"}},{"code_sha256_prefix":"8f8af080a9c99830","entry":"get_judge_idea_all_prompt","repo":"damo-nlp-sg/coi-agent","repo_kind":"official","path":"prompts/juder_prompts.py","file_url":"https://github.com/damo-nlp-sg/coi-agent/blob/HEAD/prompts/juder_prompts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8f8af080a9c99830"}},{"code_sha256_prefix":"455aefd097ad94ca","entry":"get_review_experiment_design_suggestions_prompt","repo":"damo-nlp-sg/coi-agent","repo_kind":"official","path":"prompts/review_agent_prompts.py","file_url":"https://github.com/damo-nlp-sg/coi-agent/blob/HEAD/prompts/review_agent_prompts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"455aefd097ad94ca"}},{"code_sha256_prefix":"ec479392ccee5103","entry":"get_review_search_related_paper_prompt","repo":"damo-nlp-sg/coi-agent","repo_kind":"official","path":"prompts/review_agent_prompts.py","file_url":"https://github.com/damo-nlp-sg/coi-agent/blob/HEAD/prompts/review_agent_prompts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ec479392ccee5103"}},{"code_sha256_prefix":"7480cf0c2cd3d240","entry":"get_review_suggestions_from_papers_prompt","repo":"damo-nlp-sg/coi-agent","repo_kind":"official","path":"prompts/review_agent_prompts.py","file_url":"https://github.com/damo-nlp-sg/coi-agent/blob/HEAD/prompts/review_agent_prompts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7480cf0c2cd3d240"}},{"code_sha256_prefix":"d79e88c565eff1fa","entry":"get_openai_url","repo":"damo-nlp-sg/coi-agent","repo_kind":"official","path":"LLM.py","file_url":"https://github.com/damo-nlp-sg/coi-agent/blob/HEAD/LLM.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d79e88c565eff1fa"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}