{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/large-language-models-can-automatically","title":"Large Language Models Can Automatically Engineer Features for Few-Shot Tabular Learning","arxiv_id":"2404.09491","date":"2024-04-15","proceeding":null,"authors":["Sungwon Han","Jinsung Yoon","Sercan O Arik","Tomas Pfister"],"abstract":"Large Language Models (LLMs), with their remarkable ability to tackle challenging and unseen reasoning problems, hold immense potential for tabular learning, that is vital for many real-world applications. In this paper, we propose a novel in-context learning framework, FeatLLM, which employs LLMs as feature engineers to produce an input data set that is optimally suited for tabular predictions. The generated features are used to infer class likelihood with a simple downstream machine learning model, such as linear regression and yields high performance few-shot learning. The proposed FeatLLM framework only uses this simple predictive model with the discovered features at inference time. Compared to existing LLM-based approaches, FeatLLM eliminates the need to send queries to the LLM for each sample at inference time. Moreover, it merely requires API-level access to LLMs, and overcomes prompt size limitations. As demonstrated across numerous tabular datasets from a wide range of domains, FeatLLM generates high-quality rules, significantly (10% on average) outperforming alternatives such as TabLLM and STUNT.","url_abs":"https://arxiv.org/abs/2404.09491v2","url_pdf":"https://arxiv.org/pdf/2404.09491v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"large-language-models-can-automatically","repo_url":"https://github.com/sungwon-han/featllm","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"few-shot-learning","task_name":"Few-Shot Learning"},{"task_slug":"in-context-learning","task_name":"In-Context Learning"}],"methods":[{"method_slug":"linear-regression","method_name":"Linear Regression"},{"method_slug":"set","method_name":"SET"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2404.09491","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.09491"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/Sungwon-Han/FeatLLM","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sungwon-han/featllm","reach":null}],"summary":{"ran_fixture":1,"ran_draft_wrong":3,"unverified":1},"by_repo_kind":{"official":{"samples":5,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":5,"samples":[{"code_sha256_prefix":"3101ba4997e7d9b5","entry":"evaluate","repo":"Sungwon-Han/FeatLLM","repo_kind":"official","path":"utils.py","file_url":"https://github.com/Sungwon-Han/FeatLLM/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3101ba4997e7d9b5"}},{"code_sha256_prefix":"461afa41662930fb","entry":"fill_in_templates","repo":"sungwon-han/featllm","repo_kind":"official","path":"utils.py","file_url":"https://github.com/sungwon-han/featllm/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"461afa41662930fb"}},{"code_sha256_prefix":"b72a05150d47901e","entry":"get_dataset","repo":"Sungwon-Han/FeatLLM","repo_kind":"official","path":"utils.py","file_url":"https://github.com/Sungwon-Han/FeatLLM/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b72a05150d47901e"}},{"code_sha256_prefix":"36cfd9635b49c74a","entry":"serialize","repo":"sungwon-han/featllm","repo_kind":"official","path":"utils.py","file_url":"https://github.com/sungwon-han/featllm/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"36cfd9635b49c74a"}},{"code_sha256_prefix":"f3629527c235373d","entry":"get_prompt_for_asking","repo":"sungwon-han/featllm","repo_kind":"official","path":"utils.py","file_url":"https://github.com/sungwon-han/featllm/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f3629527c235373d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}