{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/from-words-to-numbers-your-large-language","title":"From Words to Numbers: Your Large Language Model Is Secretly A Capable Regressor When Given In-Context Examples","arxiv_id":"2404.07544","date":"2024-04-11","proceeding":null,"authors":["Robert Vacareanu","Vlad-Andrei Negru","Vasile Suciu","Mihai Surdeanu"],"abstract":"We analyze how well pre-trained large language models (e.g., Llama2, GPT-4, Claude 3, etc) can do linear and non-linear regression when given in-context examples, without any additional training or gradient updates. Our findings reveal that several large language models (e.g., GPT-4, Claude 3) are able to perform regression tasks with a performance rivaling (or even outperforming) that of traditional supervised methods such as Random Forest, Bagging, or Gradient Boosting. For example, on the challenging Friedman #2 regression dataset, Claude 3 outperforms many supervised methods such as AdaBoost, SVM, Random Forest, KNN, or Gradient Boosting. We then investigate how well the performance of large language models scales with the number of in-context exemplars. We borrow from the notion of regret from online learning and empirically show that LLMs are capable of obtaining a sub-linear regret.","url_abs":"https://arxiv.org/abs/2404.07544v3","url_pdf":"https://arxiv.org/pdf/2404.07544v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"from-words-to-numbers-your-large-language","repo_url":"https://github.com/robertvacareanu/llm4regression","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"large-language-model","task_name":"Large Language Model"},{"task_slug":"regression-1","task_name":"regression"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt-4","method_name":"GPT-4"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"svm","method_name":"SVM"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2404.07544","atlas_url":"https://app.syntology.ai/?focus=2404.07544","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.07544"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/robertvacareanu/llm4regression","reach":{"status":"ok"}}],"summary":{"ran":8,"unverified":2},"by_repo_kind":{"official":{"samples":10,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":10,"samples":[{"code_sha256_prefix":"94df426ef21dc339","entry":"construct_few_shot_suffix_and_iv","repo":"robertvacareanu/llm4regression","repo_kind":"official","path":"src/regressors/prompts.py","file_url":"https://github.com/robertvacareanu/llm4regression/blob/HEAD/src/regressors/prompts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"94df426ef21dc339"}},{"code_sha256_prefix":"6031469081260f0f","entry":"fit_curves","repo":"robertvacareanu/llm4regression","repo_kind":"official","path":"analysis_utils.py","file_url":"https://github.com/robertvacareanu/llm4regression/blob/HEAD/analysis_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6031469081260f0f"}},{"code_sha256_prefix":"4e39875e810ce122","entry":"get_friedman1","repo":"robertvacareanu/llm4regression","repo_kind":"official","path":"src/dataset_utils.py","file_url":"https://github.com/robertvacareanu/llm4regression/blob/HEAD/src/dataset_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4e39875e810ce122"}},{"code_sha256_prefix":"fa7415a69e42daa4","entry":"get_regression","repo":"robertvacareanu/llm4regression","repo_kind":"official","path":"src/dataset_utils.py","file_url":"https://github.com/robertvacareanu/llm4regression/blob/HEAD/src/dataset_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fa7415a69e42daa4"}},{"code_sha256_prefix":"82e0351cf456e527","entry":"lasso","repo":"robertvacareanu/llm4regression","repo_kind":"official","path":"src/regressors/sklearn_regressors.py","file_url":"https://github.com/robertvacareanu/llm4regression/blob/HEAD/src/regressors/sklearn_regressors.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"82e0351cf456e527"}},{"code_sha256_prefix":"8345d65d7210cd50","entry":"linear_regression","repo":"robertvacareanu/llm4regression","repo_kind":"official","path":"src/regressors/sklearn_regressors.py","file_url":"https://github.com/robertvacareanu/llm4regression/blob/HEAD/src/regressors/sklearn_regressors.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8345d65d7210cd50"}},{"code_sha256_prefix":"63437ea7ae13c563","entry":"output_to_number","repo":"robertvacareanu/llm4regression","repo_kind":"official","path":"analysis_utils.py","file_url":"https://github.com/robertvacareanu/llm4regression/blob/HEAD/analysis_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"63437ea7ae13c563"}},{"code_sha256_prefix":"d46351ae9ca06fce","entry":"ridge","repo":"robertvacareanu/llm4regression","repo_kind":"official","path":"src/regressors/sklearn_regressors.py","file_url":"https://github.com/robertvacareanu/llm4regression/blob/HEAD/src/regressors/sklearn_regressors.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d46351ae9ca06fce"}},{"code_sha256_prefix":"36bbe3777460903a","entry":"get_real_estate_data","repo":"robertvacareanu/llm4regression","repo_kind":"official","path":"src/dataset_utils.py","file_url":"https://github.com/robertvacareanu/llm4regression/blob/HEAD/src/dataset_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"36bbe3777460903a"}},{"code_sha256_prefix":"59d9b24f46648543","entry":"scores","repo":"robertvacareanu/llm4regression","repo_kind":"official","path":"src/score_utils.py","file_url":"https://github.com/robertvacareanu/llm4regression/blob/HEAD/src/score_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"59d9b24f46648543"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}