{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2605-09106","title":"Fin-Bias: Comprehensive Evaluation for LLM Decision-Making under human bias in Finance Domain","arxiv_id":"2605.09106","date":"2026-05-09","proceeding":null,"authors":["Xiaoyu Hu","Jinman Zhao"],"abstract":"Large language models (LLMs) are increasingly deployed in financial contexts, raising critical concerns about reliability, alignment, and susceptibility to adversarial manipulation. While prior finance-related benchmarks assess LLMs' capabilities in stock trading, they are often restricted to small sample and fail to demonstrate LLM susceptibility to context with potential human bias. We introduce Fin-Bias (financial herding under long and uncertain financial context), a benchmark for evaluating LLM investment decision-making when faced with uncertainty and possible human-biased opinions. Fin-Bias includes 8868 long firm-specific analyst reports, including firm aspects summarized and analyzed by sophisticated analysts with investment ratings (Bullish/Neutral/Bearish) spanning from various industries. We present large language models with firm analyst reports with/without analyst investment ratings and even with 'fake' rating, to get investment ratings generated by LLMs. Our results reveal that LLMs tend to herd the explicit bias in context. We also develop a method to detect potential human opinions, which can encourage LLMs to think independently, some models even exceed human performance in predicting future stock return.","url_abs":"https://arxiv.org/abs/2605.09106","url_pdf":"https://arxiv.org/pdf/2605.09106","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2605.09106","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2605.09106"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/Xiaoyu1216/Fin-Bias","reach":null}],"summary":{"ran":4},"by_repo_kind":{"found_in_text":{"samples":4,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"678c1639adee6d2b","entry":"prepare_cot_inv_input","repo":"Xiaoyu1216/Fin-Bias","repo_kind":"found_in_text","path":"run_api.py","file_url":"https://github.com/Xiaoyu1216/Fin-Bias/blob/HEAD/run_api.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"678c1639adee6d2b"}},{"code_sha256_prefix":"5b1d00733ccfda67","entry":"prepare_model_inputs","repo":"Xiaoyu1216/Fin-Bias","repo_kind":"found_in_text","path":"run_api.py","file_url":"https://github.com/Xiaoyu1216/Fin-Bias/blob/HEAD/run_api.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5b1d00733ccfda67"}},{"code_sha256_prefix":"fcad4fe2a6eb6fda","entry":"prepare_model_inputs","repo":"Xiaoyu1216/Fin-Bias","repo_kind":"found_in_text","path":"run_vllm.py","file_url":"https://github.com/Xiaoyu1216/Fin-Bias/blob/HEAD/run_vllm.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fcad4fe2a6eb6fda"}},{"code_sha256_prefix":"d86ea5a31781b2d8","entry":"process_single_example_raw_outputs","repo":"Xiaoyu1216/Fin-Bias","repo_kind":"found_in_text","path":"run_vllm.py","file_url":"https://github.com/Xiaoyu1216/Fin-Bias/blob/HEAD/run_vllm.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d86ea5a31781b2d8"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.CL","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}