{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/evaluating-llm-adaptation-to-sociodemographic","title":"Evaluating LLM Adaptation to Sociodemographic Factors: User Profile vs. Dialogue History","arxiv_id":"2505.21362","date":"2025-05-27","proceeding":null,"authors":["Qishuai Zhong","Zongmin Li","Siqi Fan","Aixin Sun"],"abstract":"Effective engagement by large language models (LLMs) requires adapting responses to users' sociodemographic characteristics, such as age, occupation, and education level. While many real-world applications leverage dialogue history for contextualization, existing evaluations of LLMs' behavioral adaptation often focus on single-turn prompts. In this paper, we propose a framework to evaluate LLM adaptation when attributes are introduced either (1) explicitly via user profiles in the prompt or (2) implicitly through multi-turn dialogue history. We assess the consistency of model behavior across these modalities. Using a multi-agent pipeline, we construct a synthetic dataset pairing dialogue histories with distinct user profiles and employ questions from the Value Survey Module (VSM 2013) (Hofstede and Hofstede, 2016) to probe value expression. Our findings indicate that most models adjust their expressed values in response to demographic changes, particularly in age and education level, but consistency varies. Models with stronger reasoning capabilities demonstrate greater alignment, indicating the importance of reasoning in robust sociodemographic adaptation.","url_abs":"https://arxiv.org/abs/2505.21362v1","url_pdf":"https://arxiv.org/pdf/2505.21362v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"evaluating-llm-adaptation-to-sociodemographic","repo_url":"https://github.com/FerdinandZhong/model_behavior_adaption","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[],"methods":[{"method_slug":"focus","method_name":"Focus"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2505.21362","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.21362"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/FerdinandZhong/model_behavior_adaption","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":7},"by_repo_kind":{"official":{"samples":7,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"a1efc1022fe71b52","entry":"calculate_age","repo":"FerdinandZhong/model_behavior_adaption","repo_kind":"official","path":"llm_behavior_adaptation/dialogue_dataset_creation/generation_utils.py","file_url":"https://github.com/FerdinandZhong/model_behavior_adaption/blob/HEAD/llm_behavior_adaptation/dialogue_dataset_creation/generation_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a1efc1022fe71b52"}},{"code_sha256_prefix":"0c5730da42cb607a","entry":"emd_between_vectors","repo":"FerdinandZhong/model_behavior_adaption","repo_kind":"official","path":"llm_behavior_adaptation/value_measurement/distances.py","file_url":"https://github.com/FerdinandZhong/model_behavior_adaption/blob/HEAD/llm_behavior_adaptation/value_measurement/distances.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0c5730da42cb607a"}},{"code_sha256_prefix":"be6fa3e579643fe8","entry":"hellinger_distance","repo":"FerdinandZhong/model_behavior_adaption","repo_kind":"official","path":"llm_behavior_adaptation/value_measurement/formulas.py","file_url":"https://github.com/FerdinandZhong/model_behavior_adaption/blob/HEAD/llm_behavior_adaptation/value_measurement/formulas.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"be6fa3e579643fe8"}},{"code_sha256_prefix":"53e510bbdc126fec","entry":"jensen_shannon_divergence","repo":"FerdinandZhong/model_behavior_adaption","repo_kind":"official","path":"llm_behavior_adaptation/value_measurement/formulas.py","file_url":"https://github.com/FerdinandZhong/model_behavior_adaption/blob/HEAD/llm_behavior_adaptation/value_measurement/formulas.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"53e510bbdc126fec"}},{"code_sha256_prefix":"e53dd57d9bb65f76","entry":"load_json_folder","repo":"FerdinandZhong/model_behavior_adaption","repo_kind":"official","path":"llm_behavior_adaptation/dialogue_dataset_creation/generation_utils.py","file_url":"https://github.com/FerdinandZhong/model_behavior_adaption/blob/HEAD/llm_behavior_adaptation/dialogue_dataset_creation/generation_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e53dd57d9bb65f76"}},{"code_sha256_prefix":"1a62a5fc333dbad1","entry":"render_json","repo":"FerdinandZhong/model_behavior_adaption","repo_kind":"official","path":"llm_behavior_adaptation/dialogue_dataset_creation/generation_utils.py","file_url":"https://github.com/FerdinandZhong/model_behavior_adaption/blob/HEAD/llm_behavior_adaptation/dialogue_dataset_creation/generation_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1a62a5fc333dbad1"}},{"code_sha256_prefix":"411fac55497025e8","entry":"softmax","repo":"FerdinandZhong/model_behavior_adaption","repo_kind":"official","path":"llm_behavior_adaptation/value_measurement/formulas.py","file_url":"https://github.com/FerdinandZhong/model_behavior_adaption/blob/HEAD/llm_behavior_adaptation/value_measurement/formulas.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"411fac55497025e8"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}