{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/llm-based-agent-simulation-for-maternal","title":"LLM-based Agent Simulation for Maternal Health Interventions: Uncertainty Estimation and Decision-focused Evaluation","arxiv_id":"2503.22719","date":"2025-03-25","proceeding":null,"authors":["Sarah Martinson","Lingkai Kong","Cheol Woo Kim","Aparna Taneja","Milind Tambe"],"abstract":"Agent-based simulation is crucial for modeling complex human behavior, yet traditional approaches require extensive domain knowledge and large datasets. In data-scarce healthcare settings where historic and counterfactual data are limited, large language models (LLMs) offer a promising alternative by leveraging broad world knowledge. This study examines an LLM-driven simulation of a maternal mobile health program, predicting beneficiaries' listening behavior when they receive health information via automated messages (control) or live representatives (intervention). Since uncertainty quantification is critical for decision-making in health interventions, we propose an LLM epistemic uncertainty estimation method based on binary entropy across multiple samples. We enhance model robustness through ensemble approaches, improving F1 score and model calibration compared to individual models. Beyond direct evaluation, we take a decision-focused approach, demonstrating how LLM predictions inform intervention feasibility and trial implementation in data-limited settings. The proposed method extends to public health, disaster response, and other domains requiring rapid intervention assessment under severe data constraints. All code and prompts used for this work can be found at https://github.com/sarahmart/LLM-ABS-ARMMAN-prediction.","url_abs":"https://arxiv.org/abs/2503.22719v1","url_pdf":"https://arxiv.org/pdf/2503.22719v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"llm-based-agent-simulation-for-maternal","repo_url":"https://github.com/sarahmart/llm-abs-armman-prediction","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"decision-making","task_name":"Decision Making"},{"task_slug":"disaster-response","task_name":"Disaster Response"},{"task_slug":"uncertainty-quantification","task_name":"Uncertainty Quantification"},{"task_slug":"world-knowledge","task_name":"World Knowledge"},{"task_slug":null,"task_name":"counterfactual"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2503.22719","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2503.22719"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sarahmart/llm-abs-armman-prediction","reach":null}],"summary":{"ran_fixture":2,"ran_draft_wrong":2,"ran_honours":1},"by_repo_kind":{"official":{"samples":5,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"fb0e69ae8e7fe717","entry":"compute_metrics","repo":"sarahmart/llm-abs-armman-prediction","repo_kind":"official","path":"analysis/aggregation.py","file_url":"https://github.com/sarahmart/llm-abs-armman-prediction/blob/HEAD/analysis/aggregation.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fb0e69ae8e7fe717"}},{"code_sha256_prefix":"4d63de94cbc9fde0","entry":"compute_metrics_by_group","repo":"sarahmart/llm-abs-armman-prediction","repo_kind":"official","path":"analysis/aggregation.py","file_url":"https://github.com/sarahmart/llm-abs-armman-prediction/blob/HEAD/analysis/aggregation.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4d63de94cbc9fde0"}},{"code_sha256_prefix":"ade64512da6f9546","entry":"generate_prompt","repo":"sarahmart/llm-abs-armman-prediction","repo_kind":"official","path":"models/LLM_simulator.py","file_url":"https://github.com/sarahmart/llm-abs-armman-prediction/blob/HEAD/models/LLM_simulator.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ade64512da6f9546"}},{"code_sha256_prefix":"880e236bf8959bac","entry":"load_config","repo":"sarahmart/llm-abs-armman-prediction","repo_kind":"official","path":"models/LLM_simulator.py","file_url":"https://github.com/sarahmart/llm-abs-armman-prediction/blob/HEAD/models/LLM_simulator.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"880e236bf8959bac"}},{"code_sha256_prefix":"82ea9842f7a06a9f","entry":"overall_metrics_baselines","repo":"sarahmart/llm-abs-armman-prediction","repo_kind":"official","path":"analysis/aggregation.py","file_url":"https://github.com/sarahmart/llm-abs-armman-prediction/blob/HEAD/analysis/aggregation.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"82ea9842f7a06a9f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}