{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2603-25960","title":"When Chain-of-Thought Backfires: Evaluating Prompt Sensitivity in Medical Language Models","arxiv_id":"2603.25960","date":"2026-03-26","proceeding":null,"authors":["Binesh Sadanandan","Vahid Behzadan"],"abstract":"Large Language Models (LLMs) are increasingly deployed in medical settings, yet their sensitivity to prompt formatting remains poorly characterized. We evaluate MedGemma (4B and 27B parameters) on MedMCQA (4,183 questions) and PubMedQA (1,000 questions) across a broad suite of robustness tests. Our experiments reveal several concerning findings. Chain-of-Thought (CoT) prompting decreases accuracy by 5.7% compared to direct answering. Few-shot examples degrade performance by 11.9% while increasing position bias from 0.14 to 0.47. Shuffling answer options causes the model to change predictions 59.1% of the time, with accuracy dropping up to 27.4 percentage points. Front-truncating context to 50% causes accuracy to plummet below the no-context baseline, yet back-truncation preserves 97% of full-context accuracy. We further show that cloze scoring (selecting the highest log-probability option token) achieves 51.8% (4B) and 64.5% (27B), surpassing all prompting strategies and revealing that models \"know\" more than their generated text shows. Permutation voting recovers 4 percentage points over single-ordering inference. These results demonstrate that prompt engineering techniques validated on general-purpose models do not transfer to domain-specific medical LLMs, and that reliable alternatives exist.","url_abs":"https://arxiv.org/abs/2603.25960","url_pdf":"https://arxiv.org/pdf/2603.25960","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2603.25960","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2603.25960"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/UNHSAILLab/MedMCQA-Robustness-Study","reach":null}],"summary":{"unverified":9},"by_repo_kind":{"found_in_text":{"samples":9,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"f1904564c5ce1da4","entry":"bonferroni_correction","repo":"UNHSAILLab/MedMCQA-Robustness-Study","repo_kind":"found_in_text","path":"src/evaluation/significance.py","file_url":"https://github.com/UNHSAILLab/MedMCQA-Robustness-Study/blob/HEAD/src/evaluation/significance.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f1904564c5ce1da4"}},{"code_sha256_prefix":"3d7619274b44eb70","entry":"bootstrap_ci","repo":"UNHSAILLab/MedMCQA-Robustness-Study","repo_kind":"found_in_text","path":"src/evaluation/significance.py","file_url":"https://github.com/UNHSAILLab/MedMCQA-Robustness-Study/blob/HEAD/src/evaluation/significance.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3d7619274b44eb70"}},{"code_sha256_prefix":"290dc837db95a164","entry":"mcnemar_test","repo":"UNHSAILLab/MedMCQA-Robustness-Study","repo_kind":"found_in_text","path":"src/evaluation/significance.py","file_url":"https://github.com/UNHSAILLab/MedMCQA-Robustness-Study/blob/HEAD/src/evaluation/significance.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"290dc837db95a164"}},{"code_sha256_prefix":"f9ff2509e4bdcaeb","entry":"parse_mcq_answer","repo":"UNHSAILLab/MedMCQA-Robustness-Study","repo_kind":"found_in_text","path":"src/evaluation/metrics.py","file_url":"https://github.com/UNHSAILLab/MedMCQA-Robustness-Study/blob/HEAD/src/evaluation/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f9ff2509e4bdcaeb"}},{"code_sha256_prefix":"0efda3c23d604b19","entry":"parse_pubmedqa_answer","repo":"UNHSAILLab/MedMCQA-Robustness-Study","repo_kind":"found_in_text","path":"src/evaluation/metrics.py","file_url":"https://github.com/UNHSAILLab/MedMCQA-Robustness-Study/blob/HEAD/src/evaluation/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0efda3c23d604b19"}},{"code_sha256_prefix":"112204efe8deb0e0","entry":"plot_accuracy_comparison","repo":"UNHSAILLab/MedMCQA-Robustness-Study","repo_kind":"found_in_text","path":"src/evaluation/visualization.py","file_url":"https://github.com/UNHSAILLab/MedMCQA-Robustness-Study/blob/HEAD/src/evaluation/visualization.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"112204efe8deb0e0"}},{"code_sha256_prefix":"4c38c1dba8b131e8","entry":"plot_reliability_diagram","repo":"UNHSAILLab/MedMCQA-Robustness-Study","repo_kind":"found_in_text","path":"src/evaluation/visualization.py","file_url":"https://github.com/UNHSAILLab/MedMCQA-Robustness-Study/blob/HEAD/src/evaluation/visualization.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4c38c1dba8b131e8"}},{"code_sha256_prefix":"5ede6d75b429e9b5","entry":"plot_subject_heatmap","repo":"UNHSAILLab/MedMCQA-Robustness-Study","repo_kind":"found_in_text","path":"src/evaluation/visualization.py","file_url":"https://github.com/UNHSAILLab/MedMCQA-Robustness-Study/blob/HEAD/src/evaluation/visualization.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5ede6d75b429e9b5"}},{"code_sha256_prefix":"8dd6584d4fdbeb18","entry":"reliability_diagram_data","repo":"UNHSAILLab/MedMCQA-Robustness-Study","repo_kind":"found_in_text","path":"src/evaluation/calibration.py","file_url":"https://github.com/UNHSAILLab/MedMCQA-Robustness-Study/blob/HEAD/src/evaluation/calibration.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8dd6584d4fdbeb18"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.CL","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}