{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/when-meaning-stays-the-same-but-models-drift","title":"When Meaning Stays the Same, but Models Drift: Evaluating Quality of Service under Token-Level Behavioral Instability in LLMs","arxiv_id":"2506.10095","date":"2025-06-11","proceeding":null,"authors":["Xiao Li","Joel Kreuzwieser","Alan Peters"],"abstract":"We investigate how large language models respond to prompts that differ only in their token-level realization but preserve the same semantic intent, a phenomenon we call prompt variance. We propose Prompt-Based Semantic Shift (PBSS), a diagnostic framework for measuring behavioral drift in LLMs under semantically equivalent prompt rewordings. Applied to ten constrained tasks, PBSS reveals consistent, model-specific response shifts, suggesting statistical regularities linked to tokenization and decoding. These results highlight an overlooked dimension of model evaluation stability under rephrasing and suggest that tokenization strategies and decoding dynamics may contribute to post-training quality of service instability.","url_abs":"https://arxiv.org/abs/2506.10095v1","url_pdf":"https://arxiv.org/pdf/2506.10095v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"when-meaning-stays-the-same-but-models-drift","repo_url":"https://github.com/Xiao-Vandy/LLM-Prompt-Variance-Diagnostic-Analysis","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":null}],"tasks":[{"task_slug":"diagnostic","task_name":"Diagnostic"}],"methods":[{"method_slug":null,"method_name":null}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2506.10095","atlas_url":"https://app.syntology.ai/?focus=2506.10095","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2506.10095"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Xiao-Vandy/LLM-Prompt-Variance-Diagnostic-Analysis","reach":null}],"summary":{"ran_violates":1},"by_repo_kind":{"official":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"c555742c88ce8cda","entry":"compute_pbss_matrix","repo":"Xiao-Vandy/LLM-Prompt-Variance-Diagnostic-Analysis","repo_kind":"official","path":"001_pbss_case_runner.py","file_url":"https://github.com/Xiao-Vandy/LLM-Prompt-Variance-Diagnostic-Analysis/blob/HEAD/001_pbss_case_runner.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c555742c88ce8cda"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}