{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/markovian-agents-for-truthful-language","title":"Markovian Transformers for Informative Language Modeling","arxiv_id":"2404.18988","date":"2024-04-29","proceeding":null,"authors":["Scott Viteri","Max Lamparth","Peter Chatain","Clark Barrett"],"abstract":"Chain-of-Thought (CoT) reasoning often fails to faithfully reflect a language model's underlying decision process. We address this by making CoT text causally essential in a \"Markovian\" language model, factoring next-token prediction through an intermediate CoT and training it to predict future tokens independently of the original prompt. We formalize this via an \"informativeness\" objective that quantifies how much a trained CoT improves next-token predictions over a baseline. Using policy gradient, we show that Llama 3.1 8B achieves a 33.2% absolute accuracy improvement on GSM8K. Perturbation tests confirm stronger reliance on the CoT, while cross-model transfers indicate these reasoning traces generalize across interpreters. Our approach enhances both accuracy and interpretability, potentially extending CoT reasoning to arbitrarily long contexts and diverse tasks.","url_abs":"https://arxiv.org/abs/2404.18988v5","url_pdf":"https://arxiv.org/pdf/2404.18988v5.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"markovian-agents-for-truthful-language","repo_url":"https://github.com/scottviteri/markoviantraining","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"gsm8k","task_name":"GSM8K"},{"task_slug":"informativeness","task_name":"Informativeness"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"}],"methods":[{"method_slug":"llama","method_name":"LLaMA"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2404.18988","atlas_url":"https://app.syntology.ai/?focus=2404.18988","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.18988"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/scottviteri/MarkovianTraining","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/scottviteri/markoviantraining","reach":{"status":"ok"}}],"summary":{"ran_violates":1,"ran":14,"unverified":1},"by_repo_kind":{"official":{"samples":16,"ran":15,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":16,"samples":[{"code_sha256_prefix":"9fec8df5c30712a4","entry":"is_wiki_task","repo":"scottviteri/markoviantraining","repo_kind":"official","path":"src/evaluation.py","file_url":"https://github.com/scottviteri/markoviantraining/blob/HEAD/src/evaluation.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":2,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9fec8df5c30712a4"}},{"code_sha256_prefix":"589ec7d1d7e858cb","entry":"compute_loss_balance_score","repo":"scottviteri/MarkovianTraining","repo_kind":"official","path":"src/plot_training_metrics.py","file_url":"https://github.com/scottviteri/MarkovianTraining/blob/HEAD/src/plot_training_metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"589ec7d1d7e858cb"}},{"code_sha256_prefix":"b9d03cef4c0bd60b","entry":"find_all_checkpoints","repo":"scottviteri/MarkovianTraining","repo_kind":"official","path":"src/batch_calibrate_extractions.py","file_url":"https://github.com/scottviteri/MarkovianTraining/blob/HEAD/src/batch_calibrate_extractions.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b9d03cef4c0bd60b"}},{"code_sha256_prefix":"0208a40593703fe6","entry":"find_best_run_for_task","repo":"scottviteri/MarkovianTraining","repo_kind":"official","path":"src/perturbation_analysis.py","file_url":"https://github.com/scottviteri/MarkovianTraining/blob/HEAD/src/perturbation_analysis.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0208a40593703fe6"}},{"code_sha256_prefix":"942512f0aa580026","entry":"find_crossing_point","repo":"scottviteri/MarkovianTraining","repo_kind":"official","path":"src/analyze_base_logprobs.py","file_url":"https://github.com/scottviteri/MarkovianTraining/blob/HEAD/src/analyze_base_logprobs.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"942512f0aa580026"}},{"code_sha256_prefix":"3d3a57146f3f5235","entry":"format_hyperparameters_text","repo":"scottviteri/MarkovianTraining","repo_kind":"official","path":"src/plot_training_metrics.py","file_url":"https://github.com/scottviteri/MarkovianTraining/blob/HEAD/src/plot_training_metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3d3a57146f3f5235"}},{"code_sha256_prefix":"928bfd50df4e893a","entry":"get_nested_value","repo":"scottviteri/MarkovianTraining","repo_kind":"official","path":"src/create_gp_smoothed_plot.py","file_url":"https://github.com/scottviteri/MarkovianTraining/blob/HEAD/src/create_gp_smoothed_plot.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"928bfd50df4e893a"}},{"code_sha256_prefix":"138168d2f06c61b3","entry":"get_nested_value","repo":"scottviteri/MarkovianTraining","repo_kind":"official","path":"src/plot_training_metrics.py","file_url":"https://github.com/scottviteri/MarkovianTraining/blob/HEAD/src/plot_training_metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"138168d2f06c61b3"}},{"code_sha256_prefix":"11bf31b09d63e7bc","entry":"gp_smooth_with_confidence","repo":"scottviteri/MarkovianTraining","repo_kind":"official","path":"src/create_gp_smoothed_plot.py","file_url":"https://github.com/scottviteri/MarkovianTraining/blob/HEAD/src/create_gp_smoothed_plot.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"11bf31b09d63e7bc"}},{"code_sha256_prefix":"938a4c0257fde174","entry":"import_log_file","repo":"scottviteri/MarkovianTraining","repo_kind":"official","path":"src/plot_cot_answer_accuracy.py","file_url":"https://github.com/scottviteri/MarkovianTraining/blob/HEAD/src/plot_cot_answer_accuracy.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"938a4c0257fde174"}},{"code_sha256_prefix":"360a656c320a63f7","entry":"load_eval_data","repo":"scottviteri/MarkovianTraining","repo_kind":"official","path":"src/plot_alignment_scatter.py","file_url":"https://github.com/scottviteri/MarkovianTraining/blob/HEAD/src/plot_alignment_scatter.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"360a656c320a63f7"}},{"code_sha256_prefix":"6e8b3ebc90354c85","entry":"load_evaluation_results","repo":"scottviteri/MarkovianTraining","repo_kind":"official","path":"src/evaluate_cross_model.py","file_url":"https://github.com/scottviteri/MarkovianTraining/blob/HEAD/src/evaluate_cross_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6e8b3ebc90354c85"}},{"code_sha256_prefix":"314a37061a9a7a21","entry":"perturb_CoT","repo":"scottviteri/MarkovianTraining","repo_kind":"official","path":"src/perturbation_analysis.py","file_url":"https://github.com/scottviteri/MarkovianTraining/blob/HEAD/src/perturbation_analysis.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"314a37061a9a7a21"}},{"code_sha256_prefix":"5c41114c9559ce55","entry":"save_evaluation_results","repo":"scottviteri/MarkovianTraining","repo_kind":"official","path":"src/evaluate_cross_model.py","file_url":"https://github.com/scottviteri/MarkovianTraining/blob/HEAD/src/evaluate_cross_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5c41114c9559ce55"}},{"code_sha256_prefix":"097f57d1b35cac5c","entry":"smooth_curve","repo":"scottviteri/MarkovianTraining","repo_kind":"official","path":"src/analyze_base_logprobs.py","file_url":"https://github.com/scottviteri/MarkovianTraining/blob/HEAD/src/analyze_base_logprobs.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"097f57d1b35cac5c"}},{"code_sha256_prefix":"0068a24e80b41d46","entry":"calculate_token_logprobs_with_context","repo":"scottviteri/MarkovianTraining","repo_kind":"official","path":"src/analyze_base_logprobs.py","file_url":"https://github.com/scottviteri/MarkovianTraining/blob/HEAD/src/analyze_base_logprobs.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0068a24e80b41d46"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}