{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/langevin-soft-actor-critic-efficient","title":"Langevin Soft Actor-Critic: Efficient Exploration through Uncertainty-Driven Critic Learning","arxiv_id":"2501.17827","date":"2025-01-29","proceeding":null,"authors":["Haque Ishfaq","Guangyuan Wang","Sami Nur Islam","Doina Precup"],"abstract":"Existing actor-critic algorithms, which are popular for continuous control reinforcement learning (RL) tasks, suffer from poor sample efficiency due to lack of principled exploration mechanism within them. Motivated by the success of Thompson sampling for efficient exploration in RL, we propose a novel model-free RL algorithm, Langevin Soft Actor Critic (LSAC), which prioritizes enhancing critic learning through uncertainty estimation over policy optimization. LSAC employs three key innovations: approximate Thompson sampling through distributional Langevin Monte Carlo (LMC) based $Q$ updates, parallel tempering for exploring multiple modes of the posterior of the $Q$ function, and diffusion synthesized state-action samples regularized with $Q$ action gradients. Our extensive experiments demonstrate that LSAC outperforms or matches the performance of mainstream model-free RL algorithms for continuous control tasks. Notably, LSAC marks the first successful application of an LMC based Thompson sampling in continuous control tasks with continuous action spaces.","url_abs":"https://arxiv.org/abs/2501.17827v1","url_pdf":"https://arxiv.org/pdf/2501.17827v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"langevin-soft-actor-critic-efficient","repo_url":"https://github.com/hmishfaq/lsac","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"continuous-control","task_name":"Continuous Control"},{"task_slug":"efficient-exploration","task_name":"Efficient Exploration"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"},{"task_slug":"thompson-sampling","task_name":"Thompson Sampling"},{"task_slug":"continuous-control","task_name":"continuous-control"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"diffusion","method_name":"Diffusion"},{"method_slug":"experience-replay","method_name":"Experience Replay"},{"method_slug":"relu","method_name":"ReLU"},{"method_slug":"soft-actor-critic","method_name":"Soft Actor Critic"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2501.17827","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2501.17827"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/hmishfaq/LSAC","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/hmishfaq/lsac","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":2,"ran_honours":2,"ran_draft_wrong":2,"unverified":5},"by_repo_kind":{"official":{"samples":11,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"d8b8cdf71799a5fb","entry":"aSGLD","repo":"hmishfaq/lsac","repo_kind":"official","path":"lsac.py","file_url":"https://github.com/hmishfaq/lsac/blob/HEAD/lsac.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d8b8cdf71799a5fb"}},{"code_sha256_prefix":"6d24f19c56e16f4a","entry":"combined_shape","repo":"hmishfaq/lsac","repo_kind":"official","path":"lsac.py","file_url":"https://github.com/hmishfaq/lsac/blob/HEAD/lsac.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6d24f19c56e16f4a"}},{"code_sha256_prefix":"40e25432a2f48697","entry":"create_apprfunc","repo":"hmishfaq/lsac","repo_kind":"official","path":"lsac.py","file_url":"https://github.com/hmishfaq/lsac/blob/HEAD/lsac.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"40e25432a2f48697"}},{"code_sha256_prefix":"5e059d3b24974ac1","entry":"formatter","repo":"hmishfaq/lsac","repo_kind":"official","path":"lsac.py","file_url":"https://github.com/hmishfaq/lsac/blob/HEAD/lsac.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5e059d3b24974ac1"}},{"code_sha256_prefix":"e570f37628252c79","entry":"lr_lambda","repo":"hmishfaq/lsac","repo_kind":"official","path":"lsac.py","file_url":"https://github.com/hmishfaq/lsac/blob/HEAD/lsac.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e570f37628252c79"}},{"code_sha256_prefix":"568a3d72b42e3f4b","entry":"set_seed","repo":"hmishfaq/lsac","repo_kind":"official","path":"lsac.py","file_url":"https://github.com/hmishfaq/lsac/blob/HEAD/lsac.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"568a3d72b42e3f4b"}},{"code_sha256_prefix":"fcf8c24349da40d7","entry":"DiffusionBuffer","repo":"hmishfaq/lsac","repo_kind":"official","path":"lsac.py","file_url":"https://github.com/hmishfaq/lsac/blob/HEAD/lsac.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fcf8c24349da40d7"}},{"code_sha256_prefix":"af857be3d6b0ff73","entry":"LSAC","repo":"hmishfaq/lsac","repo_kind":"official","path":"lsac.py","file_url":"https://github.com/hmishfaq/lsac/blob/HEAD/lsac.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"af857be3d6b0ff73"}},{"code_sha256_prefix":"777af9a7506f47f8","entry":"Networks","repo":"hmishfaq/lsac","repo_kind":"official","path":"lsac.py","file_url":"https://github.com/hmishfaq/lsac/blob/HEAD/lsac.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"777af9a7506f47f8"}},{"code_sha256_prefix":"a80ed6021e69c6b7","entry":"adam_sgld","repo":"hmishfaq/lsac","repo_kind":"official","path":"lsac.py","file_url":"https://github.com/hmishfaq/lsac/blob/HEAD/lsac.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a80ed6021e69c6b7"}},{"code_sha256_prefix":"6f67468e83cdf046","entry":"get_apprfunc_dict","repo":"hmishfaq/lsac","repo_kind":"official","path":"lsac.py","file_url":"https://github.com/hmishfaq/lsac/blob/HEAD/lsac.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6f67468e83cdf046"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}