{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/soft-actor-critic-for-discrete-action","title":"Soft Actor-Critic for Discrete Action Settings","arxiv_id":"1910.07207","date":"2019-10-16","proceeding":null,"authors":["Petros Christodoulou"],"abstract":"Soft Actor-Critic is a state-of-the-art reinforcement learning algorithm for continuous action settings that is not applicable to discrete action settings. Many important settings involve discrete actions, however, and so here we derive an alternative version of the Soft Actor-Critic algorithm that is applicable to discrete action settings. We then show that, even without any hyperparameter tuning, it is competitive with the tuned model-free state-of-the-art on a selection of games from the Atari suite.","url_abs":"https://arxiv.org/abs/1910.07207v2","url_pdf":"https://arxiv.org/pdf/1910.07207v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"soft-actor-critic-for-discrete-action","repo_url":"https://github.com/p-christ/Deep-Reinforcement-Learning-Algorithms-with-PyTorch","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"soft-actor-critic-for-discrete-action","repo_url":"https://github.com/Bigpig4396/PyTorch-Soft-Actor-Critic-SAC","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"soft-actor-critic-for-discrete-action","repo_url":"https://github.com/Epsilon10/MARL","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"soft-actor-critic-for-discrete-action","repo_url":"https://github.com/ToolManChang/DRLseminar_code_challenge","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"soft-actor-critic-for-discrete-action","repo_url":"https://github.com/ac-93/soft-actor-critic","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":null},{"paper_slug":"soft-actor-critic-for-discrete-action","repo_url":"https://github.com/cindycia/Atari-SAC-Discrete","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"soft-actor-critic-for-discrete-action","repo_url":"https://github.com/keraJLi/rejax","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"soft-actor-critic-for-discrete-action","repo_url":"https://github.com/ku2482/rljax","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"soft-actor-critic-for-discrete-action","repo_url":"https://github.com/ku2482/rltorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"soft-actor-critic-for-discrete-action","repo_url":"https://github.com/ku2482/sac-discrete.pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"soft-actor-critic-for-discrete-action","repo_url":"https://github.com/lingweizhu/Pytorch-MunchausenActorCritic","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"soft-actor-critic-for-discrete-action","repo_url":"https://github.com/mindspore-courses/Deep-Reinforcement-Learning-Algorithms-with-MindSpore","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"mindspore","reach":{"status":"ok"}},{"paper_slug":"soft-actor-critic-for-discrete-action","repo_url":"https://github.com/BY571/SAC_discrete","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"atari-games","task_name":"Atari Games"},{"task_slug":"reinforcement-learning","task_name":"Reinforcement Learning"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"},{"task_slug":"reinforcement-learning-2","task_name":"reinforcement-learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/atari-games-on-atari-2600-alien","task":"Atari Games","dataset":"Atari 2600 Alien","model":"SAC","rank_in_archive_order":47,"of":49,"metrics":{"Score":"216.9"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-amidar","task":"Atari Games","dataset":"Atari 2600 Amidar","model":"SAC","rank_in_archive_order":48,"of":48,"metrics":{"Score":"7.9"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-assault","task":"Atari Games","dataset":"Atari 2600 Assault","model":"SAC","rank_in_archive_order":45,"of":45,"metrics":{"Score":"350"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-asterix","task":"Atari Games","dataset":"Atari 2600 Asterix","model":"SAC","rank_in_archive_order":49,"of":49,"metrics":{"Score":"272"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-battle-zone","task":"Atari Games","dataset":"Atari 2600 Battle Zone","model":"SAC","rank_in_archive_order":46,"of":47,"metrics":{"Score":"4386.7"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-beam-rider","task":"Atari Games","dataset":"Atari 2600 Beam Rider","model":"SAC","rank_in_archive_order":48,"of":49,"metrics":{"Score":"432.1"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-breakout","task":"Atari Games","dataset":"Atari 2600 Breakout","model":"SAC","rank_in_archive_order":58,"of":58,"metrics":{"Score":"0.7"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-crazy-climber","task":"Atari Games","dataset":"Atari 2600 Crazy Climber","model":"SAC","rank_in_archive_order":48,"of":49,"metrics":{"Score":"3668.7"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-enduro","task":"Atari Games","dataset":"Atari 2600 Enduro","model":"SAC","rank_in_archive_order":44,"of":48,"metrics":{"Score":"0.8"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-freeway","task":"Atari Games","dataset":"Atari 2600 Freeway","model":"SAC","rank_in_archive_order":51,"of":59,"metrics":{"Score":"4.4"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-frostbite","task":"Atari Games","dataset":"Atari 2600 Frostbite","model":"SAC","rank_in_archive_order":53,"of":53,"metrics":{"Score":"59.4"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-james-bond","task":"Atari Games","dataset":"Atari 2600 James Bond","model":"SAC","rank_in_archive_order":44,"of":45,"metrics":{"Score":"68.3"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-kangaroo","task":"Atari Games","dataset":"Atari 2600 Kangaroo","model":"SAC","rank_in_archive_order":46,"of":47,"metrics":{"Score":"29.3"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-ms-pacman","task":"Atari Games","dataset":"Atari 2600 Ms. Pacman","model":"SAC","rank_in_archive_order":45,"of":47,"metrics":{"Score":"690.9"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-pong","task":"Atari Games","dataset":"Atari 2600 Pong","model":"SAC","rank_in_archive_order":52,"of":52,"metrics":{"Score":"-20.98"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-qbert","task":"Atari Games","dataset":"Atari 2600 Q*Bert","model":"SAC","rank_in_archive_order":53,"of":57,"metrics":{"Score":"280.5"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-road-runner","task":"Atari Games","dataset":"Atari 2600 Road Runner","model":"SAC","rank_in_archive_order":42,"of":44,"metrics":{"Score":"305.3"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-seaquest","task":"Atari Games","dataset":"Atari 2600 Seaquest","model":"SAC","rank_in_archive_order":55,"of":57,"metrics":{"Score":"211.6"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-space-invaders","task":"Atari Games","dataset":"Atari 2600 Space Invaders","model":"SAC","rank_in_archive_order":54,"of":55,"metrics":{"Score":"160.8"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-up-and-down","task":"Atari Games","dataset":"Atari 2600 Up and Down","model":"SAC","rank_in_archive_order":44,"of":44,"metrics":{"Score":"250.7"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1910.07207","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1910.07207"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ku2482/rljax","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ToolManChang/DRLseminar_code_challenge","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/BY571/SAC_discrete","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mindspore-courses/Deep-Reinforcement-Learning-Algorithms-with-MindSpore","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lingweizhu/Pytorch-MunchausenActorCritic","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ac-93/soft-actor-critic","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Epsilon10/MARL","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/cindycia/Atari-SAC-Discrete","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/keraJLi/rejax","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ku2482/rltorch","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ku2482/sac-discrete.pytorch","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/p-christ/Deep-Reinforcement-Learning-Algorithms-with-PyTorch","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Bigpig4396/PyTorch-Soft-Actor-Critic-SAC","reach":{"status":"ok"}}],"summary":{"ran_honours":2,"ran_violates":1,"ran":1,"unverified":14},"by_repo_kind":{"official":{"samples":2,"ran":0,"repositories":1},"listed":{"samples":13,"ran":1,"repositories":3}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":3,"samples":[{"code_sha256_prefix":"e5cbade112f15d84","entry":"process_action","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"e5cbade112f15d84"}},{"code_sha256_prefix":"22e993f58f0d200b","entry":"process_observation","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"22e993f58f0d200b"}},{"code_sha256_prefix":"1deaa4d231ac8f53","entry":"process_reward","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"1deaa4d231ac8f53"}},{"code_sha256_prefix":"3afd917c65b80e08","entry":"to_batch","repo":"cindycia/Atari-SAC-Discrete","repo_kind":"listed","path":"rltorch/agent/utils.py","file_url":"https://github.com/cindycia/Atari-SAC-Discrete/blob/HEAD/rltorch/agent/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3afd917c65b80e08"}},{"code_sha256_prefix":"159ca677b48225e4","entry":"EpsilonGreedyPolicy","repo":"keraJLi/rejax","repo_kind":"listed","path":"src/rejax/networks.py","file_url":"https://github.com/keraJLi/rejax/blob/HEAD/src/rejax/networks.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"159ca677b48225e4"}},{"code_sha256_prefix":"6fe3b3b62f1d627e","entry":"EpsilonGreedyPolicy","repo":"keraJLi/rejax","repo_kind":"listed","path":"src/rejax/algos/iqn.py","file_url":"https://github.com/keraJLi/rejax/blob/HEAD/src/rejax/algos/iqn.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6fe3b3b62f1d627e"}},{"code_sha256_prefix":"c010a6ac8668e20d","entry":"create_dqn_base","repo":"cindycia/Atari-SAC-Discrete","repo_kind":"listed","path":"rltorch/network/builder.py","file_url":"https://github.com/cindycia/Atari-SAC-Discrete/blob/HEAD/rltorch/network/builder.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c010a6ac8668e20d"}},{"code_sha256_prefix":"95498252faaec791","entry":"create_linear_network","repo":"cindycia/Atari-SAC-Discrete","repo_kind":"listed","path":"rltorch/network/builder.py","file_url":"https://github.com/cindycia/Atari-SAC-Discrete/blob/HEAD/rltorch/network/builder.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"95498252faaec791"}},{"code_sha256_prefix":"098ce6dc4eff278c","entry":"evaluate_single","repo":"keraJLi/rejax","repo_kind":"listed","path":"src/rejax/evaluate.py","file_url":"https://github.com/keraJLi/rejax/blob/HEAD/src/rejax/evaluate.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"098ce6dc4eff278c"}},{"code_sha256_prefix":"a6efcc2f264c05c0","entry":"initialize_weights","repo":"cindycia/Atari-SAC-Discrete","repo_kind":"listed","path":"rltorch/network/builder.py","file_url":"https://github.com/cindycia/Atari-SAC-Discrete/blob/HEAD/rltorch/network/builder.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a6efcc2f264c05c0"}},{"code_sha256_prefix":"4bc698f17fe89d85","entry":"make_atari","repo":"cindycia/Atari-SAC-Discrete","repo_kind":"listed","path":"rltorch/env/atari_wrappers.py","file_url":"https://github.com/cindycia/Atari-SAC-Discrete/blob/HEAD/rltorch/env/atari_wrappers.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4bc698f17fe89d85"}},{"code_sha256_prefix":"5518bf3701f7ad8c","entry":"make_atari_game","repo":"p-christ/Deep-Reinforcement-Learning-Algorithms-with-PyTorch","repo_kind":"official","path":"environments/Atari_Environment.py","file_url":"https://github.com/p-christ/Deep-Reinforcement-Learning-Algorithms-with-PyTorch/blob/HEAD/environments/Atari_Environment.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5518bf3701f7ad8c"}},{"code_sha256_prefix":"20aa2a394b00c4d7","entry":"make_pytorch_env","repo":"cindycia/Atari-SAC-Discrete","repo_kind":"listed","path":"rltorch/env/atari_wrappers.py","file_url":"https://github.com/cindycia/Atari-SAC-Discrete/blob/HEAD/rltorch/env/atari_wrappers.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"20aa2a394b00c4d7"}},{"code_sha256_prefix":"4710cb8bbbee96dd","entry":"make_pytorch_env","repo":"ku2482/rltorch","repo_kind":"listed","path":"rltorch/env/atari_wrappers.py","file_url":"https://github.com/ku2482/rltorch/blob/HEAD/rltorch/env/atari_wrappers.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4710cb8bbbee96dd"}},{"code_sha256_prefix":"e62f6ecb2ba1c892","entry":"register_init","repo":"keraJLi/rejax","repo_kind":"listed","path":"src/rejax/algos/algorithm.py","file_url":"https://github.com/keraJLi/rejax/blob/HEAD/src/rejax/algos/algorithm.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e62f6ecb2ba1c892"}},{"code_sha256_prefix":"ea614117ba22d252","entry":"update_rms","repo":"keraJLi/rejax","repo_kind":"listed","path":"src/rejax/algos/mixins.py","file_url":"https://github.com/keraJLi/rejax/blob/HEAD/src/rejax/algos/mixins.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ea614117ba22d252"}},{"code_sha256_prefix":"f22df3c0c0b99757","entry":"wrap_deepmind","repo":"p-christ/Deep-Reinforcement-Learning-Algorithms-with-PyTorch","repo_kind":"official","path":"environments/Atari_Environment.py","file_url":"https://github.com/p-christ/Deep-Reinforcement-Learning-Algorithms-with-PyTorch/blob/HEAD/environments/Atari_Environment.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f22df3c0c0b99757"}},{"code_sha256_prefix":"4d7ad6b7f3100923","entry":"wrap_deepmind_pytorch","repo":"cindycia/Atari-SAC-Discrete","repo_kind":"listed","path":"rltorch/env/atari_wrappers.py","file_url":"https://github.com/cindycia/Atari-SAC-Discrete/blob/HEAD/rltorch/env/atari_wrappers.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4d7ad6b7f3100923"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}