{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/continuous-mdp-homomorphisms-and-homomorphic","title":"Continuous MDP Homomorphisms and Homomorphic Policy Gradient","arxiv_id":"2209.07364","date":"2022-09-15","proceeding":null,"authors":["Sahand Rezaei-Shoshtari","Rosie Zhao","Prakash Panangaden","David Meger","Doina Precup"],"abstract":"Abstraction has been widely studied as a way to improve the efficiency and generalization of reinforcement learning algorithms. In this paper, we study abstraction in the continuous-control setting. We extend the definition of MDP homomorphisms to encompass continuous actions in continuous state spaces. We derive a policy gradient theorem on the abstract MDP, which allows us to leverage approximate symmetries of the environment for policy optimization. Based on this theorem, we propose an actor-critic algorithm that is able to learn the policy and the MDP homomorphism map simultaneously, using the lax bisimulation metric. We demonstrate the effectiveness of our method on benchmark tasks in the DeepMind Control Suite. Our method's ability to utilize MDP homomorphisms for representation learning leads to improved performance when learning from pixel observations.","url_abs":"https://arxiv.org/abs/2209.07364v1","url_pdf":"https://arxiv.org/pdf/2209.07364v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"continuous-mdp-homomorphisms-and-homomorphic","repo_url":"https://github.com/sahandrez/homomorphic_policy_gradient","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"continuous-control","task_name":"Continuous Control"},{"task_slug":"policy-gradient-methods","task_name":"Policy Gradient Methods"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"},{"task_slug":"representation-learning","task_name":"Representation Learning"},{"task_slug":"continuous-control","task_name":"continuous-control"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2209.07364","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2209.07364"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/sahandrez/homomorphic_policy_gradient","reach":null}],"summary":{"ran":11,"unverified":1},"by_repo_kind":{"official":{"samples":12,"ran":11,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":12,"samples":[{"code_sha256_prefix":"5451b485b901586e","entry":"ActionEncoder","repo":"sahandrez/homomorphic_policy_gradient","repo_kind":"official","path":"agents/hpg.py","file_url":"https://github.com/sahandrez/homomorphic_policy_gradient/blob/HEAD/agents/hpg.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5451b485b901586e"}},{"code_sha256_prefix":"df6df5b49deeb8d9","entry":"DDPGCritic","repo":"sahandrez/homomorphic_policy_gradient","repo_kind":"official","path":"agents/hpg.py","file_url":"https://github.com/sahandrez/homomorphic_policy_gradient/blob/HEAD/agents/hpg.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"df6df5b49deeb8d9"}},{"code_sha256_prefix":"bd056fed8ff26278","entry":"DeterministicActor","repo":"sahandrez/homomorphic_policy_gradient","repo_kind":"official","path":"agents/hpg.py","file_url":"https://github.com/sahandrez/homomorphic_policy_gradient/blob/HEAD/agents/hpg.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bd056fed8ff26278"}},{"code_sha256_prefix":"7aed02dbcbd4346e","entry":"DeterministicTransitionModel","repo":"sahandrez/homomorphic_policy_gradient","repo_kind":"official","path":"agents/hpg.py","file_url":"https://github.com/sahandrez/homomorphic_policy_gradient/blob/HEAD/agents/hpg.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7aed02dbcbd4346e"}},{"code_sha256_prefix":"cf37e3352fb7c4a7","entry":"EnsembleOfProbabilisticTransitionModels","repo":"sahandrez/homomorphic_policy_gradient","repo_kind":"official","path":"agents/hpg.py","file_url":"https://github.com/sahandrez/homomorphic_policy_gradient/blob/HEAD/agents/hpg.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"cf37e3352fb7c4a7"}},{"code_sha256_prefix":"5765ec2739490357","entry":"PixelEncoder","repo":"sahandrez/homomorphic_policy_gradient","repo_kind":"official","path":"agents/hpg.py","file_url":"https://github.com/sahandrez/homomorphic_policy_gradient/blob/HEAD/agents/hpg.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5765ec2739490357"}},{"code_sha256_prefix":"7297a6675055e7ba","entry":"ProbabilisticTransitionModel","repo":"sahandrez/homomorphic_policy_gradient","repo_kind":"official","path":"agents/hpg.py","file_url":"https://github.com/sahandrez/homomorphic_policy_gradient/blob/HEAD/agents/hpg.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7297a6675055e7ba"}},{"code_sha256_prefix":"80aa71815083d4fb","entry":"RandomShiftsAug","repo":"sahandrez/homomorphic_policy_gradient","repo_kind":"official","path":"agents/hpg.py","file_url":"https://github.com/sahandrez/homomorphic_policy_gradient/blob/HEAD/agents/hpg.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"80aa71815083d4fb"}},{"code_sha256_prefix":"56257d35d541a64c","entry":"RewardPredictor","repo":"sahandrez/homomorphic_policy_gradient","repo_kind":"official","path":"agents/hpg.py","file_url":"https://github.com/sahandrez/homomorphic_policy_gradient/blob/HEAD/agents/hpg.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"56257d35d541a64c"}},{"code_sha256_prefix":"82dabdf8f2dac8ea","entry":"StateEncoder","repo":"sahandrez/homomorphic_policy_gradient","repo_kind":"official","path":"agents/hpg.py","file_url":"https://github.com/sahandrez/homomorphic_policy_gradient/blob/HEAD/agents/hpg.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"82dabdf8f2dac8ea"}},{"code_sha256_prefix":"a51ab55975377f26","entry":"make_transition_model","repo":"sahandrez/homomorphic_policy_gradient","repo_kind":"official","path":"agents/hpg.py","file_url":"https://github.com/sahandrez/homomorphic_policy_gradient/blob/HEAD/agents/hpg.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a51ab55975377f26"}},{"code_sha256_prefix":"452f6e823668f286","entry":"HPGAgent","repo":"sahandrez/homomorphic_policy_gradient","repo_kind":"official","path":"agents/hpg.py","file_url":"https://github.com/sahandrez/homomorphic_policy_gradient/blob/HEAD/agents/hpg.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"452f6e823668f286"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}