{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/learning-to-identify-critical-states-for","title":"Learning to Identify Critical States for Reinforcement Learning from Videos","arxiv_id":"2308.07795","date":"2023-08-15","proceeding":"ICCV 2023 1","authors":["Haozhe Liu","Mingchen Zhuge","Bing Li","Yuhui Wang","Francesco Faccio","Bernard Ghanem","Jürgen Schmidhuber"],"abstract":"Recent work on deep reinforcement learning (DRL) has pointed out that algorithmic information about good policies can be extracted from offline data which lack explicit information about executed actions. For example, videos of humans or robots may convey a lot of implicit information about rewarding action sequences, but a DRL machine that wants to profit from watching such videos must first learn by itself to identify and recognize relevant states/actions/rewards. Without relying on ground-truth annotations, our new method called Deep State Identifier learns to predict returns from episodes encoded as videos. Then it uses a kind of mask-based sensitivity analysis to extract/identify important critical states. Extensive experiments showcase our method's potential for understanding and improving agent behavior. The source code and the generated datasets are available at https://github.com/AI-Initiative-KAUST/VideoRLCS.","url_abs":"https://arxiv.org/abs/2308.07795v1","url_pdf":"https://arxiv.org/pdf/2308.07795v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"learning-to-identify-critical-states-for","repo_url":"https://github.com/ai-initiative-kaust/videorlcs","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"deep-reinforcement-learning","task_name":"Deep Reinforcement Learning"},{"task_slug":"reinforcement-learning","task_name":"Reinforcement Learning"},{"task_slug":"reinforcement-learning-2","task_name":"reinforcement-learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2308.07795","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.07795"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ai-initiative-kaust/videorlcs","reach":{"status":"ok"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/AI-Initiative-KAUST/VideoRLCS","reach":{"status":"ok"}}],"summary":{"ran":8,"ran_fixture":1},"by_repo_kind":{"official":{"samples":9,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":9,"samples":[{"code_sha256_prefix":"b406c4d00f7c7a6e","entry":"Detector","repo":"AI-Initiative-KAUST/VideoRLCS","repo_kind":"official","path":"Models/model.py","file_url":"https://github.com/AI-Initiative-KAUST/VideoRLCS/blob/HEAD/Models/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b406c4d00f7c7a6e"}},{"code_sha256_prefix":"d6169323ec29d473","entry":"Detector_Head","repo":"AI-Initiative-KAUST/VideoRLCS","repo_kind":"official","path":"Models/model.py","file_url":"https://github.com/AI-Initiative-KAUST/VideoRLCS/blob/HEAD/Models/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d6169323ec29d473"}},{"code_sha256_prefix":"a8ec9e9bd6753968","entry":"FeatureExtractor","repo":"AI-Initiative-KAUST/VideoRLCS","repo_kind":"official","path":"Models/model.py","file_url":"https://github.com/AI-Initiative-KAUST/VideoRLCS/blob/HEAD/Models/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a8ec9e9bd6753968"}},{"code_sha256_prefix":"b0f936d4d6ae3b8c","entry":"accuracy","repo":"ai-initiative-kaust/videorlcs","repo_kind":"official","path":"utils.py","file_url":"https://github.com/ai-initiative-kaust/videorlcs/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b0f936d4d6ae3b8c"}},{"code_sha256_prefix":"f0215d69a23699ed","entry":"collate_fn","repo":"ai-initiative-kaust/videorlcs","repo_kind":"official","path":"Attack/dataset.py","file_url":"https://github.com/ai-initiative-kaust/videorlcs/blob/HEAD/Attack/dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f0215d69a23699ed"}},{"code_sha256_prefix":"c94cc1b00df7300a","entry":"collate_fn","repo":"ai-initiative-kaust/videorlcs","repo_kind":"official","path":"Policy_Improvement/dataset.py","file_url":"https://github.com/ai-initiative-kaust/videorlcs/blob/HEAD/Policy_Improvement/dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c94cc1b00df7300a"}},{"code_sha256_prefix":"372e9511cee496ef","entry":"layer_init","repo":"ai-initiative-kaust/videorlcs","repo_kind":"official","path":"Policy_Improvement/Atari/atari_network.py","file_url":"https://github.com/ai-initiative-kaust/videorlcs/blob/HEAD/Policy_Improvement/Atari/atari_network.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"372e9511cee496ef"}},{"code_sha256_prefix":"4962b73d437280f2","entry":"scale_obs","repo":"ai-initiative-kaust/videorlcs","repo_kind":"official","path":"Policy_Improvement/Atari/atari_network.py","file_url":"https://github.com/ai-initiative-kaust/videorlcs/blob/HEAD/Policy_Improvement/Atari/atari_network.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4962b73d437280f2"}},{"code_sha256_prefix":"3cfc0f6223c48293","entry":"tensor_to_np","repo":"ai-initiative-kaust/videorlcs","repo_kind":"official","path":"utils.py","file_url":"https://github.com/ai-initiative-kaust/videorlcs/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3cfc0f6223c48293"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}