{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/unsupervised-learning-of-object-keypoints-for","title":"Unsupervised Learning of Object Keypoints for Perception and Control","arxiv_id":"1906.11883","date":"2019-06-19","proceeding":"NeurIPS 2019 12","authors":["Tejas Kulkarni","Ankush Gupta","Catalin Ionescu","Sebastian Borgeaud","Malcolm Reynolds","Andrew Zisserman","Volodymyr Mnih"],"abstract":"The study of object representations in computer vision has primarily focused on developing representations that are useful for image classification, object detection, or semantic segmentation as downstream tasks. In this work we aim to learn object representations that are useful for control and reinforcement learning (RL). To this end, we introduce Transporter, a neural network architecture for discovering concise geometric object representations in terms of keypoints or image-space coordinates. Our method learns from raw video frames in a fully unsupervised manner, by transporting learnt image features between video frames using a keypoint bottleneck. The discovered keypoints track objects and object parts across long time-horizons more accurately than recent similar methods. Furthermore, consistent long-term tracking enables two notable results in control domains -- (1) using the keypoint co-ordinates and corresponding image features as inputs enables highly sample-efficient reinforcement learning; (2) learning to explore by controlling keypoint locations drastically reduces the search space, enabling deep exploration (leading to states unreachable through random action exploration) without any extrinsic rewards.","url_abs":"https://arxiv.org/abs/1906.11883v2","url_pdf":"https://arxiv.org/pdf/1906.11883v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"unsupervised-learning-of-object-keypoints-for","repo_url":"https://github.com/deepmind/deepmind-research","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"tf","reach":null},{"paper_slug":"unsupervised-learning-of-object-keypoints-for","repo_url":"https://github.com/DerrickGXD/unsupervised-keypoints","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"unsupervised-learning-of-object-keypoints-for","repo_url":"https://github.com/DuaneNielsen/keypoints","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"unsupervised-learning-of-object-keypoints-for","repo_url":"https://github.com/deepmind/deepmind-research/tree/master/transporter","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":null},{"paper_slug":"unsupervised-learning-of-object-keypoints-for","repo_url":"https://github.com/ethanluoyc/transporter-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"unsupervised-learning-of-object-keypoints-for","repo_url":"https://github.com/pairlab/v-cdn","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"3d-human-action-recognition","task_name":"3D Action Recognition"},{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"object","task_name":"Object"},{"task_slug":"object-detection","task_name":"Object Detection"},{"task_slug":"reinforcement-learning","task_name":"Reinforcement Learning"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"image-classification","task_name":"image-classification"},{"task_slug":"object-detection-1","task_name":"object-detection"},{"task_slug":"reinforcement-learning-2","task_name":"reinforcement-learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1906.11883","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1906.11883"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/deepmind/deepmind-research","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/DerrickGXD/unsupervised-keypoints","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ethanluoyc/transporter-pytorch","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pairlab/v-cdn","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/DuaneNielsen/keypoints","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/deepmind/deepmind-research/tree/master/transporter","reach":null}],"summary":{"ran_fixture":2,"ran_draft_wrong":2,"unverified":7},"by_repo_kind":{"listed":{"samples":11,"ran":4,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"b23a47ea77508e05","entry":"channels_to_frame","repo":"DerrickGXD/unsupervised-keypoints","repo_kind":"listed","path":"TrainDataset.py","file_url":"https://github.com/DerrickGXD/unsupervised-keypoints/blob/HEAD/TrainDataset.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b23a47ea77508e05"}},{"code_sha256_prefix":"6cd6f0b3a526afba","entry":"get_gaussian_mean","repo":"DerrickGXD/unsupervised-keypoints","repo_kind":"listed","path":"Models.py","file_url":"https://github.com/DerrickGXD/unsupervised-keypoints/blob/HEAD/Models.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6cd6f0b3a526afba"}},{"code_sha256_prefix":"cd62068519eb2f35","entry":"loss_alex","repo":"DerrickGXD/unsupervised-keypoints","repo_kind":"listed","path":"Models.py","file_url":"https://github.com/DerrickGXD/unsupervised-keypoints/blob/HEAD/Models.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"cd62068519eb2f35"}},{"code_sha256_prefix":"6c2bf41ef195a80a","entry":"rgba_to_rgb","repo":"DerrickGXD/unsupervised-keypoints","repo_kind":"listed","path":"Models.py","file_url":"https://github.com/DerrickGXD/unsupervised-keypoints/blob/HEAD/Models.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6c2bf41ef195a80a"}},{"code_sha256_prefix":"296edbfe5d4adb67","entry":"config","repo":"DuaneNielsen/keypoints","repo_kind":"listed","path":"keypoints/config.py","file_url":"https://github.com/DuaneNielsen/keypoints/blob/HEAD/keypoints/config.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"296edbfe5d4adb67"}},{"code_sha256_prefix":"c4eadb0bceb3178a","entry":"isPD","repo":"DuaneNielsen/keypoints","repo_kind":"listed","path":"keypoints/higgham.py","file_url":"https://github.com/DuaneNielsen/keypoints/blob/HEAD/keypoints/higgham.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c4eadb0bceb3178a"}},{"code_sha256_prefix":"c7c19dfcdf23b96f","entry":"marginal_logsoftmax","repo":"DuaneNielsen/keypoints","repo_kind":"listed","path":"keypoints/models/functional.py","file_url":"https://github.com/DuaneNielsen/keypoints/blob/HEAD/keypoints/models/functional.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c7c19dfcdf23b96f"}},{"code_sha256_prefix":"dc04488a5d9bd28b","entry":"marginal_softmax","repo":"DuaneNielsen/keypoints","repo_kind":"listed","path":"keypoints/models/functional.py","file_url":"https://github.com/DuaneNielsen/keypoints/blob/HEAD/keypoints/models/functional.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dc04488a5d9bd28b"}},{"code_sha256_prefix":"92afecac079e4329","entry":"np_isPD","repo":"DuaneNielsen/keypoints","repo_kind":"listed","path":"keypoints/higgham.py","file_url":"https://github.com/DuaneNielsen/keypoints/blob/HEAD/keypoints/higgham.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"92afecac079e4329"}},{"code_sha256_prefix":"4902b87f9d94f562","entry":"np_nearestPD","repo":"DuaneNielsen/keypoints","repo_kind":"listed","path":"keypoints/higgham.py","file_url":"https://github.com/DuaneNielsen/keypoints/blob/HEAD/keypoints/higgham.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4902b87f9d94f562"}},{"code_sha256_prefix":"b80d8ae1fc2741ec","entry":"prob_to_keypoints","repo":"DuaneNielsen/keypoints","repo_kind":"listed","path":"keypoints/models/functional.py","file_url":"https://github.com/DuaneNielsen/keypoints/blob/HEAD/keypoints/models/functional.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b80d8ae1fc2741ec"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}