{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/detailed-2d-3d-joint-representation-for-human","title":"Detailed 2D-3D Joint Representation for Human-Object Interaction","arxiv_id":"2004.08154","date":"2020-04-17","proceeding":"CVPR 2020 6","authors":["Yong-Lu Li","Xinpeng Liu","Han Lu","Shiyi Wang","Junqi Liu","Jiefeng Li","Cewu Lu"],"abstract":"Human-Object Interaction (HOI) detection lies at the core of action understanding. Besides 2D information such as human/object appearance and locations, 3D pose is also usually utilized in HOI learning since its view-independence. However, rough 3D body joints just carry sparse body information and are not sufficient to understand complex interactions. Thus, we need detailed 3D body shape to go further. Meanwhile, the interacted object in 3D is also not fully studied in HOI learning. In light of these, we propose a detailed 2D-3D joint representation learning method. First, we utilize the single-view human body capture method to obtain detailed 3D body, face and hand shapes. Next, we estimate the 3D object location and size with reference to the 2D human-object spatial configuration and object category priors. Finally, a joint learning framework and cross-modal consistency tasks are proposed to learn the joint HOI representation. To better evaluate the 2D ambiguity processing capacity of models, we propose a new benchmark named Ambiguous-HOI consisting of hard ambiguous images. Extensive experiments in large-scale HOI benchmark and Ambiguous-HOI show impressive effectiveness of our method. Code and data are available at https://github.com/DirtyHarryLYL/DJ-RN.","url_abs":"https://arxiv.org/abs/2004.08154v2","url_pdf":"https://arxiv.org/pdf/2004.08154v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"detailed-2d-3d-joint-representation-for-human","repo_url":"https://github.com/DirtyHarryLYL/DJ-RN","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"action-understanding","task_name":"Action Understanding"},{"task_slug":"human-object-interaction-detection","task_name":"Human-Object Interaction Detection"},{"task_slug":"micro-action-recognition","task_name":"Micro-Action Recognition"},{"task_slug":"object","task_name":"Object"},{"task_slug":"representation-learning","task_name":"Representation Learning"}],"methods":[],"datasets_introduced":[{"slug":"ambiguous-hoi","name":"Ambiguous-HOI","full_name":""}],"methods_introduced":[],"results":[{"leaderboard":"/sota/human-object-interaction-detection-on","task":"Human-Object Interaction Detection","dataset":"Ambiguious-HOI","model":"DJ-RN","rank_in_archive_order":1,"of":2,"metrics":{"mAP":"10.37"},"uses_additional_data":false},{"leaderboard":"/sota/human-object-interaction-detection-on-hico","task":"Human-Object Interaction Detection","dataset":"HICO-DET","model":"DJ-RN","rank_in_archive_order":46,"of":55,"metrics":{"mAP":"21.34"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2004.08154","atlas_url":"https://app.syntology.ai/?focus=2004.08154","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.08154"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/DirtyHarryLYL/DJ-RN","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":8,"unverified":4},"by_repo_kind":{"official":{"samples":12,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"4c4b9fe1c3921060","entry":"Augmented_box","repo":"DirtyHarryLYL/DJ-RN","repo_kind":"official","path":"lib/ult/ult.py","file_url":"https://github.com/DirtyHarryLYL/DJ-RN/blob/HEAD/lib/ult/ult.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4c4b9fe1c3921060"}},{"code_sha256_prefix":"a945b93e1e20006e","entry":"bb_IOU","repo":"DirtyHarryLYL/DJ-RN","repo_kind":"official","path":"lib/ult/ult.py","file_url":"https://github.com/DirtyHarryLYL/DJ-RN/blob/HEAD/lib/ult/ult.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a945b93e1e20006e"}},{"code_sha256_prefix":"e7e5af90a9931839","entry":"bbox_trans","repo":"DirtyHarryLYL/DJ-RN","repo_kind":"official","path":"lib/ult/ult.py","file_url":"https://github.com/DirtyHarryLYL/DJ-RN/blob/HEAD/lib/ult/ult.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e7e5af90a9931839"}},{"code_sha256_prefix":"77951edeecdb7708","entry":"calc_hit","repo":"DirtyHarryLYL/DJ-RN","repo_kind":"official","path":"-Results/Evaluate_ambiguous.py","file_url":"https://github.com/DirtyHarryLYL/DJ-RN/blob/HEAD/-Results/Evaluate_ambiguous.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"77951edeecdb7708"}},{"code_sha256_prefix":"be7025a18e9a78f8","entry":"calc_hit","repo":"DirtyHarryLYL/DJ-RN","repo_kind":"official","path":"-Results/HICO_DET_utils.py","file_url":"https://github.com/DirtyHarryLYL/DJ-RN/blob/HEAD/-Results/HICO_DET_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"be7025a18e9a78f8"}},{"code_sha256_prefix":"9c36467d91f24d5d","entry":"getSigmoid","repo":"DirtyHarryLYL/DJ-RN","repo_kind":"official","path":"-Results/Evaluate_ambiguous.py","file_url":"https://github.com/DirtyHarryLYL/DJ-RN/blob/HEAD/-Results/Evaluate_ambiguous.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9c36467d91f24d5d"}},{"code_sha256_prefix":"434bd4b19837432c","entry":"iou","repo":"DirtyHarryLYL/DJ-RN","repo_kind":"official","path":"-Results/Evaluate_ambiguous.py","file_url":"https://github.com/DirtyHarryLYL/DJ-RN/blob/HEAD/-Results/Evaluate_ambiguous.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"434bd4b19837432c"}},{"code_sha256_prefix":"082c2f358be049b5","entry":"iou","repo":"DirtyHarryLYL/DJ-RN","repo_kind":"official","path":"-Results/HICO_DET_utils.py","file_url":"https://github.com/DirtyHarryLYL/DJ-RN/blob/HEAD/-Results/HICO_DET_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"082c2f358be049b5"}},{"code_sha256_prefix":"c83a0fe665671edc","entry":"draw_bounding_boxes","repo":"DirtyHarryLYL/DJ-RN","repo_kind":"official","path":"lib/ult/visualization.py","file_url":"https://github.com/DirtyHarryLYL/DJ-RN/blob/HEAD/lib/ult/visualization.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c83a0fe665671edc"}},{"code_sha256_prefix":"6e77405876116b07","entry":"draw_bounding_boxes_HOI","repo":"DirtyHarryLYL/DJ-RN","repo_kind":"official","path":"lib/ult/visualization.py","file_url":"https://github.com/DirtyHarryLYL/DJ-RN/blob/HEAD/lib/ult/visualization.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6e77405876116b07"}},{"code_sha256_prefix":"b5831b48920061b2","entry":"draw_bounding_boxes_PVP","repo":"DirtyHarryLYL/DJ-RN","repo_kind":"official","path":"lib/ult/visualization.py","file_url":"https://github.com/DirtyHarryLYL/DJ-RN/blob/HEAD/lib/ult/visualization.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b5831b48920061b2"}},{"code_sha256_prefix":"f4d4327d825f56f5","entry":"placeholder_inputs","repo":"DirtyHarryLYL/DJ-RN","repo_kind":"official","path":"Feature_extraction/pointnet_hico.py","file_url":"https://github.com/DirtyHarryLYL/DJ-RN/blob/HEAD/Feature_extraction/pointnet_hico.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f4d4327d825f56f5"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}