{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/on-the-utility-of-3d-hand-poses-for-action","title":"On the Utility of 3D Hand Poses for Action Recognition","arxiv_id":"2403.09805","date":"2024-03-14","proceeding":null,"authors":["Md Salman Shamil","Dibyadip Chatterjee","Fadime Sener","Shugao Ma","Angela Yao"],"abstract":"3D hand pose is an underexplored modality for action recognition. Poses are compact yet informative and can greatly benefit applications with limited compute budgets. However, poses alone offer an incomplete understanding of actions, as they cannot fully capture objects and environments with which humans interact. We propose HandFormer, a novel multimodal transformer, to efficiently model hand-object interactions. HandFormer combines 3D hand poses at a high temporal resolution for fine-grained motion modeling with sparsely sampled RGB frames for encoding scene semantics. Observing the unique characteristics of hand poses, we temporally factorize hand modeling and represent each joint by its short-term trajectories. This factorized pose representation combined with sparse RGB samples is remarkably efficient and highly accurate. Unimodal HandFormer with only hand poses outperforms existing skeleton-based methods at 5x fewer FLOPs. With RGB, we achieve new state-of-the-art performance on Assembly101 and H2O with significant improvements in egocentric action recognition.","url_abs":"https://arxiv.org/abs/2403.09805v2","url_pdf":"https://arxiv.org/pdf/2403.09805v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"on-the-utility-of-3d-hand-poses-for-action","repo_url":"https://github.com/s-shamil/HandFormer","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"3d-human-action-recognition","task_name":"3D Action Recognition"},{"task_slug":"action-recognition-in-videos","task_name":"Action Recognition"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/3d-action-recognition-on-assembly101","task":"3D Action Recognition","dataset":"Assembly101","model":"HandFormer-B/21","rank_in_archive_order":1,"of":7,"metrics":{"Actions Top-1":"41.06","Object Top-1":"51.17","Verbs Top-1":"69.23"},"uses_additional_data":false},{"leaderboard":"/sota/action-recognition-on-h2o-2-hands-and-objects","task":"Action Recognition","dataset":"H2O  (2 Hands and Objects)","model":"HandFormer-B/21x8","rank_in_archive_order":1,"of":11,"metrics":{"Actions Top-1":"93.39","Hand Pose":"3D","Object Label":"No","Object Pose":"No","RGB":"Yes"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2403.09805","atlas_url":"https://app.syntology.ai/?focus=2403.09805","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.09805"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/s-shamil/HandFormer","reach":null}],"summary":{"ran":5,"ran_draft_wrong":2,"ran_violates":1,"unverified":5},"by_repo_kind":{"official":{"samples":13,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"ea63108588c955b8","entry":"PositionalEncoding","repo":"s-shamil/HandFormer","repo_kind":"official","path":"HandFormer/models/hf_pose.py","file_url":"https://github.com/s-shamil/HandFormer/blob/HEAD/HandFormer/models/hf_pose.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ea63108588c955b8"}},{"code_sha256_prefix":"bb9b0666242adf98","entry":"TemporalConv","repo":"s-shamil/HandFormer","repo_kind":"official","path":"HandFormer/models/hf_pose.py","file_url":"https://github.com/s-shamil/HandFormer/blob/HEAD/HandFormer/models/hf_pose.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bb9b0666242adf98"}},{"code_sha256_prefix":"5bbd4da441a48e72","entry":"TransformerEncoder","repo":"s-shamil/HandFormer","repo_kind":"official","path":"HandFormer/models/hf_pose.py","file_url":"https://github.com/s-shamil/HandFormer/blob/HEAD/HandFormer/models/hf_pose.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5bbd4da441a48e72"}},{"code_sha256_prefix":"f8f7be5c3f791c68","entry":"TransformerEncoderLayer","repo":"s-shamil/HandFormer","repo_kind":"official","path":"HandFormer/models/hf_pose.py","file_url":"https://github.com/s-shamil/HandFormer/blob/HEAD/HandFormer/models/hf_pose.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f8f7be5c3f791c68"}},{"code_sha256_prefix":"12e7bd98c7ee2907","entry":"Transformer_Encoder","repo":"s-shamil/HandFormer","repo_kind":"official","path":"HandFormer/models/hf_pose.py","file_url":"https://github.com/s-shamil/HandFormer/blob/HEAD/HandFormer/models/hf_pose.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"12e7bd98c7ee2907"}},{"code_sha256_prefix":"df625bd27a59d895","entry":"_get_clones","repo":"s-shamil/HandFormer","repo_kind":"official","path":"HandFormer/models/hf_pose.py","file_url":"https://github.com/s-shamil/HandFormer/blob/HEAD/HandFormer/models/hf_pose.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"df625bd27a59d895"}},{"code_sha256_prefix":"5426f077b824bbb4","entry":"activation_factory","repo":"s-shamil/HandFormer","repo_kind":"official","path":"HandFormer/models/hf_pose.py","file_url":"https://github.com/s-shamil/HandFormer/blob/HEAD/HandFormer/models/hf_pose.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5426f077b824bbb4"}},{"code_sha256_prefix":"61e29634518807c2","entry":"normalize_tensor","repo":"s-shamil/HandFormer","repo_kind":"official","path":"HandFormer/models/hf_pose.py","file_url":"https://github.com/s-shamil/HandFormer/blob/HEAD/HandFormer/models/hf_pose.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"61e29634518807c2"}},{"code_sha256_prefix":"619c6b2598ece3bf","entry":"HF_Pose","repo":"s-shamil/HandFormer","repo_kind":"official","path":"HandFormer/models/hf_pose.py","file_url":"https://github.com/s-shamil/HandFormer/blob/HEAD/HandFormer/models/hf_pose.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"619c6b2598ece3bf"}},{"code_sha256_prefix":"0ed7906dcdd465df","entry":"MicroactionEncoder","repo":"s-shamil/HandFormer","repo_kind":"official","path":"HandFormer/models/hf_pose.py","file_url":"https://github.com/s-shamil/HandFormer/blob/HEAD/HandFormer/models/hf_pose.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0ed7906dcdd465df"}},{"code_sha256_prefix":"e5491621abfadb10","entry":"Trajectory_SelfAttention","repo":"s-shamil/HandFormer","repo_kind":"official","path":"HandFormer/models/hf_pose.py","file_url":"https://github.com/s-shamil/HandFormer/blob/HEAD/HandFormer/models/hf_pose.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e5491621abfadb10"}},{"code_sha256_prefix":"b9201db3a21104d5","entry":"Unimodal_TF","repo":"s-shamil/HandFormer","repo_kind":"official","path":"HandFormer/models/hf_pose.py","file_url":"https://github.com/s-shamil/HandFormer/blob/HEAD/HandFormer/models/hf_pose.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b9201db3a21104d5"}},{"code_sha256_prefix":"5870cc705fd8d62e","entry":"fc_init","repo":"s-shamil/HandFormer","repo_kind":"official","path":"HandFormer/models/hf_pose.py","file_url":"https://github.com/s-shamil/HandFormer/blob/HEAD/HandFormer/models/hf_pose.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5870cc705fd8d62e"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}