{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/reconstructing-hands-in-3d-with-transformers","title":"Reconstructing Hands in 3D with Transformers","arxiv_id":"2312.05251","date":"2023-12-08","proceeding":"CVPR 2024 1","authors":["Georgios Pavlakos","Dandan Shan","Ilija Radosavovic","Angjoo Kanazawa","David Fouhey","Jitendra Malik"],"abstract":"We present an approach that can reconstruct hands in 3D from monocular input. Our approach for Hand Mesh Recovery, HaMeR, follows a fully transformer-based architecture and can analyze hands with significantly increased accuracy and robustness compared to previous work. The key to HaMeR's success lies in scaling up both the data used for training and the capacity of the deep network for hand reconstruction. For training data, we combine multiple datasets that contain 2D or 3D hand annotations. For the deep model, we use a large scale Vision Transformer architecture. Our final model consistently outperforms the previous baselines on popular 3D hand pose benchmarks. To further evaluate the effect of our design in non-controlled settings, we annotate existing in-the-wild datasets with 2D hand keypoint annotations. On this newly collected dataset of annotations, HInt, we demonstrate significant improvements over existing baselines. We make our code, data and models available on the project website: https://geopavlakos.github.io/hamer/.","url_abs":"https://arxiv.org/abs/2312.05251v1","url_pdf":"https://arxiv.org/pdf/2312.05251v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"reconstructing-hands-in-3d-with-transformers","repo_url":"https://github.com/geopavlakos/hamer","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"3d-hand-pose-estimation","task_name":"3D Hand Pose Estimation"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"},{"method_slug":"vision-transformer","method_name":"Vision Transformer"}],"datasets_introduced":[{"slug":"hint-hand-interactions-in-the-wild","name":"HInt: Hand Interactions in the wild","full_name":""}],"methods_introduced":[],"results":[{"leaderboard":"/sota/3d-hand-pose-estimation-on-freihand","task":"3D Hand Pose Estimation","dataset":"FreiHAND","model":"HaMeR","rank_in_archive_order":10,"of":33,"metrics":{"PA-F@15mm":"0.990","PA-F@5mm":"0.785","PA-MPJPE":"6.0","PA-MPVPE":"5.7"},"uses_additional_data":false},{"leaderboard":"/sota/3d-hand-pose-estimation-on-hint-hand","task":"3D Hand Pose Estimation","dataset":"HInt: Hand Interactions in the wild","model":"HaMeR*","rank_in_archive_order":2,"of":10,"metrics":{"PCK@0.05 (Ego4D) All":"46.9","PCK@0.05 (Ego4D) Occ":"33.1","PCK@0.05 (Ego4D) Visible":"59.1","PCK@0.05 (New Days) All":"51.6","PCK@0.05 (NewDays) Occ":"33.2","PCK@0.05 (NewDays) Visible":"62.9","PCK@0.05 (VISOR) All":"56.5","PCK@0.05 (VISOR) Occ":"42.6","PCK@0.05 (VISOR) Visible":"66.5"},"uses_additional_data":false},{"leaderboard":"/sota/3d-hand-pose-estimation-on-hint-hand","task":"3D Hand Pose Estimation","dataset":"HInt: Hand Interactions in the wild","model":"HaMeR","rank_in_archive_order":5,"of":10,"metrics":{"PCK@0.05 (Ego4D) All":"38.9","PCK@0.05 (Ego4D) Occ":"23.0","PCK@0.05 (Ego4D) Visible":"52.0","PCK@0.05 (New Days) All":"48.0","PCK@0.05 (NewDays) Occ":"27.2","PCK@0.05 (NewDays) Visible":"60.8","PCK@0.05 (VISOR) All":"43.0","PCK@0.05 (VISOR) Occ":"25.9","PCK@0.05 (VISOR) Visible":"56.6"},"uses_additional_data":false},{"leaderboard":"/sota/3d-hand-pose-estimation-on-ho-3d","task":"3D Hand Pose Estimation","dataset":"HO-3D v2","model":"HaMeR","rank_in_archive_order":4,"of":24,"metrics":{"AUC_J":"0.846","AUC_V":"0.841","F@15mm":"0.980","F@5mm":"0.635","PA-MPJPE (mm)":"7.7","PA-MPVPE":"7.9"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2312.05251","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.05251"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/geopavlakos/hamer","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":6},"by_repo_kind":{"listed":{"samples":6,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"7b0797fbe1ca25e0","entry":"do_augmentation","repo":"geopavlakos/hamer","repo_kind":"listed","path":"hamer/datasets/utils.py","file_url":"https://github.com/geopavlakos/hamer/blob/HEAD/hamer/datasets/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7b0797fbe1ca25e0"}},{"code_sha256_prefix":"9b45eb017f731bb9","entry":"expand","repo":"geopavlakos/hamer","repo_kind":"listed","path":"hamer/datasets/image_dataset.py","file_url":"https://github.com/geopavlakos/hamer/blob/HEAD/hamer/datasets/image_dataset.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9b45eb017f731bb9"}},{"code_sha256_prefix":"c9499dc6b148256e","entry":"expand_to_aspect_ratio","repo":"geopavlakos/hamer","repo_kind":"listed","path":"hamer/datasets/utils.py","file_url":"https://github.com/geopavlakos/hamer/blob/HEAD/hamer/datasets/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c9499dc6b148256e"}},{"code_sha256_prefix":"f1ffc8911891db30","entry":"get_abs_pos","repo":"geopavlakos/hamer","repo_kind":"listed","path":"hamer/models/backbones/vit.py","file_url":"https://github.com/geopavlakos/hamer/blob/HEAD/hamer/models/backbones/vit.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f1ffc8911891db30"}},{"code_sha256_prefix":"cee6a53de87f7eaf","entry":"rotate_2d","repo":"geopavlakos/hamer","repo_kind":"listed","path":"hamer/datasets/utils.py","file_url":"https://github.com/geopavlakos/hamer/blob/HEAD/hamer/datasets/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cee6a53de87f7eaf"}},{"code_sha256_prefix":"0de9707ae7397309","entry":"vit","repo":"geopavlakos/hamer","repo_kind":"listed","path":"hamer/models/backbones/vit.py","file_url":"https://github.com/geopavlakos/hamer/blob/HEAD/hamer/models/backbones/vit.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0de9707ae7397309"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}