{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/uncertainty-aware-state-space-transformer-for","title":"Uncertainty-aware State Space Transformer for Egocentric 3D Hand Trajectory Forecasting","arxiv_id":"2307.08243","date":"2023-07-17","proceeding":"ICCV 2023 1","authors":["Wentao Bao","Lele Chen","Libing Zeng","Zhong Li","Yi Xu","Junsong Yuan","Yu Kong"],"abstract":"Hand trajectory forecasting from egocentric views is crucial for enabling a prompt understanding of human intentions when interacting with AR/VR systems. However, existing methods handle this problem in a 2D image space which is inadequate for 3D real-world applications. In this paper, we set up an egocentric 3D hand trajectory forecasting task that aims to predict hand trajectories in a 3D space from early observed RGB videos in a first-person view. To fulfill this goal, we propose an uncertainty-aware state space Transformer (USST) that takes the merits of the attention mechanism and aleatoric uncertainty within the framework of the classical state-space model. The model can be further enhanced by the velocity constraint and visual prompt tuning (VPT) on large vision transformers. Moreover, we develop an annotation workflow to collect 3D hand trajectories with high quality. Experimental results on H2O and EgoPAT3D datasets demonstrate the superiority of USST for both 2D and 3D trajectory forecasting. The code and datasets are publicly released: https://actionlab-cv.github.io/EgoHandTrajPred.","url_abs":"https://arxiv.org/abs/2307.08243v2","url_pdf":"https://arxiv.org/pdf/2307.08243v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"uncertainty-aware-state-space-transformer-for","repo_url":"https://github.com/oppo-us-research/USST","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"3d-human-pose-tracking","task_name":"3D Human Pose Tracking"},{"task_slug":"trajectory-forecasting","task_name":"Trajectory Forecasting"},{"task_slug":"trajectory-prediction","task_name":"Trajectory Prediction"},{"task_slug":"visual-prompt-tuning","task_name":"Visual Prompt Tuning"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[{"slug":"egopat3d-dt","name":"EgoPAT3D-DT","full_name":"EgoPAT3D-DT"}],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2307.08243","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2307.08243"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/oppo-us-research/USST","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":9,"unverified":4},"by_repo_kind":{"official":{"samples":13,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"dad48a20c072ff6e","entry":"denormalize_traj","repo":"oppo-us-research/USST","repo_kind":"official","path":"src/H2OLoader.py","file_url":"https://github.com/oppo-us-research/USST/blob/HEAD/src/H2OLoader.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"dad48a20c072ff6e"}},{"code_sha256_prefix":"681c38dc9a4a43ce","entry":"draw_uv_points","repo":"oppo-us-research/USST","repo_kind":"official","path":"src/utils_vis.py","file_url":"https://github.com/oppo-us-research/USST/blob/HEAD/src/utils_vis.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"681c38dc9a4a43ce"}},{"code_sha256_prefix":"a0bee2e8ddb51a58","entry":"get_depth_anchors","repo":"oppo-us-research/USST","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/oppo-us-research/USST/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a0bee2e8ddb51a58"}},{"code_sha256_prefix":"1df9d391050a4608","entry":"get_optimizer","repo":"oppo-us-research/USST","repo_kind":"official","path":"src/optimizers.py","file_url":"https://github.com/oppo-us-research/USST/blob/HEAD/src/optimizers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1df9d391050a4608"}},{"code_sha256_prefix":"6652cb820c60c46b","entry":"get_scheduler","repo":"oppo-us-research/USST","repo_kind":"official","path":"src/optimizers.py","file_url":"https://github.com/oppo-us-research/USST/blob/HEAD/src/optimizers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6652cb820c60c46b"}},{"code_sha256_prefix":"6039e11b8c330d4d","entry":"load_checkpoint","repo":"oppo-us-research/USST","repo_kind":"official","path":"src/utils_io.py","file_url":"https://github.com/oppo-us-research/USST/blob/HEAD/src/utils_io.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6039e11b8c330d4d"}},{"code_sha256_prefix":"8da2effca700d6c7","entry":"normalize_traj","repo":"oppo-us-research/USST","repo_kind":"official","path":"src/H2OLoader.py","file_url":"https://github.com/oppo-us-research/USST/blob/HEAD/src/H2OLoader.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8da2effca700d6c7"}},{"code_sha256_prefix":"4b07133043aed642","entry":"send_to_gpu","repo":"oppo-us-research/USST","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/oppo-us-research/USST/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4b07133043aed642"}},{"code_sha256_prefix":"9b2d18e6c9ff858a","entry":"world_to_camera","repo":"oppo-us-research/USST","repo_kind":"official","path":"src/H2OLoader.py","file_url":"https://github.com/oppo-us-research/USST/blob/HEAD/src/H2OLoader.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9b2d18e6c9ff858a"}},{"code_sha256_prefix":"92f4580f0daea9e6","entry":"build_dataloaders","repo":"oppo-us-research/USST","repo_kind":"official","path":"src/EgoPAT3DLoader.py","file_url":"https://github.com/oppo-us-research/USST/blob/HEAD/src/EgoPAT3DLoader.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"92f4580f0daea9e6"}},{"code_sha256_prefix":"aa4989cb736e5e1f","entry":"gather_eval_results","repo":"oppo-us-research/USST","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/oppo-us-research/USST/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"aa4989cb736e5e1f"}},{"code_sha256_prefix":"169badeb8cb30b4a","entry":"parse_configs","repo":"oppo-us-research/USST","repo_kind":"official","path":"src/config.py","file_url":"https://github.com/oppo-us-research/USST/blob/HEAD/src/config.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"169badeb8cb30b4a"}},{"code_sha256_prefix":"b09593fc01e9188e","entry":"vis_demo","repo":"oppo-us-research/USST","repo_kind":"official","path":"src/utils_vis.py","file_url":"https://github.com/oppo-us-research/USST/blob/HEAD/src/utils_vis.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b09593fc01e9188e"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}