{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/joint-hand-motion-and-interaction-hotspots","title":"Joint Hand Motion and Interaction Hotspots Prediction from Egocentric Videos","arxiv_id":"2204.01696","date":"2022-04-04","proceeding":"CVPR 2022 1","authors":["Shaowei Liu","Subarna Tripathi","Somdeb Majumdar","Xiaolong Wang"],"abstract":"We propose to forecast future hand-object interactions given an egocentric video. Instead of predicting action labels or pixels, we directly predict the hand motion trajectory and the future contact points on the next active object (i.e., interaction hotspots). This relatively low-dimensional representation provides a concrete description of future interactions. To tackle this task, we first provide an automatic way to collect trajectory and hotspots labels on large-scale data. We then use this data to train an Object-Centric Transformer (OCT) model for prediction. Our model performs hand and object interaction reasoning via the self-attention mechanism in Transformers. OCT also provides a probabilistic framework to sample the future trajectory and hotspots to handle uncertainty in prediction. We perform experiments on the Epic-Kitchens-55, Epic-Kitchens-100, and EGTEA Gaze+ datasets, and show that OCT significantly outperforms state-of-the-art approaches by a large margin. Project page is available at https://stevenlsw.github.io/hoi-forecast .","url_abs":"https://arxiv.org/abs/2204.01696v1","url_pdf":"https://arxiv.org/pdf/2204.01696v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"joint-hand-motion-and-interaction-hotspots","repo_url":"https://github.com/stevenlsw/hoi-forecast","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"object","task_name":"Object"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2204.01696","atlas_url":"https://app.syntology.ai/?focus=2204.01696","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.01696"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/stevenlsw/hoi-forecast","reach":null}],"summary":{"ran":6,"ran_draft_wrong":2,"ran_fixture":1,"unverified":5},"by_repo_kind":{"official":{"samples":13,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"59934442cfaa72c1","entry":"Decoder_PositionalEmbedding","repo":"stevenlsw/hoi-forecast","repo_kind":"official","path":"networks/transformer.py","file_url":"https://github.com/stevenlsw/hoi-forecast/blob/HEAD/networks/transformer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"59934442cfaa72c1"}},{"code_sha256_prefix":"0c020f10b0bec0e6","entry":"EncoderBlock","repo":"stevenlsw/hoi-forecast","repo_kind":"official","path":"networks/transformer.py","file_url":"https://github.com/stevenlsw/hoi-forecast/blob/HEAD/networks/transformer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0c020f10b0bec0e6"}},{"code_sha256_prefix":"89a5582c6a7ea10c","entry":"Encoder_PositionalEmbedding","repo":"stevenlsw/hoi-forecast","repo_kind":"official","path":"networks/transformer.py","file_url":"https://github.com/stevenlsw/hoi-forecast/blob/HEAD/networks/transformer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"89a5582c6a7ea10c"}},{"code_sha256_prefix":"a9bbf45f116e8c1a","entry":"MultiHeadAttention","repo":"stevenlsw/hoi-forecast","repo_kind":"official","path":"networks/transformer.py","file_url":"https://github.com/stevenlsw/hoi-forecast/blob/HEAD/networks/transformer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a9bbf45f116e8c1a"}},{"code_sha256_prefix":"78a30a44c72219c4","entry":"PositionalEncoding","repo":"stevenlsw/hoi-forecast","repo_kind":"official","path":"networks/transformer.py","file_url":"https://github.com/stevenlsw/hoi-forecast/blob/HEAD/networks/transformer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"78a30a44c72219c4"}},{"code_sha256_prefix":"7aed6286f436f90c","entry":"ScaledDotProductAttention","repo":"stevenlsw/hoi-forecast","repo_kind":"official","path":"networks/transformer.py","file_url":"https://github.com/stevenlsw/hoi-forecast/blob/HEAD/networks/transformer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7aed6286f436f90c"}},{"code_sha256_prefix":"b70634b10053e254","entry":"get_pad_mask","repo":"stevenlsw/hoi-forecast","repo_kind":"official","path":"networks/transformer.py","file_url":"https://github.com/stevenlsw/hoi-forecast/blob/HEAD/networks/transformer.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b70634b10053e254"}},{"code_sha256_prefix":"3e17308fbd05134c","entry":"get_subsequent_mask","repo":"stevenlsw/hoi-forecast","repo_kind":"official","path":"networks/transformer.py","file_url":"https://github.com/stevenlsw/hoi-forecast/blob/HEAD/networks/transformer.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3e17308fbd05134c"}},{"code_sha256_prefix":"c76d18678f19d332","entry":"trunc_normal_","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"c76d18678f19d332"}},{"code_sha256_prefix":"be0e67452fb1ba3d","entry":"Decoder","repo":"stevenlsw/hoi-forecast","repo_kind":"official","path":"networks/transformer.py","file_url":"https://github.com/stevenlsw/hoi-forecast/blob/HEAD/networks/transformer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"be0e67452fb1ba3d"}},{"code_sha256_prefix":"819242d1ab4105fb","entry":"DecoderBlock","repo":"stevenlsw/hoi-forecast","repo_kind":"official","path":"networks/transformer.py","file_url":"https://github.com/stevenlsw/hoi-forecast/blob/HEAD/networks/transformer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"819242d1ab4105fb"}},{"code_sha256_prefix":"f052bc060d415e7b","entry":"Encoder","repo":"stevenlsw/hoi-forecast","repo_kind":"official","path":"networks/transformer.py","file_url":"https://github.com/stevenlsw/hoi-forecast/blob/HEAD/networks/transformer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f052bc060d415e7b"}},{"code_sha256_prefix":"ca707a19604c5750","entry":"ObjectTransformer","repo":"stevenlsw/hoi-forecast","repo_kind":"official","path":"networks/transformer.py","file_url":"https://github.com/stevenlsw/hoi-forecast/blob/HEAD/networks/transformer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ca707a19604c5750"}},{"code_sha256_prefix":"e867e3921b4ce54e","entry":"traj_affordance_dist","repo":"stevenlsw/hoi-forecast","repo_kind":"official","path":"networks/transformer.py","file_url":"https://github.com/stevenlsw/hoi-forecast/blob/HEAD/networks/transformer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e867e3921b4ce54e"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}