{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/hypergraph-transformer-for-skeleton-based","title":"Hypergraph Transformer for Skeleton-based Action Recognition","arxiv_id":"2211.09590","date":"2022-11-17","proceeding":null,"authors":["Yuxuan Zhou","Zhi-Qi Cheng","Chao Li","Yanwen Fang","Yifeng Geng","Xuansong Xie","Margret Keuper"],"abstract":"Skeleton-based action recognition aims to recognize human actions given human joint coordinates with skeletal interconnections. By defining a graph with joints as vertices and their natural connections as edges, previous works successfully adopted Graph Convolutional networks (GCNs) to model joint co-occurrences and achieved superior performance. More recently, a limitation of GCNs is identified, i.e., the topology is fixed after training. To relax such a restriction, Self-Attention (SA) mechanism has been adopted to make the topology of GCNs adaptive to the input, resulting in the state-of-the-art hybrid models. Concurrently, attempts with plain Transformers have also been made, but they still lag behind state-of-the-art GCN-based methods due to the lack of structural prior. Unlike hybrid models, we propose a more elegant solution to incorporate the bone connectivity into Transformer via a graph distance embedding. Our embedding retains the information of skeletal structure during training, whereas GCNs merely use it for initialization. More importantly, we reveal an underlying issue of graph models in general, i.e., pairwise aggregation essentially ignores the high-order kinematic dependencies between body joints. To fill this gap, we propose a new self-attention (SA) mechanism on hypergraph, termed Hypergraph Self-Attention (HyperSA), to incorporate intrinsic higher-order relations into the model. We name the resulting model Hyperformer, and it beats state-of-the-art graph models w.r.t. accuracy and efficiency on NTU RGB+D, NTU RGB+D 120, and Northwestern-UCLA datasets.","url_abs":"https://arxiv.org/abs/2211.09590v5","url_pdf":"https://arxiv.org/pdf/2211.09590v5.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"hypergraph-transformer-for-skeleton-based","repo_url":"https://github.com/ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"action-recognition-in-videos","task_name":"Action Recognition"},{"task_slug":"skeleton-based-action-recognition","task_name":"Skeleton Based Action Recognition"}],"methods":[{"method_slug":"attention","method_name":"Attention"},{"method_slug":"hypersa","method_name":"HyperSA"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[{"slug":"hypersa","name":"HyperSA","full_name":"HyperGraph Self-Attention"}],"results":[{"leaderboard":"/sota/skeleton-based-action-recognition-on-ntu-rgbd","task":"Skeleton Based Action Recognition","dataset":"NTU RGB+D","model":"Hyperformer","rank_in_archive_order":26,"of":135,"metrics":{"Accuracy (CS)":"92.9","Accuracy (CV)":"96.5","Ensembled Modalities":"4"},"uses_additional_data":false},{"leaderboard":"/sota/skeleton-based-action-recognition-on-ntu-rgbd-1","task":"Skeleton Based Action Recognition","dataset":"NTU RGB+D 120","model":"Hyperformer","rank_in_archive_order":12,"of":83,"metrics":{"Accuracy (Cross-Setup)":"91.3","Accuracy (Cross-Subject)":"89.9","Ensembled Modalities":"4"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2211.09590","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.09590"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":6,"ran_draft_wrong":3,"ran_violates":1,"unverified":2},"by_repo_kind":{"official":{"samples":12,"ran":10,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"8b14974e8aa7320f","entry":"downsample","repo":"ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition","repo_kind":"official","path":"feeders/tools.py","file_url":"https://github.com/ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition/blob/HEAD/feeders/tools.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8b14974e8aa7320f"}},{"code_sha256_prefix":"5f5ae82020ac7ec2","entry":"edge2mat","repo":"ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition","repo_kind":"official","path":"graph/tools.py","file_url":"https://github.com/ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition/blob/HEAD/graph/tools.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5f5ae82020ac7ec2"}},{"code_sha256_prefix":"d3e8dd52d4a2755a","entry":"get_k_scale_graph","repo":"ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition","repo_kind":"official","path":"graph/tools.py","file_url":"https://github.com/ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition/blob/HEAD/graph/tools.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d3e8dd52d4a2755a"}},{"code_sha256_prefix":"5940d00ced3d0637","entry":"get_sgp_mat","repo":"ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition","repo_kind":"official","path":"graph/tools.py","file_url":"https://github.com/ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition/blob/HEAD/graph/tools.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5940d00ced3d0637"}},{"code_sha256_prefix":"ffed4f85d50832c9","entry":"import_class","repo":"ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition","repo_kind":"official","path":"feeders/feeder_ntu.py","file_url":"https://github.com/ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition/blob/HEAD/feeders/feeder_ntu.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ffed4f85d50832c9"}},{"code_sha256_prefix":"4e946d7c92be9d41","entry":"import_class","repo":"ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition","repo_kind":"official","path":"torchlight/torchlight/util.py","file_url":"https://github.com/ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition/blob/HEAD/torchlight/torchlight/util.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4e946d7c92be9d41"}},{"code_sha256_prefix":"83ed4b272b1e38d0","entry":"ngpu","repo":"ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition","repo_kind":"official","path":"torchlight/torchlight/gpu.py","file_url":"https://github.com/ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition/blob/HEAD/torchlight/torchlight/gpu.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"83ed4b272b1e38d0"}},{"code_sha256_prefix":"f017532fc389cbfe","entry":"str2bool","repo":"ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition","repo_kind":"official","path":"torchlight/torchlight/util.py","file_url":"https://github.com/ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition/blob/HEAD/torchlight/torchlight/util.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f017532fc389cbfe"}},{"code_sha256_prefix":"37939bae213ecfd4","entry":"valid_crop_resize","repo":"ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition","repo_kind":"official","path":"feeders/tools.py","file_url":"https://github.com/ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition/blob/HEAD/feeders/tools.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"37939bae213ecfd4"}},{"code_sha256_prefix":"621e08e84928c5c3","entry":"visible_gpu","repo":"ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition","repo_kind":"official","path":"torchlight/torchlight/gpu.py","file_url":"https://github.com/ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition/blob/HEAD/torchlight/torchlight/gpu.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"621e08e84928c5c3"}},{"code_sha256_prefix":"1e89b2b546737164","entry":"str2dict","repo":"ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition","repo_kind":"official","path":"torchlight/torchlight/util.py","file_url":"https://github.com/ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition/blob/HEAD/torchlight/torchlight/util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1e89b2b546737164"}},{"code_sha256_prefix":"2f7f389f0584c97e","entry":"temporal_slice","repo":"ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition","repo_kind":"official","path":"feeders/tools.py","file_url":"https://github.com/ZhouYuxuanYX/Hypergraph-Transformer-for-Skeleton-based-Action-Recognition/blob/HEAD/feeders/tools.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2f7f389f0584c97e"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}