{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/joint-relation-transformer-for-multi-person","title":"Joint-Relation Transformer for Multi-Person Motion Prediction","arxiv_id":"2308.04808","date":"2023-08-09","proceeding":"ICCV 2023 1","authors":["Qingyao Xu","Weibo Mao","Jingze Gong","Chenxin Xu","Siheng Chen","Weidi Xie","Ya zhang","Yanfeng Wang"],"abstract":"Multi-person motion prediction is a challenging problem due to the dependency of motion on both individual past movements and interactions with other people. Transformer-based methods have shown promising results on this task, but they miss the explicit relation representation between joints, such as skeleton structure and pairwise distance, which is crucial for accurate interaction modeling. In this paper, we propose the Joint-Relation Transformer, which utilizes relation information to enhance interaction modeling and improve future motion prediction. Our relation information contains the relative distance and the intra-/inter-person physical constraints. To fuse relation and joint information, we design a novel joint-relation fusion layer with relation-aware attention to update both features. Additionally, we supervise the relation information by forecasting future distance. Experiments show that our method achieves a 13.4% improvement of 900ms VIM on 3DPW-SoMoF/RC and 17.8%/12.0% improvement of 3s MPJPE on CMU-Mpcap/MuPoTS-3D dataset.","url_abs":"https://arxiv.org/abs/2308.04808v2","url_pdf":"https://arxiv.org/pdf/2308.04808v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"joint-relation-transformer-for-multi-person","repo_url":"https://github.com/mediabrain-sjtu/jrtransformer","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"prediction","task_name":"Prediction"},{"task_slug":null,"task_name":"Relation"},{"task_slug":"motion-prediction","task_name":"motion prediction"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2308.04808","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.04808"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mediabrain-sjtu/jrtransformer","reach":{"status":"ok"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/MediaBrain-SJTU/JRTransformer","reach":{"status":"ok"}}],"summary":{"ran":17,"ran_fixture":1,"unverified":3},"by_repo_kind":{"official":{"samples":21,"ran":18,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":21,"samples":[{"code_sha256_prefix":"94ca24363aab8053","entry":"Attention","repo":"MediaBrain-SJTU/JRTransformer","repo_kind":"official","path":"model/model.py","file_url":"https://github.com/MediaBrain-SJTU/JRTransformer/blob/HEAD/model/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"94ca24363aab8053"}},{"code_sha256_prefix":"7f79674712326709","entry":"Block","repo":"MediaBrain-SJTU/JRTransformer","repo_kind":"official","path":"model/model.py","file_url":"https://github.com/MediaBrain-SJTU/JRTransformer/blob/HEAD/model/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7f79674712326709"}},{"code_sha256_prefix":"4ab81933f8ad8710","entry":"MLP","repo":"MediaBrain-SJTU/JRTransformer","repo_kind":"official","path":"model/model.py","file_url":"https://github.com/MediaBrain-SJTU/JRTransformer/blob/HEAD/model/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4ab81933f8ad8710"}},{"code_sha256_prefix":"6141e32d349c6520","entry":"PositionalEmbedding","repo":"MediaBrain-SJTU/JRTransformer","repo_kind":"official","path":"model/model.py","file_url":"https://github.com/MediaBrain-SJTU/JRTransformer/blob/HEAD/model/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6141e32d349c6520"}},{"code_sha256_prefix":"242afd06c3c28368","entry":"ang2joint","repo":"mediabrain-sjtu/jrtransformer","repo_kind":"official","path":"dataset/ang2joint.py","file_url":"https://github.com/mediabrain-sjtu/jrtransformer/blob/HEAD/dataset/ang2joint.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"242afd06c3c28368"}},{"code_sha256_prefix":"aa5d767c2afee6be","entry":"batch_MPJPE","repo":"mediabrain-sjtu/jrtransformer","repo_kind":"official","path":"utils/metrics.py","file_url":"https://github.com/mediabrain-sjtu/jrtransformer/blob/HEAD/utils/metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"aa5d767c2afee6be"}},{"code_sha256_prefix":"02296c69c0428597","entry":"batch_VIM","repo":"mediabrain-sjtu/jrtransformer","repo_kind":"official","path":"utils/metrics.py","file_url":"https://github.com/mediabrain-sjtu/jrtransformer/blob/HEAD/utils/metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"02296c69c0428597"}},{"code_sha256_prefix":"88b89b16d716722e","entry":"batch_VIM_","repo":"mediabrain-sjtu/jrtransformer","repo_kind":"official","path":"utils/metrics.py","file_url":"https://github.com/mediabrain-sjtu/jrtransformer/blob/HEAD/utils/metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"88b89b16d716722e"}},{"code_sha256_prefix":"092df234b64f4cb2","entry":"batch_denormalization","repo":"mediabrain-sjtu/jrtransformer","repo_kind":"official","path":"dataset/dataset_3dpw.py","file_url":"https://github.com/mediabrain-sjtu/jrtransformer/blob/HEAD/dataset/dataset_3dpw.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"092df234b64f4cb2"}},{"code_sha256_prefix":"39eace7e2822504f","entry":"drop_path","repo":"mediabrain-sjtu/jrtransformer","repo_kind":"official","path":"model/model.py","file_url":"https://github.com/mediabrain-sjtu/jrtransformer/blob/HEAD/model/model.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"39eace7e2822504f"}},{"code_sha256_prefix":"e53e8ce70a48a8ed","entry":"get_adj","repo":"mediabrain-sjtu/jrtransformer","repo_kind":"official","path":"utils/util.py","file_url":"https://github.com/mediabrain-sjtu/jrtransformer/blob/HEAD/utils/util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e53e8ce70a48a8ed"}},{"code_sha256_prefix":"0429eac001f524e3","entry":"get_connect","repo":"mediabrain-sjtu/jrtransformer","repo_kind":"official","path":"utils/util.py","file_url":"https://github.com/mediabrain-sjtu/jrtransformer/blob/HEAD/utils/util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0429eac001f524e3"}},{"code_sha256_prefix":"19c25e3301ffd177","entry":"normalize","repo":"mediabrain-sjtu/jrtransformer","repo_kind":"official","path":"dataset/dataset_3dpw.py","file_url":"https://github.com/mediabrain-sjtu/jrtransformer/blob/HEAD/dataset/dataset_3dpw.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"19c25e3301ffd177"}},{"code_sha256_prefix":"9e93dfbacea8553c","entry":"rodrigues","repo":"mediabrain-sjtu/jrtransformer","repo_kind":"official","path":"dataset/ang2joint.py","file_url":"https://github.com/mediabrain-sjtu/jrtransformer/blob/HEAD/dataset/ang2joint.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9e93dfbacea8553c"}},{"code_sha256_prefix":"b3ac05a157b33242","entry":"rotate_Y","repo":"mediabrain-sjtu/jrtransformer","repo_kind":"official","path":"dataset/dataset_3dpw.py","file_url":"https://github.com/mediabrain-sjtu/jrtransformer/blob/HEAD/dataset/dataset_3dpw.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b3ac05a157b33242"}},{"code_sha256_prefix":"a60003739d5ccea4","entry":"rotate_Y","repo":"mediabrain-sjtu/jrtransformer","repo_kind":"official","path":"utils/util.py","file_url":"https://github.com/mediabrain-sjtu/jrtransformer/blob/HEAD/utils/util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a60003739d5ccea4"}},{"code_sha256_prefix":"fa68b44f5bf5805d","entry":"spatio_temporal_graph","repo":"mediabrain-sjtu/jrtransformer","repo_kind":"official","path":"dataset/dataset_amass.py","file_url":"https://github.com/mediabrain-sjtu/jrtransformer/blob/HEAD/dataset/dataset_amass.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fa68b44f5bf5805d"}},{"code_sha256_prefix":"f00359361920530a","entry":"with_zeros","repo":"mediabrain-sjtu/jrtransformer","repo_kind":"official","path":"dataset/ang2joint.py","file_url":"https://github.com/mediabrain-sjtu/jrtransformer/blob/HEAD/dataset/ang2joint.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f00359361920530a"}},{"code_sha256_prefix":"3fdb3598a9dd0559","entry":"JRTransformer","repo":"MediaBrain-SJTU/JRTransformer","repo_kind":"official","path":"model/model.py","file_url":"https://github.com/MediaBrain-SJTU/JRTransformer/blob/HEAD/model/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3fdb3598a9dd0559"}},{"code_sha256_prefix":"807dcdd63bf843cb","entry":"get_adj_AMASS","repo":"mediabrain-sjtu/jrtransformer","repo_kind":"official","path":"dataset/dataset_amass.py","file_url":"https://github.com/mediabrain-sjtu/jrtransformer/blob/HEAD/dataset/dataset_amass.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"807dcdd63bf843cb"}},{"code_sha256_prefix":"fe07fda014912d98","entry":"normalize_A","repo":"mediabrain-sjtu/jrtransformer","repo_kind":"official","path":"dataset/dataset_amass.py","file_url":"https://github.com/mediabrain-sjtu/jrtransformer/blob/HEAD/dataset/dataset_amass.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fe07fda014912d98"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}