{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/bridging-the-gap-a-unified-video","title":"Bridging the Gap: A Unified Video Comprehension Framework for Moment Retrieval and Highlight Detection","arxiv_id":"2311.16464","date":"2023-11-28","proceeding":"CVPR 2024 1","authors":["Yicheng Xiao","Zhuoyan Luo","Yong liu","Yue Ma","Hengwei Bian","Yatai Ji","Yujiu Yang","Xiu Li"],"abstract":"Video Moment Retrieval (MR) and Highlight Detection (HD) have attracted significant attention due to the growing demand for video analysis. Recent approaches treat MR and HD as similar video grounding problems and address them together with transformer-based architecture. However, we observe that the emphasis of MR and HD differs, with one necessitating the perception of local relationships and the other prioritizing the understanding of global contexts. Consequently, the lack of task-specific design will inevitably lead to limitations in associating the intrinsic specialty of two tasks. To tackle the issue, we propose a Unified Video COMprehension framework (UVCOM) to bridge the gap and jointly solve MR and HD effectively. By performing progressive integration on intra and inter-modality across multi-granularity, UVCOM achieves the comprehensive understanding in processing a video. Moreover, we present multi-aspect contrastive learning to consolidate the local relation modeling and global knowledge accumulation via well aligned multi-modal space. Extensive experiments on QVHighlights, Charades-STA, TACoS , YouTube Highlights and TVSum datasets demonstrate the effectiveness and rationality of UVCOM which outperforms the state-of-the-art methods by a remarkable margin.","url_abs":"https://arxiv.org/abs/2311.16464v1","url_pdf":"https://arxiv.org/pdf/2311.16464v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"bridging-the-gap-a-unified-video","repo_url":"https://github.com/easonxiao-888/uvcom","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"contrastive-learning","task_name":"Contrastive Learning"},{"task_slug":"highlight-detection","task_name":"Highlight Detection"},{"task_slug":"moment-retrieval","task_name":"Moment Retrieval"},{"task_slug":"natural-language-moment-retrieval","task_name":"Natural Language Moment Retrieval"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"action-recognition","task_name":"Temporal Action Localization"},{"task_slug":"video-grounding","task_name":"Video Grounding"}],"methods":[{"method_slug":"contrastive-learning","method_name":"Contrastive Learning"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/highlight-detection-on-tvsum","task":"Highlight Detection","dataset":"TvSum","model":"UVCOM (train from scratch)","rank_in_archive_order":5,"of":7,"metrics":{"mAP":"86.3"},"uses_additional_data":false},{"leaderboard":"/sota/highlight-detection-on-youtube-highlights","task":"Highlight Detection","dataset":"YouTube Highlights","model":"UVCOM","rank_in_archive_order":2,"of":7,"metrics":{"mAP":"77.4"},"uses_additional_data":false},{"leaderboard":"/sota/moment-retrieval-on-charades-sta","task":"Moment Retrieval","dataset":"Charades-STA","model":"UVCOM","rank_in_archive_order":14,"of":25,"metrics":{"R@1 IoU=0.5":"59.25","R@1 IoU=0.7":"36.64"},"uses_additional_data":false},{"leaderboard":"/sota/moment-retrieval-on-qvhighlights","task":"Moment Retrieval","dataset":"QVHighlights","model":"UVCOM (w/ PT ASR Captions)","rank_in_archive_order":16,"of":32,"metrics":{"R@1 IoU=0.5":"64.53","R@1 IoU=0.7":"48.31","mAP":"43.8","mAP@0.5":"64.78","mAP@0.75":"43.65"},"uses_additional_data":true},{"leaderboard":"/sota/moment-retrieval-on-qvhighlights","task":"Moment Retrieval","dataset":"QVHighlights","model":"UVCOM","rank_in_archive_order":18,"of":32,"metrics":{"R@1 IoU=0.5":"63.55","R@1 IoU=0.7":"47.47","mAP":"43.18","mAP@0.5":"63.37","mAP@0.75":"42.67"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-moment-retrieval-on-tacos","task":"Natural Language Moment Retrieval","dataset":"TACoS","model":"UVCOM","rank_in_archive_order":13,"of":13,"metrics":{"R@1,IoU=0.5":"36.39","R@1,IoU=0.7":"23.32"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2311.16464","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.16464"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/easonxiao-888/uvcom","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":7,"ran_honours":1,"ran_draft_wrong":1,"unverified":3},"by_repo_kind":{"official":{"samples":12,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"4557aa0c781bc84a","entry":"EM_RBF","repo":"easonxiao-888/uvcom","repo_kind":"official","path":"uvcom/addition.py","file_url":"https://github.com/easonxiao-888/uvcom/blob/HEAD/uvcom/addition.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4557aa0c781bc84a"}},{"code_sha256_prefix":"aa6da43d05cfaf65","entry":"all_gather","repo":"easonxiao-888/uvcom","repo_kind":"official","path":"uvcom/misc_ddp.py","file_url":"https://github.com/easonxiao-888/uvcom/blob/HEAD/uvcom/misc_ddp.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"aa6da43d05cfaf65"}},{"code_sha256_prefix":"d6ca656c20df137a","entry":"build_position_encoding","repo":"easonxiao-888/uvcom","repo_kind":"official","path":"uvcom/position_encoding.py","file_url":"https://github.com/easonxiao-888/uvcom/blob/HEAD/uvcom/position_encoding.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d6ca656c20df137a"}},{"code_sha256_prefix":"eb592047bcf7242f","entry":"calculate_l1_norm","repo":"easonxiao-888/uvcom","repo_kind":"official","path":"uvcom/addition.py","file_url":"https://github.com/easonxiao-888/uvcom/blob/HEAD/uvcom/addition.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"eb592047bcf7242f"}},{"code_sha256_prefix":"ef58ba4fdac7256e","entry":"gen_sineembed_for_position","repo":"easonxiao-888/uvcom","repo_kind":"official","path":"uvcom/CIM.py","file_url":"https://github.com/easonxiao-888/uvcom/blob/HEAD/uvcom/CIM.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ef58ba4fdac7256e"}},{"code_sha256_prefix":"6897880b6ce14c94","entry":"inverse_sigmoid","repo":"easonxiao-888/uvcom","repo_kind":"official","path":"uvcom/model.py","file_url":"https://github.com/easonxiao-888/uvcom/blob/HEAD/uvcom/model.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6897880b6ce14c94"}},{"code_sha256_prefix":"afb6efd87349dd80","entry":"nested_tensor_from_tensor_list","repo":"easonxiao-888/uvcom","repo_kind":"official","path":"uvcom/misc_ddp.py","file_url":"https://github.com/easonxiao-888/uvcom/blob/HEAD/uvcom/misc_ddp.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"afb6efd87349dd80"}},{"code_sha256_prefix":"05992505df9ef0cd","entry":"reduce_dict","repo":"easonxiao-888/uvcom","repo_kind":"official","path":"uvcom/misc_ddp.py","file_url":"https://github.com/easonxiao-888/uvcom/blob/HEAD/uvcom/misc_ddp.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"05992505df9ef0cd"}},{"code_sha256_prefix":"396d6e03b0c5d9a7","entry":"sampling","repo":"easonxiao-888/uvcom","repo_kind":"official","path":"uvcom/model.py","file_url":"https://github.com/easonxiao-888/uvcom/blob/HEAD/uvcom/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"396d6e03b0c5d9a7"}},{"code_sha256_prefix":"9e3fc74dc421fbbe","entry":"BMRW","repo":"easonxiao-888/uvcom","repo_kind":"official","path":"uvcom/addition.py","file_url":"https://github.com/easonxiao-888/uvcom/blob/HEAD/uvcom/addition.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9e3fc74dc421fbbe"}},{"code_sha256_prefix":"081c8abd57d082fd","entry":"accuracy","repo":"easonxiao-888/uvcom","repo_kind":"official","path":"uvcom/misc.py","file_url":"https://github.com/easonxiao-888/uvcom/blob/HEAD/uvcom/misc.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"081c8abd57d082fd"}},{"code_sha256_prefix":"a82091666b9db8cf","entry":"multi_head_attention_forward","repo":"easonxiao-888/uvcom","repo_kind":"official","path":"uvcom/attention.py","file_url":"https://github.com/easonxiao-888/uvcom/blob/HEAD/uvcom/attention.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a82091666b9db8cf"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}