{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/hero-hierarchical-encoder-for-video-language","title":"HERO: Hierarchical Encoder for Video+Language Omni-representation Pre-training","arxiv_id":"2005.00200","date":"2020-05-01","proceeding":"EMNLP 2020 11","authors":["Linjie Li","Yen-Chun Chen","Yu Cheng","Zhe Gan","Licheng Yu","Jingjing Liu"],"abstract":"We present HERO, a novel framework for large-scale video+language omni-representation learning. HERO encodes multimodal inputs in a hierarchical structure, where local context of a video frame is captured by a Cross-modal Transformer via multimodal fusion, and global video context is captured by a Temporal Transformer. In addition to standard Masked Language Modeling (MLM) and Masked Frame Modeling (MFM) objectives, we design two new pre-training tasks: (i) Video-Subtitle Matching (VSM), where the model predicts both global and local temporal alignment; and (ii) Frame Order Modeling (FOM), where the model predicts the right order of shuffled video frames. HERO is jointly trained on HowTo100M and large-scale TV datasets to gain deep understanding of complex social dynamics with multi-character interactions. Comprehensive experiments demonstrate that HERO achieves new state of the art on multiple benchmarks over Text-based Video/Video-moment Retrieval, Video Question Answering (QA), Video-and-language Inference and Video Captioning tasks across different domains. We also introduce two new challenging benchmarks How2QA and How2R for Video QA and Retrieval, collected from diverse video content over multimodalities.","url_abs":"https://arxiv.org/abs/2005.00200v2","url_pdf":"https://arxiv.org/pdf/2005.00200v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"hero-hierarchical-encoder-for-video-language","repo_url":"https://github.com/linjieli222/HERO","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"hero-hierarchical-encoder-for-video-language","repo_url":"https://github.com/linjieli222/hero_video_feature_extractor","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"hero-hierarchical-encoder-for-video-language","repo_url":"https://gitlab.com/grounded-sport-convai/goal-baselines","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"masked-language-modeling","task_name":"Masked Language Modeling"},{"task_slug":"moment-retrieval","task_name":"Moment Retrieval"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"representation-learning","task_name":"Representation Learning"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"video-captioning","task_name":"Video Captioning"},{"task_slug":"video-corpus-moment-retrieval","task_name":"Video Corpus Moment Retrieval"},{"task_slug":"video-question-answering","task_name":"Video Question Answering"},{"task_slug":"video-retrieval","task_name":"Video Retrieval"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"relu","method_name":"ReLU"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[{"slug":"how2qa","name":"How2QA","full_name":"How2QA"},{"slug":"how2r","name":"How2R","full_name":"How2R"}],"methods_introduced":[],"results":[{"leaderboard":"/sota/video-question-answering-on-how2qa","task":"Video Question Answering","dataset":"How2QA","model":"Hero w/ pre-training","rank_in_archive_order":5,"of":8,"metrics":{"Accuracy":"77.75"},"uses_additional_data":false},{"leaderboard":"/sota/video-question-answering-on-tvqa","task":"Video Question Answering","dataset":"TVQA","model":"Hero w/ pre-training","rank_in_archive_order":5,"of":6,"metrics":{"Accuracy":"74.24"},"uses_additional_data":false},{"leaderboard":"/sota/video-retrieval-on-tvr","task":"Video Retrieval","dataset":"TVR","model":"Hero w/ pre-training","rank_in_archive_order":1,"of":2,"metrics":{"R@1":"4.34","R@10":"13.97","R@100":"21.78"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2005.00200","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.00200"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/NVIDIA/apex","reach":{"status":"ok","spdx":"BSD-3-Clause"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/linjieli222/HERO","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/linjieli222/hero_video_feature_extractor","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://gitlab.com/grounded-sport-convai/goal-baselines","reach":null}],"summary":{"ran_draft_wrong":1,"ran":2,"ran_honours":2,"unverified":8},"by_repo_kind":{"official":{"samples":8,"ran":3,"repositories":1},"listed":{"samples":5,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":8,"samples":[{"code_sha256_prefix":"0f786c407fb1ee4c","entry":"swish","repo":"linjieli222/HERO","repo_kind":"official","path":"model/layers.py","file_url":"https://github.com/linjieli222/HERO/blob/HEAD/model/layers.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":2,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"0f786c407fb1ee4c"}},{"code_sha256_prefix":"43a229d4e3c68d02","entry":"STConv3D","repo":"linjieli222/hero_video_feature_extractor","repo_kind":"listed","path":"mil-nce/model.py","file_url":"https://github.com/linjieli222/hero_video_feature_extractor/blob/HEAD/mil-nce/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"43a229d4e3c68d02"}},{"code_sha256_prefix":"8b37c80ce67e3097","entry":"SelfGating","repo":"linjieli222/hero_video_feature_extractor","repo_kind":"listed","path":"mil-nce/model.py","file_url":"https://github.com/linjieli222/hero_video_feature_extractor/blob/HEAD/mil-nce/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8b37c80ce67e3097"}},{"code_sha256_prefix":"29af270e862dd867","entry":"gelu","repo":"linjieli222/HERO","repo_kind":"official","path":"model/layers.py","file_url":"https://github.com/linjieli222/HERO/blob/HEAD/model/layers.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"29af270e862dd867"}},{"code_sha256_prefix":"de48fec0ec9764b0","entry":"gelu_new","repo":"linjieli222/HERO","repo_kind":"official","path":"model/layers.py","file_url":"https://github.com/linjieli222/HERO/blob/HEAD/model/layers.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"de48fec0ec9764b0"}},{"code_sha256_prefix":"cf99554537d72250","entry":"InceptionBlock","repo":"linjieli222/hero_video_feature_extractor","repo_kind":"listed","path":"mil-nce/model.py","file_url":"https://github.com/linjieli222/hero_video_feature_extractor/blob/HEAD/mil-nce/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cf99554537d72250"}},{"code_sha256_prefix":"de412728921bec03","entry":"S3D","repo":"linjieli222/hero_video_feature_extractor","repo_kind":"listed","path":"mil-nce/model.py","file_url":"https://github.com/linjieli222/hero_video_feature_extractor/blob/HEAD/mil-nce/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"de412728921bec03"}},{"code_sha256_prefix":"2a1f8712954c73f8","entry":"build_model","repo":"linjieli222/hero_video_feature_extractor","repo_kind":"listed","path":"mil-nce/model.py","file_url":"https://github.com/linjieli222/hero_video_feature_extractor/blob/HEAD/mil-nce/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2a1f8712954c73f8"}},{"code_sha256_prefix":"29d0af7d06f51fd3","entry":"compute_accuracies","repo":"linjieli222/HERO","repo_kind":"official","path":"eval_videoQA.py","file_url":"https://github.com/linjieli222/HERO/blob/HEAD/eval_videoQA.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"29d0af7d06f51fd3"}},{"code_sha256_prefix":"68545d5324dc298c","entry":"compute_accuracies","repo":"linjieli222/HERO","repo_kind":"official","path":"eval_violin.py","file_url":"https://github.com/linjieli222/HERO/blob/HEAD/eval_violin.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"68545d5324dc298c"}},{"code_sha256_prefix":"c4a4d88af8bdc891","entry":"load_partial_checkpoint","repo":"linjieli222/HERO","repo_kind":"official","path":"model/modeling_utils.py","file_url":"https://github.com/linjieli222/HERO/blob/HEAD/model/modeling_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"c4a4d88af8bdc891"}},{"code_sha256_prefix":"247245b1fec40c1b","entry":"mask_logits","repo":"linjieli222/HERO","repo_kind":"official","path":"model/modeling_utils.py","file_url":"https://github.com/linjieli222/HERO/blob/HEAD/model/modeling_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"247245b1fec40c1b"}},{"code_sha256_prefix":"57947ced49b189c3","entry":"prune_linear_layer","repo":"linjieli222/HERO","repo_kind":"official","path":"model/modeling_utils.py","file_url":"https://github.com/linjieli222/HERO/blob/HEAD/model/modeling_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"57947ced49b189c3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}