{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/efficient-video-representation-learning-via","title":"EVEREST: Efficient Masked Video Autoencoder by Removing Redundant Spatiotemporal Tokens","arxiv_id":"2211.10636","date":"2022-11-19","proceeding":null,"authors":["Sunil Hwang","Jaehong Yoon","Youngwan Lee","Sung Ju Hwang"],"abstract":"Masked Video Autoencoder (MVA) approaches have demonstrated their potential by significantly outperforming previous video representation learning methods. However, they waste an excessive amount of computations and memory in predicting uninformative tokens/frames due to random masking strategies. (e.g., over 16 nodes with 128 NVIDIA A100 GPUs). To resolve this issue, we exploit the unequal information density among the patches in videos and propose EVEREST, a surprisingly efficient MVA approach for video representation learning that finds tokens containing rich motion features and discards uninformative ones during both pre-training and fine-tuning. We further present an information-intensive frame selection strategy that allows the model to focus on informative and causal frames with minimal redundancy. Our method significantly reduces the computation and memory requirements of MVA, enabling the pre-training and fine-tuning on a single machine with 8 GPUs while achieving comparable performance to computation- and memory-heavy baselines on multiple benchmarks and the uncurated Ego4D dataset. We hope that our work contributes to reducing the barrier to further research on video understanding.","url_abs":"https://arxiv.org/abs/2211.10636v6","url_pdf":"https://arxiv.org/pdf/2211.10636v6.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"efficient-video-representation-learning-via","repo_url":"https://github.com/sunilhoho/everest","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"efficient-video-representation-learning-via","repo_url":"https://github.com/sunilhoho/VideoMS","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"action-recognition-in-videos","task_name":"Action Recognition"},{"task_slug":"object-state-change-classification","task_name":"Object State Change Classification"},{"task_slug":null,"task_name":"Object State Change Classification on Ego4D"},{"task_slug":"representation-learning","task_name":"Representation Learning"},{"task_slug":"self-supervised-action-recognition","task_name":"Self-Supervised Action Recognition"},{"task_slug":"self-supervised-learning","task_name":"Self-Supervised Learning"},{"task_slug":"video-understanding","task_name":"Video Understanding"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/self-supervised-action-recognition-on-hmdb51","task":"Self-Supervised Action Recognition","dataset":"HMDB51","model":"VideoMS (ViT-B)","rank_in_archive_order":15,"of":48,"metrics":{"Frozen":"false","Pre-Training Dataset":"no extra data","Top-1 Accuracy":"65.8"},"uses_additional_data":false},{"leaderboard":"/sota/self-supervised-action-recognition-on-ucf101","task":"Self-Supervised Action Recognition","dataset":"UCF101","model":"VideoMS (ViT-B)","rank_in_archive_order":13,"of":53,"metrics":{"3-fold Accuracy":"93.4","Frozen":"false","Pre-Training Dataset":"no extra data"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2211.10636","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.10636"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/sunilhoho/EVEREST","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sunilhoho/everest","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sunilhoho/VideoMS","reach":null}],"summary":{"ran":3,"unverified":3},"by_repo_kind":{"official":{"samples":6,"ran":3,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":6,"samples":[{"code_sha256_prefix":"2fdd988be47b01b8","entry":"Block","repo":"sunilhoho/everest","repo_kind":"official","path":"modeling_pretrain.py","file_url":"https://github.com/sunilhoho/everest/blob/HEAD/modeling_pretrain.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2fdd988be47b01b8"}},{"code_sha256_prefix":"59f1eed28e030f58","entry":"PatchEmbed","repo":"sunilhoho/everest","repo_kind":"official","path":"modeling_pretrain.py","file_url":"https://github.com/sunilhoho/everest/blob/HEAD/modeling_pretrain.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"59f1eed28e030f58"}},{"code_sha256_prefix":"e42810dbfe9ff002","entry":"TubeMaskingGenerator","repo":"sunilhoho/VideoMS","repo_kind":"official","path":"masking_generator.py","file_url":"https://github.com/sunilhoho/VideoMS/blob/HEAD/masking_generator.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e42810dbfe9ff002"}},{"code_sha256_prefix":"d152666891ccca2b","entry":"PretrainVisionTransformer","repo":"sunilhoho/everest","repo_kind":"official","path":"modeling_pretrain.py","file_url":"https://github.com/sunilhoho/everest/blob/HEAD/modeling_pretrain.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d152666891ccca2b"}},{"code_sha256_prefix":"1c888f413c1310c5","entry":"PretrainVisionTransformerDecoder","repo":"sunilhoho/everest","repo_kind":"official","path":"modeling_pretrain.py","file_url":"https://github.com/sunilhoho/everest/blob/HEAD/modeling_pretrain.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1c888f413c1310c5"}},{"code_sha256_prefix":"d93efdf7b8c4ecfc","entry":"PretrainVisionTransformerEncoder","repo":"sunilhoho/everest","repo_kind":"official","path":"modeling_pretrain.py","file_url":"https://github.com/sunilhoho/everest/blob/HEAD/modeling_pretrain.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d93efdf7b8c4ecfc"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}