{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/masked-event-modeling-self-supervised","title":"Masked Event Modeling: Self-Supervised Pretraining for Event Cameras","arxiv_id":"2212.10368","date":"2022-12-20","proceeding":null,"authors":["Simon Klenk","David Bonello","Lukas Koestler","Nikita Araslanov","Daniel Cremers"],"abstract":"Event cameras asynchronously capture brightness changes with low latency, high temporal resolution, and high dynamic range. However, annotation of event data is a costly and laborious process, which limits the use of deep learning methods for classification and other semantic tasks with the event modality. To reduce the dependency on labeled event data, we introduce Masked Event Modeling (MEM), a self-supervised framework for events. Our method pretrains a neural network on unlabeled events, which can originate from any event camera recording. Subsequently, the pretrained model is finetuned on a downstream task, leading to a consistent improvement of the task accuracy. For example, our method reaches state-of-the-art classification accuracy across three datasets, N-ImageNet, N-Cars, and N-Caltech101, increasing the top-1 accuracy of previous work by significant margins. When tested on real-world event data, MEM is even superior to supervised RGB-based pretraining. The models pretrained with MEM are also label-efficient and generalize well to the dense task of semantic image segmentation.","url_abs":"https://arxiv.org/abs/2212.10368v3","url_pdf":"https://arxiv.org/pdf/2212.10368v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"masked-event-modeling-self-supervised","repo_url":"https://github.com/tum-vision/mem","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"classification-1","task_name":"Classification"},{"task_slug":"event-based-vision","task_name":"Event-based vision"},{"task_slug":"image-segmentation","task_name":"Image Segmentation"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/classification-on-n-cars","task":"Classification","dataset":"N-CARS","model":"MEM","rank_in_archive_order":1,"of":6,"metrics":{"Accuracy (%)":"98.55","Architecture":"Transformer","Representation":"Event Histogram"},"uses_additional_data":true}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2212.10368","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.10368"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/tum-vision/mem","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_violates":2,"ran":1,"ran_honours":1,"ran_draft_wrong":1,"unverified":13},"by_repo_kind":{"official":{"samples":18,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"60fff7c3c400d7ff","entry":"default","repo":"tum-vision/mem","repo_kind":"official","path":"mem/modeling_discrete_vae.py","file_url":"https://github.com/tum-vision/mem/blob/HEAD/mem/modeling_discrete_vae.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"60fff7c3c400d7ff"}},{"code_sha256_prefix":"aa5486a3650902d8","entry":"exists","repo":"tum-vision/mem","repo_kind":"official","path":"mem/modeling_discrete_vae.py","file_url":"https://github.com/tum-vision/mem/blob/HEAD/mem/modeling_discrete_vae.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"aa5486a3650902d8"}},{"code_sha256_prefix":"a3448702d6025ce6","entry":"get_num_layer_for_vit","repo":"tum-vision/mem","repo_kind":"official","path":"mem/optim_factory.py","file_url":"https://github.com/tum-vision/mem/blob/HEAD/mem/optim_factory.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a3448702d6025ce6"}},{"code_sha256_prefix":"f6be833ba409926f","entry":"pil_loader","repo":"tum-vision/mem","repo_kind":"official","path":"mem/dataset_folder.py","file_url":"https://github.com/tum-vision/mem/blob/HEAD/mem/dataset_folder.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f6be833ba409926f"}},{"code_sha256_prefix":"fc2efb619b6ee2c4","entry":"top_k","repo":"tum-vision/mem","repo_kind":"official","path":"mem/modeling_discrete_vae.py","file_url":"https://github.com/tum-vision/mem/blob/HEAD/mem/modeling_discrete_vae.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fc2efb619b6ee2c4"}},{"code_sha256_prefix":"e668ee56928fd57d","entry":"accimage_loader","repo":"tum-vision/mem","repo_kind":"official","path":"mem/dataset_folder.py","file_url":"https://github.com/tum-vision/mem/blob/HEAD/mem/dataset_folder.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e668ee56928fd57d"}},{"code_sha256_prefix":"12035a2f77d8016c","entry":"get_1d_sincos_pos_embed_from_grid","repo":"tum-vision/mem","repo_kind":"official","path":"mem/modeling_mae.py","file_url":"https://github.com/tum-vision/mem/blob/HEAD/mem/modeling_mae.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"12035a2f77d8016c"}},{"code_sha256_prefix":"3185afc3e87293ed","entry":"get_2d_sincos_pos_embed","repo":"tum-vision/mem","repo_kind":"official","path":"mem/modeling_mae.py","file_url":"https://github.com/tum-vision/mem/blob/HEAD/mem/modeling_mae.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3185afc3e87293ed"}},{"code_sha256_prefix":"f10004e059714d42","entry":"get_2d_sincos_pos_embed_from_grid","repo":"tum-vision/mem","repo_kind":"official","path":"mem/modeling_mae.py","file_url":"https://github.com/tum-vision/mem/blob/HEAD/mem/modeling_mae.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f10004e059714d42"}},{"code_sha256_prefix":"29416043c7035e7c","entry":"get_loss_scale_for_deepspeed","repo":"tum-vision/mem","repo_kind":"official","path":"mem/engine_for_finetuning.py","file_url":"https://github.com/tum-vision/mem/blob/HEAD/mem/engine_for_finetuning.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"29416043c7035e7c"}},{"code_sha256_prefix":"5d9ee4d7cd9eacc7","entry":"get_model","repo":"tum-vision/mem","repo_kind":"official","path":"mem/run_mem_pretraining.py","file_url":"https://github.com/tum-vision/mem/blob/HEAD/mem/run_mem_pretraining.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5d9ee4d7cd9eacc7"}},{"code_sha256_prefix":"0ef0019514eaaa09","entry":"get_parameter_groups","repo":"tum-vision/mem","repo_kind":"official","path":"mem/optim_factory.py","file_url":"https://github.com/tum-vision/mem/blob/HEAD/mem/optim_factory.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0ef0019514eaaa09"}},{"code_sha256_prefix":"690ba46b598729b5","entry":"get_valid_flow","repo":"tum-vision/mem","repo_kind":"official","path":"mem/dataset_folder.py","file_url":"https://github.com/tum-vision/mem/blob/HEAD/mem/dataset_folder.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"690ba46b598729b5"}},{"code_sha256_prefix":"6dd91e35fd03983c","entry":"is_valid_file_dsec","repo":"tum-vision/mem","repo_kind":"official","path":"mem/datasets.py","file_url":"https://github.com/tum-vision/mem/blob/HEAD/mem/datasets.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6dd91e35fd03983c"}},{"code_sha256_prefix":"40dede0d13c8dfae","entry":"is_valid_file_mvsec","repo":"tum-vision/mem","repo_kind":"official","path":"mem/datasets.py","file_url":"https://github.com/tum-vision/mem/blob/HEAD/mem/datasets.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"40dede0d13c8dfae"}},{"code_sha256_prefix":"96e4c97a4c23a9d1","entry":"plot_mae","repo":"tum-vision/mem","repo_kind":"official","path":"mem/engine_for_pretraining.py","file_url":"https://github.com/tum-vision/mem/blob/HEAD/mem/engine_for_pretraining.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"96e4c97a4c23a9d1"}},{"code_sha256_prefix":"bfe177b133a4d710","entry":"plot_masks_and_images","repo":"tum-vision/mem","repo_kind":"official","path":"mem/engine_for_pretraining.py","file_url":"https://github.com/tum-vision/mem/blob/HEAD/mem/engine_for_pretraining.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bfe177b133a4d710"}},{"code_sha256_prefix":"c7976ea27edc377a","entry":"train_class_batch","repo":"tum-vision/mem","repo_kind":"official","path":"mem/engine_for_finetuning.py","file_url":"https://github.com/tum-vision/mem/blob/HEAD/mem/engine_for_finetuning.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c7976ea27edc377a"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}