{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/predicting-masked-tokens-in-stochastic","title":"Stochastic positional embeddings improve masked image modeling","arxiv_id":"2308.00566","date":"2023-07-31","proceeding":null,"authors":["Amir Bar","Florian Bordes","Assaf Shocher","Mahmoud Assran","Pascal Vincent","Nicolas Ballas","Trevor Darrell","Amir Globerson","Yann Lecun"],"abstract":"Masked Image Modeling (MIM) is a promising self-supervised learning approach that enables learning from unlabeled images. Despite its recent success, learning good representations through MIM remains challenging because it requires predicting the right semantic content in accurate locations. For example, given an incomplete picture of a dog, we can guess that there is a tail, but we cannot determine its exact location. In this work, we propose to incorporate location uncertainty into MIM by using stochastic positional embeddings (StoP). Specifically, we condition the model on stochastic masked token positions drawn from a Gaussian distribution. StoP reduces overfitting to location features and guides the model toward learning features that are more robust to location uncertainties. Quantitatively, StoP improves downstream MIM performance on a variety of downstream tasks, including $+1.7\\%$ on ImageNet linear probing using ViT-B, and $+2.5\\%$ for ViT-H using $1\\%$ of the data.","url_abs":"https://arxiv.org/abs/2308.00566v2","url_pdf":"https://arxiv.org/pdf/2308.00566v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"predicting-masked-tokens-in-stochastic","repo_url":"https://github.com/amirbar/stop","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"masked-language-modeling","task_name":"Masked Language Modeling"},{"task_slug":"self-supervised-learning","task_name":"Self-Supervised Learning"},{"task_slug":"video-segmentation","task_name":"Video Segmentation"},{"task_slug":"video-semantic-segmentation","task_name":"Video Semantic Segmentation"}],"methods":[{"method_slug":"mim","method_name":"MIM"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2308.00566","atlas_url":"https://app.syntology.ai/?focus=2308.00566","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.00566"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/amirbar/stop","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran_fixture":2,"ran":5,"unverified":4},"by_repo_kind":{"official":{"samples":11,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":11,"samples":[{"code_sha256_prefix":"b0f936d4d6ae3b8c","entry":"accuracy","repo":"amirbar/stop","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/amirbar/stop/blob/HEAD/src/utils.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b0f936d4d6ae3b8c"}},{"code_sha256_prefix":"3dc19396537db789","entry":"all_reduce_mean","repo":"amirbar/stop","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/amirbar/stop/blob/HEAD/src/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3dc19396537db789"}},{"code_sha256_prefix":"4bdd36eab04c3f62","entry":"get_1d_sincos_pos_embed_from_grid","repo":"amirbar/stop","repo_kind":"official","path":"src/deit.py","file_url":"https://github.com/amirbar/stop/blob/HEAD/src/deit.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4bdd36eab04c3f62"}},{"code_sha256_prefix":"3241d318307c5d4f","entry":"get_2d_sincos_pos_embed","repo":"amirbar/stop","repo_kind":"official","path":"src/deit.py","file_url":"https://github.com/amirbar/stop/blob/HEAD/src/deit.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"3241d318307c5d4f"}},{"code_sha256_prefix":"b2c06a2a020920e1","entry":"load_pretrained","repo":"amirbar/stop","repo_kind":"official","path":"logistic_eval.py","file_url":"https://github.com/amirbar/stop/blob/HEAD/logistic_eval.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"b2c06a2a020920e1"}},{"code_sha256_prefix":"532c530f6c82d3c8","entry":"make_embeddings","repo":"amirbar/stop","repo_kind":"official","path":"logistic_eval.py","file_url":"https://github.com/amirbar/stop/blob/HEAD/logistic_eval.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"532c530f6c82d3c8"}},{"code_sha256_prefix":"a6e63a056127a5e2","entry":"make_transforms","repo":"amirbar/stop","repo_kind":"official","path":"src/data_manager.py","file_url":"https://github.com/amirbar/stop/blob/HEAD/src/data_manager.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"a6e63a056127a5e2"}},{"code_sha256_prefix":"fa865f702b4a1961","entry":"copy_imgnt_locally","repo":"amirbar/stop","repo_kind":"official","path":"src/data_manager.py","file_url":"https://github.com/amirbar/stop/blob/HEAD/src/data_manager.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"fa865f702b4a1961"}},{"code_sha256_prefix":"4fd80de79832745d","entry":"get_2d_sincos_pos_embed_from_grid","repo":"amirbar/stop","repo_kind":"official","path":"src/deit.py","file_url":"https://github.com/amirbar/stop/blob/HEAD/src/deit.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4fd80de79832745d"}},{"code_sha256_prefix":"ba1356e8ceb654d2","entry":"get_grad_norm_","repo":"amirbar/stop","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/amirbar/stop/blob/HEAD/src/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ba1356e8ceb654d2"}},{"code_sha256_prefix":"1044207f674d0fe8","entry":"init_data","repo":"amirbar/stop","repo_kind":"official","path":"src/data_manager.py","file_url":"https://github.com/amirbar/stop/blob/HEAD/src/data_manager.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"1044207f674d0fe8"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}