{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/time-masking-for-temporal-language-models","title":"Time Masking for Temporal Language Models","arxiv_id":"2110.06366","date":"2021-10-12","proceeding":null,"authors":["Guy D. Rosin","Ido Guy","Kira Radinsky"],"abstract":"Our world is constantly evolving, and so is the content on the web. Consequently, our languages, often said to mirror the world, are dynamic in nature. However, most current contextual language models are static and cannot adapt to changes over time. In this work, we propose a temporal contextual language model called TempoBERT, which uses time as an additional context of texts. Our technique is based on modifying texts with temporal information and performing time masking - specific masking for the supplementary time information. We leverage our approach for the tasks of semantic change detection and sentence time prediction, experimenting on diverse datasets in terms of time, size, genre, and language. Our extensive evaluation shows that both tasks benefit from exploiting time masking.","url_abs":"https://arxiv.org/abs/2110.06366v4","url_pdf":"https://arxiv.org/pdf/2110.06366v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"time-masking-for-temporal-language-models","repo_url":"https://github.com/guyrosin/tempobert","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"time-masking-for-temporal-language-models","repo_url":"https://github.com/guyrosin/temporal_attention","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"change-detection","task_name":"Change Detection"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"sentence","task_name":"Sentence"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2110.06366","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.06366"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/guyrosin/tempobert","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/guyrosin/temporal_attention","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_honours":1,"ran_draft_wrong":2,"unverified":9},"by_repo_kind":{"official":{"samples":3,"ran":3,"repositories":1},"listed":{"samples":9,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"f5f8f6ed8c77fc76","entry":"get_embedding","repo":"guyrosin/tempobert","repo_kind":"official","path":"semantic_change_detection.py","file_url":"https://github.com/guyrosin/tempobert/blob/HEAD/semantic_change_detection.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f5f8f6ed8c77fc76"}},{"code_sha256_prefix":"ac64b4860d21447a","entry":"tokenize_dataset_concat","repo":"guyrosin/tempobert","repo_kind":"official","path":"train_tempobert.py","file_url":"https://github.com/guyrosin/tempobert/blob/HEAD/train_tempobert.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ac64b4860d21447a"}},{"code_sha256_prefix":"e3702536b9a34fea","entry":"tokenize_dataset_line_by_line","repo":"guyrosin/tempobert","repo_kind":"official","path":"train_tempobert.py","file_url":"https://github.com/guyrosin/tempobert/blob/HEAD/train_tempobert.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e3702536b9a34fea"}},{"code_sha256_prefix":"4675a84ab7a3e543","entry":"calc_change_score_cosine_dist","repo":"guyrosin/temporal_attention","repo_kind":"listed","path":"semantic_change_detection.py","file_url":"https://github.com/guyrosin/temporal_attention/blob/HEAD/semantic_change_detection.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4675a84ab7a3e543"}},{"code_sha256_prefix":"441f6740f7809dfb","entry":"config_to_temporal","repo":"guyrosin/temporal_attention","repo_kind":"listed","path":"hf_utils.py","file_url":"https://github.com/guyrosin/temporal_attention/blob/HEAD/hf_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"441f6740f7809dfb"}},{"code_sha256_prefix":"ccd9731bc1c63e5c","entry":"get_embedding","repo":"guyrosin/temporal_attention","repo_kind":"listed","path":"semantic_change_detection.py","file_url":"https://github.com/guyrosin/temporal_attention/blob/HEAD/semantic_change_detection.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ccd9731bc1c63e5c"}},{"code_sha256_prefix":"182620e7a1d9acc4","entry":"lazyconfig_getitem","repo":"guyrosin/temporal_attention","repo_kind":"listed","path":"hf_utils.py","file_url":"https://github.com/guyrosin/temporal_attention/blob/HEAD/hf_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"182620e7a1d9acc4"}},{"code_sha256_prefix":"4948b19066b3d98c","entry":"millify","repo":"guyrosin/temporal_attention","repo_kind":"listed","path":"utils.py","file_url":"https://github.com/guyrosin/temporal_attention/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4948b19066b3d98c"}},{"code_sha256_prefix":"7dc7a4c0ec126f86","entry":"noop","repo":"guyrosin/temporal_attention","repo_kind":"listed","path":"tokenization_utils_fast.py","file_url":"https://github.com/guyrosin/temporal_attention/blob/HEAD/tokenization_utils_fast.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7dc7a4c0ec126f86"}},{"code_sha256_prefix":"6bcecd4cd8678430","entry":"prepend","repo":"guyrosin/temporal_attention","repo_kind":"listed","path":"tokenization_utils_fast.py","file_url":"https://github.com/guyrosin/temporal_attention/blob/HEAD/tokenization_utils_fast.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6bcecd4cd8678430"}},{"code_sha256_prefix":"ffe2a285827399df","entry":"prepend_tuples","repo":"guyrosin/temporal_attention","repo_kind":"listed","path":"tokenization_utils_fast.py","file_url":"https://github.com/guyrosin/temporal_attention/blob/HEAD/tokenization_utils_fast.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ffe2a285827399df"}},{"code_sha256_prefix":"7bbcb9444f86f876","entry":"search_sequence_numpy","repo":"guyrosin/temporal_attention","repo_kind":"listed","path":"utils.py","file_url":"https://github.com/guyrosin/temporal_attention/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7bbcb9444f86f876"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}