{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/future-language-modeling-from-temporal","title":"Future Language Modeling from Temporal Document History","arxiv_id":"2404.10297","date":"2024-04-16","proceeding":null,"authors":["Changmao Li","Jeffrey Flanigan"],"abstract":"Predicting the future is of great interest across many aspects of human activity. Businesses are interested in future trends, traders are interested in future stock prices, and companies are highly interested in future technological breakthroughs. While there are many automated systems for predicting future numerical data, such as weather, stock prices, and demand for products, there is relatively little work in automatically predicting textual data. Humans are interested in textual data predictions because it is a natural format for our consumption, and experts routinely make predictions in a textual format (Christensen et al., 2004; Tetlock & Gardner, 2015; Frick, 2015). However, there has been relatively little formalization of this general problem in the machine learning or natural language processing communities. To address this gap, we introduce the task of future language modeling: probabilistic modeling of texts in the future based on a temporal history of texts. To our knowledge, our work is the first work to formalize the task of predicting the future in this way. We show that it is indeed possible to build future language models that improve upon strong non-temporal language model baselines, opening the door to working on this important, and widely applicable problem.","url_abs":"https://arxiv.org/abs/2404.10297v1","url_pdf":"https://arxiv.org/pdf/2404.10297v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"future-language-modeling-from-temporal","repo_url":"https://github.com/jlab-nlp/future-language-modeling","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2404.10297","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.10297"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jlab-nlp/future-language-modeling","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":7,"ran_draft_wrong":3,"unverified":6},"by_repo_kind":{"official":{"samples":16,"ran":10,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"5066980021e52081","entry":"containsNumber","repo":"jlab-nlp/future-language-modeling","repo_kind":"official","path":"data_process/generate_extra_tokens.py","file_url":"https://github.com/jlab-nlp/future-language-modeling/blob/HEAD/data_process/generate_extra_tokens.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5066980021e52081"}},{"code_sha256_prefix":"61513ed3f2cc3ce6","entry":"normalization","repo":"jlab-nlp/future-language-modeling","repo_kind":"official","path":"model.py","file_url":"https://github.com/jlab-nlp/future-language-modeling/blob/HEAD/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"61513ed3f2cc3ce6"}},{"code_sha256_prefix":"141be3275bb0ee3e","entry":"normalization_ranged","repo":"jlab-nlp/future-language-modeling","repo_kind":"official","path":"model.py","file_url":"https://github.com/jlab-nlp/future-language-modeling/blob/HEAD/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"141be3275bb0ee3e"}},{"code_sha256_prefix":"b4da4607ebc84dec","entry":"parse_bib_with_abstract","repo":"jlab-nlp/future-language-modeling","repo_kind":"official","path":"data_process/data_process.py","file_url":"https://github.com/jlab-nlp/future-language-modeling/blob/HEAD/data_process/data_process.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b4da4607ebc84dec"}},{"code_sha256_prefix":"a766a95b9fd5ee9c","entry":"prepare_ctrl_input","repo":"jlab-nlp/future-language-modeling","repo_kind":"official","path":"generator.py","file_url":"https://github.com/jlab-nlp/future-language-modeling/blob/HEAD/generator.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a766a95b9fd5ee9c"}},{"code_sha256_prefix":"b7bf53f6b6b11a4b","entry":"prepare_xlm_input","repo":"jlab-nlp/future-language-modeling","repo_kind":"official","path":"generator.py","file_url":"https://github.com/jlab-nlp/future-language-modeling/blob/HEAD/generator.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b7bf53f6b6b11a4b"}},{"code_sha256_prefix":"80844fa7cfa6e8d8","entry":"prepare_xlnet_input","repo":"jlab-nlp/future-language-modeling","repo_kind":"official","path":"generator.py","file_url":"https://github.com/jlab-nlp/future-language-modeling/blob/HEAD/generator.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"80844fa7cfa6e8d8"}},{"code_sha256_prefix":"8c294cca9ed2f3c3","entry":"read_bib","repo":"jlab-nlp/future-language-modeling","repo_kind":"official","path":"data_process/data_process.py","file_url":"https://github.com/jlab-nlp/future-language-modeling/blob/HEAD/data_process/data_process.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8c294cca9ed2f3c3"}},{"code_sha256_prefix":"0f33c9675bbfb78d","entry":"read_gpt2_pred_data","repo":"jlab-nlp/future-language-modeling","repo_kind":"official","path":"data_process/post_processing.py","file_url":"https://github.com/jlab-nlp/future-language-modeling/blob/HEAD/data_process/post_processing.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0f33c9675bbfb78d"}},{"code_sha256_prefix":"632b87ee3b0dda8e","entry":"z_score_standardization","repo":"jlab-nlp/future-language-modeling","repo_kind":"official","path":"model.py","file_url":"https://github.com/jlab-nlp/future-language-modeling/blob/HEAD/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"632b87ee3b0dda8e"}},{"code_sha256_prefix":"49b19d90659d51a6","entry":"download_file","repo":"jlab-nlp/future-language-modeling","repo_kind":"official","path":"data_process/data_process.py","file_url":"https://github.com/jlab-nlp/future-language-modeling/blob/HEAD/data_process/data_process.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"49b19d90659d51a6"}},{"code_sha256_prefix":"d1e855d84276ec9d","entry":"evaluate_multiprocess","repo":"jlab-nlp/future-language-modeling","repo_kind":"official","path":"evaluation_tools/evaluate.py","file_url":"https://github.com/jlab-nlp/future-language-modeling/blob/HEAD/evaluation_tools/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d1e855d84276ec9d"}},{"code_sha256_prefix":"4cb9d7d0c36a3890","entry":"evaluate_single","repo":"jlab-nlp/future-language-modeling","repo_kind":"official","path":"evaluation_tools/evaluate.py","file_url":"https://github.com/jlab-nlp/future-language-modeling/blob/HEAD/evaluation_tools/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4cb9d7d0c36a3890"}},{"code_sha256_prefix":"ca7bad690a25a4c1","entry":"generate_year_vocab_representation","repo":"jlab-nlp/future-language-modeling","repo_kind":"official","path":"data_process/generate_year_vocab_representation.py","file_url":"https://github.com/jlab-nlp/future-language-modeling/blob/HEAD/data_process/generate_year_vocab_representation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ca7bad690a25a4c1"}},{"code_sha256_prefix":"1c064e135b578600","entry":"generate_year_vocab_representation_org","repo":"jlab-nlp/future-language-modeling","repo_kind":"official","path":"data_process/generate_year_vocab_representation.py","file_url":"https://github.com/jlab-nlp/future-language-modeling/blob/HEAD/data_process/generate_year_vocab_representation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1c064e135b578600"}},{"code_sha256_prefix":"7fd2c0f025d68e2e","entry":"normalized","repo":"jlab-nlp/future-language-modeling","repo_kind":"official","path":"evaluation_tools/evaluate.py","file_url":"https://github.com/jlab-nlp/future-language-modeling/blob/HEAD/evaluation_tools/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7fd2c0f025d68e2e"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}