{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/sharp-nearby-fuzzy-far-away-how-neural","title":"Sharp Nearby, Fuzzy Far Away: How Neural Language Models Use Context","arxiv_id":"1805.04623","date":"2018-05-12","proceeding":"ACL 2018 7","authors":["Urvashi Khandelwal","He He","Peng Qi","Dan Jurafsky"],"abstract":"We know very little about how neural language models (LM) use prior\nlinguistic context. In this paper, we investigate the role of context in an\nLSTM LM, through ablation studies. Specifically, we analyze the increase in\nperplexity when prior context words are shuffled, replaced, or dropped. On two\nstandard datasets, Penn Treebank and WikiText-2, we find that the model is\ncapable of using about 200 tokens of context on average, but sharply\ndistinguishes nearby context (recent 50 tokens) from the distant history. The\nmodel is highly sensitive to the order of words within the most recent\nsentence, but ignores word order in the long-range context (beyond 50 tokens),\nsuggesting the distant past is modeled only as a rough semantic field or topic.\nWe further find that the neural caching model (Grave et al., 2017b) especially\nhelps the LSTM to copy words from within this distant context. Overall, our\nanalysis not only provides a better understanding of how neural LMs use their\ncontext, but also sheds light on recent success from cache-based models.","url_abs":"http://arxiv.org/abs/1805.04623v1","url_pdf":"http://arxiv.org/pdf/1805.04623v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"sharp-nearby-fuzzy-far-away-how-neural","repo_url":"https://github.com/urvashik/lm-context-analysis","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"sentence","task_name":"Sentence"}],"methods":[{"method_slug":"lstm","method_name":"LSTM"},{"method_slug":"sigmoid-activation","method_name":"Sigmoid Activation"},{"method_slug":"tanh-activation","method_name":"Tanh Activation"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/1805.04623","atlas_url":"https://app.syntology.ai/?focus=1805.04623","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1805.04623"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/urvashik/lm-context-analysis","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_honours":1,"unverified":10},"by_repo_kind":{"official":{"samples":11,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"d38f82da04a3f93e","entry":"one_hot","repo":"urvashik/lm-context-analysis","repo_kind":"official","path":"pointer.py","file_url":"https://github.com/urvashik/lm-context-analysis/blob/HEAD/pointer.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d38f82da04a3f93e"}},{"code_sha256_prefix":"6bd10c3dd56c2b8c","entry":"batchify","repo":"urvashik/lm-context-analysis","repo_kind":"official","path":"utils.py","file_url":"https://github.com/urvashik/lm-context-analysis/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6bd10c3dd56c2b8c"}},{"code_sha256_prefix":"ebc7e86b33b13d8c","entry":"batchify_context","repo":"urvashik/lm-context-analysis","repo_kind":"official","path":"utils_context.py","file_url":"https://github.com/urvashik/lm-context-analysis/blob/HEAD/utils_context.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ebc7e86b33b13d8c"}},{"code_sha256_prefix":"e376a38c76548861","entry":"drop_pos","repo":"urvashik/lm-context-analysis","repo_kind":"official","path":"perturbations.py","file_url":"https://github.com/urvashik/lm-context-analysis/blob/HEAD/perturbations.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e376a38c76548861"}},{"code_sha256_prefix":"3c2fe307e011be7d","entry":"embedded_dropout","repo":"urvashik/lm-context-analysis","repo_kind":"official","path":"embed_regularize.py","file_url":"https://github.com/urvashik/lm-context-analysis/blob/HEAD/embed_regularize.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3c2fe307e011be7d"}},{"code_sha256_prefix":"be3874d8cf97dae0","entry":"get_batch","repo":"urvashik/lm-context-analysis","repo_kind":"official","path":"utils.py","file_url":"https://github.com/urvashik/lm-context-analysis/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"be3874d8cf97dae0"}},{"code_sha256_prefix":"7ff48303eab5927b","entry":"get_context_batch","repo":"urvashik/lm-context-analysis","repo_kind":"official","path":"utils_context.py","file_url":"https://github.com/urvashik/lm-context-analysis/blob/HEAD/utils_context.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7ff48303eab5927b"}},{"code_sha256_prefix":"24a81b705de4283b","entry":"get_vocab_all_pos","repo":"urvashik/lm-context-analysis","repo_kind":"official","path":"utils_context.py","file_url":"https://github.com/urvashik/lm-context-analysis/blob/HEAD/utils_context.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"24a81b705de4283b"}},{"code_sha256_prefix":"f4b8e8869b6d4203","entry":"keep_pos","repo":"urvashik/lm-context-analysis","repo_kind":"official","path":"perturbations.py","file_url":"https://github.com/urvashik/lm-context-analysis/blob/HEAD/perturbations.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f4b8e8869b6d4203"}},{"code_sha256_prefix":"336b12c05af1b60e","entry":"random_drop_many","repo":"urvashik/lm-context-analysis","repo_kind":"official","path":"perturbations.py","file_url":"https://github.com/urvashik/lm-context-analysis/blob/HEAD/perturbations.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"336b12c05af1b60e"}},{"code_sha256_prefix":"d98e69125467f2f0","entry":"repackage_hidden","repo":"urvashik/lm-context-analysis","repo_kind":"official","path":"utils.py","file_url":"https://github.com/urvashik/lm-context-analysis/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d98e69125467f2f0"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}