{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/neural-language-models-as-psycholinguistic","title":"Neural Language Models as Psycholinguistic Subjects: Representations of Syntactic State","arxiv_id":"1903.03260","date":"2019-03-08","proceeding":"NAACL 2019 6","authors":["Richard Futrell","Ethan Wilcox","Takashi Morita","Peng Qian","Miguel Ballesteros","Roger Levy"],"abstract":"We deploy the methods of controlled psycholinguistic experimentation to shed\nlight on the extent to which the behavior of neural network language models\nreflects incremental representations of syntactic state. To do so, we examine\nmodel behavior on artificial sentences containing a variety of syntactically\ncomplex structures. We test four models: two publicly available LSTM sequence\nmodels of English (Jozefowicz et al., 2016; Gulordava et al., 2018) trained on\nlarge datasets; an RNNG (Dyer et al., 2016) trained on a small, parsed dataset;\nand an LSTM trained on the same small corpus as the RNNG. We find evidence that\nthe LSTMs trained on large datasets represent syntactic state over large spans\nof text in a way that is comparable to the RNNG, while the LSTM trained on the\nsmall dataset does not or does so only weakly.","url_abs":"http://arxiv.org/abs/1903.03260v1","url_pdf":"http://arxiv.org/pdf/1903.03260v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"neural-language-models-as-psycholinguistic","repo_url":"https://github.com/langprocgroup/nn_syntactic_state","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"none","reach":null},{"paper_slug":"neural-language-models-as-psycholinguistic","repo_url":"https://github.com/wjurayj/garden-path-gpt2","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":null}],"tasks":[],"methods":[{"method_slug":"lstm","method_name":"LSTM"},{"method_slug":"sigmoid-activation","method_name":"Sigmoid Activation"},{"method_slug":"tanh-activation","method_name":"Tanh Activation"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1903.03260","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1903.03260"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/wjurayj/garden-path-gpt2","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/langprocgroup/nn_syntactic_state","reach":null}],"summary":{"ran_draft_wrong":3,"ran_violates":1,"unverified":1},"by_repo_kind":{"official":{"samples":5,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":5,"samples":[{"code_sha256_prefix":"302bcd1745837a31","entry":"expand_items","repo":"langprocgroup/nn_syntactic_state","repo_kind":"official","path":"digging-in-2/expand_items.py","file_url":"https://github.com/langprocgroup/nn_syntactic_state/blob/HEAD/digging-in-2/expand_items.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"302bcd1745837a31"}},{"code_sha256_prefix":"30a561fe85bc95db","entry":"expand_items","repo":"langprocgroup/nn_syntactic_state","repo_kind":"official","path":"gardenpathing-verb-ambiguity/expand_items.py","file_url":"https://github.com/langprocgroup/nn_syntactic_state/blob/HEAD/gardenpathing-verb-ambiguity/expand_items.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"30a561fe85bc95db"}},{"code_sha256_prefix":"ff842db41821f0bb","entry":"is_unk","repo":"langprocgroup/nn_syntactic_state","repo_kind":"official","path":"scripts/run_models.py","file_url":"https://github.com/langprocgroup/nn_syntactic_state/blob/HEAD/scripts/run_models.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ff842db41821f0bb"}},{"code_sha256_prefix":"b2ad8ffa712cc599","entry":"sentences","repo":"langprocgroup/nn_syntactic_state","repo_kind":"official","path":"scripts/run_models.py","file_url":"https://github.com/langprocgroup/nn_syntactic_state/blob/HEAD/scripts/run_models.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b2ad8ffa712cc599"}},{"code_sha256_prefix":"b1e4710556059603","entry":"run_model","repo":"langprocgroup/nn_syntactic_state","repo_kind":"official","path":"scripts/run_models.py","file_url":"https://github.com/langprocgroup/nn_syntactic_state/blob/HEAD/scripts/run_models.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b1e4710556059603"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}