{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/representation-learning-for-neural-population","title":"Representation learning for neural population activity with Neural Data Transformers","arxiv_id":"2108.01210","date":"2021-08-02","proceeding":null,"authors":["Joel Ye","Chethan Pandarinath"],"abstract":"Neural population activity is theorized to reflect an underlying dynamical structure. This structure can be accurately captured using state space models with explicit dynamics, such as those based on recurrent neural networks (RNNs). However, using recurrence to explicitly model dynamics necessitates sequential processing of data, slowing real-time applications such as brain-computer interfaces. Here we introduce the Neural Data Transformer (NDT), a non-recurrent alternative. We test the NDT's ability to capture autonomous dynamical systems by applying it to synthetic datasets with known dynamics and data from monkey motor cortex during a reaching task well-modeled by RNNs. The NDT models these datasets as well as state-of-the-art recurrent models. Further, its non-recurrence enables 3.9ms inference, well within the loop time of real-time applications and more than 6 times faster than recurrent baselines on the monkey reaching dataset. These results suggest that an explicit dynamics model is not necessary to model autonomous neural population dynamics. Code: https://github.com/snel-repo/neural-data-transformers","url_abs":"https://arxiv.org/abs/2108.01210v1","url_pdf":"https://arxiv.org/pdf/2108.01210v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"representation-learning-for-neural-population","repo_url":"https://github.com/snel-repo/neural-data-transformers","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"Unlicense"}}],"tasks":[{"task_slug":"representation-learning","task_name":"Representation Learning"},{"task_slug":"state-space-models","task_name":"State Space Models"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2108.01210","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2108.01210"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/snel-repo/neural-data-transformers","reach":{"status":"ok","spdx":"Unlicense"}}],"summary":{"ran_violates":1,"unverified":5},"by_repo_kind":{"official":{"samples":6,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"534905c1bdb93a5e","entry":"binary_mask_to_attn_mask","repo":"snel-repo/neural-data-transformers","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/snel-repo/neural-data-transformers/blob/HEAD/src/utils.py","link_basis":"plan_row","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Unlicense","inline_ok":true,"mcp_get_code":{"code_sha256":"534905c1bdb93a5e"}},{"code_sha256_prefix":"8adde501f873a07c","entry":"build_hp_dict","repo":"snel-repo/neural-data-transformers","repo_kind":"official","path":"ray_get_lfve.py","file_url":"https://github.com/snel-repo/neural-data-transformers/blob/HEAD/ray_get_lfve.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Unlicense","inline_ok":true,"mcp_get_code":{"code_sha256":"8adde501f873a07c"}},{"code_sha256_prefix":"19968a077e9dcdb3","entry":"exp_smooth","repo":"snel-repo/neural-data-transformers","repo_kind":"official","path":"src/runner.py","file_url":"https://github.com/snel-repo/neural-data-transformers/blob/HEAD/src/runner.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Unlicense","inline_ok":true,"mcp_get_code":{"code_sha256":"19968a077e9dcdb3"}},{"code_sha256_prefix":"ff9ea9bd21d83bd1","entry":"get_inverse_sqrt_schedule","repo":"snel-repo/neural-data-transformers","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/snel-repo/neural-data-transformers/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Unlicense","inline_ok":true,"mcp_get_code":{"code_sha256":"ff9ea9bd21d83bd1"}},{"code_sha256_prefix":"9fba08006b5c5adf","entry":"get_lightest_gpus","repo":"snel-repo/neural-data-transformers","repo_kind":"official","path":"src/runner.py","file_url":"https://github.com/snel-repo/neural-data-transformers/blob/HEAD/src/runner.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Unlicense","inline_ok":true,"mcp_get_code":{"code_sha256":"9fba08006b5c5adf"}},{"code_sha256_prefix":"87ed46df806d554a","entry":"merge_train_valid","repo":"snel-repo/neural-data-transformers","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/snel-repo/neural-data-transformers/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Unlicense","inline_ok":true,"mcp_get_code":{"code_sha256":"87ed46df806d554a"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}