{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/transformer-based-world-models-are-happy-with","title":"Transformer-based World Models Are Happy With 100k Interactions","arxiv_id":"2303.07109","date":"2023-03-13","proceeding":null,"authors":["Jan Robine","Marc Höftmann","Tobias Uelwer","Stefan Harmeling"],"abstract":"Deep neural networks have been successful in many reinforcement learning settings. However, compared to human learners they are overly data hungry. To build a sample-efficient world model, we apply a transformer to real-world episodes in an autoregressive manner: not only the compact latent states and the taken actions but also the experienced or predicted rewards are fed into the transformer, so that it can attend flexibly to all three modalities at different time steps. The transformer allows our world model to access previous states directly, instead of viewing them through a compressed recurrent state. By utilizing the Transformer-XL architecture, it is able to learn long-term dependencies while staying computationally efficient. Our transformer-based world model (TWM) generates meaningful, new experience, which is used to train a policy that outperforms previous model-free and model-based reinforcement learning algorithms on the Atari 100k benchmark.","url_abs":"https://arxiv.org/abs/2303.07109v1","url_pdf":"https://arxiv.org/pdf/2303.07109v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"transformer-based-world-models-are-happy-with","repo_url":"https://github.com/jrobine/twm","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"model-based-reinforcement-learning","task_name":"Model-based Reinforcement Learning"},{"task_slug":"reinforcement-learning","task_name":"Reinforcement Learning"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"},{"task_slug":"reinforcement-learning-2","task_name":"reinforcement-learning"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"adaptive-input-representations","method_name":"Adaptive Input Representations"},{"method_slug":"adaptive-softmax","method_name":"Adaptive Softmax"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"cosine-annealing","method_name":"Cosine Annealing"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-cosine-annealing","method_name":"Linear Warmup With Cosine Annealing"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"relu","method_name":"ReLU"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer-xl","method_name":"Transformer-XL"},{"method_slug":"variational-dropout","method_name":"Variational Dropout"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2303.07109","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.07109"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jrobine/twm","reach":null}],"summary":{"ran":6,"ran_honours":1,"ran_draft_wrong":7,"ran_violates":1,"ran_fixture":1,"unverified":9},"by_repo_kind":{"official":{"samples":25,"ran":16,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"006d91bc0bd061db","entry":"AdamOptim","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"006d91bc0bd061db"}},{"code_sha256_prefix":"e1c8c42def7db040","entry":"PositionalEncoding","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e1c8c42def7db040"}},{"code_sha256_prefix":"ea267d5f625dc1ca","entry":"RelativeMultiheadSelfAttention","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ea267d5f625dc1ca"}},{"code_sha256_prefix":"425555f50f17e87d","entry":"TransformerXLDecoder","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"425555f50f17e87d"}},{"code_sha256_prefix":"fdc181c488b365db","entry":"TransformerXLDecoderLayer","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fdc181c488b365db"}},{"code_sha256_prefix":"8cfa03dabb18e3af","entry":"_MultilayerModule","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8cfa03dabb18e3af"}},{"code_sha256_prefix":"d25a054b95990a84","entry":"_calculate_gain","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d25a054b95990a84"}},{"code_sha256_prefix":"a7250d5b635dd398","entry":"_get_initializer","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a7250d5b635dd398"}},{"code_sha256_prefix":"ab154d639b4e4ae3","entry":"combine_metrics","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ab154d639b4e4ae3"}},{"code_sha256_prefix":"5439c3b62dbdc1fc","entry":"get_activation","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5439c3b62dbdc1fc"}},{"code_sha256_prefix":"4e5362cab16fd0f9","entry":"get_norm_1d","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4e5362cab16fd0f9"}},{"code_sha256_prefix":"84eae076a18b163d","entry":"get_norm_2d","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"84eae076a18b163d"}},{"code_sha256_prefix":"fd4dd5bdc9742fc0","entry":"init_","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fd4dd5bdc9742fc0"}},{"code_sha256_prefix":"7e4ce9c6f41f3336","entry":"same_batch_shape","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7e4ce9c6f41f3336"}},{"code_sha256_prefix":"165f8bc0355a9865","entry":"same_batch_shape_time_offset","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"165f8bc0355a9865"}},{"code_sha256_prefix":"6f69e804fca3263d","entry":"update_metrics","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6f69e804fca3263d"}},{"code_sha256_prefix":"173854afd0798188","entry":"CNN","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"173854afd0798188"}},{"code_sha256_prefix":"c61718d72bba4f72","entry":"DynamicsModel","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c61718d72bba4f72"}},{"code_sha256_prefix":"103a05c5c1b23236","entry":"MLP","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"103a05c5c1b23236"}},{"code_sha256_prefix":"430454696696dc84","entry":"ObservationModel","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"430454696696dc84"}},{"code_sha256_prefix":"34a08d167acd0d24","entry":"PredictionNet","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"34a08d167acd0d24"}},{"code_sha256_prefix":"82ffe40472f6c427","entry":"TransposeCNN","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"82ffe40472f6c427"}},{"code_sha256_prefix":"dd79139c2de865e1","entry":"WorldModel","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dd79139c2de865e1"}},{"code_sha256_prefix":"426051c1e209bfef","entry":"_kaiming_uniform_","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"426051c1e209bfef"}},{"code_sha256_prefix":"862742223d0ec96a","entry":"check_no_grad","repo":"jrobine/twm","repo_kind":"official","path":"twm/world_model.py","file_url":"https://github.com/jrobine/twm/blob/HEAD/twm/world_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"862742223d0ec96a"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}