{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/how-crucial-is-transformer-in-decision","title":"How Crucial is Transformer in Decision Transformer?","arxiv_id":"2211.14655","date":"2022-11-26","proceeding":null,"authors":["Max Siebenborn","Boris Belousov","Junning Huang","Jan Peters"],"abstract":"Decision Transformer (DT) is a recently proposed architecture for Reinforcement Learning that frames the decision-making process as an auto-regressive sequence modeling problem and uses a Transformer model to predict the next action in a sequence of states, actions, and rewards. In this paper, we analyze how crucial the Transformer model is in the complete DT architecture on continuous control tasks. Namely, we replace the Transformer by an LSTM model while keeping the other parts unchanged to obtain what we call a Decision LSTM model. We compare it to DT on continuous control tasks, including pendulum swing-up and stabilization, in simulation and on physical hardware. Our experiments show that DT struggles with continuous control problems, such as inverted pendulum and Furuta pendulum stabilization. On the other hand, the proposed Decision LSTM is able to achieve expert-level performance on these tasks, in addition to learning a swing-up controller on the real system. These results suggest that the strength of the Decision Transformer for continuous control tasks may lie in the overall sequential modeling architecture and not in the Transformer per se.","url_abs":"https://arxiv.org/abs/2211.14655v1","url_pdf":"https://arxiv.org/pdf/2211.14655v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"how-crucial-is-transformer-in-decision","repo_url":"https://github.com/max7born/decision-lstm","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"continuous-control","task_name":"Continuous Control"},{"task_slug":"decision-making","task_name":"Decision Making"},{"task_slug":"continuous-control","task_name":"continuous-control"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"lstm","method_name":"LSTM"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"sigmoid-activation","method_name":"Sigmoid Activation"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"tanh-activation","method_name":"Tanh Activation"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2211.14655","atlas_url":"https://app.syntology.ai/?focus=2211.14655","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.14655"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/max7born/decision-lstm","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_fixture":1,"ran":1,"unverified":3},"by_repo_kind":{"official":{"samples":4,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"0161b27cbe3cc58d","entry":"discount_cumsum","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_fixture","verification_level":2,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"0161b27cbe3cc58d"}},{"code_sha256_prefix":"c3da8172ddacb573","entry":"evaluate_episode","repo":"max7born/decision-lstm","repo_kind":"official","path":"src/decision_transformer/evaluation/evaluate_episodes.py","file_url":"https://github.com/max7born/decision-lstm/blob/HEAD/src/decision_transformer/evaluation/evaluate_episodes.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c3da8172ddacb573"}},{"code_sha256_prefix":"f68af6f1ebefc128","entry":"evaluate_episode_rtg","repo":"max7born/decision-lstm","repo_kind":"official","path":"src/decision_transformer/evaluation/evaluate_episodes.py","file_url":"https://github.com/max7born/decision-lstm/blob/HEAD/src/decision_transformer/evaluation/evaluate_episodes.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f68af6f1ebefc128"}},{"code_sha256_prefix":"00a33466c69c5705","entry":"load_tf_weights_in_gpt2","repo":"max7born/decision-lstm","repo_kind":"official","path":"src/decision_transformer/models/trajectory_gpt2.py","file_url":"https://github.com/max7born/decision-lstm/blob/HEAD/src/decision_transformer/models/trajectory_gpt2.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"00a33466c69c5705"}},{"code_sha256_prefix":"6ae6e0a490018715","entry":"reset_quanser","repo":"max7born/decision-lstm","repo_kind":"official","path":"src/decision_transformer/evaluation/evaluate_episodes.py","file_url":"https://github.com/max7born/decision-lstm/blob/HEAD/src/decision_transformer/evaluation/evaluate_episodes.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6ae6e0a490018715"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}