{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/in-context-imitation-learning-via-next-token","title":"In-Context Imitation Learning via Next-Token Prediction","arxiv_id":"2408.15980","date":"2024-08-28","proceeding":null,"authors":["Letian Fu","Huang Huang","Gaurav Datta","Lawrence Yunliang Chen","William Chung-Ho Panitch","Fangchen Liu","Hui Li","Ken Goldberg"],"abstract":"We explore how to enhance next-token prediction models to perform in-context imitation learning on a real robot, where the robot executes new tasks by interpreting contextual information provided during the input phase, without updating its underlying policy parameters. We propose In-Context Robot Transformer (ICRT), a causal transformer that performs autoregressive prediction on sensorimotor trajectories without relying on any linguistic data or reward function. This formulation enables flexible and training-free execution of new tasks at test time, achieved by prompting the model with sensorimotor trajectories of the new task composing of image observations, actions and states tuples, collected through human teleoperation. Experiments with a Franka Emika robot demonstrate that the ICRT can adapt to new tasks specified by prompts, even in environment configurations that differ from both the prompt and the training data. In a multitask environment setup, ICRT significantly outperforms current state-of-the-art next-token prediction models in robotics on generalizing to unseen tasks. Code, checkpoints and data are available on https://icrt.dev/","url_abs":"https://arxiv.org/abs/2408.15980v2","url_pdf":"https://arxiv.org/pdf/2408.15980v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"in-context-imitation-learning-via-next-token","repo_url":"https://github.com/Max-Fu/icrt","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"imitation-learning","task_name":"Imitation Learning"},{"task_slug":"prediction","task_name":"Prediction"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2408.15980","atlas_url":"https://app.syntology.ai/?focus=2408.15980","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.15980"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Max-Fu/icrt","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":8,"ran_violates":1,"unverified":2},"by_repo_kind":{"official":{"samples":11,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"373a3280ff76d40d","entry":"apply_rotary_emb","repo":"Max-Fu/icrt","repo_kind":"official","path":"icrt/models/policy/llama.py","file_url":"https://github.com/Max-Fu/icrt/blob/HEAD/icrt/models/policy/llama.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"373a3280ff76d40d"}},{"code_sha256_prefix":"cd809ededc7ba250","entry":"get_2d_sincos_pos_embed","repo":"Max-Fu/icrt","repo_kind":"official","path":"icrt/models/backbones/pos_embed.py","file_url":"https://github.com/Max-Fu/icrt/blob/HEAD/icrt/models/backbones/pos_embed.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cd809ededc7ba250"}},{"code_sha256_prefix":"45dd229701ef23a9","entry":"get_2d_sincos_pos_embed_for_coords","repo":"Max-Fu/icrt","repo_kind":"official","path":"icrt/models/backbones/pos_embed.py","file_url":"https://github.com/Max-Fu/icrt/blob/HEAD/icrt/models/backbones/pos_embed.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"45dd229701ef23a9"}},{"code_sha256_prefix":"a9138b353bdf772d","entry":"get_default_vision_aug","repo":"Max-Fu/icrt","repo_kind":"official","path":"icrt/models/backbones/preprocess.py","file_url":"https://github.com/Max-Fu/icrt/blob/HEAD/icrt/models/backbones/preprocess.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a9138b353bdf772d"}},{"code_sha256_prefix":"14a84c2cbfebc413","entry":"precompute_freqs_cis","repo":"Max-Fu/icrt","repo_kind":"official","path":"icrt/models/policy/llama.py","file_url":"https://github.com/Max-Fu/icrt/blob/HEAD/icrt/models/policy/llama.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"14a84c2cbfebc413"}},{"code_sha256_prefix":"61fd8b1b4e07b8c9","entry":"reshape_for_broadcast","repo":"Max-Fu/icrt","repo_kind":"official","path":"icrt/models/policy/llama.py","file_url":"https://github.com/Max-Fu/icrt/blob/HEAD/icrt/models/policy/llama.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"61fd8b1b4e07b8c9"}},{"code_sha256_prefix":"7823943fccca7f70","entry":"to_pil","repo":"Max-Fu/icrt","repo_kind":"official","path":"icrt/models/backbones/preprocess.py","file_url":"https://github.com/Max-Fu/icrt/blob/HEAD/icrt/models/backbones/preprocess.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7823943fccca7f70"}},{"code_sha256_prefix":"101148492ee6fc64","entry":"undo_vision_transform","repo":"Max-Fu/icrt","repo_kind":"official","path":"icrt/models/policy/icrt_wrapper.py","file_url":"https://github.com/Max-Fu/icrt/blob/HEAD/icrt/models/policy/icrt_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"101148492ee6fc64"}},{"code_sha256_prefix":"9a73cbb66f31231c","entry":"unnormalize_fn","repo":"Max-Fu/icrt","repo_kind":"official","path":"icrt/models/backbones/preprocess.py","file_url":"https://github.com/Max-Fu/icrt/blob/HEAD/icrt/models/backbones/preprocess.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9a73cbb66f31231c"}},{"code_sha256_prefix":"b7482295ffca3e2c","entry":"get_2d_sincos_pos_embed_from_grid","repo":"Max-Fu/icrt","repo_kind":"official","path":"icrt/models/backbones/pos_embed.py","file_url":"https://github.com/Max-Fu/icrt/blob/HEAD/icrt/models/backbones/pos_embed.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b7482295ffca3e2c"}},{"code_sha256_prefix":"1a3e01d4552a3b11","entry":"handle_flash_attn","repo":"Max-Fu/icrt","repo_kind":"official","path":"icrt/models/backbones/utils.py","file_url":"https://github.com/Max-Fu/icrt/blob/HEAD/icrt/models/backbones/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1a3e01d4552a3b11"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}