{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/palm-predicting-actions-through-language","title":"Palm: Predicting Actions through Language Models @ Ego4D Long-Term Action Anticipation Challenge 2023","arxiv_id":"2306.16545","date":"2023-06-28","proceeding":null,"authors":["Daoji Huang","Otmar Hilliges","Luc van Gool","Xi Wang"],"abstract":"We present Palm, a solution to the Long-Term Action Anticipation (LTA) task utilizing vision-language and large language models. Given an input video with annotated action periods, the LTA task aims to predict possible future actions. We hypothesize that an optimal solution should capture the interdependency between past and future actions, and be able to infer future actions based on the structure and dependency encoded in the past actions. Large language models have demonstrated remarkable commonsense-based reasoning ability. Inspired by that, Palm chains an image captioning model and a large language model. It predicts future actions based on frame descriptions and action labels extracted from the input videos. Our method outperforms other participants in the EGO4D LTA challenge and achieves the best performance in terms of action prediction. Our code is available at https://github.com/DanDoge/Palm","url_abs":"https://arxiv.org/abs/2306.16545v1","url_pdf":"https://arxiv.org/pdf/2306.16545v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"palm-predicting-actions-through-language","repo_url":"https://github.com/dandoge/palm","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"action-anticipation","task_name":"Action Anticipation"},{"task_slug":"image-captioning","task_name":"Image Captioning"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"large-language-model","task_name":"Large Language Model"},{"task_slug":"long-term-action-anticipation","task_name":"Long Term Action Anticipation"}],"methods":[{"method_slug":"palm","method_name":"PaLM"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2306.16545","atlas_url":"https://app.syntology.ai/?focus=2306.16545","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.16545"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/dandoge/palm","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":1,"unverified":5},"by_repo_kind":{"official":{"samples":6,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"b07e3607b90c1e7c","entry":"round_width","repo":"dandoge/palm","repo_kind":"official","path":"ego4d_forecasting/models/video_model_builder.py","file_url":"https://github.com/dandoge/palm/blob/HEAD/ego4d_forecasting/models/video_model_builder.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b07e3607b90c1e7c"}},{"code_sha256_prefix":"bcc1cdae3bb3212c","entry":"drop_path","repo":"dandoge/palm","repo_kind":"official","path":"ego4d_forecasting/models/video_model_builder.py","file_url":"https://github.com/dandoge/palm/blob/HEAD/ego4d_forecasting/models/video_model_builder.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bcc1cdae3bb3212c"}},{"code_sha256_prefix":"dd72775d1ea00db7","entry":"get_loss_func","repo":"dandoge/palm","repo_kind":"official","path":"ego4d_forecasting/models/losses.py","file_url":"https://github.com/dandoge/palm/blob/HEAD/ego4d_forecasting/models/losses.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dd72775d1ea00db7"}},{"code_sha256_prefix":"952f23f7aff7ce7d","entry":"get_norm","repo":"dandoge/palm","repo_kind":"official","path":"ego4d_forecasting/models/batchnorm_helper.py","file_url":"https://github.com/dandoge/palm/blob/HEAD/ego4d_forecasting/models/batchnorm_helper.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"952f23f7aff7ce7d"}},{"code_sha256_prefix":"4d6991202a8d6097","entry":"get_trans_func","repo":"dandoge/palm","repo_kind":"official","path":"ego4d_forecasting/models/resnet_helper.py","file_url":"https://github.com/dandoge/palm/blob/HEAD/ego4d_forecasting/models/resnet_helper.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4d6991202a8d6097"}},{"code_sha256_prefix":"a99430fceaa1dc2d","entry":"is_detection_enabled","repo":"dandoge/palm","repo_kind":"official","path":"ego4d_forecasting/models/video_model_builder.py","file_url":"https://github.com/dandoge/palm/blob/HEAD/ego4d_forecasting/models/video_model_builder.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a99430fceaa1dc2d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}