{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/learning-the-effects-of-physical-actions-in-a","title":"Learning the Effects of Physical Actions in a Multi-modal Environment","arxiv_id":"2301.11845","date":"2023-01-27","proceeding":null,"authors":["Gautier Dagan","Frank Keller","Alex Lascarides"],"abstract":"Large Language Models (LLMs) handle physical commonsense information inadequately. As a result of being trained in a disembodied setting, LLMs often fail to predict an action's outcome in a given environment. However, predicting the effects of an action before it is executed is crucial in planning, where coherent sequences of actions are often needed to achieve a goal. Therefore, we introduce the multi-modal task of predicting the outcomes of actions solely from realistic sensory inputs (images and text). Next, we extend an LLM to model latent representations of objects to better predict action outcomes in an environment. We show that multi-modal models can capture physical commonsense when augmented with visual information. Finally, we evaluate our model's performance on novel actions and objects and find that combining modalities help models to generalize and learn physical commonsense reasoning better.","url_abs":"https://arxiv.org/abs/2301.11845v2","url_pdf":"https://arxiv.org/pdf/2301.11845v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"learning-the-effects-of-physical-actions-in-a","repo_url":"https://github.com/gautierdag/piglet-vis","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"physical-commonsense-reasoning","task_name":"Physical Commonsense Reasoning"}],"methods":[{"method_slug":"fail","method_name":"fail"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2301.11845","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2301.11845"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/gautierdag/piglet-vis","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":8},"by_repo_kind":{"official":{"samples":8,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"0dcac92ca797debf","entry":"calculate_attention_weights_per_pixel","repo":"gautierdag/piglet-vis","repo_kind":"official","path":"pigletvis/models/analysis.py","file_url":"https://github.com/gautierdag/piglet-vis/blob/HEAD/pigletvis/models/analysis.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0dcac92ca797debf"}},{"code_sha256_prefix":"6efdc3a8c781e0d6","entry":"denormalize_image","repo":"gautierdag/piglet-vis","repo_kind":"official","path":"pigletvis/dataset.py","file_url":"https://github.com/gautierdag/piglet-vis/blob/HEAD/pigletvis/dataset.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6efdc3a8c781e0d6"}},{"code_sha256_prefix":"91c0a36505acdbe3","entry":"get_actions_mapper","repo":"gautierdag/piglet-vis","repo_kind":"official","path":"pigletvis/models/mappings.py","file_url":"https://github.com/gautierdag/piglet-vis/blob/HEAD/pigletvis/models/mappings.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"91c0a36505acdbe3"}},{"code_sha256_prefix":"bf535327d6a8febe","entry":"get_cmap","repo":"gautierdag/piglet-vis","repo_kind":"official","path":"pigletvis/models/analysis.py","file_url":"https://github.com/gautierdag/piglet-vis/blob/HEAD/pigletvis/models/analysis.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bf535327d6a8febe"}},{"code_sha256_prefix":"e940eca6fbf5e7a1","entry":"get_image_feature_extractor","repo":"gautierdag/piglet-vis","repo_kind":"official","path":"pigletvis/models/image_models.py","file_url":"https://github.com/gautierdag/piglet-vis/blob/HEAD/pigletvis/models/image_models.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e940eca6fbf5e7a1"}},{"code_sha256_prefix":"15a878e324f6f04e","entry":"get_objects_mapper","repo":"gautierdag/piglet-vis","repo_kind":"official","path":"pigletvis/models/mappings.py","file_url":"https://github.com/gautierdag/piglet-vis/blob/HEAD/pigletvis/models/mappings.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"15a878e324f6f04e"}},{"code_sha256_prefix":"8dfe65bed1c3a905","entry":"load_image_from_path","repo":"gautierdag/piglet-vis","repo_kind":"official","path":"pigletvis/dataset.py","file_url":"https://github.com/gautierdag/piglet-vis/blob/HEAD/pigletvis/dataset.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8dfe65bed1c3a905"}},{"code_sha256_prefix":"b2aae85256af5069","entry":"plot_effect_grid","repo":"gautierdag/piglet-vis","repo_kind":"official","path":"pigletvis/models/analysis.py","file_url":"https://github.com/gautierdag/piglet-vis/blob/HEAD/pigletvis/models/analysis.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b2aae85256af5069"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}