{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/semantically-guided-representation-learning","title":"Semantically Guided Representation Learning For Action Anticipation","arxiv_id":"2407.02309","date":"2024-07-02","proceeding":null,"authors":["Anxhelo Diko","Danilo Avola","Bardh Prenkaj","Federico Fontana","Luigi Cinque"],"abstract":"Action anticipation is the task of forecasting future activity from a partially observed sequence of events. However, this task is exposed to intrinsic future uncertainty and the difficulty of reasoning upon interconnected actions. Unlike previous works that focus on extrapolating better visual and temporal information, we concentrate on learning action representations that are aware of their semantic interconnectivity based on prototypical action patterns and contextual co-occurrences. To this end, we propose the novel Semantically Guided Representation Learning (S-GEAR) framework. S-GEAR learns visual action prototypes and leverages language models to structure their relationship, inducing semanticity. To gather insights on S-GEAR's effectiveness, we test it on four action anticipation benchmarks, obtaining improved results compared to previous works: +3.5, +2.7, and +3.5 absolute points on Top-1 Accuracy on Epic-Kitchen 55, EGTEA Gaze+ and 50 Salads, respectively, and +0.8 on Top-5 Recall on Epic-Kitchens 100. We further observe that S-GEAR effectively transfers the geometric associations between actions from language to visual prototypes. Finally, S-GEAR opens new research frontiers in anticipation tasks by demonstrating the intricate impact of action semantic interconnectivity.","url_abs":"https://arxiv.org/abs/2407.02309v1","url_pdf":"https://arxiv.org/pdf/2407.02309v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"semantically-guided-representation-learning","repo_url":"https://github.com/ADiko1997/S-GEAR","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"action-anticipation","task_name":"Action Anticipation"},{"task_slug":"representation-learning","task_name":"Representation Learning"}],"methods":[{"method_slug":"aware","method_name":"AWARE"},{"method_slug":"focus","method_name":"Focus"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/action-anticipation-on-epic-kitchens-100","task":"Action Anticipation","dataset":"EPIC-KITCHENS-100","model":"S-GEAR","rank_in_archive_order":4,"of":9,"metrics":{"Recall@5":"19.9"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2407.02309","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.02309"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/ADiko1997/S-GEAR","reach":null}],"summary":{"ran":5,"ran_fixture":1,"unverified":2},"by_repo_kind":{"official":{"samples":8,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"f4388cc6db5bce14","entry":"DropPath","repo":"ADiko1997/S-GEAR","repo_kind":"official","path":"models/base_model_ts.py","file_url":"https://github.com/ADiko1997/S-GEAR/blob/HEAD/models/base_model_ts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f4388cc6db5bce14"}},{"code_sha256_prefix":"2c1c2614fb1bd503","entry":"ProtAttn","repo":"ADiko1997/S-GEAR","repo_kind":"official","path":"models/base_model_ts.py","file_url":"https://github.com/ADiko1997/S-GEAR/blob/HEAD/models/base_model_ts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2c1c2614fb1bd503"}},{"code_sha256_prefix":"4d0ab14d77dfb105","entry":"SelfAttention","repo":"ADiko1997/S-GEAR","repo_kind":"official","path":"models/base_model_ts.py","file_url":"https://github.com/ADiko1997/S-GEAR/blob/HEAD/models/base_model_ts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4d0ab14d77dfb105"}},{"code_sha256_prefix":"c2b345bea29462d2","entry":"TCAttention","repo":"ADiko1997/S-GEAR","repo_kind":"official","path":"models/base_model_ts.py","file_url":"https://github.com/ADiko1997/S-GEAR/blob/HEAD/models/base_model_ts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c2b345bea29462d2"}},{"code_sha256_prefix":"06f791fb49f18b6b","entry":"TopSimilarTokens","repo":"ADiko1997/S-GEAR","repo_kind":"official","path":"models/base_model_ts.py","file_url":"https://github.com/ADiko1997/S-GEAR/blob/HEAD/models/base_model_ts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"06f791fb49f18b6b"}},{"code_sha256_prefix":"314973a77b15948e","entry":"drop_path","repo":"ADiko1997/S-GEAR","repo_kind":"official","path":"models/base_model_ts.py","file_url":"https://github.com/ADiko1997/S-GEAR/blob/HEAD/models/base_model_ts.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"314973a77b15948e"}},{"code_sha256_prefix":"1710791c23c4b759","entry":"BaseModel","repo":"ADiko1997/S-GEAR","repo_kind":"official","path":"models/base_model_ts.py","file_url":"https://github.com/ADiko1997/S-GEAR/blob/HEAD/models/base_model_ts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1710791c23c4b759"}},{"code_sha256_prefix":"8c53d89c3f0e62b9","entry":"TCA","repo":"ADiko1997/S-GEAR","repo_kind":"official","path":"models/base_model_ts.py","file_url":"https://github.com/ADiko1997/S-GEAR/blob/HEAD/models/base_model_ts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8c53d89c3f0e62b9"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}