{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2508-00518","title":"Fine-grained Spatiotemporal Grounding on Egocentric Videos","arxiv_id":"2508.00518","date":"2025-08-01","proceeding":"ICCV","authors":["Shuo Liang","Yiwu Zhong","Zi-Yuan Hu","Yeyao Tao","Liwei Wang"],"abstract":"Spatiotemporal video grounding aims to localize target entities in videos based on textual queries. While existing research has made significant progress in exocentric videos, the egocentric setting remains relatively underexplored, despite its growing importance in applications such as augmented reality and robotics. In this work, we conduct a systematic analysis of the discrepancies between egocentric and exocentric videos, revealing key challenges such as shorter object durations, sparser trajectories, smaller object sizes, and larger positional shifts. To address these challenges, we introduce EgoMask, the first pixel-level benchmark for fine-grained spatiotemporal grounding in egocentric videos. It is constructed by our proposed automatic annotation pipeline, which annotates referring expressions and object masks across short-, medium-, and long-term videos. Additionally, we create EgoMask-Train, a large-scale training dataset to facilitate model development. Experiments demonstrate that the state-of-the-art spatiotemporal grounding models perform poorly on our benchmark EgoMask, but fine-tuning on EgoMask-Train yields significant improvements, while preserving performance on exocentric datasets. Our work thus provides essential resources and insights for advancing egocentric video understanding. Our code is available at https://github.com/LaVi-Lab/EgoMask .","url_abs":"https://arxiv.org/abs/2508.00518","url_pdf":"https://arxiv.org/pdf/2508.00518","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2508.00518","atlas_url":"https://app.syntology.ai/?focus=2508.00518","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2508.00518"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/LaVi-Lab/EgoMask","reach":{"status":"ok"}}],"summary":{"ran_draft_wrong":1,"ran":1,"unverified":10},"by_repo_kind":{"found_in_text":{"samples":12,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":12,"samples":[{"code_sha256_prefix":"77ff24f1cbcb4f2d","entry":"preprocess","repo":"LaVi-Lab/EgoMask","repo_kind":"found_in_text","path":"evaluation/videolisa/inference_egomask.py","file_url":"https://github.com/LaVi-Lab/EgoMask/blob/HEAD/evaluation/videolisa/inference_egomask.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"77ff24f1cbcb4f2d"}},{"code_sha256_prefix":"2362958b7e6a9591","entry":"read_json","repo":"LaVi-Lab/EgoMask","repo_kind":"found_in_text","path":"evaluation/common_utils.py","file_url":"https://github.com/LaVi-Lab/EgoMask/blob/HEAD/evaluation/common_utils.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2362958b7e6a9591"}},{"code_sha256_prefix":"c03d1bfb614ef85f","entry":"chunk_list","repo":"LaVi-Lab/EgoMask","repo_kind":"found_in_text","path":"preprocess/captioning/generate_referring_expressions.py","file_url":"https://github.com/LaVi-Lab/EgoMask/blob/HEAD/preprocess/captioning/generate_referring_expressions.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c03d1bfb614ef85f"}},{"code_sha256_prefix":"b23eee6983994c61","entry":"convert_single_format_for_save","repo":"LaVi-Lab/EgoMask","repo_kind":"found_in_text","path":"evaluation/groundedsam2/grounded_sam2.py","file_url":"https://github.com/LaVi-Lab/EgoMask/blob/HEAD/evaluation/groundedsam2/grounded_sam2.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b23eee6983994c61"}},{"code_sha256_prefix":"afd4d321111a2ff6","entry":"db_eval_iou","repo":"LaVi-Lab/EgoMask","repo_kind":"found_in_text","path":"evaluation/metrics.py","file_url":"https://github.com/LaVi-Lab/EgoMask/blob/HEAD/evaluation/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"afd4d321111a2ff6"}},{"code_sha256_prefix":"91aab049b2d138cb","entry":"ego4d_info","repo":"LaVi-Lab/EgoMask","repo_kind":"found_in_text","path":"preprocess/pre_data/egotracks_clip2vid.py","file_url":"https://github.com/LaVi-Lab/EgoMask/blob/HEAD/preprocess/pre_data/egotracks_clip2vid.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"91aab049b2d138cb"}},{"code_sha256_prefix":"cbb4106c5df06a21","entry":"get_all_frame_names","repo":"LaVi-Lab/EgoMask","repo_kind":"found_in_text","path":"evaluation/common_utils.py","file_url":"https://github.com/LaVi-Lab/EgoMask/blob/HEAD/evaluation/common_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"cbb4106c5df06a21"}},{"code_sha256_prefix":"28f231fee8b7235f","entry":"get_r2vos_accuracy","repo":"LaVi-Lab/EgoMask","repo_kind":"found_in_text","path":"evaluation/metrics.py","file_url":"https://github.com/LaVi-Lab/EgoMask/blob/HEAD/evaluation/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"28f231fee8b7235f"}},{"code_sha256_prefix":"2db69d5e1af507b2","entry":"get_r2vos_robustness","repo":"LaVi-Lab/EgoMask","repo_kind":"found_in_text","path":"evaluation/metrics.py","file_url":"https://github.com/LaVi-Lab/EgoMask/blob/HEAD/evaluation/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2db69d5e1af507b2"}},{"code_sha256_prefix":"87190ea5d639868b","entry":"get_sparse_indices","repo":"LaVi-Lab/EgoMask","repo_kind":"found_in_text","path":"evaluation/videolisa/inference_egomask.py","file_url":"https://github.com/LaVi-Lab/EgoMask/blob/HEAD/evaluation/videolisa/inference_egomask.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"87190ea5d639868b"}},{"code_sha256_prefix":"9392c06823ede9c0","entry":"select_bigest_appearance","repo":"LaVi-Lab/EgoMask","repo_kind":"found_in_text","path":"preprocess/captioning/generate_referring_expressions.py","file_url":"https://github.com/LaVi-Lab/EgoMask/blob/HEAD/preprocess/captioning/generate_referring_expressions.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9392c06823ede9c0"}},{"code_sha256_prefix":"3b244b7ef614e02b","entry":"simple_process","repo":"LaVi-Lab/EgoMask","repo_kind":"found_in_text","path":"evaluation/sa2va/inference_egomask.py","file_url":"https://github.com/LaVi-Lab/EgoMask/blob/HEAD/evaluation/sa2va/inference_egomask.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3b244b7ef614e02b"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.CV","source":"arxiv_api"},"syntology_extracted_results":null}