{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/unsupervised-activity-segmentation-by-joint","title":"Unsupervised Action Segmentation by Joint Representation Learning and Online Clustering","arxiv_id":"2105.13353","date":"2021-05-27","proceeding":"CVPR 2022 1","authors":["Sateesh Kumar","Sanjay Haresh","Awais Ahmed","Andrey Konin","M. Zeeshan Zia","Quoc-Huy Tran"],"abstract":"We present a novel approach for unsupervised activity segmentation which uses video frame clustering as a pretext task and simultaneously performs representation learning and online clustering. This is in contrast with prior works where representation learning and clustering are often performed sequentially. We leverage temporal information in videos by employing temporal optimal transport. In particular, we incorporate a temporal regularization term which preserves the temporal order of the activity into the standard optimal transport module for computing pseudo-label cluster assignments. The temporal optimal transport module enables our approach to learn effective representations for unsupervised activity segmentation. Furthermore, previous methods require storing learned features for the entire dataset before clustering them in an offline manner, whereas our approach processes one mini-batch at a time in an online manner. Extensive evaluations on three public datasets, i.e. 50-Salads, YouTube Instructions, and Breakfast, and our dataset, i.e., Desktop Assembly, show that our approach performs on par with or better than previous methods, despite having significantly less memory constraints. Our code and dataset are available on our research website: https://retrocausal.ai/research/","url_abs":"https://arxiv.org/abs/2105.13353v7","url_pdf":"https://arxiv.org/pdf/2105.13353v7.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"unsupervised-activity-segmentation-by-joint","repo_url":"https://github.com/trquhuytin/TOT-CVPR22","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"action-segmentation","task_name":"Action Segmentation"},{"task_slug":"clustering","task_name":"Clustering"},{"task_slug":"online-clustering","task_name":"Online Clustering"},{"task_slug":"pseudo-label","task_name":"Pseudo Label"},{"task_slug":"representation-learning","task_name":"Representation Learning"},{"task_slug":"unsupervised-action-segmentation","task_name":"Unsupervised Action Segmentation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/unsupervised-action-segmentation-on-50-salads","task":"Unsupervised Action Segmentation","dataset":"50 Salads","model":"TOT+TCL","rank_in_archive_order":2,"of":3,"metrics":{"Acc":"45.3","F1":"32.9"},"uses_additional_data":false},{"leaderboard":"/sota/unsupervised-action-segmentation-on-50-salads","task":"Unsupervised Action Segmentation","dataset":"50 Salads","model":"TOT","rank_in_archive_order":3,"of":3,"metrics":{"Acc":"40.6","F1":"30"},"uses_additional_data":false},{"leaderboard":"/sota/unsupervised-action-segmentation-on-breakfast","task":"Unsupervised Action Segmentation","dataset":"Breakfast","model":"TOT","rank_in_archive_order":5,"of":8,"metrics":{"Acc":"47.5","F1":"31.0","JSD":"90.2","Precision":"37.7","Recall":"26.3"},"uses_additional_data":false},{"leaderboard":"/sota/unsupervised-action-segmentation-on-breakfast","task":"Unsupervised Action Segmentation","dataset":"Breakfast","model":"TOT+TCL","rank_in_archive_order":6,"of":8,"metrics":{"Acc":"39.0","F1":"30.3","JSD":"85.6","Precision":"26.2","Recall":"36.0"},"uses_additional_data":false},{"leaderboard":"/sota/unsupervised-action-segmentation-on-ikea-asm","task":"Unsupervised Action Segmentation","dataset":"IKEA ASM","model":"TOT+TCL","rank_in_archive_order":4,"of":5,"metrics":{"Accuracy":"23.8","F1":"20.9","JSD":"79.5","Precision":"25.5","Recall":"17.7"},"uses_additional_data":false},{"leaderboard":"/sota/unsupervised-action-segmentation-on-ikea-asm","task":"Unsupervised Action Segmentation","dataset":"IKEA ASM","model":"TOT","rank_in_archive_order":5,"of":5,"metrics":{"Accuracy":"21.0","F1":"20.1","JSD":"80.0","Precision":"24.4","Recall":"17.1"},"uses_additional_data":false},{"leaderboard":"/sota/unsupervised-action-segmentation-on-youtube","task":"Unsupervised Action Segmentation","dataset":"Youtube INRIA Instructional","model":"TOT+TCL","rank_in_archive_order":4,"of":8,"metrics":{"Acc":"45.3","F1":"32.9","Precision":"40.1","Recall":"27.9"},"uses_additional_data":false},{"leaderboard":"/sota/unsupervised-action-segmentation-on-youtube","task":"Unsupervised Action Segmentation","dataset":"Youtube INRIA Instructional","model":"TOT","rank_in_archive_order":7,"of":8,"metrics":{"Acc":"40.6","F1":"30.0","Precision":"28.7","Recall":"31.4"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2105.13353","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.13353"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/trquhuytin/TOT-CVPR22","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":2},"by_repo_kind":{"official":{"samples":2,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"2afdf62a2f77e421","entry":"distributed_sinkhorn","repo":"trquhuytin/TOT-CVPR22","repo_kind":"official","path":"ute/models/training_embed.py","file_url":"https://github.com/trquhuytin/TOT-CVPR22/blob/HEAD/ute/models/training_embed.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2afdf62a2f77e421"}},{"code_sha256_prefix":"aa702c976b3ba525","entry":"get_cost_matrix","repo":"trquhuytin/TOT-CVPR22","repo_kind":"official","path":"ute/models/training_embed.py","file_url":"https://github.com/trquhuytin/TOT-CVPR22/blob/HEAD/ute/models/training_embed.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"aa702c976b3ba525"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}