{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/unified-fully-and-timestamp-supervised","title":"Unified Fully and Timestamp Supervised Temporal Action Segmentation via Sequence to Sequence Translation","arxiv_id":"2209.00638","date":"2022-09-01","proceeding":null,"authors":["Nadine Behrmann","S. Alireza Golestaneh","Zico Kolter","Juergen Gall","Mehdi Noroozi"],"abstract":"This paper introduces a unified framework for video action segmentation via sequence to sequence (seq2seq) translation in a fully and timestamp supervised setup. In contrast to current state-of-the-art frame-level prediction methods, we view action segmentation as a seq2seq translation task, i.e., mapping a sequence of video frames to a sequence of action segments. Our proposed method involves a series of modifications and auxiliary loss functions on the standard Transformer seq2seq translation model to cope with long input sequences opposed to short output sequences and relatively few videos. We incorporate an auxiliary supervision signal for the encoder via a frame-wise loss and propose a separate alignment decoder for an implicit duration prediction. Finally, we extend our framework to the timestamp supervised setting via our proposed constrained k-medoids algorithm to generate pseudo-segmentations. Our proposed framework performs consistently on both fully and timestamp supervised settings, outperforming or competing state-of-the-art on several datasets. Our code is publicly available at https://github.com/boschresearch/UVAST.","url_abs":"https://arxiv.org/abs/2209.00638v2","url_pdf":"https://arxiv.org/pdf/2209.00638v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"unified-fully-and-timestamp-supervised","repo_url":"https://github.com/boschresearch/uvast","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"unified-fully-and-timestamp-supervised","repo_url":"https://github.services.devops.takamol.support/boschresearch/UVAST","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"none","reach":null}],"tasks":[{"task_slug":"action-segmentation","task_name":"Action Segmentation"},{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"temporal-action-segmentation","task_name":"Temporal Action Segmentation"},{"task_slug":"translation","task_name":"Translation"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"lstm","method_name":"LSTM"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"seq2seq","method_name":"Seq2Seq"},{"method_slug":"sigmoid-activation","method_name":"Sigmoid Activation"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"tanh-activation","method_name":"Tanh Activation"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/action-segmentation-on-50-salads-1","task":"Action Segmentation","dataset":"50 Salads","model":"UVAST","rank_in_archive_order":7,"of":28,"metrics":{"Acc":"87.4","Edit":"83.9","F1@10%":"89.1","F1@25%":"87.6","F1@50%":"81.7"},"uses_additional_data":false},{"leaderboard":"/sota/action-segmentation-on-assembly101","task":"Action Segmentation","dataset":"Assembly101","model":"UVAST","rank_in_archive_order":5,"of":7,"metrics":{"Edit":"31.5","F1@10%":"32.1","F1@25%":"28.3","F1@50%":"20.8","MoF":"37.4"},"uses_additional_data":false},{"leaderboard":"/sota/action-segmentation-on-breakfast-1","task":"Action Segmentation","dataset":"Breakfast","model":"UVAST","rank_in_archive_order":12,"of":37,"metrics":{"Acc":"69.7","Average F1":"68.8","Edit":"77.1","F1@10%":"76.9","F1@25%":"71.5","F1@50%":"58"},"uses_additional_data":false},{"leaderboard":"/sota/action-segmentation-on-gtea-1","task":"Action Segmentation","dataset":"GTEA","model":"UVAST","rank_in_archive_order":10,"of":28,"metrics":{"Acc":"80.2","Edit":"92.1","F1@10%":"92.7","F1@25%":"91.3","F1@50%":"81"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2209.00638","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2209.00638"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/boschresearch/uvast","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.services.devops.takamol.support/boschresearch/UVAST","reach":null}],"summary":{"ran":1,"ran_draft_wrong":1},"by_repo_kind":{"official":{"samples":2,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":2,"samples":[{"code_sha256_prefix":"53957dadad8b98a6","entry":"TransformerDecoder_UVAST","repo":"boschresearch/uvast","repo_kind":"official","path":"uvast_dec.py","file_url":"https://github.com/boschresearch/uvast/blob/HEAD/uvast_dec.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"53957dadad8b98a6"}},{"code_sha256_prefix":"5c98e971f97efda9","entry":"_get_clones","repo":"boschresearch/uvast","repo_kind":"official","path":"uvast_dec.py","file_url":"https://github.com/boschresearch/uvast/blob/HEAD/uvast_dec.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"5c98e971f97efda9"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}