{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/sign-language-translation-with-transformers","title":"Better Sign Language Translation with STMC-Transformer","arxiv_id":"2004.00588","date":"2020-04-01","proceeding":"COLING 2020 8","authors":["Kayo Yin","Jesse Read"],"abstract":"Sign Language Translation (SLT) first uses a Sign Language Recognition (SLR) system to extract sign language glosses from videos. Then, a translation system generates spoken language translations from the sign language glosses. This paper focuses on the translation system and introduces the STMC-Transformer which improves on the current state-of-the-art by over 5 and 7 BLEU respectively on gloss-to-text and video-to-text translation of the PHOENIX-Weather 2014T dataset. On the ASLG-PC12 corpus, we report an increase of over 16 BLEU. We also demonstrate the problem in current methods that rely on gloss supervision. The video-to-text translation of our STMC-Transformer outperforms translation of GT glosses. This contradicts previous claims that GT gloss translation acts as an upper bound for SLT performance and reveals that glosses are an inefficient representation of sign language. For future SLT research, we therefore suggest an end-to-end training of the recognition and translation models, or using a different sign language annotation scheme.","url_abs":"https://arxiv.org/abs/2004.00588v2","url_pdf":"https://arxiv.org/pdf/2004.00588v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"sign-language-translation-with-transformers","repo_url":"https://github.com/kayoyin/transformer-slt","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"sign-language-recognition","task_name":"Sign Language Recognition"},{"task_slug":"sign-language-translation","task_name":"Sign Language Translation"},{"task_slug":"translation","task_name":"Translation"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"relu","method_name":"ReLU"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/sign-language-translation-on-aslg-pc12-1","task":"Sign Language Translation","dataset":"ASLG-PC12","model":"Transformer Ens.","rank_in_archive_order":1,"of":1,"metrics":{"BLEU-4":"82.87"},"uses_additional_data":true},{"leaderboard":"/sota/sign-language-translation-on-rwth-phoenix","task":"Sign Language Translation","dataset":"RWTH-PHOENIX-Weather 2014 T","model":"STMC+Transformer (Ens)","rank_in_archive_order":5,"of":11,"metrics":{"BLEU-4":"25.40"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2004.00588","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.00588"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kayoyin/transformer-slt","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_violates":1,"unverified":4},"by_repo_kind":{"official":{"samples":5,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"0c93830510f645a3","entry":"check_sru_requirement","repo":"kayoyin/transformer-slt","repo_kind":"official","path":"onmt/models/sru.py","file_url":"https://github.com/kayoyin/transformer-slt/blob/HEAD/onmt/models/sru.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0c93830510f645a3"}},{"code_sha256_prefix":"df1510f43e187231","entry":"average_models","repo":"kayoyin/transformer-slt","repo_kind":"official","path":"onmt/bin/average_models.py","file_url":"https://github.com/kayoyin/transformer-slt/blob/HEAD/onmt/bin/average_models.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"df1510f43e187231"}},{"code_sha256_prefix":"b60aff23d14fc8ae","entry":"collapse_copy_scores","repo":"kayoyin/transformer-slt","repo_kind":"official","path":"onmt/modules/copy_generator.py","file_url":"https://github.com/kayoyin/transformer-slt/blob/HEAD/onmt/modules/copy_generator.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b60aff23d14fc8ae"}},{"code_sha256_prefix":"f440cc28fc06b7ae","entry":"get_ctranslate2_model_spec","repo":"kayoyin/transformer-slt","repo_kind":"official","path":"onmt/bin/release_model.py","file_url":"https://github.com/kayoyin/transformer-slt/blob/HEAD/onmt/bin/release_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f440cc28fc06b7ae"}},{"code_sha256_prefix":"0c3bddd7d3eaba02","entry":"seq_linear","repo":"kayoyin/transformer-slt","repo_kind":"official","path":"onmt/modules/conv_multi_step_attention.py","file_url":"https://github.com/kayoyin/transformer-slt/blob/HEAD/onmt/modules/conv_multi_step_attention.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0c3bddd7d3eaba02"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}