{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/mt3-multi-task-multitrack-music-transcription-1","title":"MT3: Multi-Task Multitrack Music Transcription","arxiv_id":"2111.03017","date":"2021-11-04","proceeding":"ICLR 2022 4","authors":["Josh Gardner","Ian Simon","Ethan Manilow","Curtis Hawthorne","Jesse Engel"],"abstract":"Automatic Music Transcription (AMT), inferring musical notes from raw audio, is a challenging task at the core of music understanding. Unlike Automatic Speech Recognition (ASR), which typically focuses on the words of a single speaker, AMT often requires transcribing multiple instruments simultaneously, all while preserving fine-scale pitch and timing information. Further, many AMT datasets are \"low-resource\", as even expert musicians find music transcription difficult and time-consuming. Thus, prior work has focused on task-specific architectures, tailored to the individual instruments of each task. In this work, motivated by the promising results of sequence-to-sequence transfer learning for low-resource Natural Language Processing (NLP), we demonstrate that a general-purpose Transformer model can perform multi-task AMT, jointly transcribing arbitrary combinations of musical instruments across several transcription datasets. We show this unified training framework achieves high-quality transcription results across a range of datasets, dramatically improving performance for low-resource instruments (such as guitar), while preserving strong performance for abundant instruments (such as piano). Finally, by expanding the scope of AMT, we expose the need for more consistent evaluation metrics and better dataset alignment, and provide a strong baseline for this new direction of multi-task AMT.","url_abs":"https://arxiv.org/abs/2111.03017v4","url_pdf":"https://arxiv.org/pdf/2111.03017v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"mt3-multi-task-multitrack-music-transcription-1","repo_url":"https://github.com/magenta/mt3","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"jax","reach":null},{"paper_slug":"mt3-multi-task-multitrack-music-transcription-1","repo_url":"https://github.com/kunato/mt3-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"jax","reach":{"status":"ok"}},{"paper_slug":"mt3-multi-task-multitrack-music-transcription-1","repo_url":"https://github.com/mimbres/yourmt3","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok","spdx":"GPL-3.0"}}],"tasks":[{"task_slug":"automatic-speech-recognition-2","task_name":"Automatic Speech Recognition"},{"task_slug":"automatic-speech-recognition","task_name":"Automatic Speech Recognition (ASR)"},{"task_slug":"multi-instrument-music-transcription","task_name":"Multi-instrument Music Transcription"},{"task_slug":"music-transcription","task_name":"Music Transcription"},{"task_slug":"speech-recognition","task_name":"Speech Recognition"},{"task_slug":"transfer-learning","task_name":"Transfer Learning"},{"task_slug":"speech-recognition-1","task_name":"speech-recognition"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/multi-instrument-music-transcription-on-urmp","task":"Multi-instrument Music Transcription","dataset":"URMP","model":"MT3","rank_in_archive_order":2,"of":3,"metrics":{"Multi F1":"59.0"},"uses_additional_data":true},{"leaderboard":"/sota/music-transcription-on-maestro","task":"Music Transcription","dataset":"MAESTRO","model":"MT3 (single dataset)","rank_in_archive_order":8,"of":9,"metrics":{"Onset F1":"88.0"},"uses_additional_data":false},{"leaderboard":"/sota/music-transcription-on-maestro","task":"Music Transcription","dataset":"MAESTRO","model":"MT3 (multi dataset)","rank_in_archive_order":9,"of":9,"metrics":{"Onset F1":"86.0"},"uses_additional_data":true},{"leaderboard":"/sota/music-transcription-on-slakh2100","task":"Music Transcription","dataset":"Slakh2100","model":"MT3","rank_in_archive_order":5,"of":6,"metrics":{"note-level F-measure-no-offset (Fno)":"0.57"},"uses_additional_data":false},{"leaderboard":"/sota/music-transcription-on-urmp","task":"Music Transcription","dataset":"URMP","model":"MT3","rank_in_archive_order":2,"of":3,"metrics":{"Onset F1":"77.0"},"uses_additional_data":true}],"syntology":{"syntology_url":"https://syntology.ai/paper/2111.03017","atlas_url":"https://app.syntology.ai/?focus=2111.03017","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2111.03017"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/magenta/mt3","reach":null},{"provenance":"deterministic:regex_extraction","url":"https://github.com/KinWaiCheuk/ReconVAT","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mimbres/yourmt3","reach":{"status":"ok","spdx":"GPL-3.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kunato/mt3-pytorch","reach":{"status":"ok"}}],"summary":{"unverified":6},"by_repo_kind":{"official":{"samples":6,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"3bdf7ed695eae626","entry":"Decoder","repo":"magenta/mt3","repo_kind":"official","path":"mt3/network.py","file_url":"https://github.com/magenta/mt3/blob/HEAD/mt3/network.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3bdf7ed695eae626"}},{"code_sha256_prefix":"a7add228a2b5eb4c","entry":"DecoderLayer","repo":"magenta/mt3","repo_kind":"official","path":"mt3/network.py","file_url":"https://github.com/magenta/mt3/blob/HEAD/mt3/network.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a7add228a2b5eb4c"}},{"code_sha256_prefix":"2247e83f2dee9c11","entry":"Encoder","repo":"magenta/mt3","repo_kind":"official","path":"mt3/network.py","file_url":"https://github.com/magenta/mt3/blob/HEAD/mt3/network.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2247e83f2dee9c11"}},{"code_sha256_prefix":"2f90947fb1810fa8","entry":"EncoderLayer","repo":"magenta/mt3","repo_kind":"official","path":"mt3/network.py","file_url":"https://github.com/magenta/mt3/blob/HEAD/mt3/network.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2f90947fb1810fa8"}},{"code_sha256_prefix":"04ad8eef95f757d8","entry":"T5Config","repo":"magenta/mt3","repo_kind":"official","path":"mt3/network.py","file_url":"https://github.com/magenta/mt3/blob/HEAD/mt3/network.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"04ad8eef95f757d8"}},{"code_sha256_prefix":"240f4854b8ffb60f","entry":"Transformer","repo":"magenta/mt3","repo_kind":"official","path":"mt3/network.py","file_url":"https://github.com/magenta/mt3/blob/HEAD/mt3/network.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"240f4854b8ffb60f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}