{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/a-3-t-alignment-aware-acoustic-and-text","title":"A$^3$T: Alignment-Aware Acoustic and Text Pretraining for Speech Synthesis and Editing","arxiv_id":"2203.09690","date":"2022-03-18","proceeding":null,"authors":["He Bai","Renjie Zheng","Junkun Chen","Xintong Li","Mingbo Ma","Liang Huang"],"abstract":"Recently, speech representation learning has improved many speech-related tasks such as speech recognition, speech classification, and speech-to-text translation. However, all the above tasks are in the direction of speech understanding, but for the inverse direction, speech synthesis, the potential of representation learning is yet to be realized, due to the challenging nature of generating high-quality speech. To address this problem, we propose our framework, Alignment-Aware Acoustic-Text Pretraining (A$^3$T), which reconstructs masked acoustic signals with text input and acoustic-text alignment during training. In this way, the pretrained model can generate high quality reconstructed spectrogram, which can be applied to the speech editing and unseen speaker TTS directly. Experiments show A$^3$T outperforms SOTA models on speech editing, and improves multi-speaker speech synthesis without the external speaker verification model.","url_abs":"https://arxiv.org/abs/2203.09690v2","url_pdf":"https://arxiv.org/pdf/2203.09690v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"a-3-t-alignment-aware-acoustic-and-text","repo_url":"https://github.com/richardbaihe/a3t","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"a-3-t-alignment-aware-acoustic-and-text","repo_url":"https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/vctk/ernie_sat","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"paddle","reach":null}],"tasks":[{"task_slug":"representation-learning","task_name":"Representation Learning"},{"task_slug":"speaker-verification","task_name":"Speaker Verification"},{"task_slug":"speech-recognition","task_name":"Speech Recognition"},{"task_slug":"speech-representation-learning","task_name":"Speech Representation Learning"},{"task_slug":"speech-synthesis","task_name":"Speech Synthesis"},{"task_slug":"speech-to-text","task_name":"Speech-to-Text"},{"task_slug":"speech-to-text-translation","task_name":"Speech-to-Text Translation"},{"task_slug":"translation","task_name":"Translation"},{"task_slug":"speech-recognition-1","task_name":"speech-recognition"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2203.09690","atlas_url":"https://app.syntology.ai/?focus=2203.09690","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.09690"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/richardbaihe/a3t","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/vctk/ernie_sat","reach":null}],"summary":{"ran":2,"unverified":5},"by_repo_kind":{"listed":{"samples":7,"ran":2,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"eaea9b1425847241","entry":"parse_infofile","repo":"richardbaihe/a3t","repo_kind":"listed","path":"aggregate_output/sedit_decode.py","file_url":"https://github.com/richardbaihe/a3t/blob/HEAD/aggregate_output/sedit_decode.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"eaea9b1425847241"}},{"code_sha256_prefix":"7ac28de669f54554","entry":"simple_repr","repo":"PaddlePaddle/PaddleSpeech","repo_kind":"listed","path":"paddlespeech/audiotools/core/_julius.py","file_url":"https://github.com/PaddlePaddle/PaddleSpeech/blob/HEAD/paddlespeech/audiotools/core/_julius.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7ac28de669f54554"}},{"code_sha256_prefix":"bb3665ddb6b8dd2d","entry":"adadelta_eps_decay","repo":"richardbaihe/a3t","repo_kind":"listed","path":"espnet/asr/asr_utils.py","file_url":"https://github.com/richardbaihe/a3t/blob/HEAD/espnet/asr/asr_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bb3665ddb6b8dd2d"}},{"code_sha256_prefix":"1fa5ca494ed9750a","entry":"adam_lr_decay","repo":"richardbaihe/a3t","repo_kind":"listed","path":"espnet/asr/asr_utils.py","file_url":"https://github.com/richardbaihe/a3t/blob/HEAD/espnet/asr/asr_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1fa5ca494ed9750a"}},{"code_sha256_prefix":"965fb4d02c97778c","entry":"format_figure","repo":"PaddlePaddle/PaddleSpeech","repo_kind":"listed","path":"paddlespeech/audiotools/core/display.py","file_url":"https://github.com/PaddlePaddle/PaddleSpeech/blob/HEAD/paddlespeech/audiotools/core/display.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"965fb4d02c97778c"}},{"code_sha256_prefix":"4d1f42e7242859a7","entry":"r128stats","repo":"PaddlePaddle/PaddleSpeech","repo_kind":"listed","path":"paddlespeech/audiotools/core/ffmpeg.py","file_url":"https://github.com/PaddlePaddle/PaddleSpeech/blob/HEAD/paddlespeech/audiotools/core/ffmpeg.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4d1f42e7242859a7"}},{"code_sha256_prefix":"96081a60369c59e0","entry":"restore_snapshot","repo":"richardbaihe/a3t","repo_kind":"listed","path":"espnet/asr/asr_utils.py","file_url":"https://github.com/richardbaihe/a3t/blob/HEAD/espnet/asr/asr_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"96081a60369c59e0"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}