{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/speak-foreign-languages-with-your-own-voice","title":"Speak Foreign Languages with Your Own Voice: Cross-Lingual Neural Codec Language Modeling","arxiv_id":"2303.03926","date":"2023-03-07","proceeding":null,"authors":["Ziqiang Zhang","Long Zhou","Chengyi Wang","Sanyuan Chen","Yu Wu","Shujie Liu","Zhuo Chen","Yanqing Liu","Huaming Wang","Jinyu Li","Lei He","Sheng Zhao","Furu Wei"],"abstract":"We propose a cross-lingual neural codec language model, VALL-E X, for cross-lingual speech synthesis. Specifically, we extend VALL-E and train a multi-lingual conditional codec language model to predict the acoustic token sequences of the target language speech by using both the source language speech and the target language text as prompts. VALL-E X inherits strong in-context learning capabilities and can be applied for zero-shot cross-lingual text-to-speech synthesis and zero-shot speech-to-speech translation tasks. Experimental results show that it can generate high-quality speech in the target language via just one speech utterance in the source language as a prompt while preserving the unseen speaker's voice, emotion, and acoustic environment. Moreover, VALL-E X effectively alleviates the foreign accent problems, which can be controlled by a language ID. Audio samples are available at \\url{https://aka.ms/vallex}.","url_abs":"https://arxiv.org/abs/2303.03926v1","url_pdf":"https://arxiv.org/pdf/2303.03926v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"speak-foreign-languages-with-your-own-voice","repo_url":"https://github.com/plachtaa/vall-e-x","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"in-context-learning","task_name":"In-Context Learning"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"speech-synthesis","task_name":"Speech Synthesis"},{"task_slug":"speech-to-speech-translation","task_name":"Speech-to-Speech Translation"},{"task_slug":"text-to-speech","task_name":"Text to Speech"},{"task_slug":"text-to-speech-synthesis","task_name":"Text-To-Speech Synthesis"},{"task_slug":"text-to-speech-1","task_name":"text-to-speech"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2303.03926","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.03926"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/plachtaa/vall-e-x","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":4,"unverified":2},"by_repo_kind":{"listed":{"samples":6,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"982b9083b5a41368","entry":"random_cast_to_half","repo":"plachtaa/vall-e-x","repo_kind":"listed","path":"modules/scaling.py","file_url":"https://github.com/plachtaa/vall-e-x/blob/HEAD/modules/scaling.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"982b9083b5a41368"}},{"code_sha256_prefix":"1d72f492e19080e6","entry":"random_clamp","repo":"plachtaa/vall-e-x","repo_kind":"listed","path":"modules/scaling.py","file_url":"https://github.com/plachtaa/vall-e-x/blob/HEAD/modules/scaling.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1d72f492e19080e6"}},{"code_sha256_prefix":"c124aa8bd6b9ebe9","entry":"softmax","repo":"plachtaa/vall-e-x","repo_kind":"listed","path":"modules/scaling.py","file_url":"https://github.com/plachtaa/vall-e-x/blob/HEAD/modules/scaling.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c124aa8bd6b9ebe9"}},{"code_sha256_prefix":"99a311d99448f881","entry":"top_k_top_p_filtering","repo":"plachtaa/vall-e-x","repo_kind":"listed","path":"models/vallex.py","file_url":"https://github.com/plachtaa/vall-e-x/blob/HEAD/models/vallex.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"99a311d99448f881"}},{"code_sha256_prefix":"1b616e7eb0f31017","entry":"multi_head_attention_forward","repo":"plachtaa/vall-e-x","repo_kind":"listed","path":"modules/activation.py","file_url":"https://github.com/plachtaa/vall-e-x/blob/HEAD/modules/activation.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1b616e7eb0f31017"}},{"code_sha256_prefix":"660baece3071187f","entry":"topk_sampling","repo":"plachtaa/vall-e-x","repo_kind":"listed","path":"models/vallex.py","file_url":"https://github.com/plachtaa/vall-e-x/blob/HEAD/models/vallex.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"660baece3071187f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}