{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2510-04577","title":"Language Model Based Text-to-Audio Generation: Anti-Causally Aligned Collaborative Residual Transformers","arxiv_id":"2510.04577","date":"2025-10-06","proceeding":"EMNLP","authors":["Juncheng Wang","Chao Xu","Cheng Yu","Zhe Hu","Haoyu Xie","Guoqi Yu","Lei Shang","Shujun Wang"],"abstract":"While language models (LMs) paired with residual vector quantization (RVQ) tokenizers have shown promise in text-to-audio (T2A) generation, they still lag behind diffusion-based models by a non-trivial margin. We identify a critical dilemma underpinning this gap: incorporating more RVQ layers improves audio reconstruction fidelity but exceeds the generation capacity of conventional LMs. To address this, we first analyze RVQ dynamics and uncover two key limitations: 1) orthogonality of features across RVQ layers hinders effective LMs training, and 2) descending semantic richness in tokens from deeper RVQ layers exacerbates exposure bias during autoregressive decoding. Based on these insights, we propose Siren, a novel LM-based framework that employs multiple isolated transformers with causal conditioning and anti-causal alignment via reinforcement learning. Extensive experiments demonstrate that Siren outperforms both existing LM-based and diffusion-based T2A systems, achieving state-of-the-art results. By bridging the representational strengths of LMs with the fidelity demands of audio synthesis, our approach repositions LMs as competitive contenders against diffusion models in T2A tasks. Moreover, by aligning audio representations with linguistic structures, Siren facilitates a promising pathway toward unified multi-modal generation frameworks.","url_abs":"https://arxiv.org/abs/2510.04577","url_pdf":"https://arxiv.org/pdf/2510.04577","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2510.04577","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2510.04577"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/wjc2830/Siren","reach":null}],"summary":{"ran":5,"ran_honours":2,"unverified":6},"by_repo_kind":{"found_in_text":{"samples":13,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":13,"samples":[{"code_sha256_prefix":"b6cc0d213723cc49","entry":"DacCodebookLayer","repo":"wjc2830/Siren","repo_kind":"found_in_text","path":"src/siren/model.py","file_url":"https://github.com/wjc2830/Siren/blob/HEAD/src/siren/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b6cc0d213723cc49"}},{"code_sha256_prefix":"34e282922212d09b","entry":"DacEmbeddingState","repo":"wjc2830/Siren","repo_kind":"found_in_text","path":"src/siren/model.py","file_url":"https://github.com/wjc2830/Siren/blob/HEAD/src/siren/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"34e282922212d09b"}},{"code_sha256_prefix":"9669b074ff77501a","entry":"RMSNorm","repo":"wjc2830/Siren","repo_kind":"found_in_text","path":"src/siren/model.py","file_url":"https://github.com/wjc2830/Siren/blob/HEAD/src/siren/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9669b074ff77501a"}},{"code_sha256_prefix":"2259770fbb6b2e31","entry":"SirenConfig","repo":"wjc2830/Siren","repo_kind":"found_in_text","path":"src/siren/model.py","file_url":"https://github.com/wjc2830/Siren/blob/HEAD/src/siren/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2259770fbb6b2e31"}},{"code_sha256_prefix":"a71ae52495d11b8d","entry":"SwiGLU","repo":"wjc2830/Siren","repo_kind":"found_in_text","path":"src/siren/model.py","file_url":"https://github.com/wjc2830/Siren/blob/HEAD/src/siren/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a71ae52495d11b8d"}},{"code_sha256_prefix":"7730a366fecd0601","entry":"_round_up","repo":"wjc2830/Siren","repo_kind":"found_in_text","path":"src/siren/model.py","file_url":"https://github.com/wjc2830/Siren/blob/HEAD/src/siren/model.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7730a366fecd0601"}},{"code_sha256_prefix":"862fce0d4bd982cb","entry":"sinusoidal_positions","repo":"wjc2830/Siren","repo_kind":"found_in_text","path":"src/siren/model.py","file_url":"https://github.com/wjc2830/Siren/blob/HEAD/src/siren/model.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"862fce0d4bd982cb"}},{"code_sha256_prefix":"0f22591f0eeeaeb3","entry":"CausalSelfAttention","repo":"wjc2830/Siren","repo_kind":"found_in_text","path":"src/siren/model.py","file_url":"https://github.com/wjc2830/Siren/blob/HEAD/src/siren/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0f22591f0eeeaeb3"}},{"code_sha256_prefix":"cda35aded4b4d4c8","entry":"CrossAttention","repo":"wjc2830/Siren","repo_kind":"found_in_text","path":"src/siren/model.py","file_url":"https://github.com/wjc2830/Siren/blob/HEAD/src/siren/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"cda35aded4b4d4c8"}},{"code_sha256_prefix":"b532c32a7f5f8e09","entry":"CrossTransformerBlock","repo":"wjc2830/Siren","repo_kind":"found_in_text","path":"src/siren/model.py","file_url":"https://github.com/wjc2830/Siren/blob/HEAD/src/siren/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b532c32a7f5f8e09"}},{"code_sha256_prefix":"b2adb2176bc76364","entry":"DacCodebookBank","repo":"wjc2830/Siren","repo_kind":"found_in_text","path":"src/siren/model.py","file_url":"https://github.com/wjc2830/Siren/blob/HEAD/src/siren/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b2adb2176bc76364"}},{"code_sha256_prefix":"0f92e3904bec5c96","entry":"SirenExpert","repo":"wjc2830/Siren","repo_kind":"found_in_text","path":"src/siren/model.py","file_url":"https://github.com/wjc2830/Siren/blob/HEAD/src/siren/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0f92e3904bec5c96"}},{"code_sha256_prefix":"569eaf68130b8046","entry":"TransformerBlock","repo":"wjc2830/Siren","repo_kind":"found_in_text","path":"src/siren/model.py","file_url":"https://github.com/wjc2830/Siren/blob/HEAD/src/siren/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"569eaf68130b8046"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.SD","source":"arxiv_api"},"syntology_extracted_results":null}