{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/qwen-audio-advancing-universal-audio","title":"Qwen-Audio: Advancing Universal Audio Understanding via Unified Large-Scale Audio-Language Models","arxiv_id":"2311.07919","date":"2023-11-14","proceeding":null,"authors":["Yunfei Chu","Jin Xu","Xiaohuan Zhou","Qian Yang","Shiliang Zhang","Zhijie Yan","Chang Zhou","Jingren Zhou"],"abstract":"Recently, instruction-following audio-language models have received broad attention for audio interaction with humans. However, the absence of pre-trained audio models capable of handling diverse audio types and tasks has hindered progress in this field. Consequently, most existing works have only been able to support a limited range of interaction capabilities. In this paper, we develop the Qwen-Audio model and address this limitation by scaling up audio-language pre-training to cover over 30 tasks and various audio types, such as human speech, natural sounds, music, and songs, to facilitate universal audio understanding abilities. However, directly co-training all tasks and datasets can lead to interference issues, as the textual labels associated with different datasets exhibit considerable variations due to differences in task focus, language, granularity of annotation, and text structure. To overcome the one-to-many interference, we carefully design a multi-task training framework by conditioning on a sequence of hierarchical tags to the decoder for encouraging knowledge sharing and avoiding interference through shared and specified tags respectively. Remarkably, Qwen-Audio achieves impressive performance across diverse benchmark tasks without requiring any task-specific fine-tuning, surpassing its counterparts. Building upon the capabilities of Qwen-Audio, we further develop Qwen-Audio-Chat, which allows for input from various audios and text inputs, enabling multi-turn dialogues and supporting various audio-central scenarios.","url_abs":"https://arxiv.org/abs/2311.07919v2","url_pdf":"https://arxiv.org/pdf/2311.07919v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"qwen-audio-advancing-universal-audio","repo_url":"https://github.com/qwenlm/qwen-audio","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"qwen-audio-advancing-universal-audio","repo_url":"https://github.com/alibaba-damo-academy/FunASR","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"acoustic-scene-classification","task_name":"Acoustic Scene Classification"},{"task_slug":"audio-classification","task_name":"Audio Classification"},{"task_slug":"audio-captioning","task_name":"Audio captioning"},{"task_slug":"automatic-speech-recognition","task_name":"Automatic Speech Recognition (ASR)"},{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"emotion-recognition-in-conversation","task_name":"Emotion Recognition in Conversation"},{"task_slug":"instruction-following","task_name":"Instruction Following"},{"task_slug":"speech-recognition","task_name":"Speech Recognition"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/acoustic-scene-classification-on-cochlscene","task":"Acoustic Scene Classification","dataset":"CochlScene","model":"Qwen-Audio","rank_in_archive_order":2,"of":2,"metrics":{"1:1 Accuracy":"0.795"},"uses_additional_data":true},{"leaderboard":"/sota/acoustic-scene-classification-on-tut-acoustic","task":"Acoustic Scene Classification","dataset":"TUT Acoustic Scenes 2017","model":"Qwen-Audio","rank_in_archive_order":1,"of":1,"metrics":{"1:1 Accuracy":"0.649"},"uses_additional_data":true},{"leaderboard":"/sota/audio-classification-on-vocalsound","task":"Audio Classification","dataset":"VocalSound","model":"Qwen-Audio","rank_in_archive_order":2,"of":2,"metrics":{"Accuracy ":"92.89"},"uses_additional_data":true},{"leaderboard":"/sota/audio-captioning-on-clotho","task":"Audio captioning","dataset":"Clotho","model":"Qwen-Audio","rank_in_archive_order":7,"of":11,"metrics":{"CIDEr":"0.441","SPICE":"0.136","SPIDEr":"0.288"},"uses_additional_data":true},{"leaderboard":"/sota/emotion-recognition-in-conversation-on-meld","task":"Emotion Recognition in Conversation","dataset":"MELD","model":"Qwen-Audio","rank_in_archive_order":68,"of":68,"metrics":{"Accuracy":"55.70"},"uses_additional_data":true},{"leaderboard":"/sota/speech-recognition-on-aishell-1","task":"Speech Recognition","dataset":"AISHELL-1","model":"Qwen-Audio","rank_in_archive_order":3,"of":18,"metrics":{"Word Error Rate (WER)":"1.29"},"uses_additional_data":true},{"leaderboard":"/sota/speech-recognition-on-aishell-2-test-android-1","task":"Speech Recognition","dataset":"AISHELL-2 Test Android","model":"Qwen-Audio","rank_in_archive_order":1,"of":1,"metrics":{"Word Error Rate (WER)":"3.3"},"uses_additional_data":true},{"leaderboard":"/sota/speech-recognition-on-aishell-2-test-ios","task":"Speech Recognition","dataset":"AISHELL-2 Test IOS","model":"Qwen-Audio","rank_in_archive_order":1,"of":1,"metrics":{"Word Error Rate (WER)":"3.1"},"uses_additional_data":true},{"leaderboard":"/sota/speech-recognition-on-aishell-2-test-mic-1","task":"Speech Recognition","dataset":"AISHELL-2 Test Mic","model":"Qwen-Audio","rank_in_archive_order":1,"of":1,"metrics":{"Word Error Rate (WER)":"3.3"},"uses_additional_data":true},{"leaderboard":"/sota/speech-recognition-on-librispeech-test-clean","task":"Speech Recognition","dataset":"LibriSpeech test-clean","model":"Qwen-Audio","rank_in_archive_order":24,"of":64,"metrics":{"Word Error Rate (WER)":"2.0"},"uses_additional_data":false},{"leaderboard":"/sota/speech-recognition-on-librispeech-test-other","task":"Speech Recognition","dataset":"LibriSpeech test-other","model":"Qwen-Audio","rank_in_archive_order":22,"of":53,"metrics":{"Word Error Rate (WER)":"4.2"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2311.07919","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.07919"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/qwenlm/qwen-audio","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/alibaba-damo-academy/FunASR","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":1,"ran_honours":3,"ran_fixture":1,"unverified":2},"by_repo_kind":{"official":{"samples":7,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"fb0866cc3d285415","entry":"collate_fn","repo":"qwenlm/qwen-audio","repo_kind":"official","path":"eval_audio/evaluate_aqa.py","file_url":"https://github.com/qwenlm/qwen-audio/blob/HEAD/eval_audio/evaluate_aqa.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"fb0866cc3d285415"}},{"code_sha256_prefix":"d9db9f86b30f9ac0","entry":"dequantize_cache_torch","repo":"qwenlm/qwen-audio","repo_kind":"official","path":"modeling_qwen.py","file_url":"https://github.com/qwenlm/qwen-audio/blob/HEAD/modeling_qwen.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"d9db9f86b30f9ac0"}},{"code_sha256_prefix":"6e0808fc4828513a","entry":"exact_div","repo":"qwenlm/qwen-audio","repo_kind":"official","path":"audio.py","file_url":"https://github.com/qwenlm/qwen-audio/blob/HEAD/audio.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"6e0808fc4828513a"}},{"code_sha256_prefix":"3bca85a509d7b2a6","entry":"get_T_after_cnn","repo":"qwenlm/qwen-audio","repo_kind":"official","path":"audio.py","file_url":"https://github.com/qwenlm/qwen-audio/blob/HEAD/audio.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"3bca85a509d7b2a6"}},{"code_sha256_prefix":"1fec7a76f0d171a6","entry":"quantize_cache_v","repo":"qwenlm/qwen-audio","repo_kind":"official","path":"modeling_qwen.py","file_url":"https://github.com/qwenlm/qwen-audio/blob/HEAD/modeling_qwen.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"1fec7a76f0d171a6"}},{"code_sha256_prefix":"fbda921818ec02df","entry":"apply_rotary_pos_emb","repo":"qwenlm/qwen-audio","repo_kind":"official","path":"modeling_qwen.py","file_url":"https://github.com/qwenlm/qwen-audio/blob/HEAD/modeling_qwen.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"fbda921818ec02df"}},{"code_sha256_prefix":"3dbcbc5b3f57546c","entry":"load_bytesio_audio","repo":"qwenlm/qwen-audio","repo_kind":"official","path":"audio.py","file_url":"https://github.com/qwenlm/qwen-audio/blob/HEAD/audio.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"3dbcbc5b3f57546c"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}