{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/blsp-emo-towards-empathetic-large-speech","title":"BLSP-Emo: Towards Empathetic Large Speech-Language Models","arxiv_id":"2406.03872","date":"2024-06-06","proceeding":null,"authors":["Chen Wang","Minpeng Liao","Zhongqiang Huang","Junhong Wu","Chengqing Zong","Jiajun Zhang"],"abstract":"The recent release of GPT-4o showcased the potential of end-to-end multimodal models, not just in terms of low latency but also in their ability to understand and generate expressive speech with rich emotions. While the details are unknown to the open research community, it likely involves significant amounts of curated data and compute, neither of which is readily accessible. In this paper, we present BLSP-Emo (Bootstrapped Language-Speech Pretraining with Emotion support), a novel approach to developing an end-to-end speech-language model capable of understanding both semantics and emotions in speech and generate empathetic responses. BLSP-Emo utilizes existing speech recognition (ASR) and speech emotion recognition (SER) datasets through a two-stage process. The first stage focuses on semantic alignment, following recent work on pretraining speech-language models using ASR data. The second stage performs emotion alignment with the pretrained speech-language model on an emotion-aware continuation task constructed from SER data. Our experiments demonstrate that the BLSP-Emo model excels in comprehending speech and delivering empathetic responses, both in instruction-following tasks and conversations.","url_abs":"https://arxiv.org/abs/2406.03872v1","url_pdf":"https://arxiv.org/pdf/2406.03872v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"blsp-emo-towards-empathetic-large-speech","repo_url":"https://github.com/cwang621/blsp-emo","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"emotion-recognition","task_name":"Emotion Recognition"},{"task_slug":"instruction-following","task_name":"Instruction Following"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"speech-emotion-recognition","task_name":"Speech Emotion Recognition"},{"task_slug":"speech-recognition","task_name":"Speech Recognition"},{"task_slug":"speech-recognition-1","task_name":"speech-recognition"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2406.03872","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.03872"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/cwang621/blsp-emo","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":5,"ran_honours":1,"ran_fixture":1,"unverified":1},"by_repo_kind":{"official":{"samples":8,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"746ac520906d93d1","entry":"collate_tokens","repo":"cwang621/blsp-emo","repo_kind":"official","path":"generate.py","file_url":"https://github.com/cwang621/blsp-emo/blob/HEAD/generate.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"746ac520906d93d1"}},{"code_sha256_prefix":"d9db9f86b30f9ac0","entry":"dequantize_cache_torch","repo":"cwang621/blsp-emo","repo_kind":"official","path":"src/modeling_qwen.py","file_url":"https://github.com/cwang621/blsp-emo/blob/HEAD/src/modeling_qwen.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d9db9f86b30f9ac0"}},{"code_sha256_prefix":"dbbd427d8be0e03a","entry":"get_dataset","repo":"cwang621/blsp-emo","repo_kind":"official","path":"emotion_text_generation.py","file_url":"https://github.com/cwang621/blsp-emo/blob/HEAD/emotion_text_generation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"dbbd427d8be0e03a"}},{"code_sha256_prefix":"6fb5e96321718d63","entry":"get_shard_range","repo":"cwang621/blsp-emo","repo_kind":"official","path":"emotion_text_generation.py","file_url":"https://github.com/cwang621/blsp-emo/blob/HEAD/emotion_text_generation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6fb5e96321718d63"}},{"code_sha256_prefix":"dcb5bf59a6298eba","entry":"length_to_4d_attention_mask","repo":"cwang621/blsp-emo","repo_kind":"official","path":"src/modeling_utils.py","file_url":"https://github.com/cwang621/blsp-emo/blob/HEAD/src/modeling_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"dcb5bf59a6298eba"}},{"code_sha256_prefix":"89912aba5efd69fe","entry":"length_to_attention_mask","repo":"cwang621/blsp-emo","repo_kind":"official","path":"src/modeling_utils.py","file_url":"https://github.com/cwang621/blsp-emo/blob/HEAD/src/modeling_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"89912aba5efd69fe"}},{"code_sha256_prefix":"1fec7a76f0d171a6","entry":"quantize_cache_v","repo":"cwang621/blsp-emo","repo_kind":"official","path":"src/modeling_qwen.py","file_url":"https://github.com/cwang621/blsp-emo/blob/HEAD/src/modeling_qwen.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1fec7a76f0d171a6"}},{"code_sha256_prefix":"fbda921818ec02df","entry":"apply_rotary_pos_emb","repo":"cwang621/blsp-emo","repo_kind":"official","path":"src/modeling_qwen.py","file_url":"https://github.com/cwang621/blsp-emo/blob/HEAD/src/modeling_qwen.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fbda921818ec02df"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}