{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/learning-emotion-representations-from-verbal-1","title":"Learning Emotion Representations from Verbal and Nonverbal Communication","arxiv_id":"2305.13500","date":"2023-05-22","proceeding":"CVPR 2023 1","authors":["Sitao Zhang","Yimu Pan","James Z. Wang"],"abstract":"Emotion understanding is an essential but highly challenging component of artificial general intelligence. The absence of extensively annotated datasets has significantly impeded advancements in this field. We present EmotionCLIP, the first pre-training paradigm to extract visual emotion representations from verbal and nonverbal communication using only uncurated data. Compared to numerical labels or descriptions used in previous methods, communication naturally contains emotion information. Furthermore, acquiring emotion representations from communication is more congruent with the human learning process. We guide EmotionCLIP to attend to nonverbal emotion cues through subject-aware context encoding and verbal emotion cues using sentiment-guided contrastive learning. Extensive experiments validate the effectiveness and transferability of EmotionCLIP. Using merely linear-probe evaluation protocol, EmotionCLIP outperforms the state-of-the-art supervised visual emotion recognition methods and rivals many multimodal approaches across various benchmarks. We anticipate that the advent of EmotionCLIP will address the prevailing issue of data scarcity in emotion understanding, thereby fostering progress in related domains. The code and pre-trained models are available at https://github.com/Xeaver/EmotionCLIP.","url_abs":"https://arxiv.org/abs/2305.13500v1","url_pdf":"https://arxiv.org/pdf/2305.13500v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"learning-emotion-representations-from-verbal-1","repo_url":"https://github.com/xeaver/emotionclip","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"contrastive-learning","task_name":"Contrastive Learning"},{"task_slug":"emotion-recognition","task_name":"Emotion Recognition"},{"task_slug":"emotion-recognition-in-context","task_name":"Emotion Recognition in Context"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/emotion-recognition-in-context-on-bold","task":"Emotion Recognition in Context","dataset":"BoLD","model":"EmotionCLIP(linear-probe evaluation)","rank_in_archive_order":2,"of":3,"metrics":{"AUC":"69.30","Average mAP":"22.51"},"uses_additional_data":false},{"leaderboard":"/sota/emotion-recognition-in-context-on-emotic","task":"Emotion Recognition in Context","dataset":"EMOTIC","model":"EmotionCLIP(linear-probe evaluation)","rank_in_archive_order":3,"of":9,"metrics":{"mAP":"32.91"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2305.13500","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.13500"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/xeaver/emotionclip","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/Xeaver/EmotionCLIP","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":8,"ran_draft_wrong":2,"unverified":19},"by_repo_kind":{"official":{"samples":29,"ran":10,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"c31ea380cbe12461","entry":"AttentionPool2d","repo":"Xeaver/EmotionCLIP","repo_kind":"official","path":"src/models/base.py","file_url":"https://github.com/Xeaver/EmotionCLIP/blob/HEAD/src/models/base.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c31ea380cbe12461"}},{"code_sha256_prefix":"317eeb922af6df8c","entry":"Bottleneck","repo":"Xeaver/EmotionCLIP","repo_kind":"official","path":"src/models/base.py","file_url":"https://github.com/Xeaver/EmotionCLIP/blob/HEAD/src/models/base.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"317eeb922af6df8c"}},{"code_sha256_prefix":"147cc99e5a62df69","entry":"CLIPVisionCfg","repo":"Xeaver/EmotionCLIP","repo_kind":"official","path":"src/models/base.py","file_url":"https://github.com/Xeaver/EmotionCLIP/blob/HEAD/src/models/base.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"147cc99e5a62df69"}},{"code_sha256_prefix":"521eb383ee0b590c","entry":"LayerNorm","repo":"Xeaver/EmotionCLIP","repo_kind":"official","path":"src/models/base.py","file_url":"https://github.com/Xeaver/EmotionCLIP/blob/HEAD/src/models/base.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"521eb383ee0b590c"}},{"code_sha256_prefix":"72ae1815d1a81517","entry":"ResidualAttentionBlock","repo":"Xeaver/EmotionCLIP","repo_kind":"official","path":"src/models/base.py","file_url":"https://github.com/Xeaver/EmotionCLIP/blob/HEAD/src/models/base.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"72ae1815d1a81517"}},{"code_sha256_prefix":"698e12b8a3909d01","entry":"Transformer","repo":"Xeaver/EmotionCLIP","repo_kind":"official","path":"src/models/base.py","file_url":"https://github.com/Xeaver/EmotionCLIP/blob/HEAD/src/models/base.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"698e12b8a3909d01"}},{"code_sha256_prefix":"02d6e69496cfde7c","entry":"VisualHead","repo":"Xeaver/EmotionCLIP","repo_kind":"official","path":"src/models/base.py","file_url":"https://github.com/Xeaver/EmotionCLIP/blob/HEAD/src/models/base.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"02d6e69496cfde7c"}},{"code_sha256_prefix":"c857cf6246801938","entry":"VisualTransformer","repo":"Xeaver/EmotionCLIP","repo_kind":"official","path":"src/models/base.py","file_url":"https://github.com/Xeaver/EmotionCLIP/blob/HEAD/src/models/base.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c857cf6246801938"}},{"code_sha256_prefix":"ee61c52d701d0844","entry":"_ntuple","repo":"Xeaver/EmotionCLIP","repo_kind":"official","path":"src/models/base.py","file_url":"https://github.com/Xeaver/EmotionCLIP/blob/HEAD/src/models/base.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ee61c52d701d0844"}},{"code_sha256_prefix":"4414627a34d3f5a6","entry":"freeze_batch_norm_2d","repo":"Xeaver/EmotionCLIP","repo_kind":"official","path":"src/models/base.py","file_url":"https://github.com/Xeaver/EmotionCLIP/blob/HEAD/src/models/base.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4414627a34d3f5a6"}},{"code_sha256_prefix":"bd68a70f00670d88","entry":"CLIP","repo":"Xeaver/EmotionCLIP","repo_kind":"official","path":"src/models/base.py","file_url":"https://github.com/Xeaver/EmotionCLIP/blob/HEAD/src/models/base.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bd68a70f00670d88"}},{"code_sha256_prefix":"4e3b3a6334554db6","entry":"DefaultDataLoader","repo":"xeaver/emotionclip","repo_kind":"official","path":"linear_eval.py","file_url":"https://github.com/xeaver/emotionclip/blob/HEAD/linear_eval.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4e3b3a6334554db6"}},{"code_sha256_prefix":"83d96ef6b277e8b1","entry":"EmotionCLIP","repo":"Xeaver/EmotionCLIP","repo_kind":"official","path":"src/models/base.py","file_url":"https://github.com/Xeaver/EmotionCLIP/blob/HEAD/src/models/base.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"83d96ef6b277e8b1"}},{"code_sha256_prefix":"e16cbf2cae362701","entry":"ModifiedResNet","repo":"Xeaver/EmotionCLIP","repo_kind":"official","path":"src/models/base.py","file_url":"https://github.com/Xeaver/EmotionCLIP/blob/HEAD/src/models/base.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e16cbf2cae362701"}},{"code_sha256_prefix":"304f8d104c3ae85b","entry":"accuracy_from_affect2mm","repo":"xeaver/emotionclip","repo_kind":"official","path":"linear_eval.py","file_url":"https://github.com/xeaver/emotionclip/blob/HEAD/linear_eval.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"304f8d104c3ae85b"}},{"code_sha256_prefix":"22f55504153a6c4a","entry":"bbox_to_mask","repo":"xeaver/emotionclip","repo_kind":"official","path":"src/datasets/bold.py","file_url":"https://github.com/xeaver/emotionclip/blob/HEAD/src/datasets/bold.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"22f55504153a6c4a"}},{"code_sha256_prefix":"5117c4f71e3e05f2","entry":"create_index","repo":"xeaver/emotionclip","repo_kind":"official","path":"preprocessing/create_index.py","file_url":"https://github.com/xeaver/emotionclip/blob/HEAD/preprocessing/create_index.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5117c4f71e3e05f2"}},{"code_sha256_prefix":"908004750488edf4","entry":"filter_frames_by_subs","repo":"xeaver/emotionclip","repo_kind":"official","path":"preprocessing/download_videos.py","file_url":"https://github.com/xeaver/emotionclip/blob/HEAD/preprocessing/download_videos.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"908004750488edf4"}},{"code_sha256_prefix":"982a5d4308b89326","entry":"frames_to_hdf5","repo":"xeaver/emotionclip","repo_kind":"official","path":"preprocessing/frames_to_hdf5.py","file_url":"https://github.com/xeaver/emotionclip/blob/HEAD/preprocessing/frames_to_hdf5.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"982a5d4308b89326"}},{"code_sha256_prefix":"fffd8407fb58896e","entry":"get_correct_bbox","repo":"xeaver/emotionclip","repo_kind":"official","path":"src/datasets/emotic.py","file_url":"https://github.com/xeaver/emotionclip/blob/HEAD/src/datasets/emotic.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fffd8407fb58896e"}},{"code_sha256_prefix":"8fce15b73d85523c","entry":"get_word_time_range","repo":"xeaver/emotionclip","repo_kind":"official","path":"preprocessing/regroup_transcripts.py","file_url":"https://github.com/xeaver/emotionclip/blob/HEAD/preprocessing/regroup_transcripts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8fce15b73d85523c"}},{"code_sha256_prefix":"ea7b78f323af0b60","entry":"is_path_writable","repo":"xeaver/emotionclip","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/xeaver/emotionclip/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ea7b78f323af0b60"}},{"code_sha256_prefix":"617c4c27e5687338","entry":"joints_to_bbox","repo":"xeaver/emotionclip","repo_kind":"official","path":"src/datasets/bold.py","file_url":"https://github.com/xeaver/emotionclip/blob/HEAD/src/datasets/bold.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"617c4c27e5687338"}},{"code_sha256_prefix":"507b8fae3df11365","entry":"regroup_by_punc","repo":"xeaver/emotionclip","repo_kind":"official","path":"preprocessing/regroup_transcripts.py","file_url":"https://github.com/xeaver/emotionclip/blob/HEAD/preprocessing/regroup_transcripts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"507b8fae3df11365"}},{"code_sha256_prefix":"a13c7ce400fe0bd9","entry":"remove_unuseful_frames","repo":"xeaver/emotionclip","repo_kind":"official","path":"preprocessing/download_videos.py","file_url":"https://github.com/xeaver/emotionclip/blob/HEAD/preprocessing/download_videos.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a13c7ce400fe0bd9"}},{"code_sha256_prefix":"4a6f84446e0450a6","entry":"search_playlist","repo":"xeaver/emotionclip","repo_kind":"official","path":"preprocessing/search_youtube.py","file_url":"https://github.com/xeaver/emotionclip/blob/HEAD/preprocessing/search_youtube.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4a6f84446e0450a6"}},{"code_sha256_prefix":"7433c55c64017801","entry":"tar_gz_to_hdf5","repo":"xeaver/emotionclip","repo_kind":"official","path":"preprocessing/frames_to_hdf5.py","file_url":"https://github.com/xeaver/emotionclip/blob/HEAD/preprocessing/frames_to_hdf5.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7433c55c64017801"}},{"code_sha256_prefix":"e037cf75b0da50f6","entry":"timeit","repo":"xeaver/emotionclip","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/xeaver/emotionclip/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e037cf75b0da50f6"}},{"code_sha256_prefix":"6907192b23bf18d8","entry":"with_timer","repo":"xeaver/emotionclip","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/xeaver/emotionclip/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6907192b23bf18d8"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}