{"url":"/task/multimodal-emotion-recognition","name":"Multimodal Emotion Recognition","slug":"multimodal-emotion-recognition","description_markdown":"This is a leaderboard for multimodal emotion recognition on the IEMOCAP dataset. The modality abbreviations are\r\nA: Acoustic\r\nT: Text\r\nV: Visual\r\n\r\nPlease include the modality in the bracket after the model name.\r\n\r\nAll models must use standard five emotion categories and are evaluated in standard leave-one-session-out (LOSO). See the papers for references.","categories":[{"name":"Computer Vision","url":"/area/computer-vision"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":180,"papers_with_code":80,"benchmarks":7,"benchmark_tables_in_archive":7,"benchmark_tables_shown":7,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":11,"subtasks":1,"parent_tasks":1},"benchmarks":[{"leaderboard":"/sota/multimodal-emotion-recognition-on-iemocap-4","slug":"multimodal-emotion-recognition-on-iemocap-4","dataset":"IEMOCAP-4","dataset_url":null,"rows_in_archive":11,"metrics":["Weighted F1","Accuracy","F1","Weighted Recall"],"first_row_in_archive_order":{"model":"GraphSmile","paper_title":"Tracing Intricate Cues in Dialogue: Joint Graph Structure and Sentiment Dynamics for Multimodal Emotion Recognition","paper_url":"/paper/2407-21536","paper_date":"2024-07-31","arxiv_id":"2407.21536","code_links":[{"title":"lijfrank-open/GraphSmile","url":"https://github.com/lijfrank-open/GraphSmile"}],"syntology":null}},{"leaderboard":"/sota/multimodal-emotion-recognition-on-meld","slug":"multimodal-emotion-recognition-on-meld","dataset":"MELD","dataset_url":"/dataset/meld","rows_in_archive":3,"metrics":["Weighted F1","Accuracy"],"first_row_in_archive_order":{"model":"GraphSmile","paper_title":"Tracing Intricate Cues in Dialogue: Joint Graph Structure and Sentiment Dynamics for Multimodal Emotion Recognition","paper_url":"/paper/2407-21536","paper_date":"2024-07-31","arxiv_id":"2407.21536","code_links":[{"title":"lijfrank-open/GraphSmile","url":"https://github.com/lijfrank-open/GraphSmile"}],"syntology":null}},{"leaderboard":"/sota/multimodal-emotion-recognition-on-iemocap","slug":"multimodal-emotion-recognition-on-iemocap","dataset":"IEMOCAP","dataset_url":"/dataset/iemocap","rows_in_archive":2,"metrics":["Weighted F1","Accuracy"],"first_row_in_archive_order":{"model":"GraphSmile","paper_title":"Tracing Intricate Cues in Dialogue: Joint Graph Structure and Sentiment Dynamics for Multimodal Emotion Recognition","paper_url":"/paper/2407-21536","paper_date":"2024-07-31","arxiv_id":"2407.21536","code_links":[{"title":"lijfrank-open/GraphSmile","url":"https://github.com/lijfrank-open/GraphSmile"}],"syntology":null}},{"leaderboard":"/sota/multimodal-emotion-recognition-on-cmu-mosei-1","slug":"multimodal-emotion-recognition-on-cmu-mosei-1","dataset":"CMU-MOSEI-Sentiment","dataset_url":null,"rows_in_archive":1,"metrics":["Weighted F1","Accuracy"],"first_row_in_archive_order":{"model":"GraphSmile","paper_title":"Tracing Intricate Cues in Dialogue: Joint Graph Structure and Sentiment Dynamics for Multimodal Emotion Recognition","paper_url":"/paper/2407-21536","paper_date":"2024-07-31","arxiv_id":"2407.21536","code_links":[{"title":"lijfrank-open/GraphSmile","url":"https://github.com/lijfrank-open/GraphSmile"}],"syntology":null}},{"leaderboard":"/sota/multimodal-emotion-recognition-on-cmu-mosei-2","slug":"multimodal-emotion-recognition-on-cmu-mosei-2","dataset":"CMU-MOSEI-Sentiment-3","dataset_url":null,"rows_in_archive":1,"metrics":["Weighted F1","Accuracy"],"first_row_in_archive_order":{"model":"GraphSmile","paper_title":"Tracing Intricate Cues in Dialogue: Joint Graph Structure and Sentiment Dynamics for Multimodal Emotion Recognition","paper_url":"/paper/2407-21536","paper_date":"2024-07-31","arxiv_id":"2407.21536","code_links":[{"title":"lijfrank-open/GraphSmile","url":"https://github.com/lijfrank-open/GraphSmile"}],"syntology":null}},{"leaderboard":"/sota/multimodal-emotion-recognition-on-expressive","slug":"multimodal-emotion-recognition-on-expressive","dataset":"Expressive hands and faces dataset (EHF).","dataset_url":null,"rows_in_archive":1,"metrics":["v2v error"],"first_row_in_archive_order":{"model":"SMPLify-X","paper_title":"Multi-Modal Emotion recognition on IEMOCAP Dataset using Deep Learning","paper_url":"/paper/multi-modal-emotion-recognition-on-iemocap","paper_date":"2018-04-16","arxiv_id":"1804.05788","code_links":[{"title":"Samarth-Tripathi/IEMOCAP-Emotion-Detection","url":"https://github.com/Samarth-Tripathi/IEMOCAP-Emotion-Detection"},{"title":"ankurbhatia24/MULTIMODAL-EMOTION-RECOGNITION","url":"https://github.com/ankurbhatia24/MULTIMODAL-EMOTION-RECOGNITION"}],"syntology":null}},{"leaderboard":"/sota/multimodal-emotion-recognition-on-meld-1","slug":"multimodal-emotion-recognition-on-meld-1","dataset":"MELD-Sentiment","dataset_url":null,"rows_in_archive":1,"metrics":["Weighted F1","Accuracy"],"first_row_in_archive_order":{"model":"GraphSmile","paper_title":"Tracing Intricate Cues in Dialogue: Joint Graph Structure and Sentiment Dynamics for Multimodal Emotion Recognition","paper_url":"/paper/2407-21536","paper_date":"2024-07-31","arxiv_id":"2407.21536","code_links":[{"title":"lijfrank-open/GraphSmile","url":"https://github.com/lijfrank-open/GraphSmile"}],"syntology":null}}],"datasets":[{"url":"/dataset/iemocap","name":"IEMOCAP","full_name":"The Interactive Emotional Dyadic Motion Capture (IEMOCAP) Database","num_papers_in_archive":749},{"url":"/dataset/meld","name":"MELD","full_name":"Multimodal EmotionLines Dataset","num_papers_in_archive":289},{"url":"/dataset/cmu-mosei","name":"CMU-MOSEI","full_name":"","num_papers_in_archive":190},{"url":"/dataset/emotic","name":"EMOTIC","full_name":"EMOTIons in Context","num_papers_in_archive":38},{"url":"/dataset/cped","name":"CPED","full_name":"Chinese Personalized and Emotional Dialogue","num_papers_in_archive":15},{"url":"/dataset/deap","name":"DEAP","full_name":"","num_papers_in_archive":11},{"url":"/dataset/resd","name":"RESD","full_name":"Russian Emotional Speech Dialogs with annotated text","num_papers_in_archive":3},{"url":"/dataset/aesi","name":"AESI","full_name":"Athens Emotional States Inventory","num_papers_in_archive":1},{"url":"/dataset/ses","name":"SES","full_name":"Spanish Emotional Speech","num_papers_in_archive":1},{"url":"/dataset/werewolf-xl","name":"Werewolf-XL","full_name":"","num_papers_in_archive":1},{"url":"/dataset/storytelling-video-dataset","name":"Video Dataset","full_name":"Storytelling Video Dataset (Russian, Emotion, Gesture, Speech)","num_papers_in_archive":0}],"subtasks":[{"url":"/task/video-emotion-detection","name":"Video Emotion Detection"}],"parent_tasks":[{"url":"/task/emotion-recognition","name":"Emotion Recognition"}],"papers":{"order":"repositories listed in the archive (desc), then date (desc); the archive holds no stars","population":"papers tagged with this task that list at least one repository in the archive","shown":30,"of":80,"tagged_in_all":180,"items":[{"url":"/paper/multimodal-speech-emotion-recognition-and","title":"Multimodal Speech Emotion Recognition and Ambiguity Resolution","date":"2019-04-12","arxiv_id":"1904.06022","repositories_listed":5,"syntology":null},{"url":"/paper/multimodal-speech-emotion-recognition-using","title":"Multimodal Speech Emotion Recognition Using Audio and Text","date":"2018-10-10","arxiv_id":"1810.04635","repositories_listed":4,"syntology":null},{"url":"/paper/mer-2023-multi-label-learning-modality","title":"MER 2023: Multi-label Learning, Modality Robustness, and Semi-Supervised Learning","date":"2023-04-18","arxiv_id":"2304.08981","repositories_listed":3,"syntology":null},{"url":"/paper/audio-text-sentiment-analysis-using-deep","title":"Complementary Fusion of Multi-Features and Multi-Modalities in Sentiment Analysis","date":"2019-04-17","arxiv_id":"1904.08138","repositories_listed":3,"syntology":null},{"url":"/paper/m2se-a-multistage-multitask-instruction","title":"EmoVerse: Exploring Multimodal Large Language Models for Sentiment and Emotion Understanding","date":"2024-12-11","arxiv_id":"2412.08049","repositories_listed":2,"syntology":null},{"url":"/paper/text-and-feature-based-models-for-compound","title":"Textualized and Feature-based Models for Compound Multimodal Emotion Recognition in the Wild","date":"2024-07-17","arxiv_id":"2407.12927","repositories_listed":2,"syntology":{"n":4,"n_ran":3,"n_unverified":1,"n_pointer_only":4}},{"url":"/paper/emotion-llama-multimodal-emotion-recognition","title":"Emotion-LLaMA: Multimodal Emotion Recognition and Reasoning with Instruction Tuning","date":"2024-06-17","arxiv_id":"2406.11161","repositories_listed":2,"syntology":{"n":2,"n_ran":1,"n_unverified":1,"n_pointer_only":0}},{"url":"/paper/mer-2024-semi-supervised-learning-noise","title":"MER 2024: Semi-Supervised Learning, Noise Robustness, and Open-Vocabulary Multimodal Emotion Recognition","date":"2024-04-26","arxiv_id":"2404.17113","repositories_listed":2,"syntology":{"n":9,"n_ran":5,"n_unverified":4,"n_pointer_only":9}},{"url":"/paper/cogmen-contextualized-gnn-based-multimodal","title":"COGMEN: COntextualized GNN based Multimodal Emotion recognitioN","date":"2022-05-05","arxiv_id":"2205.02455","repositories_listed":2,"syntology":null},{"url":"/paper/emotion-recognition-in-audio-and-video-using","title":"Emotion Recognition in Audio and Video Using Deep Neural Networks","date":"2020-06-15","arxiv_id":"2006.08129","repositories_listed":2,"syntology":null},{"url":"/paper/dialoguernn-an-attentive-rnn-for-emotion","title":"DialogueRNN: An Attentive RNN for Emotion Detection in Conversations","date":"2018-11-01","arxiv_id":"1811.00405","repositories_listed":2,"syntology":null},{"url":"/paper/multi-modal-emotion-recognition-on-iemocap","title":"Multi-Modal Emotion recognition on IEMOCAP Dataset using Deep Learning","date":"2018-04-16","arxiv_id":"1804.05788","repositories_listed":2,"syntology":null},{"url":"/paper/context-dependent-sentiment-analysis-in-user","title":"Context-Dependent Sentiment Analysis in User-Generated Videos","date":"2017-07-01","arxiv_id":null,"repositories_listed":2,"syntology":null},{"url":"/paper/end-to-end-multimodal-emotion-recognition","title":"End-to-End Multimodal Emotion Recognition using Deep Neural Networks","date":"2017-04-27","arxiv_id":"1704.08619","repositories_listed":2,"syntology":null},{"url":"/paper/towards-robust-multimodal-emotion-recognition","title":"Towards Robust Multimodal Emotion Recognition under Missing Modalities and Distribution Shifts","date":"2025-06-12","arxiv_id":"2506.10452","repositories_listed":1,"syntology":null},{"url":"/paper/tacfn-transformer-based-adaptive-cross-modal","title":"TACFN: Transformer-based Adaptive Cross-modal Fusion Network for Multimodal Emotion Recognition","date":"2025-05-10","arxiv_id":"2505.06536","repositories_listed":1,"syntology":null},{"url":"/paper/feature-based-dual-visual-feature-extraction","title":"Feature-Based Dual Visual Feature Extraction Model for Compound Multimodal Emotion Recognition","date":"2025-03-21","arxiv_id":"2503.17453","repositories_listed":1,"syntology":null},{"url":"/paper/r1-omni-explainable-omni-multimodal-emotion","title":"R1-Omni: Explainable Omni-Multimodal Emotion Recognition with Reinforcement Learning","date":"2025-03-07","arxiv_id":"2503.05379","repositories_listed":1,"syntology":{"n":3,"n_ran":0,"n_unverified":3,"n_pointer_only":3}},{"url":"/paper/latent-distribution-decoupling-a","title":"Latent Distribution Decoupling: A Probabilistic Framework for Uncertainty-Aware Multimodal Emotion Recognition","date":"2025-02-19","arxiv_id":"2502.13954","repositories_listed":1,"syntology":null},{"url":"/paper/milmer-a-framework-for-multiple-instance","title":"Milmer: a Framework for Multiple Instance Learning based Multimodal Emotion Recognition","date":"2025-02-01","arxiv_id":"2502.00547","repositories_listed":1,"syntology":null},{"url":"/paper/sdr-gnn-spectral-domain-reconstruction-graph","title":"SDR-GNN: Spectral Domain Reconstruction Graph Neural Network for Incomplete Multimodal Learning in Conversational Emotion Recognition","date":"2024-11-29","arxiv_id":"2411.19822","repositories_listed":1,"syntology":null},{"url":"/paper/hierarchical-hypercomplex-network-for","title":"Hierarchical Hypercomplex Network for Multimodal Emotion Recognition","date":"2024-09-13","arxiv_id":"2409.09194","repositories_listed":1,"syntology":null},{"url":"/paper/phemonet-a-multimodal-network-for","title":"PHemoNet: A Multimodal Network for Physiological Signals","date":"2024-09-13","arxiv_id":"2410.00010","repositories_listed":1,"syntology":null},{"url":"/paper/recent-trends-of-multimodal-affective","title":"Recent Trends of Multimodal Affective Computing: A Survey from NLP Perspective","date":"2024-09-11","arxiv_id":"2409.07388","repositories_listed":1,"syntology":null},{"url":"/paper/leveraging-contrastive-learning-and-self","title":"Leveraging Contrastive Learning and Self-Training for Multimodal Emotion Recognition with Limited Labeled Samples","date":"2024-08-23","arxiv_id":"2409.04447","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_unverified":0,"n_pointer_only":0}},{"url":"/paper/sztu-cmu-at-mer2024-improving-emotion-llama","title":"SZTU-CMU at MER2024: Improving Emotion-LLaMA with Conv-Attention for Multimodal Emotion Recognition","date":"2024-08-20","arxiv_id":"2408.10500","repositories_listed":1,"syntology":{"n":8,"n_ran":5,"n_unverified":3,"n_pointer_only":0}},{"url":"/paper/enhancing-modal-fusion-by-alignment-and-label","title":"Enhancing Modal Fusion by Alignment and Label Matching for Multimodal Emotion Recognition","date":"2024-08-18","arxiv_id":"2408.09438","repositories_listed":1,"syntology":null},{"url":"/paper/multi-teacher-privileged-knowledge","title":"Multi Teacher Privileged Knowledge Distillation for Multimodal Expression Recognition","date":"2024-08-16","arxiv_id":"2408.09035","repositories_listed":1,"syntology":null},{"url":"/paper/2407-21536","title":"Tracing Intricate Cues in Dialogue: Joint Graph Structure and Sentiment Dynamics for Multimodal Emotion Recognition","date":"2024-07-31","arxiv_id":"2407.21536","repositories_listed":1,"syntology":null},{"url":"/paper/multimodal-emotion-recognition-using-audio","title":"Multimodal Emotion Recognition using Audio-Video Transformer Fusion with Cross Attention","date":"2024-07-26","arxiv_id":"2407.18552","repositories_listed":1,"syntology":null}],"syntology_records":6,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-09-24T18:15:14+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"}},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"}}