{"url":"/task/multimodal-sentiment-analysis","name":"Multimodal Sentiment Analysis","slug":"multimodal-sentiment-analysis","description_markdown":"Multimodal sentiment analysis is the task of performing sentiment analysis with multiple data sources - e.g. a camera feed of someone's face and their recorded speech.\r\n\r\n<span style=\"color:grey; opacity: 0.6\">( Image credit: [ICON: Interactive Conversational Memory Network\r\nfor Multimodal Emotion Detection](https://www.aclweb.org/anthology/D18-1280.pdf) )</span>","categories":[{"name":"Computer Vision","url":"/area/computer-vision"},{"name":"Natural Language Processing","url":"/area/natural-language-processing"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":202,"papers_with_code":93,"benchmarks":5,"benchmark_tables_in_archive":5,"benchmark_tables_shown":5,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":7,"subtasks":0,"parent_tasks":1},"benchmarks":[{"leaderboard":"/sota/multimodal-sentiment-analysis-on-cmu-mosei-1","slug":"multimodal-sentiment-analysis-on-cmu-mosei-1","dataset":"CMU-MOSEI","dataset_url":"/dataset/cmu-mosei","rows_in_archive":15,"metrics":["Accuracy","MAE","F1","Acc-5","Acc-7","Corr"],"first_row_in_archive_order":{"model":"SeMUL-PCD","paper_title":"Multi-label Emotion Analysis in Conversation via Multimodal Knowledge Distillation","paper_url":"/paper/multi-label-emotion-analysis-in-conversation","paper_date":"2023-10-27","arxiv_id":null,"code_links":[],"syntology":null}},{"leaderboard":"/sota/multimodal-sentiment-analysis-on-cmu-mosi","slug":"multimodal-sentiment-analysis-on-cmu-mosi","dataset":"CMU-MOSI","dataset_url":"/dataset/cmu-mosi","rows_in_archive":12,"metrics":["F1","Acc-7","Acc-2","Corr","MAE","F1-score (Weighted)","Acc-5"],"first_row_in_archive_order":{"model":"MMML","paper_title":"Multimodal Multi-loss Fusion Network for Sentiment Analysis","paper_url":"/paper/multi-modality-multi-loss-fusion-network","paper_date":"2023-08-01","arxiv_id":"2308.00264","code_links":[{"title":"zehuiwu/MMML","url":"https://github.com/zehuiwu/MMML"}],"syntology":null}},{"leaderboard":"/sota/multimodal-sentiment-analysis-on-mosi","slug":"multimodal-sentiment-analysis-on-mosi","dataset":"MOSI","dataset_url":"/dataset/multimodal-opinionlevel-sentiment-intensity","rows_in_archive":11,"metrics":["Accuracy","F1 score"],"first_row_in_archive_order":{"model":"MMML","paper_title":"Multimodal Multi-loss Fusion Network for Sentiment Analysis","paper_url":"/paper/multi-modality-multi-loss-fusion-network","paper_date":"2023-08-01","arxiv_id":"2308.00264","code_links":[{"title":"zehuiwu/MMML","url":"https://github.com/zehuiwu/MMML"}],"syntology":null}},{"leaderboard":"/sota/multimodal-sentiment-analysis-on-b-t4sa","slug":"multimodal-sentiment-analysis-on-b-t4sa","dataset":"B-T4SA","dataset_url":"/dataset/b-t4sa","rows_in_archive":7,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"AutoML-Based Fusion Approach","paper_title":"An AutoML-based Approach to Multimodal Image Sentiment Analysis","paper_url":"/paper/an-automl-based-approach-to-multimodal-image","paper_date":"2021-02-16","arxiv_id":"2102.08092","code_links":[],"syntology":null}},{"leaderboard":"/sota/multimodal-sentiment-analysis-on-ch-sims","slug":"multimodal-sentiment-analysis-on-ch-sims","dataset":"CH-SIMS","dataset_url":"/dataset/ch-sims","rows_in_archive":2,"metrics":["F1","MAE","CORR","Acc-2","Acc-3","Acc-5"],"first_row_in_archive_order":{"model":"MMML","paper_title":"Multimodal Multi-loss Fusion Network for Sentiment Analysis","paper_url":"/paper/multi-modality-multi-loss-fusion-network","paper_date":"2023-08-01","arxiv_id":"2308.00264","code_links":[{"title":"zehuiwu/MMML","url":"https://github.com/zehuiwu/MMML"}],"syntology":null}}],"datasets":[{"url":"/dataset/cmu-mosei","name":"CMU-MOSEI","full_name":"","num_papers_in_archive":190},{"url":"/dataset/multimodal-opinionlevel-sentiment-intensity","name":"Multimodal Opinionlevel Sentiment Intensity","full_name":"MOSI","num_papers_in_archive":69},{"url":"/dataset/ch-sims","name":"CH-SIMS","full_name":"CH-SIMS","num_papers_in_archive":25},{"url":"/dataset/cmu-mosi","name":"CMU-MOSI","full_name":"Multimodal Corpus of Sentiment Intensity","num_papers_in_archive":16},{"url":"/dataset/muse-car","name":"MuSe-CaR","full_name":"Multimodal Sentiment Analysis in Car Reviews","num_papers_in_archive":9},{"url":"/dataset/memotion-analysis","name":"SemEval-2020 Task-8","full_name":"","num_papers_in_archive":6},{"url":"/dataset/b-t4sa","name":"B-T4SA","full_name":"B-T4SA","num_papers_in_archive":1}],"subtasks":[],"parent_tasks":[{"url":"/task/sentiment-analysis","name":"Sentiment Analysis"}],"papers":{"order":"repositories listed in the archive (desc), then date (desc); the archive holds no stars","population":"papers tagged with this task that list at least one repository in the archive","shown":30,"of":93,"tagged_in_all":202,"items":[{"url":"/paper/words-can-shift-dynamically-adjusting-word","title":"Words Can Shift: Dynamically Adjusting Word Representations Using Nonverbal Behaviors","date":"2018-11-23","arxiv_id":"1811.09362","repositories_listed":5,"syntology":{"n":2,"n_ran":0,"n_unverified":2,"n_pointer_only":0}},{"url":"/paper/190600295","title":"Multimodal Transformer for Unaligned Multimodal Language Sequences","date":"2019-06-01","arxiv_id":"1906.00295","repositories_listed":4,"syntology":{"n":17,"n_ran":8,"n_unverified":9,"n_pointer_only":10}},{"url":"/paper/multimodal-speech-emotion-recognition-using","title":"Multimodal Speech Emotion Recognition Using Audio and Text","date":"2018-10-10","arxiv_id":"1810.04635","repositories_listed":4,"syntology":null},{"url":"/paper/tvlt-textless-vision-language-transformer","title":"TVLT: Textless Vision-Language Transformer","date":"2022-09-28","arxiv_id":"2209.14156","repositories_listed":3,"syntology":{"n":4,"n_ran":3,"n_unverified":1,"n_pointer_only":0}},{"url":"/paper/m-sena-an-integrated-platform-for-multimodal","title":"M-SENA: An Integrated Platform for Multimodal Sentiment Analysis","date":"2022-03-23","arxiv_id":"2203.12441","repositories_listed":3,"syntology":null},{"url":"/paper/audio-text-sentiment-analysis-using-deep","title":"Complementary Fusion of Multi-Features and Multi-Modalities in Sentiment Analysis","date":"2019-04-17","arxiv_id":"1904.08138","repositories_listed":3,"syntology":null},{"url":"/paper/efficient-low-rank-multimodal-fusion-with","title":"Efficient Low-rank Multimodal Fusion with Modality-Specific Factors","date":"2018-05-31","arxiv_id":"1806.00064","repositories_listed":3,"syntology":null},{"url":"/paper/m2se-a-multistage-multitask-instruction","title":"EmoVerse: Exploring Multimodal Large Language Models for Sentiment and Emotion Understanding","date":"2024-12-11","arxiv_id":"2412.08049","repositories_listed":2,"syntology":null},{"url":"/paper/2407-21054","title":"Sentiment Reasoning for Healthcare","date":"2024-07-24","arxiv_id":"2407.21054","repositories_listed":2,"syntology":null},{"url":"/paper/unisa-unified-generative-framework-for-1","title":"UniSA: Unified Generative Framework for Sentiment Analysis","date":"2023-09-04","arxiv_id":"2309.01339","repositories_listed":2,"syntology":{"n":8,"n_ran":6,"n_unverified":2,"n_pointer_only":0}},{"url":"/paper/improving-multimodal-fusion-with-hierarchical","title":"Improving Multimodal Fusion with Hierarchical Mutual Information Maximization for Multimodal Sentiment Analysis","date":"2021-09-01","arxiv_id":"2109.00412","repositories_listed":2,"syntology":{"n":7,"n_ran":4,"n_unverified":3,"n_pointer_only":0}},{"url":"/paper/bi-bimodal-modality-fusion-for-correlation","title":"Bi-Bimodal Modality Fusion for Correlation-Controlled Multimodal Sentiment Analysis","date":"2021-07-28","arxiv_id":"2107.13669","repositories_listed":2,"syntology":{"n":2,"n_ran":2,"n_unverified":0,"n_pointer_only":0}},{"url":"/paper/learning-modality-specific-representations","title":"Learning Modality-Specific Representations with Self-Supervised Multi-Task Learning for Multimodal Sentiment Analysis","date":"2021-02-09","arxiv_id":"2102.04830","repositories_listed":2,"syntology":{"n":6,"n_ran":4,"n_unverified":2,"n_pointer_only":0}},{"url":"/paper/misa-modality-invariant-and-specific","title":"MISA: Modality-Invariant and -Specific Representations for Multimodal Sentiment Analysis","date":"2020-05-07","arxiv_id":"2005.03545","repositories_listed":2,"syntology":{"n":10,"n_ran":2,"n_unverified":8,"n_pointer_only":0}},{"url":"/paper/found-in-translation-learning-robust-joint","title":"Found in Translation: Learning Robust Joint Representations by Cyclic Translations Between Modalities","date":"2018-12-19","arxiv_id":"1812.07809","repositories_listed":2,"syntology":null},{"url":"/paper/multimodal-sentiment-analysis-to-explore-the","title":"Multimodal Sentiment Analysis To Explore the Structure of Emotions","date":"2018-05-25","arxiv_id":"1805.10205","repositories_listed":2,"syntology":null},{"url":"/paper/multimodal-sentiment-analysis-with-word-level","title":"Multimodal Sentiment Analysis with Word-Level Fusion and Reinforcement Learning","date":"2018-02-03","arxiv_id":"1802.00924","repositories_listed":2,"syntology":null},{"url":"/paper/multi-attention-recurrent-network-for-human","title":"Multi-attention Recurrent Network for Human Communication Comprehension","date":"2018-02-03","arxiv_id":"1802.00923","repositories_listed":2,"syntology":null},{"url":"/paper/tensor-fusion-network-for-multimodal","title":"Tensor Fusion Network for Multimodal Sentiment Analysis","date":"2017-07-23","arxiv_id":"1707.07250","repositories_listed":2,"syntology":null},{"url":"/paper/context-dependent-sentiment-analysis-in-user","title":"Context-Dependent Sentiment Analysis in User-Generated Videos","date":"2017-07-01","arxiv_id":null,"repositories_listed":2,"syntology":null},{"url":"/paper/catch-cognitive-assessment-through-cookie","title":"CAtCh: Cognitive Assessment through Cookie Thief","date":"2025-06-07","arxiv_id":"2506.06603","repositories_listed":1,"syntology":null},{"url":"/paper/mse-adapter-a-lightweight-plugin-endowing","title":"MSE-Adapter: A Lightweight Plugin Endowing LLMs with the Capability to Perform Multimodal Sentiment Analysis and Emotion Recognition","date":"2025-02-18","arxiv_id":"2502.12478","repositories_listed":1,"syntology":null},{"url":"/paper/multi-modality-collaborative-learning-for","title":"Multi-Modality Collaborative Learning for Sentiment Analysis","date":"2025-01-21","arxiv_id":"2501.12424","repositories_listed":1,"syntology":null},{"url":"/paper/dlf-disentangled-language-focused-multimodal","title":"DLF: Disentangled-Language-Focused Multimodal Sentiment Analysis","date":"2024-12-16","arxiv_id":"2412.12225","repositories_listed":1,"syntology":{"n":6,"n_ran":0,"n_unverified":6,"n_pointer_only":0}},{"url":"/paper/semi-iin-semi-supervised-intra-inter-modal","title":"Semi-IIN: Semi-supervised Intra-inter modal Interaction Learning Network for Multimodal Sentiment Analysis","date":"2024-12-13","arxiv_id":"2412.09784","repositories_listed":1,"syntology":null},{"url":"/paper/bridging-the-gap-for-test-time-multimodal","title":"Bridging the Gap for Test-Time Multimodal Sentiment Analysis","date":"2024-12-10","arxiv_id":"2412.07121","repositories_listed":1,"syntology":{"n":10,"n_ran":5,"n_unverified":5,"n_pointer_only":0}},{"url":"/paper/enhancing-multimodal-sentiment-analysis-for","title":"Enhancing Multimodal Sentiment Analysis for Missing Modality through Self-Distillation and Unified Modality Cross-Attention","date":"2024-10-19","arxiv_id":"2410.15029","repositories_listed":1,"syntology":null},{"url":"/paper/knowledge-guided-dynamic-modality-attention","title":"Knowledge-Guided Dynamic Modality Attention Fusion Framework for Multimodal Sentiment Analysis","date":"2024-10-06","arxiv_id":"2410.04491","repositories_listed":1,"syntology":null},{"url":"/paper/towards-robust-multimodal-sentiment-analysis","title":"Towards Robust Multimodal Sentiment Analysis with Incomplete Data","date":"2024-09-30","arxiv_id":"2409.20012","repositories_listed":1,"syntology":{"n":14,"n_ran":10,"n_unverified":4,"n_pointer_only":0}},{"url":"/paper/recent-trends-of-multimodal-affective","title":"Recent Trends of Multimodal Affective Computing: A Survey from NLP Perspective","date":"2024-09-11","arxiv_id":"2409.07388","repositories_listed":1,"syntology":null}],"syntology_records":11,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-09-24T18:15:14+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"}},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"}}