{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/mast-multimodal-abstractive-summarization","title":"MAST: Multimodal Abstractive Summarization with Trimodal Hierarchical Attention","arxiv_id":"2010.08021","date":"2020-10-15","proceeding":"EMNLP (nlpbt) 2020 11","authors":["Aman Khullar","Udit Arora"],"abstract":"This paper presents MAST, a new model for Multimodal Abstractive Text Summarization that utilizes information from all three modalities -- text, audio and video -- in a multimodal video. Prior work on multimodal abstractive text summarization only utilized information from the text and video modalities. We examine the usefulness and challenges of deriving information from the audio modality and present a sequence-to-sequence trimodal hierarchical attention-based model that overcomes these challenges by letting the model pay more attention to the text modality. MAST outperforms the current state of the art model (video-text) by 2.51 points in terms of Content F1 score and 1.00 points in terms of Rouge-L score on the How2 dataset for multimodal language understanding.","url_abs":"https://arxiv.org/abs/2010.08021v1","url_pdf":"https://arxiv.org/pdf/2010.08021v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"mast-multimodal-abstractive-summarization","repo_url":"https://github.com/amankhullar/mast","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"abstractive-text-summarization","task_name":"Abstractive Text Summarization"},{"task_slug":"multimodal-abstractive-text-summarization","task_name":"Multimodal Abstractive Text Summarization"},{"task_slug":"text-summarization","task_name":"Text Summarization"}],"methods":[{"method_slug":"lstm","method_name":"LSTM"},{"method_slug":"seq2seq","method_name":"Seq2Seq"},{"method_slug":"sigmoid-activation","method_name":"Sigmoid Activation"},{"method_slug":"tanh-activation","method_name":"Tanh Activation"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/multimodal-abstractive-text-summarization-on","task":"Multimodal Abstractive Text Summarization","dataset":"How2 300h","model":"MAST","rank_in_archive_order":1,"of":1,"metrics":{"ROUGE-L":"43.23"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2010.08021","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}