{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/a-modular-approach-for-multimodal","title":"A Modular Approach for Multimodal Summarization of TV Shows","arxiv_id":"2403.03823","date":"2024-03-06","proceeding":null,"authors":["Louis Mahon","Mirella Lapata"],"abstract":"In this paper we address the task of summarizing television shows, which touches key areas in AI research: complex reasoning, multiple modalities, and long narratives. We present a modular approach where separate components perform specialized sub-tasks which we argue affords greater flexibility compared to end-to-end methods. Our modules involve detecting scene boundaries, reordering scenes so as to minimize the number of cuts between different events, converting visual information to text, summarizing the dialogue in each scene, and fusing the scene summaries into a final summary for the entire episode. We also present a new metric, PRISMA (Precision and Recall EvaluatIon of Summary FActs), to measure both precision and recall of generated summaries, which we decompose into atomic facts. Tested on the recently released SummScreen3D dataset, our method produces higher quality summaries than comparison models, as measured with ROUGE and our new fact-based metric, and as assessed by human evaluators.","url_abs":"https://arxiv.org/abs/2403.03823v9","url_pdf":"https://arxiv.org/pdf/2403.03823v9.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"a-modular-approach-for-multimodal","repo_url":"https://github.com/lou1sm/modular_multimodal_summarization","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2403.03823","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.03823"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/shmsw25/FActScore","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":8,"ran_draft_wrong":1,"unverified":3},"by_repo_kind":{"found_in_text":{"samples":12,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"510528f4db1006bc","entry":"best_demos","repo":"shmsw25/FActScore","repo_kind":"found_in_text","path":"factscore/atomic_facts.py","file_url":"https://github.com/shmsw25/FActScore/blob/HEAD/factscore/atomic_facts.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"510528f4db1006bc"}},{"code_sha256_prefix":"6c0eab0d68ea3036","entry":"call_ChatGPT","repo":"shmsw25/FActScore","repo_kind":"found_in_text","path":"factscore/openai_lm.py","file_url":"https://github.com/shmsw25/FActScore/blob/HEAD/factscore/openai_lm.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6c0eab0d68ea3036"}},{"code_sha256_prefix":"e5bb87d92ea99a8b","entry":"call_GPT3","repo":"shmsw25/FActScore","repo_kind":"found_in_text","path":"factscore/openai_lm.py","file_url":"https://github.com/shmsw25/FActScore/blob/HEAD/factscore/openai_lm.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e5bb87d92ea99a8b"}},{"code_sha256_prefix":"f06c53e363e1ebaa","entry":"get_memory_footprint","repo":"shmsw25/FActScore","repo_kind":"found_in_text","path":"factscore/utils.py","file_url":"https://github.com/shmsw25/FActScore/blob/HEAD/factscore/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f06c53e363e1ebaa"}},{"code_sha256_prefix":"8eb846a8e76a79b8","entry":"is_invalid_paragraph_ppl","repo":"shmsw25/FActScore","repo_kind":"found_in_text","path":"factscore/abstain_detection.py","file_url":"https://github.com/shmsw25/FActScore/blob/HEAD/factscore/abstain_detection.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8eb846a8e76a79b8"}},{"code_sha256_prefix":"672bd38d830be48d","entry":"is_invalid_ppl","repo":"shmsw25/FActScore","repo_kind":"found_in_text","path":"factscore/abstain_detection.py","file_url":"https://github.com/shmsw25/FActScore/blob/HEAD/factscore/abstain_detection.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"672bd38d830be48d"}},{"code_sha256_prefix":"6a96435eba311b08","entry":"normalize_answer","repo":"shmsw25/FActScore","repo_kind":"found_in_text","path":"factscore/atomic_facts.py","file_url":"https://github.com/shmsw25/FActScore/blob/HEAD/factscore/atomic_facts.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"code_sha256_prefix":"3635046105ae0fce","entry":"remove_citation","repo":"shmsw25/FActScore","repo_kind":"found_in_text","path":"factscore/abstain_detection.py","file_url":"https://github.com/shmsw25/FActScore/blob/HEAD/factscore/abstain_detection.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3635046105ae0fce"}},{"code_sha256_prefix":"13867f872a481bd3","entry":"text_to_sentences","repo":"shmsw25/FActScore","repo_kind":"found_in_text","path":"factscore/atomic_facts.py","file_url":"https://github.com/shmsw25/FActScore/blob/HEAD/factscore/atomic_facts.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"13867f872a481bd3"}},{"code_sha256_prefix":"7e4c366c4dbf9568","entry":"convert_model_to_int8_on_gpu","repo":"shmsw25/FActScore","repo_kind":"found_in_text","path":"factscore/utils.py","file_url":"https://github.com/shmsw25/FActScore/blob/HEAD/factscore/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7e4c366c4dbf9568"}},{"code_sha256_prefix":"02c83d89394be1c7","entry":"recover_instruct_llama","repo":"shmsw25/FActScore","repo_kind":"found_in_text","path":"factscore/download_data.py","file_url":"https://github.com/shmsw25/FActScore/blob/HEAD/factscore/download_data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"02c83d89394be1c7"}},{"code_sha256_prefix":"925c59b7e44138ab","entry":"softmax","repo":"shmsw25/FActScore","repo_kind":"found_in_text","path":"factscore/npm.py","file_url":"https://github.com/shmsw25/FActScore/blob/HEAD/factscore/npm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"925c59b7e44138ab"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}