{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/nus-hlt-report-for-activitynet-challenge-2021","title":"NUS-HLT Report for ActivityNet Challenge 2021 AVA (Speaker)","arxiv_id":null,"date":"2021-06-01","proceeding":"The ActivityNet Large-Scale Activity Recognition Challenge Workshop, CVPR 2021 6","authors":["Ruijie Tao","Zexu Pan","Rohan Kumar Das","Xinyuan Qian","Mike Zheng Shou","Haizhou Li"],"abstract":"Active speaker detection (ASD) seeks to detect who is speaking in a visual scene of one or more speakers. The successful ASD depends on accurate interpretation of short-term and long-term audio and visual information, as well as audiovisual interaction. Unlike the prior work where systems makedecision instantaneously using short-term features, we propose a novel framework, named TalkNet, that makes decision by taking both short-term and long-term features into consideration. TalkNet consists of audio and visual temporal encoders for feature representation, audio-visual cross-attention mechanism for\r\ninter-modality interaction, and a self-attention mechanism to capture long-term speaking evidence. The experiments demonstrate that TalkNet achieves 3.5% and 3.0% improvement over the state-of-the-art systems on the AVA-ActiveSpeaker validation and test dataset, respectively. We will release the codes, the models and data logs.","url_abs":"http://research.google.com/ava/2021/S3_NUS_Report_AVA_ActiveSpeaker_2021.pdf","url_pdf":"http://research.google.com/ava/2021/S3_NUS_Report_AVA_ActiveSpeaker_2021.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"nus-hlt-report-for-activitynet-challenge-2021","repo_url":"https://github.com/TaoRuijie/TalkNet_ASD","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"active-speaker-detection","task_name":"Active Speaker Detection"},{"task_slug":"audio-visual-active-speaker-detection","task_name":"Audio-Visual Active Speaker Detection"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/audio-visual-active-speaker-detection-on-ava","task":"Audio-Visual Active Speaker Detection","dataset":"AVA-ActiveSpeaker","model":"TalkNet","rank_in_archive_order":12,"of":20,"metrics":{"validation mean average precision":"92.3%"},"uses_additional_data":false}],"syntology":{"atlas_url":null,"mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}