{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/do-you-remember-dense-video-captioning-with","title":"Do You Remember? Dense Video Captioning with Cross-Modal Memory Retrieval","arxiv_id":"2404.07610","date":"2024-04-11","proceeding":"CVPR 2024 1","authors":["Minkuk Kim","Hyeon Bae Kim","Jinyoung Moon","Jinwoo Choi","Seong Tae Kim"],"abstract":"There has been significant attention to the research on dense video captioning, which aims to automatically localize and caption all events within untrimmed video. Several studies introduce methods by designing dense video captioning as a multitasking problem of event localization and event captioning to consider inter-task relations. However, addressing both tasks using only visual input is challenging due to the lack of semantic content. In this study, we address this by proposing a novel framework inspired by the cognitive information processing of humans. Our model utilizes external memory to incorporate prior knowledge. The memory retrieval method is proposed with cross-modal video-to-text matching. To effectively incorporate retrieved text features, the versatile encoder and the decoder with visual and textual cross-attention modules are designed. Comparative experiments have been conducted to show the effectiveness of the proposed method on ActivityNet Captions and YouCook2 datasets. Experimental results show promising performance of our model without extensive pretraining from a large video dataset.","url_abs":"https://arxiv.org/abs/2404.07610v1","url_pdf":"https://arxiv.org/pdf/2404.07610v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"do-you-remember-dense-video-captioning-with","repo_url":"https://github.com/ailab-kyunghee/cm2_dvc","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"dense-video-captioning","task_name":"Dense Video Captioning"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"text-matching","task_name":"Text Matching"},{"task_slug":"video-captioning","task_name":"Video Captioning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/dense-video-captioning-on-activitynet","task":"Dense Video Captioning","dataset":"ActivityNet Captions","model":"CM²","rank_in_archive_order":8,"of":12,"metrics":{"BLEU4":"2.38","CIDEr":"33.01","F1":"55.21","METEOR":"8.55","Precision":"56.81","Recall":"53.71","SODA":"6.18"},"uses_additional_data":false},{"leaderboard":"/sota/dense-video-captioning-on-youcook2","task":"Dense Video Captioning","dataset":"YouCook2","model":"CM²","rank_in_archive_order":4,"of":7,"metrics":{"BLEU4":"1.63","CIDEr":"31.66","F1":"28.43","METEOR":"6.08","Precision":"33.38","Recall":"24.76","SODA":"5.34"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2404.07610","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.07610"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ailab-kyunghee/cm2_dvc","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/ailabkyunghee/CM2_DVC","reach":{"status":"gone","observed_at":"2026-09-17","how":"tree_404+repo_404"}}],"summary":{"ran":1,"ran_draft_wrong":1,"unverified":1},"by_repo_kind":{"official":{"samples":3,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"65a13ac177e73bcd","entry":"regression_loss","repo":"ailab-kyunghee/cm2_dvc","repo_kind":"official","path":"cm2/criterion.py","file_url":"https://github.com/ailab-kyunghee/cm2_dvc/blob/HEAD/cm2/criterion.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"65a13ac177e73bcd"}},{"code_sha256_prefix":"5c0711aada67957e","entry":"sigmoid_focal_loss","repo":"ailab-kyunghee/cm2_dvc","repo_kind":"official","path":"cm2/criterion.py","file_url":"https://github.com/ailab-kyunghee/cm2_dvc/blob/HEAD/cm2/criterion.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5c0711aada67957e"}},{"code_sha256_prefix":"594f41589a9b1b8a","entry":"cross_entropy_with_gaussian_mask","repo":"ailab-kyunghee/cm2_dvc","repo_kind":"official","path":"cm2/criterion.py","file_url":"https://github.com/ailab-kyunghee/cm2_dvc/blob/HEAD/cm2/criterion.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"594f41589a9b1b8a"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}