{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/text-video-retrieval-with-disentangled","title":"Text-Video Retrieval with Disentangled Conceptualization and Set-to-Set Alignment","arxiv_id":"2305.12218","date":"2023-05-20","proceeding":null,"authors":["Peng Jin","Hao Li","Zesen Cheng","Jinfa Huang","Zhennan Wang","Li Yuan","Chang Liu","Jie Chen"],"abstract":"Text-video retrieval is a challenging cross-modal task, which aims to align visual entities with natural language descriptions. Current methods either fail to leverage the local details or are computationally expensive. What's worse, they fail to leverage the heterogeneous concepts in data. In this paper, we propose the Disentangled Conceptualization and Set-to-set Alignment (DiCoSA) to simulate the conceptualizing and reasoning process of human beings. For disentangled conceptualization, we divide the coarse feature into multiple latent factors related to semantic concepts. For set-to-set alignment, where a set of visual concepts correspond to a set of textual concepts, we propose an adaptive pooling method to aggregate semantic concepts to address the partial matching. In particular, since we encode concepts independently in only a few dimensions, DiCoSA is superior at efficiency and granularity, ensuring fine-grained interactions using a similar computational complexity as coarse-grained alignment. Extensive experiments on five datasets, including MSR-VTT, LSMDC, MSVD, ActivityNet, and DiDeMo, demonstrate that our method outperforms the existing state-of-the-art methods.","url_abs":"https://arxiv.org/abs/2305.12218v1","url_pdf":"https://arxiv.org/pdf/2305.12218v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"text-video-retrieval-with-disentangled","repo_url":"https://github.com/jpthu17/dicosa","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"text-video-retrieval-with-disentangled","repo_url":"https://github.com/jpthu17/HBI","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"text-video-retrieval-with-disentangled","repo_url":"https://github.com/jpthu17/diffusionret","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"text-video-retrieval-with-disentangled","repo_url":"https://github.com/jpthu17/emcl","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"video-retrieval","task_name":"Video Retrieval"}],"methods":[{"method_slug":"align","method_name":"ALIGN"},{"method_slug":"fail","method_name":"fail"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2305.12218","atlas_url":"https://app.syntology.ai/?focus=2305.12218","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.12218"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jpthu17/dicosa","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jpthu17/HBI","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jpthu17/emcl","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jpthu17/diffusionret","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/jpthu17/DiCoSA","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_draft_wrong":6,"ran_honours":2,"ran_fixture":1,"unverified":14},"by_repo_kind":{"official":{"samples":16,"ran":7,"repositories":1},"listed":{"samples":7,"ran":2,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"0f786c407fb1ee4c","entry":"swish","repo":"jpthu17/DiCoSA","repo_kind":"official","path":"tvr/models/until_module.py","file_url":"https://github.com/jpthu17/DiCoSA/blob/HEAD/tvr/models/until_module.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":2,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0f786c407fb1ee4c"}},{"code_sha256_prefix":"98f385d847636a3e","entry":"basic_clean","repo":"jpthu17/DiCoSA","repo_kind":"official","path":"tvr/models/tokenization_clip.py","file_url":"https://github.com/jpthu17/DiCoSA/blob/HEAD/tvr/models/tokenization_clip.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"98f385d847636a3e"}},{"code_sha256_prefix":"e54804982d1e6d22","entry":"euclidean_dist","repo":"jpthu17/DiCoSA","repo_kind":"official","path":"tvr/models/until_module.py","file_url":"https://github.com/jpthu17/DiCoSA/blob/HEAD/tvr/models/until_module.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e54804982d1e6d22"}},{"code_sha256_prefix":"50e1ffed03f484ec","entry":"gelu","repo":"jpthu17/DiCoSA","repo_kind":"official","path":"tvr/models/until_module.py","file_url":"https://github.com/jpthu17/DiCoSA/blob/HEAD/tvr/models/until_module.py","link_basis":"plan_row","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"50e1ffed03f484ec"}},{"code_sha256_prefix":"d919ae32e5e4e616","entry":"get_pairs","repo":"jpthu17/DiCoSA","repo_kind":"official","path":"tvr/models/tokenization_clip.py","file_url":"https://github.com/jpthu17/DiCoSA/blob/HEAD/tvr/models/tokenization_clip.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d919ae32e5e4e616"}},{"code_sha256_prefix":"0e49170b7c8f60fd","entry":"merge_tokens","repo":"jpthu17/HBI","repo_kind":"listed","path":"HBI/models/cluster.py","file_url":"https://github.com/jpthu17/HBI/blob/HEAD/HBI/models/cluster.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0e49170b7c8f60fd"}},{"code_sha256_prefix":"af64ec220e8bcdbc","entry":"url_to_filename","repo":"jpthu17/diffusionret","repo_kind":"listed","path":"DiffusionRet/models/file_utils.py","file_url":"https://github.com/jpthu17/diffusionret/blob/HEAD/DiffusionRet/models/file_utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"af64ec220e8bcdbc"}},{"code_sha256_prefix":"e0960afdb0d64aa7","entry":"warmup_linear","repo":"jpthu17/DiCoSA","repo_kind":"official","path":"tvr/models/optimization.py","file_url":"https://github.com/jpthu17/DiCoSA/blob/HEAD/tvr/models/optimization.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e0960afdb0d64aa7"}},{"code_sha256_prefix":"9542161e9640b858","entry":"whitespace_clean","repo":"jpthu17/DiCoSA","repo_kind":"official","path":"tvr/models/tokenization_clip.py","file_url":"https://github.com/jpthu17/DiCoSA/blob/HEAD/tvr/models/tokenization_clip.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9542161e9640b858"}},{"code_sha256_prefix":"8bc3e8d24dc52e8b","entry":"CTM","repo":"jpthu17/HBI","repo_kind":"listed","path":"HBI/models/cluster.py","file_url":"https://github.com/jpthu17/HBI/blob/HEAD/HBI/models/cluster.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8bc3e8d24dc52e8b"}},{"code_sha256_prefix":"8b50e3e4f063bce8","entry":"TokenConv","repo":"jpthu17/HBI","repo_kind":"listed","path":"HBI/models/cluster.py","file_url":"https://github.com/jpthu17/HBI/blob/HEAD/HBI/models/cluster.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8b50e3e4f063bce8"}},{"code_sha256_prefix":"349d780dd4a89a37","entry":"cached_path","repo":"jpthu17/diffusionret","repo_kind":"listed","path":"DiffusionRet/models/file_utils.py","file_url":"https://github.com/jpthu17/diffusionret/blob/HEAD/DiffusionRet/models/file_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"349d780dd4a89a37"}},{"code_sha256_prefix":"ab028ff1220c10c1","entry":"dataloader_activity_train","repo":"jpthu17/diffusionret","repo_kind":"listed","path":"DiffusionRet/dataloaders/data_dataloaders.py","file_url":"https://github.com/jpthu17/diffusionret/blob/HEAD/DiffusionRet/dataloaders/data_dataloaders.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ab028ff1220c10c1"}},{"code_sha256_prefix":"b6070984a76e9bf3","entry":"dataloader_msrvtt_test","repo":"jpthu17/DiCoSA","repo_kind":"official","path":"tvr/dataloaders/data_dataloaders.py","file_url":"https://github.com/jpthu17/DiCoSA/blob/HEAD/tvr/dataloaders/data_dataloaders.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b6070984a76e9bf3"}},{"code_sha256_prefix":"780f499e676f8267","entry":"dataloader_msrvtt_train","repo":"jpthu17/DiCoSA","repo_kind":"official","path":"tvr/dataloaders/data_dataloaders.py","file_url":"https://github.com/jpthu17/DiCoSA/blob/HEAD/tvr/dataloaders/data_dataloaders.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"780f499e676f8267"}},{"code_sha256_prefix":"70d56cc28b0d7a14","entry":"dataloader_msrvtt_train_test","repo":"jpthu17/DiCoSA","repo_kind":"official","path":"tvr/dataloaders/data_dataloaders.py","file_url":"https://github.com/jpthu17/DiCoSA/blob/HEAD/tvr/dataloaders/data_dataloaders.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"70d56cc28b0d7a14"}},{"code_sha256_prefix":"db0ac56aaf6e35e6","entry":"filename_to_url","repo":"jpthu17/diffusionret","repo_kind":"listed","path":"DiffusionRet/models/file_utils.py","file_url":"https://github.com/jpthu17/diffusionret/blob/HEAD/DiffusionRet/models/file_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"db0ac56aaf6e35e6"}},{"code_sha256_prefix":"91e7fe98724c5bad","entry":"get_args","repo":"jpthu17/DiCoSA","repo_kind":"official","path":"main_retrieval.py","file_url":"https://github.com/jpthu17/DiCoSA/blob/HEAD/main_retrieval.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"91e7fe98724c5bad"}},{"code_sha256_prefix":"d227e7f9a9ea755e","entry":"load_stopwords","repo":"jpthu17/DiCoSA","repo_kind":"official","path":"tvr/dataloaders/dataloader_activitynet_retrieval.py","file_url":"https://github.com/jpthu17/DiCoSA/blob/HEAD/tvr/dataloaders/dataloader_activitynet_retrieval.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d227e7f9a9ea755e"}},{"code_sha256_prefix":"1d11677b554a969d","entry":"remove_stopwords","repo":"jpthu17/DiCoSA","repo_kind":"official","path":"tvr/dataloaders/dataloader_activitynet_retrieval.py","file_url":"https://github.com/jpthu17/DiCoSA/blob/HEAD/tvr/dataloaders/dataloader_activitynet_retrieval.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1d11677b554a969d"}},{"code_sha256_prefix":"8952edef143f2747","entry":"set_seed_logger","repo":"jpthu17/DiCoSA","repo_kind":"official","path":"main_retrieval.py","file_url":"https://github.com/jpthu17/DiCoSA/blob/HEAD/main_retrieval.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8952edef143f2747"}},{"code_sha256_prefix":"59e4e730a9dc5dc6","entry":"warmup_constant","repo":"jpthu17/DiCoSA","repo_kind":"official","path":"tvr/models/optimization.py","file_url":"https://github.com/jpthu17/DiCoSA/blob/HEAD/tvr/models/optimization.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"59e4e730a9dc5dc6"}},{"code_sha256_prefix":"20b3d25ba1b796e5","entry":"warmup_cosine","repo":"jpthu17/DiCoSA","repo_kind":"official","path":"tvr/models/optimization.py","file_url":"https://github.com/jpthu17/DiCoSA/blob/HEAD/tvr/models/optimization.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"20b3d25ba1b796e5"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}