{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/separating-skills-and-concepts-for-novel-1","title":"Separating Skills and Concepts for Novel Visual Question Answering","arxiv_id":"2107.09106","date":"2021-07-19","proceeding":"CVPR 2021 1","authors":["Spencer Whitehead","Hui Wu","Heng Ji","Rogerio Feris","Kate Saenko"],"abstract":"Generalization to out-of-distribution data has been a problem for Visual Question Answering (VQA) models. To measure generalization to novel questions, we propose to separate them into \"skills\" and \"concepts\". \"Skills\" are visual tasks, such as counting or attribute recognition, and are applied to \"concepts\" mentioned in the question, such as objects and people. VQA methods should be able to compose skills and concepts in novel ways, regardless of whether the specific composition has been seen in training, yet we demonstrate that existing models have much to improve upon towards handling new compositions. We present a novel method for learning to compose skills and concepts that separates these two factors implicitly within a model by learning grounded concept representations and disentangling the encoding of skills from that of concepts. We enforce these properties with a novel contrastive learning procedure that does not rely on external annotations and can be learned from unlabeled image-question pairs. Experiments demonstrate the effectiveness of our approach for improving compositional and grounding performance.","url_abs":"https://arxiv.org/abs/2107.09106v1","url_pdf":"https://arxiv.org/pdf/2107.09106v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"separating-skills-and-concepts-for-novel-1","repo_url":"https://github.com/SpencerWhitehead/novelvqa","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"separating-skills-and-concepts-for-novel-1","repo_url":"https://github.com/MalvinaNikandrou/contvqa","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"attribute","task_name":"Attribute"},{"task_slug":"contrastive-learning","task_name":"Contrastive Learning"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"}],"methods":[{"method_slug":"contrastive-learning","method_name":"Contrastive Learning"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2107.09106","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2107.09106"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/SpencerWhitehead/novelvqa","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/MalvinaNikandrou/contvqa","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":8},"by_repo_kind":{"official":{"samples":1,"ran":0,"repositories":1},"listed":{"samples":7,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"fbfb2ee28ffecc81","entry":"get_categories_per_task","repo":"MalvinaNikandrou/contvqa","repo_kind":"listed","path":"src/contvqa/get_image_based_settings.py","file_url":"https://github.com/MalvinaNikandrou/contvqa/blob/HEAD/src/contvqa/get_image_based_settings.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fbfb2ee28ffecc81"}},{"code_sha256_prefix":"a0692efb554bc382","entry":"get_num_of_samples_per_task","repo":"MalvinaNikandrou/contvqa","repo_kind":"listed","path":"src/contvqa/get_dataset_statistics.py","file_url":"https://github.com/MalvinaNikandrou/contvqa/blob/HEAD/src/contvqa/get_dataset_statistics.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a0692efb554bc382"}},{"code_sha256_prefix":"b6bf5a95f3801883","entry":"get_optim","repo":"SpencerWhitehead/novelvqa","repo_kind":"official","path":"core/model/optim.py","file_url":"https://github.com/SpencerWhitehead/novelvqa/blob/HEAD/core/model/optim.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b6bf5a95f3801883"}},{"code_sha256_prefix":"88571e8825d16071","entry":"get_question_ids_to_questions","repo":"MalvinaNikandrou/contvqa","repo_kind":"listed","path":"src/contvqa/plot_distributions.py","file_url":"https://github.com/MalvinaNikandrou/contvqa/blob/HEAD/src/contvqa/plot_distributions.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"88571e8825d16071"}},{"code_sha256_prefix":"f71c20e0c28c07ae","entry":"prep_ans","repo":"MalvinaNikandrou/contvqa","repo_kind":"listed","path":"src/contvqa/answer_preprocessing.py","file_url":"https://github.com/MalvinaNikandrou/contvqa/blob/HEAD/src/contvqa/answer_preprocessing.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f71c20e0c28c07ae"}},{"code_sha256_prefix":"7201dbcd85dd597d","entry":"process_digit_article","repo":"MalvinaNikandrou/contvqa","repo_kind":"listed","path":"src/contvqa/answer_preprocessing.py","file_url":"https://github.com/MalvinaNikandrou/contvqa/blob/HEAD/src/contvqa/answer_preprocessing.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7201dbcd85dd597d"}},{"code_sha256_prefix":"389046e117d0b05c","entry":"process_punctuation","repo":"MalvinaNikandrou/contvqa","repo_kind":"listed","path":"src/contvqa/answer_preprocessing.py","file_url":"https://github.com/MalvinaNikandrou/contvqa/blob/HEAD/src/contvqa/answer_preprocessing.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"389046e117d0b05c"}},{"code_sha256_prefix":"a5e5d18d0c38ef2a","entry":"read_json","repo":"MalvinaNikandrou/contvqa","repo_kind":"listed","path":"src/contvqa/common.py","file_url":"https://github.com/MalvinaNikandrou/contvqa/blob/HEAD/src/contvqa/common.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a5e5d18d0c38ef2a"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}