{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/table-and-image-generation-for-investigating","title":"Table and Image Generation for Investigating Knowledge of Entities in Pre-trained Vision and Language Models","arxiv_id":"2306.02115","date":"2023-06-03","proceeding":null,"authors":["Hidetaka Kamigaito","Katsuhiko Hayashi","Taro Watanabe"],"abstract":"In this paper, we propose a table and image generation task to verify how the knowledge about entities acquired from natural language is retained in Vision & Language (V&L) models. This task consists of two parts: the first is to generate a table containing knowledge about an entity and its related image, and the second is to generate an image from an entity with a caption and a table containing related knowledge of the entity. In both tasks, the model must know the entities used to perform the generation properly. We created the Wikipedia Table and Image Generation (WikiTIG) dataset from about 200,000 infoboxes in English Wikipedia articles to perform the proposed tasks. We evaluated the performance on the tasks with respect to the above research question using the V&L model OFA, which has achieved state-of-the-art results in multiple tasks. Experimental results show that OFA forgets part of its entity knowledge by pre-training as a complement to improve the performance of image related tasks.","url_abs":"https://arxiv.org/abs/2306.02115v2","url_pdf":"https://arxiv.org/pdf/2306.02115v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"table-and-image-generation-for-investigating","repo_url":"https://github.com/kamigaito/wikitig","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"articles","task_name":"Articles"},{"task_slug":"image-generation","task_name":"Image Generation"}],"methods":[{"method_slug":"ofa","method_name":"OFA"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2306.02115","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.02115"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/OFA-Sys/OFA","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/kamigaito/WikiTIG","reach":{"status":"ok"}}],"summary":{"ran":4,"ran_draft_wrong":2,"unverified":5},"by_repo_kind":{"found_in_text":{"samples":11,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"f18199d037514305","entry":"BatchNorm2d","repo":"OFA-Sys/OFA","repo_kind":"found_in_text","path":"models/ofa/unify_transformer.py","file_url":"https://github.com/OFA-Sys/OFA/blob/HEAD/models/ofa/unify_transformer.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f18199d037514305"}},{"code_sha256_prefix":"d9def42110729a85","entry":"conv1x1","repo":"OFA-Sys/OFA","repo_kind":"found_in_text","path":"models/ofa/resnet.py","file_url":"https://github.com/OFA-Sys/OFA/blob/HEAD/models/ofa/resnet.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"d9def42110729a85"}},{"code_sha256_prefix":"160bb14bd76201b4","entry":"conv3x3","repo":"OFA-Sys/OFA","repo_kind":"found_in_text","path":"models/ofa/resnet.py","file_url":"https://github.com/OFA-Sys/OFA/blob/HEAD/models/ofa/resnet.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"160bb14bd76201b4"}},{"code_sha256_prefix":"52471eacb56db726","entry":"drop_path","repo":"OFA-Sys/OFA","repo_kind":"found_in_text","path":"models/ofa/resnet.py","file_url":"https://github.com/OFA-Sys/OFA/blob/HEAD/models/ofa/resnet.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"52471eacb56db726"}},{"code_sha256_prefix":"fba339cb809b27fe","entry":"make_image_bucket_position","repo":"OFA-Sys/OFA","repo_kind":"found_in_text","path":"models/ofa/unify_transformer.py","file_url":"https://github.com/OFA-Sys/OFA/blob/HEAD/models/ofa/unify_transformer.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fba339cb809b27fe"}},{"code_sha256_prefix":"c0b56d6f87906655","entry":"make_token_bucket_position","repo":"OFA-Sys/OFA","repo_kind":"found_in_text","path":"models/ofa/unify_transformer.py","file_url":"https://github.com/OFA-Sys/OFA/blob/HEAD/models/ofa/unify_transformer.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c0b56d6f87906655"}},{"code_sha256_prefix":"e1397e09cb4c249d","entry":"construct_rdrop_sample","repo":"OFA-Sys/OFA","repo_kind":"found_in_text","path":"criterions/label_smoothed_encouraging_loss.py","file_url":"https://github.com/OFA-Sys/OFA/blob/HEAD/criterions/label_smoothed_encouraging_loss.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"e1397e09cb4c249d"}},{"code_sha256_prefix":"01478032e19f9e68","entry":"custom_to_pil","repo":"OFA-Sys/OFA","repo_kind":"found_in_text","path":"criterions/clip_scst_loss.py","file_url":"https://github.com/OFA-Sys/OFA/blob/HEAD/criterions/clip_scst_loss.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"01478032e19f9e68"}},{"code_sha256_prefix":"e7939f8552fd8729","entry":"kl_loss","repo":"OFA-Sys/OFA","repo_kind":"found_in_text","path":"criterions/label_smoothed_encouraging_loss.py","file_url":"https://github.com/OFA-Sys/OFA/blob/HEAD/criterions/label_smoothed_encouraging_loss.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"e7939f8552fd8729"}},{"code_sha256_prefix":"cbd8615e3eba731b","entry":"label_smoothed_nll_loss","repo":"OFA-Sys/OFA","repo_kind":"found_in_text","path":"criterions/label_smoothed_encouraging_loss.py","file_url":"https://github.com/OFA-Sys/OFA/blob/HEAD/criterions/label_smoothed_encouraging_loss.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"cbd8615e3eba731b"}},{"code_sha256_prefix":"b0d5d75f27b8ce4f","entry":"scst_loss","repo":"OFA-Sys/OFA","repo_kind":"found_in_text","path":"criterions/clip_scst_loss.py","file_url":"https://github.com/OFA-Sys/OFA/blob/HEAD/criterions/clip_scst_loss.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"b0d5d75f27b8ce4f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}