{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/improving-generation-and-evaluation-of-visual","title":"Improving Generation and Evaluation of Visual Stories via Semantic Consistency","arxiv_id":"2105.10026","date":"2021-05-20","proceeding":"NAACL 2021 4","authors":["Adyasha Maharana","Darryl Hannan","Mohit Bansal"],"abstract":"Story visualization is an under-explored task that falls at the intersection of many important research directions in both computer vision and natural language processing. In this task, given a series of natural language captions which compose a story, an agent must generate a sequence of images that correspond to the captions. Prior work has introduced recurrent generative models which outperform text-to-image synthesis models on this task. However, there is room for improvement of generated images in terms of visual quality, coherence and relevance. We present a number of improvements to prior modeling approaches, including (1) the addition of a dual learning framework that utilizes video captioning to reinforce the semantic alignment between the story and generated images, (2) a copy-transform mechanism for sequentially-consistent story visualization, and (3) MART-based transformers to model complex interactions between frames. We present ablation studies to demonstrate the effect of each of these techniques on the generative power of the model for both individual images as well as the entire narrative. Furthermore, due to the complexity and generative nature of the task, standard evaluation metrics do not accurately reflect performance. Therefore, we also provide an exploration of evaluation metrics for the model, focused on aspects of the generated frames such as the presence/quality of generated characters, the relevance to captions, and the diversity of the generated images. We also present correlation experiments of our proposed automated metrics with human evaluations. Code and data available at: https://github.com/adymaharana/StoryViz","url_abs":"https://arxiv.org/abs/2105.10026v1","url_pdf":"https://arxiv.org/pdf/2105.10026v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"improving-generation-and-evaluation-of-visual","repo_url":"https://github.com/adymaharana/StoryViz","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"image-generation","task_name":"Image Generation"},{"task_slug":"story-visualization","task_name":"Story Visualization"},{"task_slug":"video-captioning","task_name":"Video Captioning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2105.10026","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.10026"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/adymaharana/StoryViz","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":3,"unverified":11},"by_repo_kind":{"official":{"samples":14,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"19379807ba70daaa","entry":"conv1x1","repo":"adymaharana/StoryViz","repo_kind":"official","path":"dcsgan/model.py","file_url":"https://github.com/adymaharana/StoryViz/blob/HEAD/dcsgan/model.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"19379807ba70daaa"}},{"code_sha256_prefix":"fac5364e2f53c6db","entry":"conv3x3","repo":"adymaharana/StoryViz","repo_kind":"official","path":"dcsgan/model.py","file_url":"https://github.com/adymaharana/StoryViz/blob/HEAD/dcsgan/model.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fac5364e2f53c6db"}},{"code_sha256_prefix":"721bc7234edf9845","entry":"upBlock","repo":"adymaharana/StoryViz","repo_kind":"official","path":"dcsgan/model.py","file_url":"https://github.com/adymaharana/StoryViz/blob/HEAD/dcsgan/model.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"721bc7234edf9845"}},{"code_sha256_prefix":"ea9c44af20afefdb","entry":"cal_performance","repo":"adymaharana/StoryViz","repo_kind":"official","path":"train_mart.py","file_url":"https://github.com/adymaharana/StoryViz/blob/HEAD/train_mart.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ea9c44af20afefdb"}},{"code_sha256_prefix":"1a92c2ee141e707f","entry":"calculate_activation_statistics","repo":"adymaharana/StoryViz","repo_kind":"official","path":"vfid/fid_score.py","file_url":"https://github.com/adymaharana/StoryViz/blob/HEAD/vfid/fid_score.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1a92c2ee141e707f"}},{"code_sha256_prefix":"eec8014aaf0ca309","entry":"calculate_activation_statistics","repo":"adymaharana/StoryViz","repo_kind":"official","path":"vfid/vfid_score.py","file_url":"https://github.com/adymaharana/StoryViz/blob/HEAD/vfid/vfid_score.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"eec8014aaf0ca309"}},{"code_sha256_prefix":"b71963b3facc5fb5","entry":"calculate_frechet_distance","repo":"adymaharana/StoryViz","repo_kind":"official","path":"vfid/fid_score.py","file_url":"https://github.com/adymaharana/StoryViz/blob/HEAD/vfid/fid_score.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b71963b3facc5fb5"}},{"code_sha256_prefix":"1e9b9a9b96ddfe5f","entry":"conv1x1","repo":"adymaharana/StoryViz","repo_kind":"official","path":"dcsgan/GLAttention.py","file_url":"https://github.com/adymaharana/StoryViz/blob/HEAD/dcsgan/GLAttention.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1e9b9a9b96ddfe5f"}},{"code_sha256_prefix":"bff619bf94913212","entry":"extract_img_features","repo":"adymaharana/StoryViz","repo_kind":"official","path":"train_mart.py","file_url":"https://github.com/adymaharana/StoryViz/blob/HEAD/train_mart.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bff619bf94913212"}},{"code_sha256_prefix":"93adcaa7f08a2399","entry":"fid_score","repo":"adymaharana/StoryViz","repo_kind":"official","path":"vfid/fid_score.py","file_url":"https://github.com/adymaharana/StoryViz/blob/HEAD/vfid/fid_score.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"93adcaa7f08a2399"}},{"code_sha256_prefix":"43e375fe6435c36d","entry":"fid_score","repo":"adymaharana/StoryViz","repo_kind":"official","path":"vfid/vfid_score.py","file_url":"https://github.com/adymaharana/StoryViz/blob/HEAD/vfid/vfid_score.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"43e375fe6435c36d"}},{"code_sha256_prefix":"4dcdc9386ef6f6f5","entry":"func_attention","repo":"adymaharana/StoryViz","repo_kind":"official","path":"dcsgan/GLAttention.py","file_url":"https://github.com/adymaharana/StoryViz/blob/HEAD/dcsgan/GLAttention.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4dcdc9386ef6f6f5"}},{"code_sha256_prefix":"d6b963ff0de4fe07","entry":"initialize_model","repo":"adymaharana/StoryViz","repo_kind":"official","path":"train_classifier.py","file_url":"https://github.com/adymaharana/StoryViz/blob/HEAD/train_classifier.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d6b963ff0de4fe07"}},{"code_sha256_prefix":"16898e129b726c1b","entry":"train_model","repo":"adymaharana/StoryViz","repo_kind":"official","path":"train_classifier.py","file_url":"https://github.com/adymaharana/StoryViz/blob/HEAD/train_classifier.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"16898e129b726c1b"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}