{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/character-centric-story-visualization-via","title":"Character-Centric Story Visualization via Visual Planning and Token Alignment","arxiv_id":"2210.08465","date":"2022-10-16","proceeding":null,"authors":["Hong Chen","Rujun Han","Te-Lin Wu","Hideki Nakayama","Nanyun Peng"],"abstract":"Story visualization advances the traditional text-to-image generation by enabling multiple image generation based on a complete story. This task requires machines to 1) understand long text inputs and 2) produce a globally consistent image sequence that illustrates the contents of the story. A key challenge of consistent story visualization is to preserve characters that are essential in stories. To tackle the challenge, we propose to adapt a recent work that augments Vector-Quantized Variational Autoencoders (VQ-VAE) with a text-tovisual-token (transformer) architecture. Specifically, we modify the text-to-visual-token module with a two-stage framework: 1) character token planning model that predicts the visual tokens for characters only; 2) visual token completion model that generates the remaining visual token sequence, which is sent to VQ-VAE for finalizing image generations. To encourage characters to appear in the images, we further train the two-stage framework with a character-token alignment objective. Extensive experiments and evaluations demonstrate that the proposed method excels at preserving characters and can produce higher quality image sequences compared with the strong baselines. Codes can be found in https://github.com/sairin1202/VP-CSV","url_abs":"https://arxiv.org/abs/2210.08465v4","url_pdf":"https://arxiv.org/pdf/2210.08465v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"character-centric-story-visualization-via","repo_url":"https://github.com/pluslabnlp/vp-csv","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"character-centric-story-visualization-via","repo_url":"https://github.com/sairin1202/vp-csv","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"image-generation","task_name":"Image Generation"},{"task_slug":"story-visualization","task_name":"Story Visualization"},{"task_slug":"text-to-image-generation-1","task_name":"Text to Image Generation"},{"task_slug":"text-to-image-generation","task_name":"Text-to-Image Generation"}],"methods":[{"method_slug":"vq-vae","method_name":"VQ-VAE"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2210.08465","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.08465"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sairin1202/vp-csv","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pluslabnlp/vp-csv","reach":{"status":"ok"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/PlusLabNLP/VP-CSV","reach":{"status":"ok"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/adymaharana/VLCStoryGan","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":8,"ran_honours":1,"ran_fixture":2,"ran_draft_wrong":1,"unverified":9},"by_repo_kind":{"found_in_text":{"samples":21,"ran":12,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"79728760d660d983","entry":"BertEmbeddings","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"79728760d660d983"}},{"code_sha256_prefix":"455739c5ad6b2387","entry":"BertIntermediate","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"455739c5ad6b2387"}},{"code_sha256_prefix":"a8ea3a32f7622e56","entry":"DynamicFilterLayer","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a8ea3a32f7622e56"}},{"code_sha256_prefix":"118186316f75b11b","entry":"LxmertAttention","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"118186316f75b11b"}},{"code_sha256_prefix":"37566e964d58aad9","entry":"LxmertAttentionOutput","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"37566e964d58aad9"}},{"code_sha256_prefix":"f5111c56952b1ddc","entry":"MartPooler","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f5111c56952b1ddc"}},{"code_sha256_prefix":"b6f03dfe01742a92","entry":"PositionEncoding","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b6f03dfe01742a92"}},{"code_sha256_prefix":"d434aa3ac7f6993a","entry":"ResBlock","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d434aa3ac7f6993a"}},{"code_sha256_prefix":"a3abf3fe697fabd0","entry":"gelu","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a3abf3fe697fabd0"}},{"code_sha256_prefix":"93af85ca91baf7b0","entry":"make_pad_shifted_mask","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"93af85ca91baf7b0"}},{"code_sha256_prefix":"cd0b5f41868c631f","entry":"make_shifted_mask","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cd0b5f41868c631f"}},{"code_sha256_prefix":"721bc7234edf9845","entry":"upBlock","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"721bc7234edf9845"}},{"code_sha256_prefix":"7758e2dc60da7479","entry":"BertAttention","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7758e2dc60da7479"}},{"code_sha256_prefix":"38a29e27a1641775","entry":"BertEncoderWithMemory","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"38a29e27a1641775"}},{"code_sha256_prefix":"3b66ce72af6145fa","entry":"BertLayerWithMemory","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3b66ce72af6145fa"}},{"code_sha256_prefix":"e5006dcf91e21556","entry":"BertSelfAttention","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e5006dcf91e21556"}},{"code_sha256_prefix":"99751ef923ab069c","entry":"CA_NET","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"99751ef923ab069c"}},{"code_sha256_prefix":"44aa26a7f2f9a571","entry":"MemoryUpdater","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"44aa26a7f2f9a571"}},{"code_sha256_prefix":"b72cbd35cd593c73","entry":"NEXT_STAGE_G","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b72cbd35cd593c73"}},{"code_sha256_prefix":"eddb0906b50d313d","entry":"StoryMartGAN","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"eddb0906b50d313d"}},{"code_sha256_prefix":"4b14e2379269d142","entry":"make_video_only_mask","repo":"adymaharana/VLCStoryGan","repo_kind":"found_in_text","path":"vlcgan/model.py","file_url":"https://github.com/adymaharana/VLCStoryGan/blob/HEAD/vlcgan/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4b14e2379269d142"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}