{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/learning-to-forecast-and-refine-residual","title":"Learning to Forecast and Refine Residual Motion for Image-to-Video Generation","arxiv_id":"1807.09951","date":"2018-07-26","proceeding":"ECCV 2018 9","authors":["Long Zhao","Xi Peng","Yu Tian","Mubbasir Kapadia","Dimitris Metaxas"],"abstract":"We consider the problem of image-to-video translation, where an input image\nis translated into an output video containing motions of a single object.\nRecent methods for such problems typically train transformation networks to\ngenerate future frames conditioned on the structure sequence. Parallel work has\nshown that short high-quality motions can be generated by spatiotemporal\ngenerative networks that leverage temporal knowledge from the training data. We\ncombine the benefits of both approaches and propose a two-stage generation\nframework where videos are generated from structures and then refined by\ntemporal signals. To model motions more efficiently, we train networks to learn\nresidual motion between the current and future frames, which avoids learning\nmotion-irrelevant details. We conduct extensive experiments on two\nimage-to-video translation tasks: facial expression retargeting and human pose\nforecasting. Superior results over the state-of-the-art methods on both tasks\ndemonstrate the effectiveness of our approach.","url_abs":"http://arxiv.org/abs/1807.09951v1","url_pdf":"http://arxiv.org/pdf/1807.09951v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"learning-to-forecast-and-refine-residual","repo_url":"https://github.com/garyzhao/FRGAN","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"human-pose-forecasting","task_name":"Human Pose Forecasting"},{"task_slug":"image-to-video","task_name":"Image to Video Generation"},{"task_slug":"translation","task_name":"Translation"},{"task_slug":"video-generation","task_name":"Video Generation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/1807.09951","atlas_url":"https://app.syntology.ai/?focus=1807.09951","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1807.09951"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/garyzhao/FRGAN","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":15},"by_repo_kind":{"listed":{"samples":15,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"8df37ce3b5bac7f3","entry":"add_sn","repo":"garyzhao/FRGAN","repo_kind":"listed","path":"common/utils.py","file_url":"https://github.com/garyzhao/FRGAN/blob/HEAD/common/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8df37ce3b5bac7f3"}},{"code_sha256_prefix":"caf28d4f019dee35","entry":"compute_bbox_size","repo":"garyzhao/FRGAN","repo_kind":"listed","path":"dfa/face.py","file_url":"https://github.com/garyzhao/FRGAN/blob/HEAD/dfa/face.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"caf28d4f019dee35"}},{"code_sha256_prefix":"b228caa6744033c7","entry":"crop_img","repo":"garyzhao/FRGAN","repo_kind":"listed","path":"dfa/face.py","file_url":"https://github.com/garyzhao/FRGAN/blob/HEAD/dfa/face.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b228caa6744033c7"}},{"code_sha256_prefix":"685fda3ca7f39c7b","entry":"load_dataset_split","repo":"garyzhao/FRGAN","repo_kind":"listed","path":"common/io.py","file_url":"https://github.com/garyzhao/FRGAN/blob/HEAD/common/io.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"685fda3ca7f39c7b"}},{"code_sha256_prefix":"7ae9c8bee34b0711","entry":"loss_norm_l1","repo":"garyzhao/FRGAN","repo_kind":"listed","path":"common/utils.py","file_url":"https://github.com/garyzhao/FRGAN/blob/HEAD/common/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7ae9c8bee34b0711"}},{"code_sha256_prefix":"0e9607194d198cdc","entry":"mobilenet","repo":"garyzhao/FRGAN","repo_kind":"listed","path":"dfa/mobilenet_v1.py","file_url":"https://github.com/garyzhao/FRGAN/blob/HEAD/dfa/mobilenet_v1.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0e9607194d198cdc"}},{"code_sha256_prefix":"2e691025cd2d1017","entry":"mobilenet_1","repo":"garyzhao/FRGAN","repo_kind":"listed","path":"dfa/mobilenet_v1.py","file_url":"https://github.com/garyzhao/FRGAN/blob/HEAD/dfa/mobilenet_v1.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2e691025cd2d1017"}},{"code_sha256_prefix":"d3aea641fac06744","entry":"mobilenet_2","repo":"garyzhao/FRGAN","repo_kind":"listed","path":"dfa/mobilenet_v1.py","file_url":"https://github.com/garyzhao/FRGAN/blob/HEAD/dfa/mobilenet_v1.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d3aea641fac06744"}},{"code_sha256_prefix":"b14f92d78ef0a0f2","entry":"parse_param","repo":"garyzhao/FRGAN","repo_kind":"listed","path":"dfa/inference.py","file_url":"https://github.com/garyzhao/FRGAN/blob/HEAD/dfa/inference.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b14f92d78ef0a0f2"}},{"code_sha256_prefix":"f3b964c1506b9216","entry":"pil_load_image","repo":"garyzhao/FRGAN","repo_kind":"listed","path":"common/io.py","file_url":"https://github.com/garyzhao/FRGAN/blob/HEAD/common/io.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f3b964c1506b9216"}},{"code_sha256_prefix":"921113cf2a16c965","entry":"plot_overlap","repo":"garyzhao/FRGAN","repo_kind":"listed","path":"common/logger.py","file_url":"https://github.com/garyzhao/FRGAN/blob/HEAD/common/logger.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"921113cf2a16c965"}},{"code_sha256_prefix":"1f60ac47d218051a","entry":"predict_vertices","repo":"garyzhao/FRGAN","repo_kind":"listed","path":"dfa/inference.py","file_url":"https://github.com/garyzhao/FRGAN/blob/HEAD/dfa/inference.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1f60ac47d218051a"}},{"code_sha256_prefix":"d8f125be3a977929","entry":"reconstruct_vertex","repo":"garyzhao/FRGAN","repo_kind":"listed","path":"dfa/inference.py","file_url":"https://github.com/garyzhao/FRGAN/blob/HEAD/dfa/inference.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d8f125be3a977929"}},{"code_sha256_prefix":"fd56da3a7da6c57e","entry":"torch_to_pil_image","repo":"garyzhao/FRGAN","repo_kind":"listed","path":"common/io.py","file_url":"https://github.com/garyzhao/FRGAN/blob/HEAD/common/io.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fd56da3a7da6c57e"}},{"code_sha256_prefix":"e8920dc9bd595394","entry":"visualize_norm","repo":"garyzhao/FRGAN","repo_kind":"listed","path":"common/utils.py","file_url":"https://github.com/garyzhao/FRGAN/blob/HEAD/common/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e8920dc9bd595394"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}