{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/very-deep-vaes-generalize-autoregressive-1","title":"Very Deep VAEs Generalize Autoregressive Models and Can Outperform Them on Images","arxiv_id":"2011.10650","date":"2020-11-20","proceeding":"ICLR 2021 1","authors":["Rewon Child"],"abstract":"We present a hierarchical VAE that, for the first time, generates samples quickly while outperforming the PixelCNN in log-likelihood on all natural image benchmarks. We begin by observing that, in theory, VAEs can actually represent autoregressive models, as well as faster, better models if they exist, when made sufficiently deep. Despite this, autoregressive models have historically outperformed VAEs in log-likelihood. We test if insufficient depth explains why by scaling a VAE to greater stochastic depth than previously explored and evaluating it CIFAR-10, ImageNet, and FFHQ. In comparison to the PixelCNN, these very deep VAEs achieve higher likelihoods, use fewer parameters, generate samples thousands of times faster, and are more easily applied to high-resolution images. Qualitative studies suggest this is because the VAE learns efficient hierarchical visual representations. We release our source code and models at https://github.com/openai/vdvae.","url_abs":"https://arxiv.org/abs/2011.10650v2","url_pdf":"https://arxiv.org/pdf/2011.10650v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"very-deep-vaes-generalize-autoregressive-1","repo_url":"https://github.com/openai/vdvae","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"very-deep-vaes-generalize-autoregressive-1","repo_url":"https://github.com/ericl122333/latent-vae","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"very-deep-vaes-generalize-autoregressive-1","repo_url":"https://github.com/j-towns/vdvae-jax","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"unanswered"}},{"paper_slug":"very-deep-vaes-generalize-autoregressive-1","repo_url":"https://github.com/lupalab/posterior-matching","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":null},{"paper_slug":"very-deep-vaes-generalize-autoregressive-1","repo_url":"https://github.com/plai-group/ipa","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"unanswered"}},{"paper_slug":"very-deep-vaes-generalize-autoregressive-1","repo_url":"https://github.com/tcl9876/visual-vae","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":{"status":"unanswered"}},{"paper_slug":"very-deep-vaes-generalize-autoregressive-1","repo_url":"https://github.com/vvvm23/vdvae","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"very-deep-vaes-generalize-autoregressive-1","repo_url":"https://github.com/EugenHotaj/pytorch-generative/blob/master/pytorch_generative/models/vae/vd_vae.py","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"image-generation","task_name":"Image Generation"}],"methods":[{"method_slug":"hierarchical-vae","method_name":"Hierarchical VAE"},{"method_slug":"stochastic-depth","method_name":"Stochastic Depth"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/image-generation-on-ffhq-1024-x-1024","task":"Image Generation","dataset":"FFHQ 1024 x 1024","model":"Very Deep VAE","rank_in_archive_order":20,"of":20,"metrics":{"bits/dimension":"2.42"},"uses_additional_data":false},{"leaderboard":"/sota/image-generation-on-ffhq-256-x-256","task":"Image Generation","dataset":"FFHQ 256 x 256","model":"Very Deep VAE","rank_in_archive_order":50,"of":51,"metrics":{"bits/dimension":"0.61"},"uses_additional_data":false},{"leaderboard":"/sota/image-generation-on-imagenet-32x32","task":"Image Generation","dataset":"ImageNet 32x32","model":"Very Deep VAE","rank_in_archive_order":24,"of":35,"metrics":{"bpd":"3.8"},"uses_additional_data":false},{"leaderboard":"/sota/image-generation-on-imagenet-64x64","task":"Image Generation","dataset":"ImageNet 64x64","model":"Very Deep VAE","rank_in_archive_order":48,"of":65,"metrics":{"Bits per dim":"3.52"},"uses_additional_data":false}],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2011.10650","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2011.10650"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lupalab/posterior-matching","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/openai/vdvae","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/tcl9876/visual-vae","reach":{"status":"unanswered"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ericl122333/latent-vae","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/j-towns/vdvae-jax","reach":{"status":"unanswered"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/plai-group/ipa","reach":{"status":"unanswered"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/EugenHotaj/pytorch-generative/blob/master/pytorch_generative/models/vae/vd_vae.py","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/vvvm23/vdvae","reach":null}],"summary":{"ran_draft_wrong":3,"ran_fixture":3,"ran_honours":2,"unverified":1},"by_repo_kind":{"official":{"samples":4,"ran":4,"repositories":1},"listed":{"samples":4,"ran":3,"repositories":3}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"a169b353d342ad7f","entry":"add_vae_arguments","repo":"openai/vdvae","repo_kind":"official","path":"hps.py","file_url":"https://github.com/openai/vdvae/blob/HEAD/hps.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a169b353d342ad7f"}},{"code_sha256_prefix":"31e670f6fff8614d","entry":"draw_gaussian_diag_samples","repo":"vvvm23/vdvae","repo_kind":"listed","path":"vae.py","file_url":"https://github.com/vvvm23/vdvae/blob/HEAD/vae.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"31e670f6fff8614d"}},{"code_sha256_prefix":"f8752a89ed52495c","entry":"gaussian_analytical_kl","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"f8752a89ed52495c"}},{"code_sha256_prefix":"2ec4359e4ba42b26","entry":"get_evenly_spaced_indices","repo":"ericl122333/latent-vae","repo_kind":"listed","path":"vae/model.py","file_url":"https://github.com/ericl122333/latent-vae/blob/HEAD/vae/model.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2ec4359e4ba42b26"}},{"code_sha256_prefix":"070ddf6b065864d3","entry":"get_resolutions","repo":"ericl122333/latent-vae","repo_kind":"listed","path":"vae/model.py","file_url":"https://github.com/ericl122333/latent-vae/blob/HEAD/vae/model.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"070ddf6b065864d3"}},{"code_sha256_prefix":"9419abf027f0115d","entry":"get_width_settings","repo":"openai/vdvae","repo_kind":"official","path":"vae.py","file_url":"https://github.com/openai/vdvae/blob/HEAD/vae.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9419abf027f0115d"}},{"code_sha256_prefix":"dd095f22dc8304f0","entry":"pad_channels","repo":"openai/vdvae","repo_kind":"official","path":"vae.py","file_url":"https://github.com/openai/vdvae/blob/HEAD/vae.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dd095f22dc8304f0"}},{"code_sha256_prefix":"95d3df9fe1921426","entry":"parse_layer_string","repo":"openai/vdvae","repo_kind":"official","path":"vae.py","file_url":"https://github.com/openai/vdvae/blob/HEAD/vae.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"95d3df9fe1921426"}},{"code_sha256_prefix":"6e32683806d3201d","entry":"reproduce","repo":"EugenHotaj/pytorch-generative","repo_kind":"listed","path":"pytorch_generative/models/vae/vd_vae.py","file_url":"https://github.com/EugenHotaj/pytorch-generative/blob/HEAD/pytorch_generative/models/vae/vd_vae.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6e32683806d3201d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}