{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/variational-mixture-of-experts-autoencoders","title":"Variational Mixture-of-Experts Autoencoders for Multi-Modal Deep Generative Models","arxiv_id":"1911.03393","date":"2019-11-08","proceeding":"NeurIPS 2019 12","authors":["Yuge Shi","N. Siddharth","Brooks Paige","Philip H. S. Torr"],"abstract":"Learning generative models that span multiple data modalities, such as vision and language, is often motivated by the desire to learn more useful, generalisable representations that faithfully capture common underlying factors between the modalities. In this work, we characterise successful learning of such models as the fulfillment of four criteria: i) implicit latent decomposition into shared and private subspaces, ii) coherent joint generation over all modalities, iii) coherent cross-generation across individual modalities, and iv) improved model learning for individual modalities through multi-modal integration. Here, we propose a mixture-of-experts multimodal variational autoencoder (MMVAE) to learn generative models on different sets of modalities, including a challenging image-language dataset, and demonstrate its ability to satisfy all four criteria, both qualitatively and quantitatively.","url_abs":"https://arxiv.org/abs/1911.03393v1","url_pdf":"https://arxiv.org/pdf/1911.03393v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"variational-mixture-of-experts-autoencoders","repo_url":"https://github.com/iffsid/mmvae","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"variational-mixture-of-experts-autoencoders","repo_url":"https://github.com/gabinsane/multi-vaes-in-robotics","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"CC0-1.0"}},{"paper_slug":"variational-mixture-of-experts-autoencoders","repo_url":"https://github.com/gabinsane/multimodal-vae-comparison","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"mixture-of-experts","task_name":"Mixture-of-Experts"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/1911.03393","atlas_url":"https://app.syntology.ai/?focus=1911.03393","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1911.03393"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/gabinsane/multi-vaes-in-robotics","reach":{"status":"ok","spdx":"CC0-1.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/gabinsane/multimodal-vae-comparison","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/iffsid/mmvae","reach":null}],"summary":{"ran":6,"ran_honours":1,"unverified":1},"by_repo_kind":{"listed":{"samples":7,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"d880f1304c23dd2f","entry":"calculate_angle_2d","repo":"gabinsane/multi-vaes-in-robotics","repo_kind":"listed","path":"models/lanro_datasets.py","file_url":"https://github.com/gabinsane/multi-vaes-in-robotics/blob/HEAD/models/lanro_datasets.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"CC0-1.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d880f1304c23dd2f"}},{"code_sha256_prefix":"5cb57a536ad8fefc","entry":"expand_layer","repo":"gabinsane/multi-vaes-in-robotics","repo_kind":"listed","path":"models/nn_modules.py","file_url":"https://github.com/gabinsane/multi-vaes-in-robotics/blob/HEAD/models/nn_modules.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"CC0-1.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5cb57a536ad8fefc"}},{"code_sha256_prefix":"6871b0c6b992f16a","entry":"extra_hidden_layer","repo":"gabinsane/multi-vaes-in-robotics","repo_kind":"listed","path":"models/decoders.py","file_url":"https://github.com/gabinsane/multi-vaes-in-robotics/blob/HEAD/models/decoders.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"CC0-1.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6871b0c6b992f16a"}},{"code_sha256_prefix":"6a65d95c230d85e5","entry":"extra_hidden_layer","repo":"gabinsane/multi-vaes-in-robotics","repo_kind":"listed","path":"models/encoders.py","file_url":"https://github.com/gabinsane/multi-vaes-in-robotics/blob/HEAD/models/encoders.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"CC0-1.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6a65d95c230d85e5"}},{"code_sha256_prefix":"2e896473f863acba","entry":"make_layers_resnet_encoder_feature_compressor","repo":"gabinsane/multi-vaes-in-robotics","repo_kind":"listed","path":"models/nn_modules.py","file_url":"https://github.com/gabinsane/multi-vaes-in-robotics/blob/HEAD/models/nn_modules.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"CC0-1.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2e896473f863acba"}},{"code_sha256_prefix":"57f9e0450d5d2489","entry":"make_res_block_encoder_feature_compressor","repo":"gabinsane/multi-vaes-in-robotics","repo_kind":"listed","path":"models/nn_modules.py","file_url":"https://github.com/gabinsane/multi-vaes-in-robotics/blob/HEAD/models/nn_modules.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"CC0-1.0","inline_ok":true,"mcp_get_code":{"code_sha256":"57f9e0450d5d2489"}},{"code_sha256_prefix":"49f906d6c8bb6264","entry":"unpack_data_mlp","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"49f906d6c8bb6264"}},{"code_sha256_prefix":"8e437cf31d760b56","entry":"object_pos","repo":"gabinsane/multi-vaes-in-robotics","repo_kind":"listed","path":"models/lanro_datasets.py","file_url":"https://github.com/gabinsane/multi-vaes-in-robotics/blob/HEAD/models/lanro_datasets.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"CC0-1.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8e437cf31d760b56"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}