{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/meta-dataset-a-dataset-of-datasets-for","title":"Meta-Dataset: A Dataset of Datasets for Learning to Learn from Few Examples","arxiv_id":"1903.03096","date":"2019-03-07","proceeding":"ICLR 2020 1","authors":["Eleni Triantafillou","Tyler Zhu","Vincent Dumoulin","Pascal Lamblin","Utku Evci","Kelvin Xu","Ross Goroshin","Carles Gelada","Kevin Swersky","Pierre-Antoine Manzagol","Hugo Larochelle"],"abstract":"Few-shot classification refers to learning a classifier for new classes given only a few examples. While a plethora of models have emerged to tackle it, we find the procedure and datasets that are used to assess their progress lacking. To address this limitation, we propose Meta-Dataset: a new benchmark for training and evaluating models that is large-scale, consists of diverse datasets, and presents more realistic tasks. We experiment with popular baselines and meta-learners on Meta-Dataset, along with a competitive method that we propose. We analyze performance as a function of various characteristics of test tasks and examine the models' ability to leverage diverse training sources for improving their generalization. We also propose a new set of baselines for quantifying the benefit of meta-learning in Meta-Dataset. Our extensive experimentation has uncovered important research challenges and we hope to inspire work in these directions.","url_abs":"https://arxiv.org/abs/1903.03096v4","url_pdf":"https://arxiv.org/pdf/1903.03096v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"meta-dataset-a-dataset-of-datasets-for","repo_url":"https://github.com/google-research/meta-dataset","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"tf","reach":null},{"paper_slug":"meta-dataset-a-dataset-of-datasets-for","repo_url":"https://github.com/cambridge-mlg/cnaps","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"meta-dataset-a-dataset-of-datasets-for","repo_url":"https://github.com/ebadrian/metadl","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"meta-dataset-a-dataset-of-datasets-for","repo_url":"https://github.com/gebob19/REPTILE-Metalearning","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"meta-dataset-a-dataset-of-datasets-for","repo_url":"https://github.com/gebob19/cscd94-metalearning","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"meta-dataset-a-dataset-of-datasets-for","repo_url":"https://github.com/gebob19/cscd94_metalearning","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"meta-dataset-a-dataset-of-datasets-for","repo_url":"https://github.com/gokyeongryeol/MAHA","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"meta-dataset-a-dataset-of-datasets-for","repo_url":"https://github.com/gomerudo/nas-dmrl","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"meta-dataset-a-dataset-of-datasets-for","repo_url":"https://github.com/jimzai/deta","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"meta-dataset-a-dataset-of-datasets-for","repo_url":"https://github.com/nobody-1617/deta","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"meta-dataset-a-dataset-of-datasets-for","repo_url":"https://github.com/tmlr-group/CoPA","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":null},{"paper_slug":"meta-dataset-a-dataset-of-datasets-for","repo_url":"https://github.com/tmlr-group/mokd","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"meta-dataset-a-dataset-of-datasets-for","repo_url":"https://github.com/mboudiaf/pytorch-meta-dataset","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"meta-dataset-a-dataset-of-datasets-for","repo_url":"https://github.com/peymanbateni/simple-cnaps","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"meta-dataset-a-dataset-of-datasets-for","repo_url":"https://github.com/plai-group/simple-cnaps","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"few-shot-image-classification","task_name":"Few-Shot Image Classification"},{"task_slug":"classification","task_name":"General Classification"},{"task_slug":"meta-learning","task_name":"Meta-Learning"}],"methods":[],"datasets_introduced":[{"slug":"meta-dataset","name":"Meta-Dataset","full_name":""}],"methods_introduced":[],"results":[{"leaderboard":"/sota/few-shot-image-classification-on-meta-dataset","task":"Few-Shot Image Classification","dataset":"Meta-Dataset","model":"fo-Proto-MAML","rank_in_archive_order":16,"of":22,"metrics":{"Accuracy":"63.428"},"uses_additional_data":false},{"leaderboard":"/sota/few-shot-image-classification-on-meta-dataset","task":"Few-Shot Image Classification","dataset":"Meta-Dataset","model":"Finetune","rank_in_archive_order":18,"of":22,"metrics":{"Accuracy":"58.758"},"uses_additional_data":false},{"leaderboard":"/sota/few-shot-image-classification-on-meta-dataset","task":"Few-Shot Image Classification","dataset":"Meta-Dataset","model":"k-NN","rank_in_archive_order":21,"of":22,"metrics":{"Accuracy":"54.319"},"uses_additional_data":false},{"leaderboard":"/sota/few-shot-image-classification-on-meta-dataset-1","task":"Few-Shot Image Classification","dataset":"Meta-Dataset Rank","model":"fo-Proto-MAML","rank_in_archive_order":7,"of":13,"metrics":{"Mean Rank":"6.65"},"uses_additional_data":false},{"leaderboard":"/sota/few-shot-image-classification-on-meta-dataset-1","task":"Few-Shot Image Classification","dataset":"Meta-Dataset Rank","model":"Finetune","rank_in_archive_order":9,"of":13,"metrics":{"Mean Rank":"8.7"},"uses_additional_data":false},{"leaderboard":"/sota/few-shot-image-classification-on-meta-dataset-1","task":"Few-Shot Image Classification","dataset":"Meta-Dataset Rank","model":"k-NN","rank_in_archive_order":12,"of":13,"metrics":{"Mean Rank":"10.85"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/1903.03096","atlas_url":"https://app.syntology.ai/?focus=1903.03096","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1903.03096"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/google-research/meta-dataset","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/tmlr-group/mokd","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/gebob19/REPTILE-Metalearning","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mboudiaf/pytorch-meta-dataset","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/peymanbateni/simple-cnaps","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/plai-group/simple-cnaps","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/cambridge-mlg/cnaps","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/gebob19/cscd94_metalearning","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ebadrian/metadl","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jimzai/deta","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/gomerudo/nas-dmrl","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/nobody-1617/deta","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/gokyeongryeol/MAHA","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/gebob19/cscd94-metalearning","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/tmlr-group/CoPA","reach":null}],"summary":{"ran_fixture":1,"ran_honours":1,"ran":6,"ran_draft_wrong":2,"unverified":1},"by_repo_kind":{"official":{"samples":3,"ran":3,"repositories":1},"listed":{"samples":8,"ran":7,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"6b4ce423f84ce9bc","entry":"compute_class_precision","repo":"google-research/meta-dataset","repo_kind":"official","path":"meta_dataset/analyze.py","file_url":"https://github.com/google-research/meta-dataset/blob/HEAD/meta_dataset/analyze.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6b4ce423f84ce9bc"}},{"code_sha256_prefix":"6a9728ff66fde324","entry":"compute_episode_accuracy","repo":"google-research/meta-dataset","repo_kind":"official","path":"meta_dataset/analyze.py","file_url":"https://github.com/google-research/meta-dataset/blob/HEAD/meta_dataset/analyze.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6a9728ff66fde324"}},{"code_sha256_prefix":"ab96a11f341b7378","entry":"conv_net","repo":"ebadrian/metadl","repo_kind":"listed","path":"baselines/maml/helper.py","file_url":"https://github.com/ebadrian/metadl/blob/HEAD/baselines/maml/helper.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ab96a11f341b7378"}},{"code_sha256_prefix":"568b7bd83c95b525","entry":"extract_elapsed_time","repo":"ebadrian/metadl","repo_kind":"listed","path":"metadl/core/scoring/scoring.py","file_url":"https://github.com/ebadrian/metadl/blob/HEAD/metadl/core/scoring/scoring.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"568b7bd83c95b525"}},{"code_sha256_prefix":"e8e595468bf6e7e6","entry":"get_logger","repo":"ebadrian/metadl","repo_kind":"listed","path":"metadl/core/parent_scoring/parent_scoring.py","file_url":"https://github.com/ebadrian/metadl/blob/HEAD/metadl/core/parent_scoring/parent_scoring.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e8e595468bf6e7e6"}},{"code_sha256_prefix":"b2efd9150822b76f","entry":"get_shot_to_precision","repo":"google-research/meta-dataset","repo_kind":"official","path":"meta_dataset/analyze.py","file_url":"https://github.com/google-research/meta-dataset/blob/HEAD/meta_dataset/analyze.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b2efd9150822b76f"}},{"code_sha256_prefix":"5c8ffb087acc6095","entry":"is_one_hot_vector","repo":"ebadrian/metadl","repo_kind":"listed","path":"metadl/core/scoring/scoring.py","file_url":"https://github.com/ebadrian/metadl/blob/HEAD/metadl/core/scoring/scoring.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5c8ffb087acc6095"}},{"code_sha256_prefix":"9ff47b55c732dbfd","entry":"moving_average","repo":"nobody-1617/deta","repo_kind":"listed","path":"meta_dataset/analysis/select_best_model.py","file_url":"https://github.com/nobody-1617/deta/blob/HEAD/meta_dataset/analysis/select_best_model.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9ff47b55c732dbfd"}},{"code_sha256_prefix":"eb381c3ecf39155c","entry":"read_curve","repo":"ebadrian/metadl","repo_kind":"listed","path":"metadl/core/parent_scoring/parent_scoring.py","file_url":"https://github.com/ebadrian/metadl/blob/HEAD/metadl/core/parent_scoring/parent_scoring.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"eb381c3ecf39155c"}},{"code_sha256_prefix":"2f30e207b048cceb","entry":"read_score","repo":"ebadrian/metadl","repo_kind":"listed","path":"metadl/core/parent_scoring/parent_scoring.py","file_url":"https://github.com/ebadrian/metadl/blob/HEAD/metadl/core/parent_scoring/parent_scoring.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2f30e207b048cceb"}},{"code_sha256_prefix":"cd5ee3f06fa23554","entry":"NwayKshot_accuracy","repo":"ebadrian/metadl","repo_kind":"listed","path":"metadl/core/scoring/scoring.py","file_url":"https://github.com/ebadrian/metadl/blob/HEAD/metadl/core/scoring/scoring.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cd5ee3f06fa23554"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}