{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/large-scale-learning-of-general-visual","title":"Big Transfer (BiT): General Visual Representation Learning","arxiv_id":"1912.11370","date":"2019-12-24","proceeding":"ECCV 2020 8","authors":["Alexander Kolesnikov","Lucas Beyer","Xiaohua Zhai","Joan Puigcerver","Jessica Yung","Sylvain Gelly","Neil Houlsby"],"abstract":"Transfer of pre-trained representations improves sample efficiency and simplifies hyperparameter tuning when training deep neural networks for vision. We revisit the paradigm of pre-training on large supervised datasets and fine-tuning the model on a target task. We scale up pre-training, and propose a simple recipe that we call Big Transfer (BiT). By combining a few carefully selected components, and transferring using a simple heuristic, we achieve strong performance on over 20 datasets. BiT performs well across a surprisingly wide range of data regimes -- from 1 example per class to 1M total examples. BiT achieves 87.5% top-1 accuracy on ILSVRC-2012, 99.4% on CIFAR-10, and 76.3% on the 19 task Visual Task Adaptation Benchmark (VTAB). On small datasets, BiT attains 76.8% on ILSVRC-2012 with 10 examples per class, and 97.0% on CIFAR-10 with 10 examples per class. We conduct detailed analysis of the main components that lead to high transfer performance.","url_abs":"https://arxiv.org/abs/1912.11370v3","url_pdf":"https://arxiv.org/pdf/1912.11370v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"large-scale-learning-of-general-visual","repo_url":"https://github.com/google-research/big_transfer","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"large-scale-learning-of-general-visual","repo_url":"https://github.com/SoojungYang/supervised_pretraining_GN_WS","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"large-scale-learning-of-general-visual","repo_url":"https://github.com/batsresearch/taglets","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"large-scale-learning-of-general-visual","repo_url":"https://github.com/bethgelab/InDomainGeneralizationBenchmark","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"large-scale-learning-of-general-visual","repo_url":"https://github.com/sayakpaul/A-Barebones-Image-Retrieval-System","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"large-scale-learning-of-general-visual","repo_url":"https://github.com/sayakpaul/FunMatch-Distillation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"large-scale-learning-of-general-visual","repo_url":"https://github.com/2024-MindSpore-1/Code2/tree/main/model-1/bit","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null},{"paper_slug":"large-scale-learning-of-general-visual","repo_url":"https://github.com/MS-Mind/MS-Code-02/tree/main/configs/bit","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"large-scale-learning-of-general-visual","repo_url":"https://github.com/hw666666666666/BigTransfer","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[{"task_slug":"few-shot-learning","task_name":"Few-Shot Learning"},{"task_slug":"fine-grained-image-classification","task_name":"Fine-Grained Image Classification"},{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"out-of-distribution-generalization","task_name":"Out-of-Distribution Generalization"},{"task_slug":"representation-learning","task_name":"Representation Learning"}],"methods":[{"method_slug":"1x1-convolution","method_name":"1x1 Convolution"},{"method_slug":"average-pooling","method_name":"Average Pooling"},{"method_slug":"bottleneck-residual-block","method_name":"Bottleneck Residual Block"},{"method_slug":"convolution","method_name":"Convolution"},{"method_slug":"fixres","method_name":"FixRes"},{"method_slug":"global-average-pooling","method_name":"Global Average Pooling"},{"method_slug":"group-normalization","method_name":"Group Normalization"},{"method_slug":"kaiming-initialization","method_name":"Kaiming Initialization"},{"method_slug":"max-pooling","method_name":"Max Pooling"},{"method_slug":"mixup","method_name":"Mixup"},{"method_slug":"randomhorizontalflip","method_name":"Random Horizontal Flip"},{"method_slug":"random-resized-crop","method_name":"Random Resized Crop"},{"method_slug":"relu","method_name":"ReLU"},{"method_slug":"residual-block","method_name":"Residual Block"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"sgd-with-momentum","method_name":"SGD with Momentum"},{"method_slug":"weight-standardization","method_name":"Weight Standardization"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/fine-grained-image-classification-on-oxford","task":"Fine-Grained Image Classification","dataset":"Oxford 102 Flowers","model":"BiT-L (ResNet)","rank_in_archive_order":2,"of":25,"metrics":{"Accuracy":"99.63%","Top-1 Error Rate":"0.37"},"uses_additional_data":true},{"leaderboard":"/sota/fine-grained-image-classification-on-oxford","task":"Fine-Grained Image Classification","dataset":"Oxford 102 Flowers","model":"BiT-M (ResNet)","rank_in_archive_order":4,"of":25,"metrics":{"Accuracy":"99.30%","Top-1 Error Rate":"0.70"},"uses_additional_data":true},{"leaderboard":"/sota/fine-grained-image-classification-on-oxford-2","task":"Fine-Grained Image Classification","dataset":"Oxford-IIIT Pets","model":"BiT-L (ResNet)","rank_in_archive_order":2,"of":19,"metrics":{"Accuracy":"96.62","Top-1 Error Rate":"3.38%"},"uses_additional_data":false},{"leaderboard":"/sota/fine-grained-image-classification-on-oxford-2","task":"Fine-Grained Image Classification","dataset":"Oxford-IIIT Pets","model":"BiT-M (ResNet)","rank_in_archive_order":5,"of":19,"metrics":{"Accuracy":"94.47","Top-1 Error Rate":"5.53%"},"uses_additional_data":false},{"leaderboard":"/sota/image-classification-on-cifar-10","task":"Image Classification","dataset":"CIFAR-10","model":"BiT-L (ResNet)","rank_in_archive_order":7,"of":265,"metrics":{"Percentage correct":"99.37"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-cifar-10","task":"Image Classification","dataset":"CIFAR-10","model":"BiT-M (ResNet)","rank_in_archive_order":26,"of":265,"metrics":{"Percentage correct":"98.91"},"uses_additional_data":false},{"leaderboard":"/sota/image-classification-on-cifar-100","task":"Image Classification","dataset":"CIFAR-100","model":"BiT-L (ResNet)","rank_in_archive_order":8,"of":211,"metrics":{"Percentage correct":"93.51"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-cifar-100","task":"Image Classification","dataset":"CIFAR-100","model":"BiT-M (ResNet)","rank_in_archive_order":15,"of":211,"metrics":{"Percentage correct":"92.17"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-flowers-102","task":"Image Classification","dataset":"Flowers-102","model":"BiT-L (ResNet)","rank_in_archive_order":8,"of":52,"metrics":{"Accuracy":"99.63"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-flowers-102","task":"Image Classification","dataset":"Flowers-102","model":"BiT-M (ResNet)","rank_in_archive_order":12,"of":52,"metrics":{"Accuracy":"99.30"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-imagenet","task":"Image Classification","dataset":"ImageNet","model":"BiT-L (ResNet)","rank_in_archive_order":80,"of":1060,"metrics":{"Top 1 Accuracy":"87.54%","Top 5 Accuracy":"98.46"},"uses_additional_data":false},{"leaderboard":"/sota/image-classification-on-imagenet","task":"Image Classification","dataset":"ImageNet","model":"BiT-M (ResNet)","rank_in_archive_order":237,"of":1060,"metrics":{"Number of params":"928M","Top 1 Accuracy":"85.39%"},"uses_additional_data":false},{"leaderboard":"/sota/image-classification-on-imagenet-real","task":"Image Classification","dataset":"ImageNet ReaL","model":"BiT-L","rank_in_archive_order":17,"of":57,"metrics":{"Accuracy":"90.54%","Params":"928M"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-imagenet-real","task":"Image Classification","dataset":"ImageNet ReaL","model":"BiT-M","rank_in_archive_order":25,"of":57,"metrics":{"Accuracy":"89.02%"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-objectnet","task":"Image Classification","dataset":"ObjectNet","model":"BiT-L (ResNet-152x4)","rank_in_archive_order":21,"of":106,"metrics":{"Top-1 Accuracy":"58.7","Top-5 Accuracy":"80"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-objectnet","task":"Image Classification","dataset":"ObjectNet","model":"BiT-M (ResNet-152x4)","rank_in_archive_order":33,"of":106,"metrics":{"Top-1 Accuracy":"47.0","Top-5 Accuracy":"69"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-objectnet","task":"Image Classification","dataset":"ObjectNet","model":"BiT-S (ResNet-152x4)","rank_in_archive_order":50,"of":106,"metrics":{"Top-1 Accuracy":"36.0","Top-5 Accuracy":"57"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-objectnet-bounding","task":"Image Classification","dataset":"ObjectNet (Bounding Box)","model":"BiT-L (ResNet)","rank_in_archive_order":1,"of":4,"metrics":{"Top 5 Accuracy":"85.1"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-objectnet-bounding","task":"Image Classification","dataset":"ObjectNet (Bounding Box)","model":"BiT-M (ResNet)","rank_in_archive_order":2,"of":4,"metrics":{"Top 5 Accuracy":"76.0"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-objectnet-bounding","task":"Image Classification","dataset":"ObjectNet (Bounding Box)","model":"BiT-S (ResNet)","rank_in_archive_order":3,"of":4,"metrics":{"Top 5 Accuracy":"64.4"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-omnibenchmark","task":"Image Classification","dataset":"OmniBenchmark","model":"BiT-M","rank_in_archive_order":7,"of":22,"metrics":{"Average Top-1 Accuracy":"40.4"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-vtab-1k-1","task":"Image Classification","dataset":"VTAB-1k","model":"BiT-L (50 hypers/task)","rank_in_archive_order":2,"of":34,"metrics":{"Top-1 Accuracy":"78.72"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-vtab-1k-1","task":"Image Classification","dataset":"VTAB-1k","model":"BiT-L","rank_in_archive_order":6,"of":34,"metrics":{"Top-1 Accuracy":"76.3"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-vtab-1k-1","task":"Image Classification","dataset":"VTAB-1k","model":"BiT-M","rank_in_archive_order":11,"of":34,"metrics":{"Top-1 Accuracy":"70.6"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-vtab-1k-1","task":"Image Classification","dataset":"VTAB-1k","model":"BiT-S","rank_in_archive_order":15,"of":34,"metrics":{"Top-1 Accuracy":"66.9"},"uses_additional_data":true}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1912.11370","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1912.11370"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sayakpaul/FunMatch-Distillation","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/batsresearch/taglets","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/MS-Mind/MS-Code-02/tree/main/configs/bit","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/2024-MindSpore-1/Code2/tree/main/model-1/bit","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/SoojungYang/supervised_pretraining_GN_WS","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/bethgelab/InDomainGeneralizationBenchmark","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/google-research/big_transfer","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/hw666666666666/BigTransfer","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sayakpaul/A-Barebones-Image-Retrieval-System","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":2,"ran_violates":1,"unverified":7},"by_repo_kind":{"listed":{"samples":10,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"ee80cf467065076b","entry":"conv1x1","repo":"batsresearch/taglets","repo_kind":"listed","path":"taglets/models/resnetv2.py","file_url":"https://github.com/batsresearch/taglets/blob/HEAD/taglets/models/resnetv2.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ee80cf467065076b"}},{"code_sha256_prefix":"f01b8d3901f0b289","entry":"conv3x3","repo":"batsresearch/taglets","repo_kind":"listed","path":"taglets/models/resnetv2.py","file_url":"https://github.com/batsresearch/taglets/blob/HEAD/taglets/models/resnetv2.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f01b8d3901f0b289"}},{"code_sha256_prefix":"7023eb86801bb3a3","entry":"tf2th","repo":"batsresearch/taglets","repo_kind":"listed","path":"taglets/models/resnetv2.py","file_url":"https://github.com/batsresearch/taglets/blob/HEAD/taglets/models/resnetv2.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7023eb86801bb3a3"}},{"code_sha256_prefix":"00301e057c30f413","entry":"Brier_Score_AMCL","repo":"batsresearch/taglets","repo_kind":"listed","path":"taglets/labelmodel/amcl_helper.py","file_url":"https://github.com/batsresearch/taglets/blob/HEAD/taglets/labelmodel/amcl_helper.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"00301e057c30f413"}},{"code_sha256_prefix":"081ac982970319a1","entry":"Brier_loss_linear","repo":"batsresearch/taglets","repo_kind":"listed","path":"taglets/labelmodel/amcl_helper.py","file_url":"https://github.com/batsresearch/taglets/blob/HEAD/taglets/labelmodel/amcl_helper.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"081ac982970319a1"}},{"code_sha256_prefix":"174d041dba5d15c6","entry":"get_data","repo":"batsresearch/taglets","repo_kind":"listed","path":"taglets/labelmodel/amcl.py","file_url":"https://github.com/batsresearch/taglets/blob/HEAD/taglets/labelmodel/amcl.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"174d041dba5d15c6"}},{"code_sha256_prefix":"25fc1400e3579d6f","entry":"get_data","repo":"batsresearch/taglets","repo_kind":"listed","path":"taglets/labelmodel/run_labelmodel.py","file_url":"https://github.com/batsresearch/taglets/blob/HEAD/taglets/labelmodel/run_labelmodel.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"25fc1400e3579d6f"}},{"code_sha256_prefix":"fbc44ea7cbcc39ee","entry":"get_test_data","repo":"batsresearch/taglets","repo_kind":"listed","path":"taglets/labelmodel/amcl.py","file_url":"https://github.com/batsresearch/taglets/blob/HEAD/taglets/labelmodel/amcl.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fbc44ea7cbcc39ee"}},{"code_sha256_prefix":"9f7ae362c69b5fbf","entry":"get_test_data","repo":"batsresearch/taglets","repo_kind":"listed","path":"taglets/labelmodel/run_labelmodel.py","file_url":"https://github.com/batsresearch/taglets/blob/HEAD/taglets/labelmodel/run_labelmodel.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9f7ae362c69b5fbf"}},{"code_sha256_prefix":"4b28a291d70a1b4b","entry":"mse","repo":"batsresearch/taglets","repo_kind":"listed","path":"taglets/labelmodel/amcl_helper.py","file_url":"https://github.com/batsresearch/taglets/blob/HEAD/taglets/labelmodel/amcl_helper.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4b28a291d70a1b4b"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}