{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/co-training-improves-prompt-based-learning","title":"Co-training Improves Prompt-based Learning for Large Language Models","arxiv_id":"2202.00828","date":"2022-02-02","proceeding":null,"authors":["Hunter Lang","Monica Agrawal","Yoon Kim","David Sontag"],"abstract":"We demonstrate that co-training (Blum & Mitchell, 1998) can improve the performance of prompt-based learning by using unlabeled data. While prompting has emerged as a promising paradigm for few-shot and zero-shot learning, it is often brittle and requires much larger models compared to the standard supervised setup. We find that co-training makes it possible to improve the original prompt model and at the same time learn a smaller, downstream task-specific model. In the case where we only have partial access to a prompt model (e.g., output probabilities from GPT-3 (Brown et al., 2020)) we learn a calibration model over the prompt outputs. When we have full access to the prompt model's gradients but full finetuning remains prohibitively expensive (e.g., T0 (Sanh et al., 2021)), we learn a set of soft prompt continuous vectors to iteratively update the prompt model. We find that models trained in this manner can significantly improve performance on challenging datasets where there is currently a large gap between prompt-based learning and fully-supervised models.","url_abs":"https://arxiv.org/abs/2202.00828v1","url_pdf":"https://arxiv.org/pdf/2202.00828v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"co-training-improves-prompt-based-learning","repo_url":"https://github.com/clinicalml/cotrain-prompting","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"zero-shot-learning","task_name":"Zero-Shot Learning"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"cosine-annealing","method_name":"Cosine Annealing"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt-3","method_name":"GPT-3"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-cosine-annealing","method_name":"Linear Warmup With Cosine Annealing"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2202.00828","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2202.00828"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/clinicalml/cotrain-prompting","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":3,"unverified":7},"by_repo_kind":{"official":{"samples":10,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"86cd0bb77fbf123f","entry":"fast_walsh_hadamard_torched","repo":"clinicalml/cotrain-prompting","repo_kind":"official","path":"src/models/intrinsic.py","file_url":"https://github.com/clinicalml/cotrain-prompting/blob/HEAD/src/models/intrinsic.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"86cd0bb77fbf123f"}},{"code_sha256_prefix":"93fee8766c612b43","entry":"modify_with_bitfit","repo":"clinicalml/cotrain-prompting","repo_kind":"official","path":"src/models/bitfit.py","file_url":"https://github.com/clinicalml/cotrain-prompting/blob/HEAD/src/models/bitfit.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"93fee8766c612b43"}},{"code_sha256_prefix":"ceef2f830a0ed0cd","entry":"modify_with_lora","repo":"clinicalml/cotrain-prompting","repo_kind":"official","path":"src/models/lora.py","file_url":"https://github.com/clinicalml/cotrain-prompting/blob/HEAD/src/models/lora.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ceef2f830a0ed0cd"}},{"code_sha256_prefix":"1e24166d952ce221","entry":"fastfood_vars","repo":"clinicalml/cotrain-prompting","repo_kind":"official","path":"src/models/intrinsic.py","file_url":"https://github.com/clinicalml/cotrain-prompting/blob/HEAD/src/models/intrinsic.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1e24166d952ce221"}},{"code_sha256_prefix":"8437c16cba343b4d","entry":"get_adapter","repo":"clinicalml/cotrain-prompting","repo_kind":"official","path":"src/models/adapters.py","file_url":"https://github.com/clinicalml/cotrain-prompting/blob/HEAD/src/models/adapters.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8437c16cba343b4d"}},{"code_sha256_prefix":"837d5f20bc5563c5","entry":"get_conf_inds","repo":"clinicalml/cotrain-prompting","repo_kind":"official","path":"src/utils/cotrain_utils.py","file_url":"https://github.com/clinicalml/cotrain-prompting/blob/HEAD/src/utils/cotrain_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"837d5f20bc5563c5"}},{"code_sha256_prefix":"94f3895aa6257dfb","entry":"get_conf_inds_minppc","repo":"clinicalml/cotrain-prompting","repo_kind":"official","path":"src/utils/cotrain_utils.py","file_url":"https://github.com/clinicalml/cotrain-prompting/blob/HEAD/src/utils/cotrain_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"94f3895aa6257dfb"}},{"code_sha256_prefix":"73fb194c40a4254a","entry":"get_conf_inds_per_class","repo":"clinicalml/cotrain-prompting","repo_kind":"official","path":"src/utils/cotrain_utils.py","file_url":"https://github.com/clinicalml/cotrain-prompting/blob/HEAD/src/utils/cotrain_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"73fb194c40a4254a"}},{"code_sha256_prefix":"e76a3ca72fd3d2d5","entry":"modify_with_adapters","repo":"clinicalml/cotrain-prompting","repo_kind":"official","path":"src/models/adapters.py","file_url":"https://github.com/clinicalml/cotrain-prompting/blob/HEAD/src/models/adapters.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e76a3ca72fd3d2d5"}},{"code_sha256_prefix":"ff477f580a9ac93f","entry":"random_vars","repo":"clinicalml/cotrain-prompting","repo_kind":"official","path":"src/models/intrinsic.py","file_url":"https://github.com/clinicalml/cotrain-prompting/blob/HEAD/src/models/intrinsic.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ff477f580a9ac93f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}