{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/mind-your-format-towards-consistent","title":"Mind Your Format: Towards Consistent Evaluation of In-Context Learning Improvements","arxiv_id":"2401.06766","date":"2024-01-12","proceeding":null,"authors":["Anton Voronov","Lena Wolf","Max Ryabinin"],"abstract":"Large language models demonstrate a remarkable capability for learning to solve new tasks from a few examples. The prompt template, or the way the input examples are formatted to obtain the prompt, is an important yet often overlooked aspect of in-context learning. In this work, we conduct a comprehensive study of the template format's influence on the in-context learning performance. We evaluate the impact of the prompt template across 21 models (from 770M to 70B parameters) and 4 standard classification datasets. We show that a poor choice of the template can reduce the performance of the strongest models and inference methods to a random guess level. More importantly, the best templates do not transfer between different setups and even between models of the same family. Our findings show that the currently prevalent approach to evaluation, which ignores template selection, may give misleading results due to different templates in different works. As a first step towards mitigating this issue, we propose Template Ensembles that aggregate model predictions across several templates. This simple test-time augmentation boosts average performance while being robust to the choice of random set of templates.","url_abs":"https://arxiv.org/abs/2401.06766v3","url_pdf":"https://arxiv.org/pdf/2401.06766v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"mind-your-format-towards-consistent","repo_url":"https://github.com/yandex-research/mind-your-format","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"in-context-learning","task_name":"In-Context Learning"}],"methods":[{"method_slug":"set","method_name":"SET"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2401.06766","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.06766"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yandex-research/mind-your-format","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":6,"unverified":4},"by_repo_kind":{"official":{"samples":10,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"ac21fd94625fab06","entry":"find_current_run","repo":"yandex-research/mind-your-format","repo_kind":"official","path":"utils.py","file_url":"https://github.com/yandex-research/mind-your-format/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ac21fd94625fab06"}},{"code_sha256_prefix":"39c1250cde8fa4e6","entry":"get_examples","repo":"yandex-research/mind-your-format","repo_kind":"official","path":"examples.py","file_url":"https://github.com/yandex-research/mind-your-format/blob/HEAD/examples.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"39c1250cde8fa4e6"}},{"code_sha256_prefix":"8cde6f9b2356a263","entry":"get_loss","repo":"yandex-research/mind-your-format","repo_kind":"official","path":"evaluate.py","file_url":"https://github.com/yandex-research/mind-your-format/blob/HEAD/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8cde6f9b2356a263"}},{"code_sha256_prefix":"88bc6dc454b35322","entry":"get_results_pd","repo":"yandex-research/mind-your-format","repo_kind":"official","path":"utils.py","file_url":"https://github.com/yandex-research/mind-your-format/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"88bc6dc454b35322"}},{"code_sha256_prefix":"4784a7ec29b42cc7","entry":"get_results_torch","repo":"yandex-research/mind-your-format","repo_kind":"official","path":"utils.py","file_url":"https://github.com/yandex-research/mind-your-format/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4784a7ec29b42cc7"}},{"code_sha256_prefix":"6047b9a8b85a7ea6","entry":"load_examples","repo":"yandex-research/mind-your-format","repo_kind":"official","path":"examples.py","file_url":"https://github.com/yandex-research/mind-your-format/blob/HEAD/examples.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6047b9a8b85a7ea6"}},{"code_sha256_prefix":"aba3849d1ad68f1f","entry":"classify","repo":"yandex-research/mind-your-format","repo_kind":"official","path":"evaluate.py","file_url":"https://github.com/yandex-research/mind-your-format/blob/HEAD/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"aba3849d1ad68f1f"}},{"code_sha256_prefix":"a1f9ff8d28378db4","entry":"load_generator","repo":"yandex-research/mind-your-format","repo_kind":"official","path":"models.py","file_url":"https://github.com/yandex-research/mind-your-format/blob/HEAD/models.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a1f9ff8d28378db4"}},{"code_sha256_prefix":"21c03a2d2acf838e","entry":"load_split_dataset","repo":"yandex-research/mind-your-format","repo_kind":"official","path":"data.py","file_url":"https://github.com/yandex-research/mind-your-format/blob/HEAD/data.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"21c03a2d2acf838e"}},{"code_sha256_prefix":"5b3e1a3d02d90a79","entry":"predict","repo":"yandex-research/mind-your-format","repo_kind":"official","path":"evaluate.py","file_url":"https://github.com/yandex-research/mind-your-format/blob/HEAD/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5b3e1a3d02d90a79"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}