{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/gollie-annotation-guidelines-improve-zero","title":"GoLLIE: Annotation Guidelines improve Zero-Shot Information-Extraction","arxiv_id":"2310.03668","date":"2023-10-05","proceeding":null,"authors":["Oscar Sainz","Iker García-Ferrero","Rodrigo Agerri","Oier Lopez de Lacalle","German Rigau","Eneko Agirre"],"abstract":"Large Language Models (LLMs) combined with instruction tuning have made significant progress when generalizing to unseen tasks. However, they have been less successful in Information Extraction (IE), lagging behind task-specific models. Typically, IE tasks are characterized by complex annotation guidelines that describe the task and give examples to humans. Previous attempts to leverage such information have failed, even with the largest models, as they are not able to follow the guidelines out of the box. In this paper, we propose GoLLIE (Guideline-following Large Language Model for IE), a model able to improve zero-shot results on unseen IE tasks by virtue of being fine-tuned to comply with annotation guidelines. Comprehensive evaluation empirically demonstrates that GoLLIE is able to generalize to and follow unseen guidelines, outperforming previous attempts at zero-shot information extraction. The ablation study shows that detailed guidelines are key for good results.","url_abs":"https://arxiv.org/abs/2310.03668v5","url_pdf":"https://arxiv.org/pdf/2310.03668v5.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"gollie-annotation-guidelines-improve-zero","repo_url":"https://github.com/hitz-zentroa/gollie","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"event-argument-extraction","task_name":"Event Argument Extraction"},{"task_slug":"event-extraction","task_name":"Event Extraction"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"large-language-model","task_name":"Large Language Model"},{"task_slug":"low-resource-named-entity-recognition","task_name":"Low Resource Named Entity Recognition"},{"task_slug":"named-entity-recognition-1","task_name":"Named Entity Recognition"},{"task_slug":"named-entity-recognition-ner","task_name":"Named Entity Recognition (NER)"},{"task_slug":"relation-extraction","task_name":"Relation Extraction"},{"task_slug":"zero-shot-event-extraction","task_name":"Zero-shot Event Extraction"},{"task_slug":"zero-shot-named-entity-recognition-ner","task_name":"Zero-shot Named Entity Recognition (NER)"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/named-entity-recognition-on-ace-2005","task":"Named Entity Recognition (NER)","dataset":"ACE 2005","model":"GoLLIE","rank_in_archive_order":2,"of":20,"metrics":{"F1":"89.6"},"uses_additional_data":true},{"leaderboard":"/sota/named-entity-recognition-ner-on-bc5cdr","task":"Named Entity Recognition (NER)","dataset":"BC5CDR","model":"GoLLIE","rank_in_archive_order":12,"of":16,"metrics":{"F1":"88.4"},"uses_additional_data":true},{"leaderboard":"/sota/named-entity-recognition-ner-on-conll-2003","task":"Named Entity Recognition (NER)","dataset":"CoNLL 2003 (English)","model":"GoLLIE","rank_in_archive_order":29,"of":73,"metrics":{"F1":"93.1"},"uses_additional_data":true},{"leaderboard":"/sota/named-entity-recognition-ner-on-ncbi-disease","task":"Named Entity Recognition (NER)","dataset":"NCBI-disease","model":"GoLLIE","rank_in_archive_order":22,"of":26,"metrics":{"F1":"86.5"},"uses_additional_data":true},{"leaderboard":"/sota/named-entity-recognition-on-wnut-2017","task":"Named Entity Recognition (NER)","dataset":"WNUT 2017","model":"GoLLIE","rank_in_archive_order":11,"of":23,"metrics":{"F1":"54.3"},"uses_additional_data":true},{"leaderboard":"/sota/relation-extraction-on-ace-2005","task":"Relation Extraction","dataset":"ACE 2005","model":"GoLLIE","rank_in_archive_order":4,"of":30,"metrics":{"RE Micro F1":"70.1"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-named-entity-recognition-ner-on","task":"Zero-shot Named Entity Recognition (NER)","dataset":"Broad Twitter Corpus","model":"GoLLIE","rank_in_archive_order":2,"of":2,"metrics":{"Entity F1":"51.4"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-named-entity-recognition-ner-on-1","task":"Zero-shot Named Entity Recognition (NER)","dataset":"CrossNER","model":"GoLLIE","rank_in_archive_order":2,"of":4,"metrics":{"AI":"61.6","Literature":"62.7","Music":"68.4","Politics":"60.2","Science":"56.3"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-named-entity-recognition-ner-on-2","task":"Zero-shot Named Entity Recognition (NER)","dataset":"HarveyNER","model":"GoLLIE","rank_in_archive_order":1,"of":2,"metrics":{"Entity F1":"41.3"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-named-entity-recognition-ner-on-3","task":"Zero-shot Named Entity Recognition (NER)","dataset":"WikiEvents","model":"GoLLIE","rank_in_archive_order":1,"of":2,"metrics":{"Entity F1":"81.3"},"uses_additional_data":true}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2310.03668","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.03668"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/hitz-zentroa/GoLLIE","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/hitz-zentroa/gollie","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":17,"ran_draft_wrong":1,"unverified":7},"by_repo_kind":{"official":{"samples":25,"ran":18,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"c8bee51bf997140e","entry":"FlashRotaryEmbedding","repo":"hitz-zentroa/gollie","repo_kind":"official","path":"src/model/patch_models/modeling_flash_llama.py","file_url":"https://github.com/hitz-zentroa/gollie/blob/HEAD/src/model/patch_models/modeling_flash_llama.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c8bee51bf997140e"}},{"code_sha256_prefix":"3e5e1a1a6ae02184","entry":"LlamaRMSNorm","repo":"hitz-zentroa/gollie","repo_kind":"official","path":"src/model/patch_models/modeling_flash_llama.py","file_url":"https://github.com/hitz-zentroa/gollie/blob/HEAD/src/model/patch_models/modeling_flash_llama.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3e5e1a1a6ae02184"}},{"code_sha256_prefix":"a31cb9b49aa3fb29","entry":"clean_guidelines","repo":"hitz-zentroa/GoLLIE","repo_kind":"official","path":"src/paraphrase/utils.py","file_url":"https://github.com/hitz-zentroa/GoLLIE/blob/HEAD/src/paraphrase/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a31cb9b49aa3fb29"}},{"code_sha256_prefix":"ef50b7a39a5c925f","entry":"find_all_linear_names","repo":"hitz-zentroa/GoLLIE","repo_kind":"official","path":"src/model/model_utils.py","file_url":"https://github.com/hitz-zentroa/GoLLIE/blob/HEAD/src/model/model_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ef50b7a39a5c925f"}},{"code_sha256_prefix":"25ec67d8318f45ea","entry":"fix_prompt_outputs","repo":"hitz-zentroa/GoLLIE","repo_kind":"official","path":"src/evaluate.py","file_url":"https://github.com/hitz-zentroa/GoLLIE/blob/HEAD/src/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"25ec67d8318f45ea"}},{"code_sha256_prefix":"40c63137e2539113","entry":"get_class","repo":"hitz-zentroa/GoLLIE","repo_kind":"official","path":"src/evaluate.py","file_url":"https://github.com/hitz-zentroa/GoLLIE/blob/HEAD/src/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"40c63137e2539113"}},{"code_sha256_prefix":"d5e035b43805bee4","entry":"get_class","repo":"hitz-zentroa/GoLLIE","repo_kind":"official","path":"src/generate_data.py","file_url":"https://github.com/hitz-zentroa/GoLLIE/blob/HEAD/src/generate_data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d5e035b43805bee4"}},{"code_sha256_prefix":"47660e151147cb9e","entry":"get_device_map","repo":"hitz-zentroa/GoLLIE","repo_kind":"official","path":"src/model/load_model.py","file_url":"https://github.com/hitz-zentroa/GoLLIE/blob/HEAD/src/model/load_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"47660e151147cb9e"}},{"code_sha256_prefix":"d4149ebae2786013","entry":"get_num_return_sentences","repo":"hitz-zentroa/GoLLIE","repo_kind":"official","path":"src/paraphrase/utils.py","file_url":"https://github.com/hitz-zentroa/GoLLIE/blob/HEAD/src/paraphrase/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d4149ebae2786013"}},{"code_sha256_prefix":"f32e00b404c9eec4","entry":"get_trainable_parameters","repo":"hitz-zentroa/GoLLIE","repo_kind":"official","path":"src/model/model_utils.py","file_url":"https://github.com/hitz-zentroa/GoLLIE/blob/HEAD/src/model/model_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f32e00b404c9eec4"}},{"code_sha256_prefix":"67c3b2ef9e5c371c","entry":"prepare_data","repo":"hitz-zentroa/GoLLIE","repo_kind":"official","path":"src/dataset/dataset.py","file_url":"https://github.com/hitz-zentroa/GoLLIE/blob/HEAD/src/dataset/dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"67c3b2ef9e5c371c"}},{"code_sha256_prefix":"4fa8a9b51ecfca0c","entry":"prepare_model_for_kbit_training","repo":"hitz-zentroa/GoLLIE","repo_kind":"official","path":"src/model/model_utils.py","file_url":"https://github.com/hitz-zentroa/GoLLIE/blob/HEAD/src/model/model_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4fa8a9b51ecfca0c"}},{"code_sha256_prefix":"02c4c2306e3a0610","entry":"remove_hallucinations","repo":"hitz-zentroa/GoLLIE","repo_kind":"official","path":"src/evaluate.py","file_url":"https://github.com/hitz-zentroa/GoLLIE/blob/HEAD/src/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"02c4c2306e3a0610"}},{"code_sha256_prefix":"349dce8e37362617","entry":"rmsnorm_func","repo":"hitz-zentroa/gollie","repo_kind":"official","path":"src/model/patch_models/modeling_flash_llama.py","file_url":"https://github.com/hitz-zentroa/gollie/blob/HEAD/src/model/patch_models/modeling_flash_llama.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"349dce8e37362617"}},{"code_sha256_prefix":"d2660d11a380300e","entry":"to_bilou_encoding","repo":"hitz-zentroa/GoLLIE","repo_kind":"official","path":"src/tasks/label_encoding.py","file_url":"https://github.com/hitz-zentroa/GoLLIE/blob/HEAD/src/tasks/label_encoding.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d2660d11a380300e"}},{"code_sha256_prefix":"6c7118a2f5cc50e4","entry":"to_iob2_encoding","repo":"hitz-zentroa/GoLLIE","repo_kind":"official","path":"src/tasks/label_encoding.py","file_url":"https://github.com/hitz-zentroa/GoLLIE/blob/HEAD/src/tasks/label_encoding.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6c7118a2f5cc50e4"}},{"code_sha256_prefix":"f709c942540a746b","entry":"to_iob_encoding","repo":"hitz-zentroa/GoLLIE","repo_kind":"official","path":"src/tasks/label_encoding.py","file_url":"https://github.com/hitz-zentroa/GoLLIE/blob/HEAD/src/tasks/label_encoding.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f709c942540a746b"}},{"code_sha256_prefix":"e05fd99ba3c452ba","entry":"update_guidelines","repo":"hitz-zentroa/GoLLIE","repo_kind":"official","path":"src/paraphrase/utils.py","file_url":"https://github.com/hitz-zentroa/GoLLIE/blob/HEAD/src/paraphrase/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e05fd99ba3c452ba"}},{"code_sha256_prefix":"04267dd6afc2cfd2","entry":"LlamaAttention","repo":"hitz-zentroa/gollie","repo_kind":"official","path":"src/model/patch_models/modeling_flash_llama.py","file_url":"https://github.com/hitz-zentroa/gollie/blob/HEAD/src/model/patch_models/modeling_flash_llama.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"04267dd6afc2cfd2"}},{"code_sha256_prefix":"71e315c10d6d705e","entry":"LlamaDecoderLayer","repo":"hitz-zentroa/gollie","repo_kind":"official","path":"src/model/patch_models/modeling_flash_llama.py","file_url":"https://github.com/hitz-zentroa/gollie/blob/HEAD/src/model/patch_models/modeling_flash_llama.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"71e315c10d6d705e"}},{"code_sha256_prefix":"fb6a17240fa2036c","entry":"LlamaForCausalLM","repo":"hitz-zentroa/gollie","repo_kind":"official","path":"src/model/patch_models/modeling_flash_llama.py","file_url":"https://github.com/hitz-zentroa/gollie/blob/HEAD/src/model/patch_models/modeling_flash_llama.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fb6a17240fa2036c"}},{"code_sha256_prefix":"b6c0bf260825919c","entry":"LlamaModel","repo":"hitz-zentroa/gollie","repo_kind":"official","path":"src/model/patch_models/modeling_flash_llama.py","file_url":"https://github.com/hitz-zentroa/gollie/blob/HEAD/src/model/patch_models/modeling_flash_llama.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b6c0bf260825919c"}},{"code_sha256_prefix":"d28dcf85671df49b","entry":"batch_tokenization","repo":"hitz-zentroa/GoLLIE","repo_kind":"official","path":"src/dataset/dataset.py","file_url":"https://github.com/hitz-zentroa/GoLLIE/blob/HEAD/src/dataset/dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d28dcf85671df49b"}},{"code_sha256_prefix":"bfca1181a79266f6","entry":"load_model","repo":"hitz-zentroa/GoLLIE","repo_kind":"official","path":"src/model/load_model.py","file_url":"https://github.com/hitz-zentroa/GoLLIE/blob/HEAD/src/model/load_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bfca1181a79266f6"}},{"code_sha256_prefix":"44b3ce4579f7806b","entry":"repeat_kv","repo":"hitz-zentroa/gollie","repo_kind":"official","path":"src/model/patch_models/modeling_flash_llama.py","file_url":"https://github.com/hitz-zentroa/gollie/blob/HEAD/src/model/patch_models/modeling_flash_llama.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"44b3ce4579f7806b"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}