{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/on-the-fly-definition-augmentation-of-llms","title":"On-the-fly Definition Augmentation of LLMs for Biomedical NER","arxiv_id":"2404.00152","date":"2024-03-29","proceeding":null,"authors":["Monica Munnangi","Sergey Feldman","Byron C Wallace","Silvio Amir","Tom Hope","Aakanksha Naik"],"abstract":"Despite their general capabilities, LLMs still struggle on biomedical NER tasks, which are difficult due to the presence of specialized terminology and lack of training data. In this work we set out to improve LLM performance on biomedical NER in limited data settings via a new knowledge augmentation approach which incorporates definitions of relevant concepts on-the-fly. During this process, to provide a test bed for knowledge augmentation, we perform a comprehensive exploration of prompting strategies. Our experiments show that definition augmentation is useful for both open source and closed LLMs. For example, it leads to a relative improvement of 15\\% (on average) in GPT-4 performance (F1) across all (six) of our test datasets. We conduct extensive ablations and analyses to demonstrate that our performance improvements stem from adding relevant definitional knowledge. We find that careful prompting strategies also improve LLM performance, allowing them to outperform fine-tuned language models in few-shot settings. To facilitate future research in this direction, we release our code at https://github.com/allenai/beacon.","url_abs":"https://arxiv.org/abs/2404.00152v2","url_pdf":"https://arxiv.org/pdf/2404.00152v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"on-the-fly-definition-augmentation-of-llms","repo_url":"https://github.com/allenai/beacon","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"cg","task_name":"NER"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt-4","method_name":"GPT-4"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"set","method_name":"SET"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2404.00152","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.00152"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/allenai/beacon","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":4,"unverified":3},"by_repo_kind":{"official":{"samples":7,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"7e25397694232c4d","entry":"convert_dict_to_list_of_dicts","repo":"allenai/beacon","repo_kind":"official","path":"finetuning_data/conll_formatting.py","file_url":"https://github.com/allenai/beacon/blob/HEAD/finetuning_data/conll_formatting.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7e25397694232c4d"}},{"code_sha256_prefix":"85b409d4935c8b83","entry":"generate_text","repo":"allenai/beacon","repo_kind":"official","path":"calls/openai_call.py","file_url":"https://github.com/allenai/beacon/blob/HEAD/calls/openai_call.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"85b409d4935c8b83"}},{"code_sha256_prefix":"9df10872433e5698","entry":"jsonNotFormattedCorrectly","repo":"allenai/beacon","repo_kind":"official","path":"calls/openai_call.py","file_url":"https://github.com/allenai/beacon/blob/HEAD/calls/openai_call.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9df10872433e5698"}},{"code_sha256_prefix":"93075ac4da9d7fb7","entry":"take","repo":"allenai/beacon","repo_kind":"official","path":"finetuning_data/conll_formatting.py","file_url":"https://github.com/allenai/beacon/blob/HEAD/finetuning_data/conll_formatting.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"93075ac4da9d7fb7"}},{"code_sha256_prefix":"a76a469ab43921fd","entry":"convert_offsets_to_word_level","repo":"allenai/beacon","repo_kind":"official","path":"finetuning_data/conll_formatting.py","file_url":"https://github.com/allenai/beacon/blob/HEAD/finetuning_data/conll_formatting.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a76a469ab43921fd"}},{"code_sha256_prefix":"e2271ea9bc72c3cd","entry":"jsonNotFormattedCorrectly","repo":"allenai/beacon","repo_kind":"official","path":"calls/claude_call.py","file_url":"https://github.com/allenai/beacon/blob/HEAD/calls/claude_call.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e2271ea9bc72c3cd"}},{"code_sha256_prefix":"d6230ab1c41b1c09","entry":"jsonNotFormattedCorrectly","repo":"allenai/beacon","repo_kind":"official","path":"calls/llama_call.py","file_url":"https://github.com/allenai/beacon/blob/HEAD/calls/llama_call.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d6230ab1c41b1c09"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}