{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/learning-to-compute-word-embeddings-on-the","title":"Learning to Compute Word Embeddings On the Fly","arxiv_id":"1706.00286","date":"2017-06-01","proceeding":"ICLR 2018 1","authors":["Dzmitry Bahdanau","Tom Bosc","Stanisław Jastrzębski","Edward Grefenstette","Pascal Vincent","Yoshua Bengio"],"abstract":"Words in natural language follow a Zipfian distribution whereby some words\nare frequent but most are rare. Learning representations for words in the \"long\ntail\" of this distribution requires enormous amounts of data. Representations\nof rare words trained directly on end tasks are usually poor, requiring us to\npre-train embeddings on external data, or treat all rare words as\nout-of-vocabulary words with a unique representation. We provide a method for\npredicting embeddings of rare words on the fly from small amounts of auxiliary\ndata with a network trained end-to-end for the downstream task. We show that\nthis improves results against baselines where embeddings are trained on the end\ntask for reading comprehension, recognizing textual entailment and language\nmodeling.","url_abs":"http://arxiv.org/abs/1706.00286v3","url_pdf":"http://arxiv.org/pdf/1706.00286v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"natural-language-inference","task_name":"Natural Language Inference"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"reading-comprehension","task_name":"Reading Comprehension"},{"task_slug":"word-embeddings","task_name":"Word Embeddings"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/question-answering-on-squad11","task":"Question Answering","dataset":"SQuAD1.1","model":"OTF dict+spelling (single)","rank_in_archive_order":183,"of":213,"metrics":{"EM":"64.083","F1":"73.056"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-squad11","task":"Question Answering","dataset":"SQuAD1.1","model":"OTF spelling (single)","rank_in_archive_order":185,"of":213,"metrics":{"EM":"62.897","F1":"72.016"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-squad11","task":"Question Answering","dataset":"SQuAD1.1","model":"OTF spelling+lemma (single)","rank_in_archive_order":186,"of":213,"metrics":{"EM":"62.604","F1":"71.968"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-squad11-dev","task":"Question Answering","dataset":"SQuAD1.1 dev","model":"OTF dict+spelling (single)","rank_in_archive_order":48,"of":55,"metrics":{"EM":"63.06"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/1706.00286","atlas_url":"https://app.syntology.ai/?focus=1706.00286","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}