{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/simple-entity-centric-questions-challenge","title":"Simple Entity-Centric Questions Challenge Dense Retrievers","arxiv_id":"2109.08535","date":"2021-09-17","proceeding":"EMNLP 2021 11","authors":["Christopher Sciavolino","Zexuan Zhong","Jinhyuk Lee","Danqi Chen"],"abstract":"Open-domain question answering has exploded in popularity recently due to the success of dense retrieval models, which have surpassed sparse models using only a few supervised training examples. However, in this paper, we demonstrate current dense models are not yet the holy grail of retrieval. We first construct EntityQuestions, a set of simple, entity-rich questions based on facts from Wikidata (e.g., \"Where was Arve Furset born?\"), and observe that dense retrievers drastically underperform sparse methods. We investigate this issue and uncover that dense retrievers can only generalize to common entities unless the question pattern is explicitly observed during training. We discuss two simple solutions towards addressing this critical problem. First, we demonstrate that data augmentation is unable to fix the generalization problem. Second, we argue a more robust passage encoder helps facilitate better question adaptation using specialized question encoders. We hope our work can shed light on the challenges in creating a robust, universal dense retriever that works well across different input distributions.","url_abs":"https://arxiv.org/abs/2109.08535v3","url_pdf":"https://arxiv.org/pdf/2109.08535v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"simple-entity-centric-questions-challenge","repo_url":"https://github.com/princeton-nlp/entityquestions","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"data-augmentation","task_name":"Data Augmentation"},{"task_slug":"open-domain-question-answering","task_name":"Open-Domain Question Answering"},{"task_slug":"passage-retrieval","task_name":"Passage Retrieval"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"retrieval","task_name":"Retrieval"}],"methods":[],"datasets_introduced":[{"slug":"entityquestions","name":"EntityQuestions","full_name":""}],"methods_introduced":[],"results":[{"leaderboard":"/sota/passage-retrieval-on-entityquestions","task":"Passage Retrieval","dataset":"EntityQuestions","model":"BM25","rank_in_archive_order":3,"of":7,"metrics":{"Recall@20":"0.720"},"uses_additional_data":false},{"leaderboard":"/sota/passage-retrieval-on-entityquestions","task":"Passage Retrieval","dataset":"EntityQuestions","model":"DPR-multi","rank_in_archive_order":6,"of":7,"metrics":{"Recall@20":"0.567"},"uses_additional_data":false},{"leaderboard":"/sota/passage-retrieval-on-entityquestions","task":"Passage Retrieval","dataset":"EntityQuestions","model":"DPR-NQ","rank_in_archive_order":7,"of":7,"metrics":{"Recall@20":"0.497"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2109.08535","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.08535"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/princeton-nlp/entityquestions","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/princeton-nlp/EntityQuestions","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":1,"unverified":5},"by_repo_kind":{"official":{"samples":6,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"7d5cfda09688b35a","entry":"search","repo":"princeton-nlp/entityquestions","repo_kind":"official","path":"bm25/bm25_retriever.py","file_url":"https://github.com/princeton-nlp/entityquestions/blob/HEAD/bm25/bm25_retriever.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7d5cfda09688b35a"}},{"code_sha256_prefix":"feb9ebd7b055510a","entry":"has_answer_field","repo":"princeton-nlp/EntityQuestions","repo_kind":"official","path":"utils/has_answer_fn.py","file_url":"https://github.com/princeton-nlp/EntityQuestions/blob/HEAD/utils/has_answer_fn.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"feb9ebd7b055510a"}},{"code_sha256_prefix":"cdaed747cfe39693","entry":"normalize","repo":"princeton-nlp/EntityQuestions","repo_kind":"official","path":"utils/has_answer_fn.py","file_url":"https://github.com/princeton-nlp/EntityQuestions/blob/HEAD/utils/has_answer_fn.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cdaed747cfe39693"}},{"code_sha256_prefix":"e7b94494907985b3","entry":"read_file","repo":"princeton-nlp/EntityQuestions","repo_kind":"official","path":"utils/ion.py","file_url":"https://github.com/princeton-nlp/EntityQuestions/blob/HEAD/utils/ion.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e7b94494907985b3"}},{"code_sha256_prefix":"9ad3e5e941b7cf41","entry":"read_json","repo":"princeton-nlp/EntityQuestions","repo_kind":"official","path":"utils/ion.py","file_url":"https://github.com/princeton-nlp/EntityQuestions/blob/HEAD/utils/ion.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9ad3e5e941b7cf41"}},{"code_sha256_prefix":"9b6d4e8dece9a1b5","entry":"read_jsonl","repo":"princeton-nlp/EntityQuestions","repo_kind":"official","path":"utils/ion.py","file_url":"https://github.com/princeton-nlp/EntityQuestions/blob/HEAD/utils/ion.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9b6d4e8dece9a1b5"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}