{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/sd-qa-spoken-dialectal-question-answering-for","title":"SD-QA: Spoken Dialectal Question Answering for the Real World","arxiv_id":"2109.12072","date":"2021-09-24","proceeding":"Findings (EMNLP) 2021 11","authors":["Fahim Faisal","Sharlina Keshava","Md Mahfuz ibn Alam","Antonios Anastasopoulos"],"abstract":"Question answering (QA) systems are now available through numerous commercial applications for a wide variety of domains, serving millions of users that interact with them via speech interfaces. However, current benchmarks in QA research do not account for the errors that speech recognition models might introduce, nor do they consider the language variations (dialects) of the users. To address this gap, we augment an existing QA dataset to construct a multi-dialect, spoken QA benchmark on five languages (Arabic, Bengali, English, Kiswahili, Korean) with more than 68k audio prompts in 24 dialects from 255 speakers. We provide baseline results showcasing the real-world performance of QA systems and analyze the effect of language variety and other sensitive speaker attributes on downstream performance. Last, we study the fairness of the ASR and QA models with respect to the underlying user populations. The dataset, model outputs, and code for reproducing all our experiments are available: https://github.com/ffaisal93/SD-QA.","url_abs":"https://arxiv.org/abs/2109.12072v1","url_pdf":"https://arxiv.org/pdf/2109.12072v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"sd-qa-spoken-dialectal-question-answering-for","repo_url":"https://github.com/ffaisal93/sd-qa","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"fairness","task_name":"Fairness"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"speech-recognition","task_name":"Speech Recognition"},{"task_slug":"speech-recognition-1","task_name":"speech-recognition"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2109.12072","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.12072"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ffaisal93/sd-qa","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":22},"by_repo_kind":{"official":{"samples":22,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"7671754d0cee7d11","entry":"byte_len","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"baselines/tydiqa/baseline/data.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/baselines/tydiqa/baseline/data.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7671754d0cee7d11"}},{"code_sha256_prefix":"787fbf63d8c1ab07","entry":"byte_slice","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"baselines/tydiqa/tydi_eval.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/baselines/tydiqa/tydi_eval.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"787fbf63d8c1ab07"}},{"code_sha256_prefix":"5e27cb5b15ab895f","entry":"byte_slice","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"baselines/tydiqa/baseline/data.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/baselines/tydiqa/baseline/data.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5e27cb5b15ab895f"}},{"code_sha256_prefix":"a5845e18acb8844b","entry":"byte_str","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"baselines/tydiqa/baseline/data.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/baselines/tydiqa/baseline/data.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a5845e18acb8844b"}},{"code_sha256_prefix":"7ef49e1793449f8b","entry":"compute_partial_match_scores","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"baselines/tydiqa/eval_utils.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/baselines/tydiqa/eval_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7ef49e1793449f8b"}},{"code_sha256_prefix":"ea2aa48fb12d0a30","entry":"dir_check","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"experiments/utils.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/experiments/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ea2aa48fb12d0a30"}},{"code_sha256_prefix":"fd24ba8d7375d425","entry":"extract_answers","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"extract_qa_from_jsonl.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/extract_qa_from_jsonl.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fd24ba8d7375d425"}},{"code_sha256_prefix":"d19afd0251c48ecd","entry":"find_audio_dir","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"sdqa_triples_from_jsonl.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/sdqa_triples_from_jsonl.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d19afd0251c48ecd"}},{"code_sha256_prefix":"51d3bc0cfa6415a3","entry":"gelu","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"baselines/tydiqa/baseline/bert/modeling.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/baselines/tydiqa/baseline/bert/modeling.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"51d3bc0cfa6415a3"}},{"code_sha256_prefix":"217fe96a5945a42d","entry":"get_activation","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"baselines/tydiqa/baseline/bert/modeling.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/baselines/tydiqa/baseline/bert/modeling.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"217fe96a5945a42d"}},{"code_sha256_prefix":"7bc46f4950599a87","entry":"get_assignment_map_from_checkpoint","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"baselines/tydiqa/baseline/bert/modeling.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/baselines/tydiqa/baseline/bert/modeling.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7bc46f4950599a87"}},{"code_sha256_prefix":"d60c3869c082ce69","entry":"get_metadata","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"experiments/utils.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/experiments/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d60c3869c082ce69"}},{"code_sha256_prefix":"da7b79cb32612610","entry":"load_jsonl_example_ids","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"analyze_data_links.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/analyze_data_links.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"da7b79cb32612610"}},{"code_sha256_prefix":"50ed7d3089187d7d","entry":"load_metadata_ids","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"analyze_data_links.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/analyze_data_links.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"50ed7d3089187d7d"}},{"code_sha256_prefix":"5d1dc7f1ffae5ff9","entry":"load_txt_ids","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"analyze_data_links.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/analyze_data_links.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5d1dc7f1ffae5ff9"}},{"code_sha256_prefix":"91c7ebc9939cc186","entry":"read_reference","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"experiments/transcript_replace.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/experiments/transcript_replace.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"91c7ebc9939cc186"}},{"code_sha256_prefix":"e5609b0f6acf5d82","entry":"safe_average","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"baselines/tydiqa/eval_utils.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/baselines/tydiqa/eval_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e5609b0f6acf5d82"}},{"code_sha256_prefix":"999c2750713a6692","entry":"safe_divide","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"baselines/tydiqa/eval_utils.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/baselines/tydiqa/eval_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"999c2750713a6692"}},{"code_sha256_prefix":"606f24554fa9223b","entry":"sanitize_field","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"extract_qa_from_jsonl.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/extract_qa_from_jsonl.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"606f24554fa9223b"}},{"code_sha256_prefix":"812e406d9764d77d","entry":"utf8_slice","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"extract_qa_from_jsonl.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/extract_qa_from_jsonl.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"812e406d9764d77d"}},{"code_sha256_prefix":"f4a930be2113e081","entry":"utf8_slice","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"sdqa_triples_from_jsonl.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/sdqa_triples_from_jsonl.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f4a930be2113e081"}},{"code_sha256_prefix":"dad8041c5ab85e66","entry":"write_out","repo":"ffaisal93/sd-qa","repo_kind":"official","path":"experiments/transcript_replace.py","file_url":"https://github.com/ffaisal93/sd-qa/blob/HEAD/experiments/transcript_replace.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"dad8041c5ab85e66"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}