{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/on-evaluation-of-bangla-word-analogies","title":"On Evaluation of Bangla Word Analogies","arxiv_id":"2304.04613","date":"2023-04-10","proceeding":null,"authors":["Mousumi Akter","Souvika Sarkar","Shubhra Kanti Karmaker"],"abstract":"This paper presents a high-quality dataset for evaluating the quality of Bangla word embeddings, which is a fundamental task in the field of Natural Language Processing (NLP). Despite being the 7th most-spoken language in the world, Bangla is a low-resource language and popular NLP models fail to perform well. Developing a reliable evaluation test set for Bangla word embeddings are crucial for benchmarking and guiding future research. We provide a Mikolov-style word analogy evaluation set specifically for Bangla, with a sample size of 16678, as well as a translated and curated version of the Mikolov dataset, which contains 10594 samples for cross-lingual research. Our experiments with different state-of-the-art embedding models reveal that Bangla has its own unique characteristics, and current embeddings for Bangla still struggle to achieve high accuracy on both datasets. We suggest that future research should focus on training models with larger datasets and considering the unique morphological characteristics of Bangla. This study represents the first step towards building a reliable NLP system for the Bangla language1.","url_abs":"https://arxiv.org/abs/2304.04613v1","url_pdf":"https://arxiv.org/pdf/2304.04613v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[],"tasks":[{"task_slug":"benchmarking","task_name":"Benchmarking"},{"task_slug":"word-embeddings","task_name":"Word Embeddings"}],"methods":[{"method_slug":"test","method_name":"Test"},{"method_slug":"fail","method_name":"fail"}],"datasets_introduced":[{"slug":"bangla-word-analogy-1","name":"Bangla Word Analogy","full_name":"Bangla Word Analogy"}],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2304.04613","atlas_url":"https://app.syntology.ai/?focus=2304.04613","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.04613"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/facebookresearch/fastText","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":6,"ran_draft_wrong":1,"unverified":4},"by_repo_kind":{"found_in_text":{"samples":11,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":11,"samples":[{"code_sha256_prefix":"5d48a5764718f390","entry":"find_nearest_neighbor","repo":"facebookresearch/fastText","repo_kind":"found_in_text","path":"python/fasttext_module/fasttext/util/util.py","file_url":"https://github.com/facebookresearch/fastText/blob/HEAD/python/fasttext_module/fasttext/util/util.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"5d48a5764718f390"}},{"code_sha256_prefix":"c2a80677a6cb6044","entry":"guess_target_name","repo":"facebookresearch/fastText","repo_kind":"found_in_text","path":"reduce_model.py","file_url":"https://github.com/facebookresearch/fastText/blob/HEAD/reduce_model.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"c2a80677a6cb6044"}},{"code_sha256_prefix":"831f6e244ecea2bb","entry":"idx","repo":"facebookresearch/fastText","repo_kind":"found_in_text","path":"alignment/utils.py","file_url":"https://github.com/facebookresearch/fastText/blob/HEAD/alignment/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"831f6e244ecea2bb"}},{"code_sha256_prefix":"6f28ff1eccefb093","entry":"procrustes","repo":"facebookresearch/fastText","repo_kind":"found_in_text","path":"alignment/utils.py","file_url":"https://github.com/facebookresearch/fastText/blob/HEAD/alignment/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"6f28ff1eccefb093"}},{"code_sha256_prefix":"f697966cbe929024","entry":"read_args","repo":"facebookresearch/fastText","repo_kind":"found_in_text","path":"python/fasttext_module/fasttext/FastText.py","file_url":"https://github.com/facebookresearch/fastText/blob/HEAD/python/fasttext_module/fasttext/FastText.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"f697966cbe929024"}},{"code_sha256_prefix":"e4a10d52bc6d1601","entry":"sqrt_eig","repo":"facebookresearch/fastText","repo_kind":"found_in_text","path":"alignment/unsup_align.py","file_url":"https://github.com/facebookresearch/fastText/blob/HEAD/alignment/unsup_align.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"e4a10d52bc6d1601"}},{"code_sha256_prefix":"5c3bcf5ae2042c99","entry":"test","repo":"facebookresearch/fastText","repo_kind":"found_in_text","path":"python/fasttext_module/fasttext/util/util.py","file_url":"https://github.com/facebookresearch/fastText/blob/HEAD/python/fasttext_module/fasttext/util/util.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"5c3bcf5ae2042c99"}},{"code_sha256_prefix":"f685d8c430dfac70","entry":"getknn","repo":"facebookresearch/fastText","repo_kind":"found_in_text","path":"alignment/align.py","file_url":"https://github.com/facebookresearch/fastText/blob/HEAD/alignment/align.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"f685d8c430dfac70"}},{"code_sha256_prefix":"5b38e4065c54800a","entry":"load_vectors","repo":"facebookresearch/fastText","repo_kind":"found_in_text","path":"alignment/utils.py","file_url":"https://github.com/facebookresearch/fastText/blob/HEAD/alignment/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"5b38e4065c54800a"}},{"code_sha256_prefix":"d484b37352fc8959","entry":"rcsls","repo":"facebookresearch/fastText","repo_kind":"found_in_text","path":"alignment/unsup_multialign.py","file_url":"https://github.com/facebookresearch/fastText/blob/HEAD/alignment/unsup_multialign.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"d484b37352fc8959"}},{"code_sha256_prefix":"3ab9d330ea24c434","entry":"reduce_model","repo":"facebookresearch/fastText","repo_kind":"found_in_text","path":"python/fasttext_module/fasttext/util/util.py","file_url":"https://github.com/facebookresearch/fastText/blob/HEAD/python/fasttext_module/fasttext/util/util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"3ab9d330ea24c434"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}