{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2603-01691","title":"Building a Strong Instruction Language Model for a Less-Resourced Language","arxiv_id":"2603.01691","date":"2026-03-02","proceeding":null,"authors":["Domen Vreš","Tjaša Arčon","Timotej Petrič","Dario Vajda","Marko Robnik-Šikonja","Iztok Lebar Bajec"],"abstract":"Large language models (LLMs) have become an essential tool for natural language processing and artificial intelligence in general. Current open-source models are primarily trained on English texts, resulting in poorer performance on less-resourced languages and cultures. We present a set of methodological approaches necessary for the successful adaptation of an LLM to a less-resourced language, and demonstrate them using the Slovene language. We present GaMS3-12B, a generative model for Slovene with 12 billion parameters, and demonstrate that it is the best-performing open-source model for Slovene within its parameter range. We adapted the model to the Slovene language using three-stage continual pre-training of the Gemma 3 model, followed by two-stage supervised fine-tuning (SFT). We trained the model on a combination of 140B Slovene, English, Bosnian, Serbian, and Croatian pretraining tokens, and over 200 thousand English and Slovene SFT examples. We evaluate GaMS3-12B on the Slovenian-LLM-Eval datasets, English-to-Slovene translation, and the Slovene LLM arena. We show that the described model outperforms 12B Gemma 3 across all three scenarios and performs comparably to much larger commercial GPT-4o in the Slovene LLM arena, achieving a win rate of over 60 %.","url_abs":"https://arxiv.org/abs/2603.01691","url_pdf":"https://arxiv.org/pdf/2603.01691","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2603.01691","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2603.01691"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/NVIDIA-NeMo/NeMo","reach":{"status":"ok","spdx":"Apache-2.0"}},{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/datalab-to/marker","reach":null},{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/GaMS-Team/local_ocr","reach":null}],"summary":{"unverified":16},"by_repo_kind":{"found_in_text":{"samples":16,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"15774ca6895e9baa","entry":"extract_content_from_tag","repo":"GaMS-Team/local_ocr","repo_kind":"found_in_text","path":"postprocess_markdown/utils.py","file_url":"https://github.com/GaMS-Team/local_ocr/blob/HEAD/postprocess_markdown/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"15774ca6895e9baa"}},{"code_sha256_prefix":"98a5e3c65904b8da","entry":"generate_responses","repo":"GaMS-Team/local_ocr","repo_kind":"found_in_text","path":"postprocess_markdown/classify_pages.py","file_url":"https://github.com/GaMS-Team/local_ocr/blob/HEAD/postprocess_markdown/classify_pages.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"98a5e3c65904b8da"}},{"code_sha256_prefix":"a45378cd91f5e6d9","entry":"generate_responses","repo":"GaMS-Team/local_ocr","repo_kind":"found_in_text","path":"postprocess_markdown/merge_pages.py","file_url":"https://github.com/GaMS-Team/local_ocr/blob/HEAD/postprocess_markdown/merge_pages.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a45378cd91f5e6d9"}},{"code_sha256_prefix":"08a3ead54a6bcf17","entry":"get_number_from_file","repo":"GaMS-Team/local_ocr","repo_kind":"found_in_text","path":"postprocess_markdown/classify_pages.py","file_url":"https://github.com/GaMS-Team/local_ocr/blob/HEAD/postprocess_markdown/classify_pages.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"08a3ead54a6bcf17"}},{"code_sha256_prefix":"1e23e5c57ca6fbe9","entry":"get_page_prompt","repo":"GaMS-Team/local_ocr","repo_kind":"found_in_text","path":"nanonets_ocr/ocr_with_vllm.py","file_url":"https://github.com/GaMS-Team/local_ocr/blob/HEAD/nanonets_ocr/ocr_with_vllm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1e23e5c57ca6fbe9"}},{"code_sha256_prefix":"ee6b631841b07773","entry":"ocr_page","repo":"GaMS-Team/local_ocr","repo_kind":"found_in_text","path":"chatgpt_ocr/chatgpt_ocr.py","file_url":"https://github.com/GaMS-Team/local_ocr/blob/HEAD/chatgpt_ocr/chatgpt_ocr.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ee6b631841b07773"}},{"code_sha256_prefix":"2ae1456ceaa9bf01","entry":"page_to_image","repo":"GaMS-Team/local_ocr","repo_kind":"found_in_text","path":"llama4_ocr/ocr_with_local_server.py","file_url":"https://github.com/GaMS-Team/local_ocr/blob/HEAD/llama4_ocr/ocr_with_local_server.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2ae1456ceaa9bf01"}},{"code_sha256_prefix":"fdef91e5cde260ec","entry":"page_to_image","repo":"GaMS-Team/local_ocr","repo_kind":"found_in_text","path":"nanonets_ocr/ocr_with_transformers.py","file_url":"https://github.com/GaMS-Team/local_ocr/blob/HEAD/nanonets_ocr/ocr_with_transformers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fdef91e5cde260ec"}},{"code_sha256_prefix":"25e0991010208c40","entry":"page_to_image","repo":"GaMS-Team/local_ocr","repo_kind":"found_in_text","path":"nanonets_ocr/ocr_with_vllm.py","file_url":"https://github.com/GaMS-Team/local_ocr/blob/HEAD/nanonets_ocr/ocr_with_vllm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"25e0991010208c40"}},{"code_sha256_prefix":"c7f87eb0059ed979","entry":"page_to_image","repo":"GaMS-Team/local_ocr","repo_kind":"found_in_text","path":"chatgpt_ocr/chatgpt_ocr.py","file_url":"https://github.com/GaMS-Team/local_ocr/blob/HEAD/chatgpt_ocr/chatgpt_ocr.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c7f87eb0059ed979"}},{"code_sha256_prefix":"98b689870c1001ec","entry":"preprocess_image","repo":"GaMS-Team/local_ocr","repo_kind":"found_in_text","path":"tesseract_ocr/tesseract_OCR.py","file_url":"https://github.com/GaMS-Team/local_ocr/blob/HEAD/tesseract_ocr/tesseract_OCR.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"98b689870c1001ec"}},{"code_sha256_prefix":"7e629f856ba12aa0","entry":"process_batch","repo":"GaMS-Team/local_ocr","repo_kind":"found_in_text","path":"postprocess_markdown/classify_pages.py","file_url":"https://github.com/GaMS-Team/local_ocr/blob/HEAD/postprocess_markdown/classify_pages.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7e629f856ba12aa0"}},{"code_sha256_prefix":"ce57ccba583cfc9f","entry":"process_batch","repo":"GaMS-Team/local_ocr","repo_kind":"found_in_text","path":"postprocess_markdown/merge_pages.py","file_url":"https://github.com/GaMS-Team/local_ocr/blob/HEAD/postprocess_markdown/merge_pages.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ce57ccba583cfc9f"}},{"code_sha256_prefix":"4688d9e2055e2e04","entry":"process_too_long_prompt","repo":"GaMS-Team/local_ocr","repo_kind":"found_in_text","path":"postprocess_markdown/merge_pages.py","file_url":"https://github.com/GaMS-Team/local_ocr/blob/HEAD/postprocess_markdown/merge_pages.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4688d9e2055e2e04"}},{"code_sha256_prefix":"848422699e20d1e3","entry":"send_image_to_model","repo":"GaMS-Team/local_ocr","repo_kind":"found_in_text","path":"llama4_ocr/ocr_with_local_server.py","file_url":"https://github.com/GaMS-Team/local_ocr/blob/HEAD/llama4_ocr/ocr_with_local_server.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"848422699e20d1e3"}},{"code_sha256_prefix":"ee3e3eb2d845f727","entry":"send_image_to_model","repo":"GaMS-Team/local_ocr","repo_kind":"found_in_text","path":"nanonets_ocr/ocr_with_transformers.py","file_url":"https://github.com/GaMS-Team/local_ocr/blob/HEAD/nanonets_ocr/ocr_with_transformers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ee3e3eb2d845f727"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.LG","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}