{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/paligemma-2-a-family-of-versatile-vlms-for","title":"PaliGemma 2: A Family of Versatile VLMs for Transfer","arxiv_id":"2412.03555","date":"2024-12-04","proceeding":null,"authors":["Andreas Steiner","André Susano Pinto","Michael Tschannen","Daniel Keysers","Xiao Wang","Yonatan Bitton","Alexey Gritsenko","Matthias Minderer","Anthony Sherbondy","Shangbang Long","Siyang Qin","Reeve Ingle","Emanuele Bugliarello","Sahar Kazemzadeh","Thomas Mesnard","Ibrahim Alabdulmohsin","Lucas Beyer","Xiaohua Zhai"],"abstract":"PaliGemma 2 is an upgrade of the PaliGemma open Vision-Language Model (VLM) based on the Gemma 2 family of language models. We combine the SigLIP-So400m vision encoder that was also used by PaliGemma with the whole range of Gemma 2 models, from the 2B one all the way up to the 27B model. We train these models at three resolutions (224px, 448px, and 896px) in multiple stages to equip them with broad knowledge for transfer via fine-tuning. The resulting family of base models covering different model sizes and resolutions allows us to investigate factors impacting transfer performance (such as learning rate) and to analyze the interplay between the type of task, model size, and resolution. We further increase the number and breadth of transfer tasks beyond the scope of PaliGemma including different OCR-related tasks such as table structure recognition, molecular structure recognition, music score recognition, as well as long fine-grained captioning and radiography report generation, on which PaliGemma 2 obtains state-of-the-art results.","url_abs":"https://arxiv.org/abs/2412.03555v1","url_pdf":"https://arxiv.org/pdf/2412.03555v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"paligemma-2-a-family-of-versatile-vlms-for","repo_url":"https://github.com/kyutai-labs/moshivis","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"optical-character-recognition","task_name":"Optical Character Recognition (OCR)"}],"methods":[{"method_slug":"base","method_name":"BASE"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2412.03555","atlas_url":"https://app.syntology.ai/?focus=2412.03555","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2412.03555"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kyutai-labs/moshivis","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":4,"unverified":3},"by_repo_kind":{"listed":{"samples":7,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"a11e8571b42d5bf8","entry":"colorize","repo":"kyutai-labs/moshivis","repo_kind":"listed","path":"kyuteye_mlx/kyuteye_mlx/local_web.py","file_url":"https://github.com/kyutai-labs/moshivis/blob/HEAD/kyuteye_mlx/kyuteye_mlx/local_web.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a11e8571b42d5bf8"}},{"code_sha256_prefix":"fcb8bc1bcc0e936a","entry":"compile_pattern","repo":"kyutai-labs/moshivis","repo_kind":"listed","path":"ssvd/utils.py","file_url":"https://github.com/kyutai-labs/moshivis/blob/HEAD/ssvd/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fcb8bc1bcc0e936a"}},{"code_sha256_prefix":"250e3cffe0703509","entry":"maybe_shorten_caption","repo":"kyutai-labs/moshivis","repo_kind":"listed","path":"ssvd/utils.py","file_url":"https://github.com/kyutai-labs/moshivis/blob/HEAD/ssvd/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"250e3cffe0703509"}},{"code_sha256_prefix":"e6c9ef475f43f2f7","entry":"preprocess_pixelprose_captions","repo":"kyutai-labs/moshivis","repo_kind":"listed","path":"ssvd/utils.py","file_url":"https://github.com/kyutai-labs/moshivis/blob/HEAD/ssvd/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e6c9ef475f43f2f7"}},{"code_sha256_prefix":"e65b1f630e98892d","entry":"get_model_file","repo":"kyutai-labs/moshivis","repo_kind":"listed","path":"kyuteye_mlx/kyuteye_mlx/local_web.py","file_url":"https://github.com/kyutai-labs/moshivis/blob/HEAD/kyuteye_mlx/kyuteye_mlx/local_web.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e65b1f630e98892d"}},{"code_sha256_prefix":"b75902c10eb95a50","entry":"get_pipeline","repo":"kyutai-labs/moshivis","repo_kind":"listed","path":"ssvd/generate.py","file_url":"https://github.com/kyutai-labs/moshivis/blob/HEAD/ssvd/generate.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b75902c10eb95a50"}},{"code_sha256_prefix":"d98ebfa2e7a5c6d3","entry":"hf_hub_download","repo":"kyutai-labs/moshivis","repo_kind":"listed","path":"kyuteye_mlx/kyuteye_mlx/local_web.py","file_url":"https://github.com/kyutai-labs/moshivis/blob/HEAD/kyuteye_mlx/kyuteye_mlx/local_web.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d98ebfa2e7a5c6d3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}