{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/large-language-models-as-visual-cross-domain","title":"VLLaVO: Mitigating Visual Gap through LLMs","arxiv_id":"2401.03253","date":"2024-01-06","proceeding":null,"authors":["Shuhao Chen","Yulong Zhang","Weisen Jiang","Jiangang Lu","Yu Zhang"],"abstract":"Recent advances achieved by deep learning models rely on the independent and identically distributed assumption, hindering their applications in real-world scenarios with domain shifts. To tackle this issue, cross-domain learning aims at extracting domain-invariant knowledge to reduce the domain shift between training and testing data. However, in visual cross-domain learning, traditional methods concentrate solely on the image modality, disregarding the potential benefits of incorporating the text modality. In this work, we propose VLLaVO, combining Vision language models and Large Language models as Visual cross-dOmain learners. VLLaVO uses vision-language models to convert images into detailed textual descriptions. A large language model is then finetuned on textual descriptions of the source/target domain generated by a designed instruction template. Extensive experimental results under domain generalization and unsupervised domain adaptation settings demonstrate the effectiveness of the proposed method.","url_abs":"https://arxiv.org/abs/2401.03253v2","url_pdf":"https://arxiv.org/pdf/2401.03253v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"large-language-models-as-visual-cross-domain","repo_url":"https://github.com/LL-a-VO/VLLaVO","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"domain-adaptation","task_name":"Domain Adaptation"},{"task_slug":"domain-generalization","task_name":"Domain Generalization"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"large-language-model","task_name":"Large Language Model"},{"task_slug":"unsupervised-domain-adaptation","task_name":"Unsupervised Domain Adaptation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2401.03253","atlas_url":"https://app.syntology.ai/?focus=2401.03253","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.03253"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/LL-a-VO/VLLaVO","reach":{"status":"ok"}}],"summary":{"ran_honours":1,"ran":6,"unverified":6},"by_repo_kind":{"official":{"samples":13,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":13,"samples":[{"code_sha256_prefix":"20922c79a0929bbf","entry":"flatten","repo":"LL-a-VO/VLLaVO","repo_kind":"official","path":"lens/model.py","file_url":"https://github.com/LL-a-VO/VLLaVO/blob/HEAD/lens/model.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":2,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"20922c79a0929bbf"}},{"code_sha256_prefix":"f95424e47545245d","entry":"create_dataloader","repo":"LL-a-VO/VLLaVO","repo_kind":"official","path":"lens/utils.py","file_url":"https://github.com/LL-a-VO/VLLaVO/blob/HEAD/lens/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f95424e47545245d"}},{"code_sha256_prefix":"1d11345966a0f999","entry":"create_sampler","repo":"LL-a-VO/VLLaVO","repo_kind":"official","path":"lens/utils.py","file_url":"https://github.com/LL-a-VO/VLLaVO/blob/HEAD/lens/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1d11345966a0f999"}},{"code_sha256_prefix":"6ca5611d6608bbc9","entry":"extract_modules","repo":"LL-a-VO/VLLaVO","repo_kind":"official","path":"script/classification_llama.py","file_url":"https://github.com/LL-a-VO/VLLaVO/blob/HEAD/script/classification_llama.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6ca5611d6608bbc9"}},{"code_sha256_prefix":"86c58be91e935ce3","entry":"get_dataset_domain_name","repo":"LL-a-VO/VLLaVO","repo_kind":"official","path":"utils/universal_utils.py","file_url":"https://github.com/LL-a-VO/VLLaVO/blob/HEAD/utils/universal_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"86c58be91e935ce3"}},{"code_sha256_prefix":"672bfa09d232c963","entry":"get_prompt_split","repo":"LL-a-VO/VLLaVO","repo_kind":"official","path":"script/zeroShot_llama.py","file_url":"https://github.com/LL-a-VO/VLLaVO/blob/HEAD/script/zeroShot_llama.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"672bfa09d232c963"}},{"code_sha256_prefix":"54a4b8663745c56f","entry":"most_similar_item","repo":"LL-a-VO/VLLaVO","repo_kind":"official","path":"utils/universal_utils.py","file_url":"https://github.com/LL-a-VO/VLLaVO/blob/HEAD/utils/universal_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"54a4b8663745c56f"}},{"code_sha256_prefix":"20b4c3e3fe9a2903","entry":"get_dataset","repo":"LL-a-VO/VLLaVO","repo_kind":"official","path":"script/finetune_llama.py","file_url":"https://github.com/LL-a-VO/VLLaVO/blob/HEAD/script/finetune_llama.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"20b4c3e3fe9a2903"}},{"code_sha256_prefix":"7108ccc674bf9d6e","entry":"get_dataset_classes","repo":"LL-a-VO/VLLaVO","repo_kind":"official","path":"utils/universal_utils.py","file_url":"https://github.com/LL-a-VO/VLLaVO/blob/HEAD/utils/universal_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7108ccc674bf9d6e"}},{"code_sha256_prefix":"2c4a12d43a47a863","entry":"get_dataset_pseudo","repo":"LL-a-VO/VLLaVO","repo_kind":"official","path":"script/two_stage_finetune_llama.py","file_url":"https://github.com/LL-a-VO/VLLaVO/blob/HEAD/script/two_stage_finetune_llama.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2c4a12d43a47a863"}},{"code_sha256_prefix":"3b956935072fd84f","entry":"get_llm_model","repo":"LL-a-VO/VLLaVO","repo_kind":"official","path":"lens/utils.py","file_url":"https://github.com/LL-a-VO/VLLaVO/blob/HEAD/lens/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3b956935072fd84f"}},{"code_sha256_prefix":"085e93ac862a43d3","entry":"parse_data_file","repo":"LL-a-VO/VLLaVO","repo_kind":"official","path":"script/descriptions_extractor.py","file_url":"https://github.com/LL-a-VO/VLLaVO/blob/HEAD/script/descriptions_extractor.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"085e93ac862a43d3"}},{"code_sha256_prefix":"70fb50a19a9639f7","entry":"refine_descriptions","repo":"LL-a-VO/VLLaVO","repo_kind":"official","path":"script/classification_llama.py","file_url":"https://github.com/LL-a-VO/VLLaVO/blob/HEAD/script/classification_llama.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"70fb50a19a9639f7"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}