{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/teochat-a-large-vision-language-assistant-for","title":"TEOChat: A Large Vision-Language Assistant for Temporal Earth Observation Data","arxiv_id":"2410.06234","date":"2024-10-08","proceeding":null,"authors":["Jeremy Andrew Irvin","Emily Ruoyu Liu","Joyce Chuyi Chen","Ines Dormoy","Jinyoung Kim","Samar Khanna","Zhuo Zheng","Stefano Ermon"],"abstract":"Large vision and language assistants have enabled new capabilities for interpreting natural images. These approaches have recently been adapted to earth observation data, but they are only able to handle single image inputs, limiting their use for many real-world tasks. In this work, we develop a new vision and language assistant called TEOChat that can engage in conversations about temporal sequences of earth observation data. To train TEOChat, we curate an instruction-following dataset composed of many single image and temporal tasks including building change and damage assessment, semantic change detection, and temporal scene classification. We show that TEOChat can perform a wide variety of spatial and temporal reasoning tasks, substantially outperforming previous vision and language assistants, and even achieving comparable or better performance than several specialist models trained to perform specific tasks. Furthermore, TEOChat achieves impressive zero-shot performance on a change detection and change question answering dataset, outperforms GPT-4o and Gemini 1.5 Pro on multiple temporal tasks, and exhibits stronger single image capabilities than a comparable single image instruction-following model on scene classification, visual question answering, and captioning. We publicly release our data, model, and code at https://github.com/ermongroup/TEOChat .","url_abs":"https://arxiv.org/abs/2410.06234v2","url_pdf":"https://arxiv.org/pdf/2410.06234v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"teochat-a-large-vision-language-assistant-for","repo_url":"https://github.com/ermongroup/teochat","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"change-detection","task_name":"Change Detection"},{"task_slug":"earth-observation","task_name":"Earth Observation"},{"task_slug":"instruction-following","task_name":"Instruction Following"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"scene-classification","task_name":"Scene Classification"},{"task_slug":"temporal-sequences","task_name":"Temporal Sequences"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2410.06234","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.06234"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/ermongroup/TEOChat","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ermongroup/teochat","reach":null}],"summary":{"ran_honours":2,"ran_violates":1,"ran_fixture":1,"unverified":5},"by_repo_kind":{"official":{"samples":9,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"308cba720b940398","entry":"extract_bboxes","repo":"ermongroup/TEOChat","repo_kind":"official","path":"videollava/eval/inference.py","file_url":"https://github.com/ermongroup/TEOChat/blob/HEAD/videollava/eval/inference.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"308cba720b940398"}},{"code_sha256_prefix":"9f989312bc8c0b11","entry":"extract_box_sequences","repo":"ermongroup/TEOChat","repo_kind":"official","path":"videollava/serve/teochat_demo.py","file_url":"https://github.com/ermongroup/TEOChat/blob/HEAD/videollava/serve/teochat_demo.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9f989312bc8c0b11"}},{"code_sha256_prefix":"0ee429e410684a36","entry":"get_bbox_in_polyline_format","repo":"ermongroup/TEOChat","repo_kind":"official","path":"videollava/serve/teochat_demo.py","file_url":"https://github.com/ermongroup/TEOChat/blob/HEAD/videollava/serve/teochat_demo.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0ee429e410684a36"}},{"code_sha256_prefix":"37438be3369e6493","entry":"is_overlapping","repo":"ermongroup/TEOChat","repo_kind":"official","path":"videollava/serve/teochat_demo.py","file_url":"https://github.com/ermongroup/TEOChat/blob/HEAD/videollava/serve/teochat_demo.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"37438be3369e6493"}},{"code_sha256_prefix":"f60b6c476c6273c2","entry":"LlavaLlamaForCausalLM","repo":"ermongroup/teochat","repo_kind":"official","path":"videollava/model/language_model/llava_llama.py","file_url":"https://github.com/ermongroup/teochat/blob/HEAD/videollava/model/language_model/llava_llama.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f60b6c476c6273c2"}},{"code_sha256_prefix":"bcec4a0354378c68","entry":"LlavaLlamaModel","repo":"ermongroup/teochat","repo_kind":"official","path":"videollava/model/language_model/llava_llama.py","file_url":"https://github.com/ermongroup/teochat/blob/HEAD/videollava/model/language_model/llava_llama.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bcec4a0354378c68"}},{"code_sha256_prefix":"10ce3e7babead476","entry":"LlavaMetaModel","repo":"ermongroup/teochat","repo_kind":"official","path":"videollava/model/language_model/llava_llama.py","file_url":"https://github.com/ermongroup/teochat/blob/HEAD/videollava/model/language_model/llava_llama.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"10ce3e7babead476"}},{"code_sha256_prefix":"182c873357c1da2f","entry":"build_image_tower","repo":"ermongroup/teochat","repo_kind":"official","path":"videollava/model/language_model/llava_llama.py","file_url":"https://github.com/ermongroup/teochat/blob/HEAD/videollava/model/language_model/llava_llama.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"182c873357c1da2f"}},{"code_sha256_prefix":"806829873ac85173","entry":"build_video_tower","repo":"ermongroup/teochat","repo_kind":"official","path":"videollava/model/language_model/llava_llama.py","file_url":"https://github.com/ermongroup/teochat/blob/HEAD/videollava/model/language_model/llava_llama.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"806829873ac85173"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}