{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/meerkat-audio-visual-large-language-model-for","title":"Meerkat: Audio-Visual Large Language Model for Grounding in Space and Time","arxiv_id":"2407.01851","date":"2024-07-01","proceeding":null,"authors":["Sanjoy Chowdhury","Sayan Nag","Subhrajyoti Dasgupta","Jun Chen","Mohamed Elhoseiny","Ruohan Gao","Dinesh Manocha"],"abstract":"Leveraging Large Language Models' remarkable proficiency in text-based tasks, recent works on Multi-modal LLMs (MLLMs) extend them to other modalities like vision and audio. However, the progress in these directions has been mostly focused on tasks that only require a coarse-grained understanding of the audio-visual semantics. We present Meerkat, an audio-visual LLM equipped with a fine-grained understanding of image and audio both spatially and temporally. With a new modality alignment module based on optimal transport and a cross-attention module that enforces audio-visual consistency, Meerkat can tackle challenging tasks such as audio referred image grounding, image guided audio temporal localization, and audio-visual fact-checking. Moreover, we carefully curate a large dataset AVFIT that comprises 3M instruction tuning samples collected from open-source datasets, and introduce MeerkatBench that unifies five challenging audio-visual tasks. We achieve state-of-the-art performance on all these downstream tasks with a relative improvement of up to 37.12%.","url_abs":"https://arxiv.org/abs/2407.01851v2","url_pdf":"https://arxiv.org/pdf/2407.01851v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"meerkat-audio-visual-large-language-model-for","repo_url":"https://github.com/schowdhury671/meerkat","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"audio-visual-question-answering-music-avqa-v2","task_name":"AUDIO-VISUAL QUESTION ANSWERING (MUSIC-AVQA-v2.0)"},{"task_slug":"fact-checking","task_name":"Fact Checking"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"large-language-model","task_name":"Large Language Model"},{"task_slug":"temporal-localization","task_name":"Temporal Localization"}],"methods":[{"method_slug":"concatenated-skip-connection","method_name":"Concatenated Skip Connection"},{"method_slug":"softmax","method_name":"Softmax"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/audio-visual-question-answering-music-avqa-v2","task":"AUDIO-VISUAL QUESTION ANSWERING (MUSIC-AVQA-v2.0)","dataset":"MUSIC-AVQA v2.0","model":"Meerkat","rank_in_archive_order":1,"of":5,"metrics":{"Accuracy":"79.15"},"uses_additional_data":true}],"syntology":{"syntology_url":"https://syntology.ai/paper/2407.01851","atlas_url":"https://app.syntology.ai/?focus=2407.01851","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.01851"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/schowdhury671/meerkat","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":6,"ran_fixture":1,"ran_draft_wrong":1,"unverified":4},"by_repo_kind":{"official":{"samples":12,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"2e29aff3235aa10d","entry":"IPOT_distance_torch","repo":"schowdhury671/meerkat","repo_kind":"official","path":"ot.py","file_url":"https://github.com/schowdhury671/meerkat/blob/HEAD/ot.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2e29aff3235aa10d"}},{"code_sha256_prefix":"b5722793bedbd27d","entry":"IPOT_torch","repo":"schowdhury671/meerkat","repo_kind":"official","path":"ot.py","file_url":"https://github.com/schowdhury671/meerkat/blob/HEAD/ot.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b5722793bedbd27d"}},{"code_sha256_prefix":"c5acd613a38384de","entry":"apply_rotary_pos_emb","repo":"schowdhury671/meerkat","repo_kind":"official","path":"modeling.py","file_url":"https://github.com/schowdhury671/meerkat/blob/HEAD/modeling.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c5acd613a38384de"}},{"code_sha256_prefix":"d79fd2ac61421292","entry":"cost_matrix_torch","repo":"schowdhury671/meerkat","repo_kind":"official","path":"ot.py","file_url":"https://github.com/schowdhury671/meerkat/blob/HEAD/ot.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d79fd2ac61421292"}},{"code_sha256_prefix":"cd8d3c38c1c00782","entry":"create_positional_encoding","repo":"schowdhury671/meerkat","repo_kind":"official","path":"modeling.py","file_url":"https://github.com/schowdhury671/meerkat/blob/HEAD/modeling.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cd8d3c38c1c00782"}},{"code_sha256_prefix":"85940fcc885772a0","entry":"draw_samples","repo":"schowdhury671/meerkat","repo_kind":"official","path":"llm_trainer.py","file_url":"https://github.com/schowdhury671/meerkat/blob/HEAD/llm_trainer.py","link_basis":"plan_row","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"85940fcc885772a0"}},{"code_sha256_prefix":"41e656ac2d002178","entry":"prepare_model_for_training","repo":"schowdhury671/meerkat","repo_kind":"official","path":"run_clm_llms.py","file_url":"https://github.com/schowdhury671/meerkat/blob/HEAD/run_clm_llms.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"41e656ac2d002178"}},{"code_sha256_prefix":"e03d53ba9d4f9ae5","entry":"rotate_half","repo":"schowdhury671/meerkat","repo_kind":"official","path":"modeling.py","file_url":"https://github.com/schowdhury671/meerkat/blob/HEAD/modeling.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e03d53ba9d4f9ae5"}},{"code_sha256_prefix":"14a47047d9a6be42","entry":"load_datasets","repo":"schowdhury671/meerkat","repo_kind":"official","path":"run_clm_llms.py","file_url":"https://github.com/schowdhury671/meerkat/blob/HEAD/run_clm_llms.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"14a47047d9a6be42"}},{"code_sha256_prefix":"5a4fa70c9f9c5fb7","entry":"load_datasets","repo":"schowdhury671/meerkat","repo_kind":"official","path":"run_clm_llms_inference.py","file_url":"https://github.com/schowdhury671/meerkat/blob/HEAD/run_clm_llms_inference.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5a4fa70c9f9c5fb7"}},{"code_sha256_prefix":"219ec7b58f19dfd1","entry":"preprocess_alpaca_to_tensor_dataset","repo":"schowdhury671/meerkat","repo_kind":"official","path":"preprocess_data_supervised.py","file_url":"https://github.com/schowdhury671/meerkat/blob/HEAD/preprocess_data_supervised.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"219ec7b58f19dfd1"}},{"code_sha256_prefix":"042def84c77ef6b4","entry":"preprocess_vqa2_to_tensor_dataset","repo":"schowdhury671/meerkat","repo_kind":"official","path":"preprocess_data_supervised.py","file_url":"https://github.com/schowdhury671/meerkat/blob/HEAD/preprocess_data_supervised.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"042def84c77ef6b4"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}