{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/jmi-at-semeval-2024-task-3-two-step-approach","title":"JMI at SemEval 2024 Task 3: Two-step approach for multimodal ECAC using in-context learning with GPT and instruction-tuned Llama models","arxiv_id":"2403.04798","date":"2024-03-05","proceeding":null,"authors":["Arefa","Mohammed Abbas Ansari","Chandni Saxena","Tanvir Ahmad"],"abstract":"This paper presents our system development for SemEval-2024 Task 3: \"The Competition of Multimodal Emotion Cause Analysis in Conversations\". Effectively capturing emotions in human conversations requires integrating multiple modalities such as text, audio, and video. However, the complexities of these diverse modalities pose challenges for developing an efficient multimodal emotion cause analysis (ECA) system. Our proposed approach addresses these challenges by a two-step framework. We adopt two different approaches in our implementation. In Approach 1, we employ instruction-tuning with two separate Llama 2 models for emotion and cause prediction. In Approach 2, we use GPT-4V for conversation-level video description and employ in-context learning with annotated conversation using GPT 3.5. Our system wins rank 4, and system ablation experiments demonstrate that our proposed solutions achieve significant performance gains. All the experimental codes are available on Github.","url_abs":"https://arxiv.org/abs/2403.04798v2","url_pdf":"https://arxiv.org/pdf/2403.04798v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"jmi-at-semeval-2024-task-3-two-step-approach","repo_url":"https://github.com/cmooncs/semeval-2024_multimodal_ecpe","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"in-context-learning","task_name":"In-Context Learning"},{"task_slug":"video-description","task_name":"Video Description"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"cosine-annealing","method_name":"Cosine Annealing"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"discriminative-fine-tuning","method_name":"Discriminative Fine-Tuning"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt","method_name":"GPT"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-cosine-annealing","method_name":"Linear Warmup With Cosine Annealing"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2403.04798","atlas_url":"https://app.syntology.ai/?focus=2403.04798","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.04798"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/cmooncs/semeval-2024_multimodal_ecpe","reach":{"status":"ok"}}],"summary":{"ran":5,"unverified":7},"by_repo_kind":{"official":{"samples":12,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":12,"samples":[{"code_sha256_prefix":"a52a136c6b8fabb0","entry":"audioCaptions","repo":"cmooncs/semeval-2024_multimodal_ecpe","repo_kind":"official","path":"Llama2/generate_input.py","file_url":"https://github.com/cmooncs/semeval-2024_multimodal_ecpe/blob/HEAD/Llama2/generate_input.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a52a136c6b8fabb0"}},{"code_sha256_prefix":"9bc274c8923f748f","entry":"convert_list_to_tensor","repo":"cmooncs/semeval-2024_multimodal_ecpe","repo_kind":"official","path":"old_experiments/emotion_recognition/utils.py","file_url":"https://github.com/cmooncs/semeval-2024_multimodal_ecpe/blob/HEAD/old_experiments/emotion_recognition/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9bc274c8923f748f"}},{"code_sha256_prefix":"736360d234ede39c","entry":"create_json","repo":"cmooncs/semeval-2024_multimodal_ecpe","repo_kind":"official","path":"Llama2/generate_input.py","file_url":"https://github.com/cmooncs/semeval-2024_multimodal_ecpe/blob/HEAD/Llama2/generate_input.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"736360d234ede39c"}},{"code_sha256_prefix":"a5d799e609df955b","entry":"get_stacked_tensor","repo":"cmooncs/semeval-2024_multimodal_ecpe","repo_kind":"official","path":"old_experiments/emotion_recognition/utils.py","file_url":"https://github.com/cmooncs/semeval-2024_multimodal_ecpe/blob/HEAD/old_experiments/emotion_recognition/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a5d799e609df955b"}},{"code_sha256_prefix":"22f07976823a323c","entry":"videoCaption","repo":"cmooncs/semeval-2024_multimodal_ecpe","repo_kind":"official","path":"Llama2/generate_input.py","file_url":"https://github.com/cmooncs/semeval-2024_multimodal_ecpe/blob/HEAD/Llama2/generate_input.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"22f07976823a323c"}},{"code_sha256_prefix":"7f4ace96e2d9557b","entry":"bert_batch_preprocessing","repo":"cmooncs/semeval-2024_multimodal_ecpe","repo_kind":"official","path":"old_experiments/emotion_recognition/data_loader2.py","file_url":"https://github.com/cmooncs/semeval-2024_multimodal_ecpe/blob/HEAD/old_experiments/emotion_recognition/data_loader2.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7f4ace96e2d9557b"}},{"code_sha256_prefix":"58020c488a30bc05","entry":"bert_batch_preprocessing","repo":"cmooncs/semeval-2024_multimodal_ecpe","repo_kind":"official","path":"old_experiments/pair_prediction/data_loader2.py","file_url":"https://github.com/cmooncs/semeval-2024_multimodal_ecpe/blob/HEAD/old_experiments/pair_prediction/data_loader2.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"58020c488a30bc05"}},{"code_sha256_prefix":"e7d8472ae55da9e6","entry":"build_inference_data","repo":"cmooncs/semeval-2024_multimodal_ecpe","repo_kind":"official","path":"old_experiments/emotion_recognition/data_loader2.py","file_url":"https://github.com/cmooncs/semeval-2024_multimodal_ecpe/blob/HEAD/old_experiments/emotion_recognition/data_loader2.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e7d8472ae55da9e6"}},{"code_sha256_prefix":"4d316fa28f335d9c","entry":"build_inference_data","repo":"cmooncs/semeval-2024_multimodal_ecpe","repo_kind":"official","path":"old_experiments/pair_prediction/data_loader2.py","file_url":"https://github.com/cmooncs/semeval-2024_multimodal_ecpe/blob/HEAD/old_experiments/pair_prediction/data_loader2.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4d316fa28f335d9c"}},{"code_sha256_prefix":"3483d0eafe69ea3e","entry":"build_train_data","repo":"cmooncs/semeval-2024_multimodal_ecpe","repo_kind":"official","path":"old_experiments/emotion_recognition/data_loader2.py","file_url":"https://github.com/cmooncs/semeval-2024_multimodal_ecpe/blob/HEAD/old_experiments/emotion_recognition/data_loader2.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3483d0eafe69ea3e"}},{"code_sha256_prefix":"84427cf16236bc84","entry":"build_train_data","repo":"cmooncs/semeval-2024_multimodal_ecpe","repo_kind":"official","path":"old_experiments/pair_prediction/data_loader2.py","file_url":"https://github.com/cmooncs/semeval-2024_multimodal_ecpe/blob/HEAD/old_experiments/pair_prediction/data_loader2.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"84427cf16236bc84"}},{"code_sha256_prefix":"a58806a330cb6df1","entry":"init_wandb","repo":"cmooncs/semeval-2024_multimodal_ecpe","repo_kind":"official","path":"old_experiments/pair_prediction/wrapper_new.py","file_url":"https://github.com/cmooncs/semeval-2024_multimodal_ecpe/blob/HEAD/old_experiments/pair_prediction/wrapper_new.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a58806a330cb6df1"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}