{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/multimodal-analogical-reasoning-over","title":"Multimodal Analogical Reasoning over Knowledge Graphs","arxiv_id":"2210.00312","date":"2022-10-01","proceeding":null,"authors":["Ningyu Zhang","Lei LI","Xiang Chen","Xiaozhuan Liang","Shumin Deng","Huajun Chen"],"abstract":"Analogical reasoning is fundamental to human cognition and holds an important place in various fields. However, previous studies mainly focus on single-modal analogical reasoning and ignore taking advantage of structure knowledge. Notably, the research in cognitive psychology has demonstrated that information from multimodal sources always brings more powerful cognitive transfer than single modality sources. To this end, we introduce the new task of multimodal analogical reasoning over knowledge graphs, which requires multimodal reasoning ability with the help of background knowledge. Specifically, we construct a Multimodal Analogical Reasoning dataSet (MARS) and a multimodal knowledge graph MarKG. We evaluate with multimodal knowledge graph embedding and pre-trained Transformer baselines, illustrating the potential challenges of the proposed task. We further propose a novel model-agnostic Multimodal analogical reasoning framework with Transformer (MarT) motivated by the structure mapping theory, which can obtain better performance. Code and datasets are available in https://github.com/zjunlp/MKG_Analogy.","url_abs":"https://arxiv.org/abs/2210.00312v4","url_pdf":"https://arxiv.org/pdf/2210.00312v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"multimodal-analogical-reasoning-over","repo_url":"https://github.com/zjunlp/MKG_Analogy","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"multimodal-analogical-reasoning-over","repo_url":"https://github.com/zjunlp/MKGformer","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"graph-embedding","task_name":"Graph Embedding"},{"task_slug":"knowledge-graph-embedding","task_name":"Knowledge Graph Embedding"},{"task_slug":"knowledge-graphs","task_name":"Knowledge Graphs"},{"task_slug":"multimodal-reasoning","task_name":"Multimodal Reasoning"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[{"slug":"mars-multimodal-analogical-reasoning-dataset","name":"MARS (Multimodal Analogical Reasoning dataSet)","full_name":""},{"slug":"markg","name":"MarKG","full_name":"Multimodal analogical reasoning Knowledge Graph"}],"methods_introduced":[],"results":[{"leaderboard":"/sota/knowledge-graphs-on-mars-multimodal","task":"Knowledge Graphs","dataset":"MARS (Multimodal Analogical Reasoning dataSet)","model":"MarT_MKGformer","rank_in_archive_order":1,"of":8,"metrics":{"MRR":"0.341"},"uses_additional_data":false},{"leaderboard":"/sota/knowledge-graphs-on-mars-multimodal","task":"Knowledge Graphs","dataset":"MARS (Multimodal Analogical Reasoning dataSet)","model":"MKGformer","rank_in_archive_order":2,"of":8,"metrics":{"MRR":"0.321"},"uses_additional_data":false},{"leaderboard":"/sota/knowledge-graphs-on-mars-multimodal","task":"Knowledge Graphs","dataset":"MARS (Multimodal Analogical Reasoning dataSet)","model":"MarT_FLAVA","rank_in_archive_order":3,"of":8,"metrics":{"MRR":"0.288"},"uses_additional_data":false},{"leaderboard":"/sota/knowledge-graphs-on-mars-multimodal","task":"Knowledge Graphs","dataset":"MARS (Multimodal Analogical Reasoning dataSet)","model":"ViLBERT","rank_in_archive_order":4,"of":8,"metrics":{"MRR":"0.287"},"uses_additional_data":false},{"leaderboard":"/sota/knowledge-graphs-on-mars-multimodal","task":"Knowledge Graphs","dataset":"MARS (Multimodal Analogical Reasoning dataSet)","model":"IKRL (ANALOGY)","rank_in_archive_order":5,"of":8,"metrics":{"MRR":"0.283"},"uses_additional_data":false},{"leaderboard":"/sota/knowledge-graphs-on-mars-multimodal","task":"Knowledge Graphs","dataset":"MARS (Multimodal Analogical Reasoning dataSet)","model":"IKRL","rank_in_archive_order":6,"of":8,"metrics":{"MRR":"0.274"},"uses_additional_data":false},{"leaderboard":"/sota/knowledge-graphs-on-mars-multimodal","task":"Knowledge Graphs","dataset":"MARS (Multimodal Analogical Reasoning dataSet)","model":"ViLT","rank_in_archive_order":7,"of":8,"metrics":{"MRR":"0.257"},"uses_additional_data":false},{"leaderboard":"/sota/knowledge-graphs-on-mars-multimodal","task":"Knowledge Graphs","dataset":"MARS (Multimodal Analogical Reasoning dataSet)","model":"TransAE","rank_in_archive_order":8,"of":8,"metrics":{"MRR":"0.223"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2210.00312","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.00312"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/zjunlp/MKGformer","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/zjunlp/MKG_Analogy","reach":null}],"summary":{"ran":2,"ran_draft_wrong":3,"ran_violates":1,"ran_honours":1,"unverified":20},"by_repo_kind":{"official":{"samples":15,"ran":5,"repositories":1},"listed":{"samples":12,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"afce1111b619c932","entry":"CLIPMLP","repo":"zjunlp/MKG_Analogy","repo_kind":"official","path":"MarT/models/modeling_clip.py","file_url":"https://github.com/zjunlp/MKG_Analogy/blob/HEAD/MarT/models/modeling_clip.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"afce1111b619c932"}},{"code_sha256_prefix":"83268e2e1fee13be","entry":"CLIPVisionEmbeddings","repo":"zjunlp/MKG_Analogy","repo_kind":"official","path":"MarT/models/modeling_clip.py","file_url":"https://github.com/zjunlp/MKG_Analogy/blob/HEAD/MarT/models/modeling_clip.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"83268e2e1fee13be"}},{"code_sha256_prefix":"4b1fb3ae6a6d23fd","entry":"_expand_mask","repo":"zjunlp/MKG_Analogy","repo_kind":"official","path":"MarT/models/modeling_clip.py","file_url":"https://github.com/zjunlp/MKG_Analogy/blob/HEAD/MarT/models/modeling_clip.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4b1fb3ae6a6d23fd"}},{"code_sha256_prefix":"e5b89b1f72c319ed","entry":"clip_loss","repo":"zjunlp/MKGformer","repo_kind":"listed","path":"MKG/models/modeling_clip.py","file_url":"https://github.com/zjunlp/MKGformer/blob/HEAD/MKG/models/modeling_clip.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e5b89b1f72c319ed"}},{"code_sha256_prefix":"1aa7f7b75b2418fa","entry":"clip_loss","repo":"zjunlp/MKG_Analogy","repo_kind":"official","path":"MarT/models/modeling_clip.py","file_url":"https://github.com/zjunlp/MKG_Analogy/blob/HEAD/MarT/models/modeling_clip.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1aa7f7b75b2418fa"}},{"code_sha256_prefix":"4675779aec92d6c0","entry":"contrastive_loss","repo":"zjunlp/MKGformer","repo_kind":"listed","path":"MKG/models/modeling_clip.py","file_url":"https://github.com/zjunlp/MKGformer/blob/HEAD/MKG/models/modeling_clip.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4675779aec92d6c0"}},{"code_sha256_prefix":"f2e5f28608115fa6","entry":"contrastive_loss","repo":"zjunlp/MKG_Analogy","repo_kind":"official","path":"MarT/models/modeling_clip.py","file_url":"https://github.com/zjunlp/MKG_Analogy/blob/HEAD/MarT/models/modeling_clip.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f2e5f28608115fa6"}},{"code_sha256_prefix":"bc71a7b0ac64e193","entry":"CLIPAttention","repo":"zjunlp/MKG_Analogy","repo_kind":"official","path":"MarT/models/modeling_clip.py","file_url":"https://github.com/zjunlp/MKG_Analogy/blob/HEAD/MarT/models/modeling_clip.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bc71a7b0ac64e193"}},{"code_sha256_prefix":"eac07c4625c7ecf0","entry":"CLIPBaseModelOutput","repo":"zjunlp/MKG_Analogy","repo_kind":"official","path":"MarT/models/modeling_clip.py","file_url":"https://github.com/zjunlp/MKG_Analogy/blob/HEAD/MarT/models/modeling_clip.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"eac07c4625c7ecf0"}},{"code_sha256_prefix":"0a895d6880258edf","entry":"CLIPBaseModelOutputWithPooling","repo":"zjunlp/MKG_Analogy","repo_kind":"official","path":"MarT/models/modeling_clip.py","file_url":"https://github.com/zjunlp/MKG_Analogy/blob/HEAD/MarT/models/modeling_clip.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0a895d6880258edf"}},{"code_sha256_prefix":"89fb5ce36cf33f90","entry":"CLIPEncoder","repo":"zjunlp/MKG_Analogy","repo_kind":"official","path":"MarT/models/modeling_clip.py","file_url":"https://github.com/zjunlp/MKG_Analogy/blob/HEAD/MarT/models/modeling_clip.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"89fb5ce36cf33f90"}},{"code_sha256_prefix":"d8f1abf8ccc978b3","entry":"CLIPEncoderLayer","repo":"zjunlp/MKG_Analogy","repo_kind":"official","path":"MarT/models/modeling_clip.py","file_url":"https://github.com/zjunlp/MKG_Analogy/blob/HEAD/MarT/models/modeling_clip.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d8f1abf8ccc978b3"}},{"code_sha256_prefix":"27ff4c9021371f00","entry":"CLIPModel","repo":"zjunlp/MKG_Analogy","repo_kind":"official","path":"MarT/models/modeling_clip.py","file_url":"https://github.com/zjunlp/MKG_Analogy/blob/HEAD/MarT/models/modeling_clip.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"27ff4c9021371f00"}},{"code_sha256_prefix":"c1560a0097321925","entry":"CLIPOutput","repo":"zjunlp/MKG_Analogy","repo_kind":"official","path":"MarT/models/modeling_clip.py","file_url":"https://github.com/zjunlp/MKG_Analogy/blob/HEAD/MarT/models/modeling_clip.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c1560a0097321925"}},{"code_sha256_prefix":"f27778de07c0b042","entry":"CLIPTextEmbeddings","repo":"zjunlp/MKG_Analogy","repo_kind":"official","path":"MarT/models/modeling_clip.py","file_url":"https://github.com/zjunlp/MKG_Analogy/blob/HEAD/MarT/models/modeling_clip.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f27778de07c0b042"}},{"code_sha256_prefix":"cb9f127c9449b83d","entry":"CLIPTextTransformer","repo":"zjunlp/MKG_Analogy","repo_kind":"official","path":"MarT/models/modeling_clip.py","file_url":"https://github.com/zjunlp/MKG_Analogy/blob/HEAD/MarT/models/modeling_clip.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cb9f127c9449b83d"}},{"code_sha256_prefix":"255891e0b510d846","entry":"CLIPVisionTransformer","repo":"zjunlp/MKG_Analogy","repo_kind":"official","path":"MarT/models/modeling_clip.py","file_url":"https://github.com/zjunlp/MKG_Analogy/blob/HEAD/MarT/models/modeling_clip.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"255891e0b510d846"}},{"code_sha256_prefix":"e5c96daa2363c6ff","entry":"acc","repo":"zjunlp/MKGformer","repo_kind":"listed","path":"MKG/lit_models/utils.py","file_url":"https://github.com/zjunlp/MKGformer/blob/HEAD/MKG/lit_models/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e5c96daa2363c6ff"}},{"code_sha256_prefix":"4aaee21eb89b469e","entry":"decode","repo":"zjunlp/MKGformer","repo_kind":"listed","path":"MKG/lit_models/transformer.py","file_url":"https://github.com/zjunlp/MKGformer/blob/HEAD/MKG/lit_models/transformer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4aaee21eb89b469e"}},{"code_sha256_prefix":"a0b34bcb573fbd1c","entry":"get_entity_spans_pre_processing","repo":"zjunlp/MKGformer","repo_kind":"listed","path":"MKG/models/utils.py","file_url":"https://github.com/zjunlp/MKGformer/blob/HEAD/MKG/models/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a0b34bcb573fbd1c"}},{"code_sha256_prefix":"dd48a2143b53be6f","entry":"get_extended_attention_mask","repo":"zjunlp/MKGformer","repo_kind":"listed","path":"MKG/models/modeling_unimo.py","file_url":"https://github.com/zjunlp/MKGformer/blob/HEAD/MKG/models/modeling_unimo.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dd48a2143b53be6f"}},{"code_sha256_prefix":"14ea78cc67ff4f00","entry":"get_head_mask","repo":"zjunlp/MKGformer","repo_kind":"listed","path":"MKG/models/modeling_unimo.py","file_url":"https://github.com/zjunlp/MKGformer/blob/HEAD/MKG/models/modeling_unimo.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"14ea78cc67ff4f00"}},{"code_sha256_prefix":"fadc10fc2c451c92","entry":"label_smoothed_nll_loss","repo":"zjunlp/MKGformer","repo_kind":"listed","path":"MKG/models/utils.py","file_url":"https://github.com/zjunlp/MKGformer/blob/HEAD/MKG/models/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fadc10fc2c451c92"}},{"code_sha256_prefix":"cca3a1dc3db5bbf1","entry":"lmap","repo":"zjunlp/MKGformer","repo_kind":"listed","path":"MKG/lit_models/transformer.py","file_url":"https://github.com/zjunlp/MKGformer/blob/HEAD/MKG/lit_models/transformer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cca3a1dc3db5bbf1"}},{"code_sha256_prefix":"d235c954a3acaf08","entry":"multilabel_categorical_crossentropy","repo":"zjunlp/MKGformer","repo_kind":"listed","path":"MKG/lit_models/transformer.py","file_url":"https://github.com/zjunlp/MKGformer/blob/HEAD/MKG/lit_models/transformer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d235c954a3acaf08"}},{"code_sha256_prefix":"a698ba8988b64507","entry":"rank_score","repo":"zjunlp/MKGformer","repo_kind":"listed","path":"MKG/lit_models/utils.py","file_url":"https://github.com/zjunlp/MKGformer/blob/HEAD/MKG/lit_models/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a698ba8988b64507"}},{"code_sha256_prefix":"67aa808378a82feb","entry":"replace_return_docstrings","repo":"zjunlp/MKGformer","repo_kind":"listed","path":"MKG/models/utils.py","file_url":"https://github.com/zjunlp/MKGformer/blob/HEAD/MKG/models/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"67aa808378a82feb"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}