{"url":"/task/multimodal-text-and-image-classification","name":"Multimodal Text and Image Classification","slug":"multimodal-text-and-image-classification","description_markdown":"Classification with both source Image and Text","categories":[{"name":"Natural Language Processing","url":"/area/natural-language-processing"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":7,"papers_with_code":5,"benchmarks":3,"benchmark_tables_in_archive":3,"benchmark_tables_shown":3,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":4,"subtasks":2,"parent_tasks":1},"benchmarks":[{"leaderboard":"/sota/multimodal-text-and-image-classification-on-1","slug":"multimodal-text-and-image-classification-on-1","dataset":"Food-101","dataset_url":"/dataset/food-101","rows_in_archive":2,"metrics":["Accuracy (%)"],"first_row_in_archive_order":{"model":"Early Fusion (Bert + InceptionV3)","paper_title":"Image and Text fusion for UPMC Food-101 \\\\using BERT and CNNs","paper_url":"/paper/image-and-text-fusion-for-upmc-food-101-using","paper_date":"2020-12-17","arxiv_id":null,"code_links":[{"title":"artelab/Image-and-Text-fusion-for-UPMC-Food-101-using-BERT-and-CNNs","url":"https://github.com/artelab/Image-and-Text-fusion-for-UPMC-Food-101-using-BERT-and-CNNs"}],"syntology":null}},{"leaderboard":"/sota/multimodal-text-and-image-classification-on","slug":"multimodal-text-and-image-classification-on","dataset":"CUB-200-2011","dataset_url":"/dataset/cub-200-2011","rows_in_archive":1,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Two Branch Network (Text - Bert + Image - Nts-Net)","paper_title":"Are These Birds Similar: Learning Branched Networks for Fine-grained Representations","paper_url":"/paper/are-these-birds-similar-learning-branched","paper_date":"2020-01-16","arxiv_id":null,"code_links":[{"title":"nicolalandro/ntsnet-cub200","url":"https://github.com/nicolalandro/ntsnet-cub200"},{"title":"Mind23-2/MindCode-101","url":"https://github.com/Mind23-2/MindCode-101/tree/main/ntsnet"},{"title":"artelabsuper/ivcnz-2019-bird","url":"https://gitlab.com/artelabsuper/ivcnz-2019-bird"}],"syntology":null}},{"leaderboard":"/sota/multimodal-text-and-image-classification-on-2","slug":"multimodal-text-and-image-classification-on-2","dataset":"CD18","dataset_url":"/dataset/cd18","rows_in_archive":1,"metrics":["Accuracy","F-measure (%)"],"first_row_in_archive_order":{"model":"Convolutional image feature extraction and dense concatenating","paper_title":"Multimodal price prediction","paper_url":"/paper/multimodal-price-prediction","paper_date":"2020-07-09","arxiv_id":"2007.05056","code_links":[],"syntology":null}}],"datasets":[{"url":"/dataset/cub-200-2011","name":"CUB-200-2011","full_name":"Caltech-UCSD Birds-200-2011","num_papers_in_archive":2235},{"url":"/dataset/food-101","name":"Food-101","full_name":"","num_papers_in_archive":805},{"url":"/dataset/quilt-1m","name":"QUILT-1M","full_name":"","num_papers_in_archive":17},{"url":"/dataset/cd18","name":"CD18","full_name":"Cellphone Dataset with 18 Features","num_papers_in_archive":1}],"subtasks":[{"url":"/task/image-sentence-alignment","name":"image-sentence alignment"},{"url":"/task/open-world-social-event-classification","name":"Open-World Social Event Classification"}],"parent_tasks":[{"url":"/task/multimodal-deep-learning","name":"Multimodal Deep Learning"}],"papers":{"order":"repositories listed in the archive (desc), then date (desc); the archive holds no stars","population":"papers tagged with this task that list at least one repository in the archive","shown":5,"of":5,"tagged_in_all":7,"items":[{"url":"/paper/are-these-birds-similar-learning-branched","title":"Are These Birds Similar: Learning Branched Networks for Fine-grained Representations","date":"2020-01-16","arxiv_id":null,"repositories_listed":3,"syntology":null},{"url":"/paper/harmonic-nas-hardware-aware-multimodal-neural","title":"Harmonic-NAS: Hardware-Aware Multimodal Neural Architecture Search on Resource-constrained Devices","date":"2023-09-12","arxiv_id":"2309.06612","repositories_listed":1,"syntology":{"n":2,"n_ran":0,"n_unverified":2,"n_pointer_only":2}},{"url":"/paper/detecting-hate-speech-in-memes-using","title":"Detecting Hate Speech in Memes Using Multimodal Deep Learning Approaches: Prize-winning solution to Hateful Memes Challenge","date":"2020-12-23","arxiv_id":"2012.12975","repositories_listed":1,"syntology":null},{"url":"/paper/image-and-text-fusion-for-upmc-food-101-using","title":"Image and Text fusion for UPMC Food-101 \\\\using BERT and CNNs","date":"2020-12-17","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/analysis-of-social-media-data-using","title":"Analysis of Social Media Data using Multimodal Deep Learning for Disaster Response","date":"2020-04-14","arxiv_id":"2004.11838","repositories_listed":1,"syntology":null}],"syntology_records":1,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-09-24T18:15:14+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"}},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"}}