{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/revamping-cross-modal-recipe-retrieval-with","title":"Revamping Cross-Modal Recipe Retrieval with Hierarchical Transformers and Self-supervised Learning","arxiv_id":"2103.13061","date":"2021-03-24","proceeding":"CVPR 2021 1","authors":["Amaia Salvador","Erhan Gundogdu","Loris Bazzani","Michael Donoser"],"abstract":"Cross-modal recipe retrieval has recently gained substantial attention due to the importance of food in people's lives, as well as the availability of vast amounts of digital cooking recipes and food images to train machine learning models. In this work, we revisit existing approaches for cross-modal recipe retrieval and propose a simplified end-to-end model based on well established and high performing encoders for text and images. We introduce a hierarchical recipe Transformer which attentively encodes individual recipe components (titles, ingredients and instructions). Further, we propose a self-supervised loss function computed on top of pairs of individual recipe components, which is able to leverage semantic relationships within recipes, and enables training using both image-recipe and recipe-only samples. We conduct a thorough analysis and ablation studies to validate our design choices. As a result, our proposed method achieves state-of-the-art performance in the cross-modal recipe retrieval task on the Recipe1M dataset. We make code and models publicly available.","url_abs":"https://arxiv.org/abs/2103.13061v1","url_pdf":"https://arxiv.org/pdf/2103.13061v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"revamping-cross-modal-recipe-retrieval-with","repo_url":"https://github.com/amzn/image-to-recipe-transformers","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"cross-modal-retrieval","task_name":"Cross-Modal Retrieval"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"self-supervised-learning","task_name":"Self-Supervised Learning"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/cross-modal-retrieval-on-recipe1m","task":"Cross-Modal Retrieval","dataset":"Recipe1M","model":"H-T","rank_in_archive_order":6,"of":9,"metrics":{"Image-to-text R@1":"60.0","Text-to-image R@1":"60.3"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2103.13061","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2103.13061"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/amzn/image-to-recipe-transformers","reach":null}],"summary":{"ran":3,"unverified":1},"by_repo_kind":{"official":{"samples":4,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"f46bd2e85f26c092","entry":"LearnedPositionalEncoding","repo":"amzn/image-to-recipe-transformers","repo_kind":"official","path":"src/models.py","file_url":"https://github.com/amzn/image-to-recipe-transformers/blob/HEAD/src/models.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f46bd2e85f26c092"}},{"code_sha256_prefix":"b26d6c273c4d32f3","entry":"RecipeTransformerEncoder","repo":"amzn/image-to-recipe-transformers","repo_kind":"official","path":"src/models.py","file_url":"https://github.com/amzn/image-to-recipe-transformers/blob/HEAD/src/models.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b26d6c273c4d32f3"}},{"code_sha256_prefix":"8d3b2bbe6944e636","entry":"SingleTransformerEncoder","repo":"amzn/image-to-recipe-transformers","repo_kind":"official","path":"src/models.py","file_url":"https://github.com/amzn/image-to-recipe-transformers/blob/HEAD/src/models.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8d3b2bbe6944e636"}},{"code_sha256_prefix":"fe2eba13dedc4953","entry":"AvgPoolSequence","repo":"amzn/image-to-recipe-transformers","repo_kind":"official","path":"src/models.py","file_url":"https://github.com/amzn/image-to-recipe-transformers/blob/HEAD/src/models.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fe2eba13dedc4953"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}