{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/show-and-tell-lessons-learned-from-the-2015","title":"Show and Tell: Lessons learned from the 2015 MSCOCO Image Captioning Challenge","arxiv_id":"1609.06647","date":"2016-09-21","proceeding":null,"authors":["Oriol Vinyals","Alexander Toshev","Samy Bengio","Dumitru Erhan"],"abstract":"Automatically describing the content of an image is a fundamental problem in\nartificial intelligence that connects computer vision and natural language\nprocessing. In this paper, we present a generative model based on a deep\nrecurrent architecture that combines recent advances in computer vision and\nmachine translation and that can be used to generate natural sentences\ndescribing an image. The model is trained to maximize the likelihood of the\ntarget description sentence given the training image. Experiments on several\ndatasets show the accuracy of the model and the fluency of the language it\nlearns solely from image descriptions. Our model is often quite accurate, which\nwe verify both qualitatively and quantitatively. Finally, given the recent\nsurge of interest in this task, a competition was organized in 2015 using the\nnewly released COCO dataset. We describe and analyze the various improvements\nwe applied to our own baseline and show the resulting performance in the\ncompetition, which we won ex-aequo with a team from Microsoft Research, and\nprovide an open source implementation in TensorFlow.","url_abs":"http://arxiv.org/abs/1609.06647v1","url_pdf":"http://arxiv.org/pdf/1609.06647v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"show-and-tell-lessons-learned-from-the-2015","repo_url":"https://github.com/21-projects-for-deep-learning/image2text","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"show-and-tell-lessons-learned-from-the-2015","repo_url":"https://github.com/ChiZhangRIT/BNRHN","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":null},{"paper_slug":"show-and-tell-lessons-learned-from-the-2015","repo_url":"https://github.com/HughKu/Im2txt","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"show-and-tell-lessons-learned-from-the-2015","repo_url":"https://github.com/Jayanthipalanichamy/ElucidateTheScene","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"show-and-tell-lessons-learned-from-the-2015","repo_url":"https://github.com/Muennighoff/Im2txt","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"show-and-tell-lessons-learned-from-the-2015","repo_url":"https://github.com/ShroukAbozeid/ImageCaptionByVoice","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"show-and-tell-lessons-learned-from-the-2015","repo_url":"https://github.com/SophiaYuSophiaYu/ImageCaption","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"show-and-tell-lessons-learned-from-the-2015","repo_url":"https://github.com/ag169/Image-Captioning","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"show-and-tell-lessons-learned-from-the-2015","repo_url":"https://github.com/b2renger/Runway-experiment-img2txt2img2txt","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"show-and-tell-lessons-learned-from-the-2015","repo_url":"https://github.com/brandontrabucco/im2txt_express","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"show-and-tell-lessons-learned-from-the-2015","repo_url":"https://github.com/brandontrabucco/im2txt_match","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"show-and-tell-lessons-learned-from-the-2015","repo_url":"https://github.com/dgonzalez-ri/neural-visual-storyteller","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"show-and-tell-lessons-learned-from-the-2015","repo_url":"https://github.com/dianaglzrico/neural-visual-storyteller","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"show-and-tell-lessons-learned-from-the-2015","repo_url":"https://github.com/purvaten/punny_captions","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"show-and-tell-lessons-learned-from-the-2015","repo_url":"https://github.com/puxinhe/im2txt_v3","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"show-and-tell-lessons-learned-from-the-2015","repo_url":"https://github.com/sovit-123/Deep-Learning-Image-Captioning","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"show-and-tell-lessons-learned-from-the-2015","repo_url":"https://github.com/stoensin/IC","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"show-and-tell-lessons-learned-from-the-2015","repo_url":"https://github.com/tensorflow/models/tree/master/research/im2txt","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":null},{"paper_slug":"show-and-tell-lessons-learned-from-the-2015","repo_url":"https://github.com/viplazylmht/image-captioning-inference","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"show-and-tell-lessons-learned-from-the-2015","repo_url":"https://github.com/vladsandulescu/hatefulmemes","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"unanswered"}}],"tasks":[{"task_slug":"image-captioning","task_name":"Image Captioning"},{"task_slug":"sentence","task_name":"Sentence"},{"task_slug":"translation","task_name":"Translation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/1609.06647","atlas_url":"https://app.syntology.ai/?focus=1609.06647","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1609.06647"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/HughKu/Im2txt","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/viplazylmht/image-captioning-inference","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/vladsandulescu/hatefulmemes","reach":{"status":"unanswered"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/tensorflow/models/tree/master/research/im2txt","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ag169/Image-Captioning","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/21-projects-for-deep-learning/image2text","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/stoensin/IC","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/purvaten/punny_captions","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ChiZhangRIT/BNRHN","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/brandontrabucco/im2txt_match","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/b2renger/Runway-experiment-img2txt2img2txt","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Muennighoff/Im2txt","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/puxinhe/im2txt_v3","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/SophiaYuSophiaYu/ImageCaption","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Jayanthipalanichamy/ElucidateTheScene","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sovit-123/Deep-Learning-Image-Captioning","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/brandontrabucco/im2txt_express","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/dgonzalez-ri/neural-visual-storyteller","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/dianaglzrico/neural-visual-storyteller","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ShroukAbozeid/ImageCaptionByVoice","reach":{"status":"ok"}}],"summary":{"ran_draft_wrong":3,"ran_fixture":1,"unverified":11},"by_repo_kind":{"listed":{"samples":15,"ran":4,"repositories":4}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"1f6cc35d04b1e568","entry":"beam_search_inference","repo":"ag169/Image-Captioning","repo_kind":"listed","path":"eval_caption.py","file_url":"https://github.com/ag169/Image-Captioning/blob/HEAD/eval_caption.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1f6cc35d04b1e568"}},{"code_sha256_prefix":"a640c50faf0d8a9d","entry":"forward_with_loss","repo":"ag169/Image-Captioning","repo_kind":"listed","path":"train_caption.py","file_url":"https://github.com/ag169/Image-Captioning/blob/HEAD/train_caption.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a640c50faf0d8a9d"}},{"code_sha256_prefix":"9305318ce8a2d74f","entry":"inference","repo":"ag169/Image-Captioning","repo_kind":"listed","path":"train_caption.py","file_url":"https://github.com/ag169/Image-Captioning/blob/HEAD/train_caption.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9305318ce8a2d74f"}},{"code_sha256_prefix":"83bce32a04182448","entry":"inference","repo":"ag169/Image-Captioning","repo_kind":"listed","path":"eval_caption.py","file_url":"https://github.com/ag169/Image-Captioning/blob/HEAD/eval_caption.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"83bce32a04182448"}},{"code_sha256_prefix":"43a082fdefbc3753","entry":"batch_with_dynamic_pad","repo":"brandontrabucco/im2txt_express","repo_kind":"listed","path":"im2txt_express/ops/inputs.py","file_url":"https://github.com/brandontrabucco/im2txt_express/blob/HEAD/im2txt_express/ops/inputs.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"43a082fdefbc3753"}},{"code_sha256_prefix":"9d746b47756500fc","entry":"batch_with_dynamic_pad","repo":"dgonzalez-ri/neural-visual-storyteller","repo_kind":"listed","path":"inputs.py","file_url":"https://github.com/dgonzalez-ri/neural-visual-storyteller/blob/HEAD/inputs.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9d746b47756500fc"}},{"code_sha256_prefix":"c019732c3cedf8f2","entry":"distort_image","repo":"brandontrabucco/im2txt_express","repo_kind":"listed","path":"im2txt_express/ops/image_processing.py","file_url":"https://github.com/brandontrabucco/im2txt_express/blob/HEAD/im2txt_express/ops/image_processing.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c019732c3cedf8f2"}},{"code_sha256_prefix":"f883de80a5b8761f","entry":"get_filename","repo":"ChiZhangRIT/BNRHN","repo_kind":"listed","path":"generate_captions.py","file_url":"https://github.com/ChiZhangRIT/BNRHN/blob/HEAD/generate_captions.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f883de80a5b8761f"}},{"code_sha256_prefix":"7feab14292d065d3","entry":"get_id","repo":"ChiZhangRIT/BNRHN","repo_kind":"listed","path":"generate_captions.py","file_url":"https://github.com/ChiZhangRIT/BNRHN/blob/HEAD/generate_captions.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7feab14292d065d3"}},{"code_sha256_prefix":"f44861975300132f","entry":"parse_sequence_example","repo":"brandontrabucco/im2txt_express","repo_kind":"listed","path":"im2txt_express/ops/inputs.py","file_url":"https://github.com/brandontrabucco/im2txt_express/blob/HEAD/im2txt_express/ops/inputs.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f44861975300132f"}},{"code_sha256_prefix":"603dfaf835c546b7","entry":"parse_sequence_example","repo":"dgonzalez-ri/neural-visual-storyteller","repo_kind":"listed","path":"inputs.py","file_url":"https://github.com/dgonzalez-ri/neural-visual-storyteller/blob/HEAD/inputs.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"603dfaf835c546b7"}},{"code_sha256_prefix":"7fea30b908ef4610","entry":"prefetch_input_data","repo":"brandontrabucco/im2txt_express","repo_kind":"listed","path":"im2txt_express/ops/inputs.py","file_url":"https://github.com/brandontrabucco/im2txt_express/blob/HEAD/im2txt_express/ops/inputs.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7fea30b908ef4610"}},{"code_sha256_prefix":"2972f031ce3b83bd","entry":"prefetch_input_data","repo":"dgonzalez-ri/neural-visual-storyteller","repo_kind":"listed","path":"inputs.py","file_url":"https://github.com/dgonzalez-ri/neural-visual-storyteller/blob/HEAD/inputs.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2972f031ce3b83bd"}},{"code_sha256_prefix":"00dd21191623ca8c","entry":"process_image","repo":"brandontrabucco/im2txt_express","repo_kind":"listed","path":"im2txt_express/ops/image_processing.py","file_url":"https://github.com/brandontrabucco/im2txt_express/blob/HEAD/im2txt_express/ops/image_processing.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"00dd21191623ca8c"}},{"code_sha256_prefix":"6a98e984239dce16","entry":"train","repo":"ag169/Image-Captioning","repo_kind":"listed","path":"train_caption.py","file_url":"https://github.com/ag169/Image-Captioning/blob/HEAD/train_caption.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6a98e984239dce16"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}