{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/image-to-markup-generation-with-coarse-to","title":"Image-to-Markup Generation with Coarse-to-Fine Attention","arxiv_id":"1609.04938","date":"2016-09-16","proceeding":"ICML 2017 8","authors":["Yuntian Deng","Anssi Kanervisto","Jeffrey Ling","Alexander M. Rush"],"abstract":"We present a neural encoder-decoder model to convert images into\npresentational markup based on a scalable coarse-to-fine attention mechanism.\nOur method is evaluated in the context of image-to-LaTeX generation, and we\nintroduce a new dataset of real-world rendered mathematical expressions paired\nwith LaTeX markup. We show that unlike neural OCR techniques using CTC-based\nmodels, attention-based approaches can tackle this non-standard OCR task. Our\napproach outperforms classical mathematical OCR systems by a large margin on\nin-domain rendered data, and, with pretraining, also performs well on\nout-of-domain handwritten data. To reduce the inference complexity associated\nwith the attention-based approaches, we introduce a new coarse-to-fine\nattention layer that selects a support region before applying attention.","url_abs":"http://arxiv.org/abs/1609.04938v2","url_pdf":"http://arxiv.org/pdf/1609.04938v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"image-to-markup-generation-with-coarse-to","repo_url":"https://github.com/harvardnlp/im2markup","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"torch","reach":null},{"paper_slug":"image-to-markup-generation-with-coarse-to","repo_url":"https://github.com/BachoSeven/stellestelline","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"image-to-markup-generation-with-coarse-to","repo_url":"https://github.com/CT2001/image-to-latex","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"image-to-markup-generation-with-coarse-to","repo_url":"https://github.com/CT2001/image-to-latex-biology","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"image-to-markup-generation-with-coarse-to","repo_url":"https://github.com/aspnetcs/myim2latex-tensorflow-docker","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"image-to-markup-generation-with-coarse-to","repo_url":"https://github.com/da03/Attention-OCR","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":null},{"paper_slug":"image-to-markup-generation-with-coarse-to","repo_url":"https://github.com/jvpoulos/Attention-OCR","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"image-to-markup-generation-with-coarse-to","repo_url":"https://github.com/kingyiusuen/image-to-latex","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"image-to-markup-generation-with-coarse-to","repo_url":"https://github.com/lukas-blecher/LaTeX-OCR","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"image-to-markup-generation-with-coarse-to","repo_url":"https://github.com/lukecage0/implementation-of-Im2Latex","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"image-to-markup-generation-with-coarse-to","repo_url":"https://github.com/opennmt/im2text","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"image-to-markup-generation-with-coarse-to","repo_url":"https://github.com/LinXueyuanStdio/LaTeX_OCR_PRO","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"tf","reach":{"status":"ok","spdx":"GPL-3.0"}},{"paper_slug":"image-to-markup-generation-with-coarse-to","repo_url":"https://github.com/tuanio/image2latex","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"image-to-markup-generation-with-coarse-to","repo_url":"https://github.com/untrix/im2latex","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"tf","reach":{"status":"ok","spdx":"AGPL-3.0"}}],"tasks":[{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"optical-character-recognition","task_name":"Optical Character Recognition (OCR)"}],"methods":[],"datasets_introduced":[{"slug":"im2latex-100k","name":"im2latex-100k","full_name":""}],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/1609.04938","atlas_url":"https://app.syntology.ai/?focus=1609.04938","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1609.04938"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lukecage0/implementation-of-Im2Latex","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/CT2001/image-to-latex","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/CT2001/image-to-latex-biology","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/aspnetcs/myim2latex-tensorflow-docker","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/BachoSeven/stellestelline","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/da03/Attention-OCR","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/opennmt/im2text","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/LinXueyuanStdio/LaTeX_OCR_PRO","reach":{"status":"ok","spdx":"GPL-3.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jvpoulos/Attention-OCR","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/harvardnlp/im2markup","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/untrix/im2latex","reach":{"status":"ok","spdx":"AGPL-3.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kingyiusuen/image-to-latex","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/tuanio/image2latex","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lukas-blecher/LaTeX-OCR","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":1,"unverified":12},"by_repo_kind":{"official":{"samples":1,"ran":0,"repositories":1},"listed":{"samples":12,"ran":1,"repositories":3}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"5f977334a9330cac","entry":"process_args","repo":"da03/Attention-OCR","repo_kind":"listed","path":"src/launcher.py","file_url":"https://github.com/da03/Attention-OCR/blob/HEAD/src/launcher.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5f977334a9330cac"}},{"code_sha256_prefix":"5bfcddf02c8fcf58","entry":"FMM_func","repo":"CT2001/image-to-latex","repo_kind":"listed","path":"data-preprocess/no_chinese.py","file_url":"https://github.com/CT2001/image-to-latex/blob/HEAD/data-preprocess/no_chinese.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5bfcddf02c8fcf58"}},{"code_sha256_prefix":"d3fb41b62eaedad7","entry":"find_first","repo":"CT2001/image-to-latex","repo_kind":"listed","path":"image_to_latex/models/resnet_transformer.py","file_url":"https://github.com/CT2001/image-to-latex/blob/HEAD/image_to_latex/models/resnet_transformer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d3fb41b62eaedad7"}},{"code_sha256_prefix":"9fae5025ebafa98e","entry":"generate_square_subsequent_mask","repo":"CT2001/image-to-latex","repo_kind":"listed","path":"image_to_latex/models/resnet_transformer.py","file_url":"https://github.com/CT2001/image-to-latex/blob/HEAD/image_to_latex/models/resnet_transformer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9fae5025ebafa98e"}},{"code_sha256_prefix":"ec532e8f70289f38","entry":"get_wmt_enfr_dev_set","repo":"jvpoulos/Attention-OCR","repo_kind":"listed","path":"src/model/data_utils.py","file_url":"https://github.com/jvpoulos/Attention-OCR/blob/HEAD/src/model/data_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ec532e8f70289f38"}},{"code_sha256_prefix":"0a7b6ca2e2bf4659","entry":"get_wmt_enfr_train_set","repo":"jvpoulos/Attention-OCR","repo_kind":"listed","path":"src/model/data_utils.py","file_url":"https://github.com/jvpoulos/Attention-OCR/blob/HEAD/src/model/data_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0a7b6ca2e2bf4659"}},{"code_sha256_prefix":"3ca4a31edc44c233","entry":"max_2x1pool","repo":"jvpoulos/Attention-OCR","repo_kind":"listed","path":"src/model/cnn.py","file_url":"https://github.com/jvpoulos/Attention-OCR/blob/HEAD/src/model/cnn.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3ca4a31edc44c233"}},{"code_sha256_prefix":"32711cdafa63eda4","entry":"max_2x2pool","repo":"jvpoulos/Attention-OCR","repo_kind":"listed","path":"src/model/cnn.py","file_url":"https://github.com/jvpoulos/Attention-OCR/blob/HEAD/src/model/cnn.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"32711cdafa63eda4"}},{"code_sha256_prefix":"43ea785c6d00f3e4","entry":"maybe_download","repo":"jvpoulos/Attention-OCR","repo_kind":"listed","path":"src/model/data_utils.py","file_url":"https://github.com/jvpoulos/Attention-OCR/blob/HEAD/src/model/data_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"43ea785c6d00f3e4"}},{"code_sha256_prefix":"ae08cb323bcbd595","entry":"process_args","repo":"harvardnlp/im2markup","repo_kind":"official","path":"scripts/preprocessing/preprocess_images.py","file_url":"https://github.com/harvardnlp/im2markup/blob/HEAD/scripts/preprocessing/preprocess_images.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ae08cb323bcbd595"}},{"code_sha256_prefix":"e1ee189649199b8f","entry":"process_args","repo":"jvpoulos/Attention-OCR","repo_kind":"listed","path":"src/launcher.py","file_url":"https://github.com/jvpoulos/Attention-OCR/blob/HEAD/src/launcher.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e1ee189649199b8f"}},{"code_sha256_prefix":"f44847e8b1c4bd91","entry":"rnn_decoder","repo":"jvpoulos/Attention-OCR","repo_kind":"listed","path":"src/model/seq2seq.py","file_url":"https://github.com/jvpoulos/Attention-OCR/blob/HEAD/src/model/seq2seq.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f44847e8b1c4bd91"}},{"code_sha256_prefix":"213cd44559181629","entry":"var_random","repo":"jvpoulos/Attention-OCR","repo_kind":"listed","path":"src/model/cnn.py","file_url":"https://github.com/jvpoulos/Attention-OCR/blob/HEAD/src/model/cnn.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"213cd44559181629"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}