{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/umic-an-unreferenced-metric-for-image","title":"UMIC: An Unreferenced Metric for Image Captioning via Contrastive Learning","arxiv_id":"2106.14019","date":"2021-06-26","proceeding":"ACL 2021 5","authors":["Hwanhee Lee","Seunghyun Yoon","Franck Dernoncourt","Trung Bui","Kyomin Jung"],"abstract":"Despite the success of various text generation metrics such as BERTScore, it is still difficult to evaluate the image captions without enough reference captions due to the diversity of the descriptions. In this paper, we introduce a new metric UMIC, an Unreferenced Metric for Image Captioning which does not require reference captions to evaluate image captions. Based on Vision-and-Language BERT, we train UMIC to discriminate negative captions via contrastive learning. Also, we observe critical problems of the previous benchmark dataset (i.e., human annotations) on image captioning metric, and introduce a new collection of human annotations on the generated captions. We validate UMIC on four datasets, including our new dataset, and show that UMIC has a higher correlation than all previous metrics that require multiple references. We release the benchmark dataset and pre-trained models to compute the UMIC.","url_abs":"https://arxiv.org/abs/2106.14019v1","url_pdf":"https://arxiv.org/pdf/2106.14019v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"umic-an-unreferenced-metric-for-image","repo_url":"https://github.com/hwanheelee1993/UMIC","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"contrastive-learning","task_name":"Contrastive Learning"},{"task_slug":"diversity","task_name":"Diversity"},{"task_slug":"image-captioning","task_name":"Image Captioning"},{"task_slug":"text-generation","task_name":"Text Generation"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2106.14019","atlas_url":"https://app.syntology.ai/?focus=2106.14019","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.14019"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/hwanheelee1993/UMIC","reach":null}],"summary":{"ran":6,"ran_honours":1,"unverified":7},"by_repo_kind":{"official":{"samples":14,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"eb099be49425d23f","entry":"BertAttention","repo":"hwanheelee1993/UMIC","repo_kind":"official","path":"model/ce.py","file_url":"https://github.com/hwanheelee1993/UMIC/blob/HEAD/model/ce.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"eb099be49425d23f"}},{"code_sha256_prefix":"f7d5bced2dfc431a","entry":"BertIntermediate","repo":"hwanheelee1993/UMIC","repo_kind":"official","path":"model/ce.py","file_url":"https://github.com/hwanheelee1993/UMIC/blob/HEAD/model/ce.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f7d5bced2dfc431a"}},{"code_sha256_prefix":"97991514f03a073e","entry":"BertOutput","repo":"hwanheelee1993/UMIC","repo_kind":"official","path":"model/ce.py","file_url":"https://github.com/hwanheelee1993/UMIC/blob/HEAD/model/ce.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"97991514f03a073e"}},{"code_sha256_prefix":"ef95c976ac657dd1","entry":"UniterConfig","repo":"hwanheelee1993/UMIC","repo_kind":"official","path":"model/ce.py","file_url":"https://github.com/hwanheelee1993/UMIC/blob/HEAD/model/ce.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ef95c976ac657dd1"}},{"code_sha256_prefix":"5795fe073f560736","entry":"UniterImageEmbeddings","repo":"hwanheelee1993/UMIC","repo_kind":"official","path":"model/ce.py","file_url":"https://github.com/hwanheelee1993/UMIC/blob/HEAD/model/ce.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5795fe073f560736"}},{"code_sha256_prefix":"bb1fe468c41db642","entry":"UniterTextEmbeddings","repo":"hwanheelee1993/UMIC","repo_kind":"official","path":"model/ce.py","file_url":"https://github.com/hwanheelee1993/UMIC/blob/HEAD/model/ce.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bb1fe468c41db642"}},{"code_sha256_prefix":"c2f0705ed8e6f7e6","entry":"gelu","repo":"hwanheelee1993/UMIC","repo_kind":"official","path":"model/ce.py","file_url":"https://github.com/hwanheelee1993/UMIC/blob/HEAD/model/ce.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c2f0705ed8e6f7e6"}},{"code_sha256_prefix":"5216511bc6458b3b","entry":"BertLayer","repo":"hwanheelee1993/UMIC","repo_kind":"official","path":"model/ce.py","file_url":"https://github.com/hwanheelee1993/UMIC/blob/HEAD/model/ce.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5216511bc6458b3b"}},{"code_sha256_prefix":"50a06ca454859f0c","entry":"BertSelfAttention","repo":"hwanheelee1993/UMIC","repo_kind":"official","path":"model/ce.py","file_url":"https://github.com/hwanheelee1993/UMIC/blob/HEAD/model/ce.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"50a06ca454859f0c"}},{"code_sha256_prefix":"10c57c28582d7a4d","entry":"BertSelfOutput","repo":"hwanheelee1993/UMIC","repo_kind":"official","path":"model/ce.py","file_url":"https://github.com/hwanheelee1993/UMIC/blob/HEAD/model/ce.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"10c57c28582d7a4d"}},{"code_sha256_prefix":"5b623a6c129b4007","entry":"UniterEncoder","repo":"hwanheelee1993/UMIC","repo_kind":"official","path":"model/ce.py","file_url":"https://github.com/hwanheelee1993/UMIC/blob/HEAD/model/ce.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5b623a6c129b4007"}},{"code_sha256_prefix":"913d7d2bb3aa0978","entry":"UniterForCaptioningMetric","repo":"hwanheelee1993/UMIC","repo_kind":"official","path":"model/ce.py","file_url":"https://github.com/hwanheelee1993/UMIC/blob/HEAD/model/ce.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"913d7d2bb3aa0978"}},{"code_sha256_prefix":"b9c1d1743cf759e5","entry":"UniterModel","repo":"hwanheelee1993/UMIC","repo_kind":"official","path":"model/ce.py","file_url":"https://github.com/hwanheelee1993/UMIC/blob/HEAD/model/ce.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b9c1d1743cf759e5"}},{"code_sha256_prefix":"24ede76832eaa584","entry":"UniterPreTrainedModel","repo":"hwanheelee1993/UMIC","repo_kind":"official","path":"model/ce.py","file_url":"https://github.com/hwanheelee1993/UMIC/blob/HEAD/model/ce.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"24ede76832eaa584"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}