{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/oscar-object-semantics-aligned-pre-training","title":"Oscar: Object-Semantics Aligned Pre-training for Vision-Language Tasks","arxiv_id":"2004.06165","date":"2020-04-13","proceeding":"ECCV 2020 8","authors":["Xiujun Li","Xi Yin","Chunyuan Li","Pengchuan Zhang","Xiao-Wei Hu","Lei Zhang","Lijuan Wang","Houdong Hu","Li Dong","Furu Wei","Yejin Choi","Jianfeng Gao"],"abstract":"Large-scale pre-training methods of learning cross-modal representations on image-text pairs are becoming popular for vision-language tasks. While existing methods simply concatenate image region features and text features as input to the model to be pre-trained and use self-attention to learn image-text semantic alignments in a brute force manner, in this paper, we propose a new learning method Oscar (Object-Semantics Aligned Pre-training), which uses object tags detected in images as anchor points to significantly ease the learning of alignments. Our method is motivated by the observation that the salient objects in an image can be accurately detected, and are often mentioned in the paired text. We pre-train an Oscar model on the public corpus of 6.5 million text-image pairs, and fine-tune it on downstream tasks, creating new state-of-the-arts on six well-established vision-language understanding and generation tasks.","url_abs":"https://arxiv.org/abs/2004.06165v5","url_pdf":"https://arxiv.org/pdf/2004.06165v5.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"oscar-object-semantics-aligned-pre-training","repo_url":"https://github.com/microsoft/Oscar","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"oscar-object-semantics-aligned-pre-training","repo_url":"https://github.com/ThanThoai/Visual-Question-Answering_Vietnamese","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"oscar-object-semantics-aligned-pre-training","repo_url":"https://github.com/milvlg/rosita","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"oscar-object-semantics-aligned-pre-training","repo_url":"https://github.com/rmokady/clip_prefix_caption","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"cross-modal-retrieval","task_name":"Cross-Modal Retrieval"},{"task_slug":"image-captioning","task_name":"Image Captioning"},{"task_slug":"image-retrieval","task_name":"Image Retrieval"},{"task_slug":"image-text-matching","task_name":"Image-text matching"},{"task_slug":"image-to-text-retrieval","task_name":"Image-to-Text Retrieval"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"}],"methods":[{"method_slug":"oscar","method_name":"OSCAR"}],"datasets_introduced":[],"methods_introduced":[{"slug":"oscar","name":"OSCAR","full_name":"OSCAR"}],"results":[{"leaderboard":"/sota/cross-modal-retrieval-on-coco-2014","task":"Cross-Modal Retrieval","dataset":"COCO 2014","model":"Oscar","rank_in_archive_order":19,"of":36,"metrics":{"Image-to-text R@1":"73.5","Image-to-text R@10":"96.0","Image-to-text R@5":"92.2","Text-to-image R@1":"57.5","Text-to-image R@10":"89.8","Text-to-image R@5":"82.8"},"uses_additional_data":true},{"leaderboard":"/sota/image-captioning-on-coco-captions","task":"Image Captioning","dataset":"COCO Captions","model":"Oscar","rank_in_archive_order":11,"of":41,"metrics":{"BLEU-4":"41.7","CIDER":"140","METEOR":"30.6","SPICE":"24.5"},"uses_additional_data":false},{"leaderboard":"/sota/image-captioning-on-nocaps-val-overall","task":"Image Captioning","dataset":"nocaps-val-overall","model":"OSCAR","rank_in_archive_order":11,"of":11,"metrics":{"CIDEr":"80.9","Pretrain (#images)":"345M","SPICE":"11.3"},"uses_additional_data":false},{"leaderboard":"/sota/image-retrieval-on-coco","task":"Image Retrieval","dataset":"COCO (Common Objects in Context)","model":"Oscar","rank_in_archive_order":6,"of":6,"metrics":{"Recall@10":"98.3"},"uses_additional_data":false},{"leaderboard":"/sota/image-text-matching-on-commercialadsdataset","task":"Image-text matching","dataset":"CommercialAdsDataset","model":"OSCAR","rank_in_archive_order":4,"of":8,"metrics":{"ADD(S) AUC":"87.45"},"uses_additional_data":false},{"leaderboard":"/sota/image-to-text-retrieval-on-coco","task":"Image-to-Text Retrieval","dataset":"COCO (Common Objects in Context)","model":"Oscar","rank_in_archive_order":7,"of":9,"metrics":{"Recall@10":"99.8"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-vqa-v2-test-dev","task":"Visual Question Answering (VQA)","dataset":"VQA v2 test-dev","model":"Oscar","rank_in_archive_order":20,"of":56,"metrics":{"Accuracy":"73.82"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2004.06165","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.06165"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/microsoft/Oscar","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ThanThoai/Visual-Question-Answering_Vietnamese","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/milvlg/rosita","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/rmokady/clip_prefix_caption","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":7,"ran_honours":1,"ran_draft_wrong":3,"unverified":12},"by_repo_kind":{"official":{"samples":1,"ran":1,"repositories":1},"listed":{"samples":22,"ran":10,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"bb6ea058ef7c09e1","entry":"Backbone","repo":"milvlg/rosita","repo_kind":"listed","path":"rosita/modeling/pretrain_tasks/rosita.py","file_url":"https://github.com/milvlg/rosita/blob/HEAD/rosita/modeling/pretrain_tasks/rosita.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bb6ea058ef7c09e1"}},{"code_sha256_prefix":"1e4945df552f6045","entry":"FeedForward","repo":"milvlg/rosita","repo_kind":"listed","path":"rosita/modeling/pretrain_tasks/rosita.py","file_url":"https://github.com/milvlg/rosita/blob/HEAD/rosita/modeling/pretrain_tasks/rosita.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1e4945df552f6045"}},{"code_sha256_prefix":"fcab94bf5f3f29d5","entry":"LayerNorm","repo":"milvlg/rosita","repo_kind":"listed","path":"rosita/modeling/pretrain_tasks/rosita.py","file_url":"https://github.com/milvlg/rosita/blob/HEAD/rosita/modeling/pretrain_tasks/rosita.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fcab94bf5f3f29d5"}},{"code_sha256_prefix":"14d39fe285bd537f","entry":"Pooler","repo":"milvlg/rosita","repo_kind":"listed","path":"rosita/modeling/pretrain_tasks/rosita.py","file_url":"https://github.com/milvlg/rosita/blob/HEAD/rosita/modeling/pretrain_tasks/rosita.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"14d39fe285bd537f"}},{"code_sha256_prefix":"1e2c2b0e7e10968a","entry":"TextEmbeddings","repo":"milvlg/rosita","repo_kind":"listed","path":"rosita/modeling/pretrain_tasks/rosita.py","file_url":"https://github.com/milvlg/rosita/blob/HEAD/rosita/modeling/pretrain_tasks/rosita.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1e2c2b0e7e10968a"}},{"code_sha256_prefix":"b0d904c1669cb4b0","entry":"VisualEmbeddings","repo":"milvlg/rosita","repo_kind":"listed","path":"rosita/modeling/pretrain_tasks/rosita.py","file_url":"https://github.com/milvlg/rosita/blob/HEAD/rosita/modeling/pretrain_tasks/rosita.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b0d904c1669cb4b0"}},{"code_sha256_prefix":"8d619e54294fb0b7","entry":"gelu","repo":"milvlg/rosita","repo_kind":"listed","path":"rosita/modeling/pretrain_tasks/rosita.py","file_url":"https://github.com/milvlg/rosita/blob/HEAD/rosita/modeling/pretrain_tasks/rosita.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8d619e54294fb0b7"}},{"code_sha256_prefix":"e17f1049b8148f71","entry":"make_data_sampler","repo":"microsoft/Oscar","repo_kind":"official","path":"oscar/run_captioning.py","file_url":"https://github.com/microsoft/Oscar/blob/HEAD/oscar/run_captioning.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e17f1049b8148f71"}},{"code_sha256_prefix":"e766a0a3063c15d0","entry":"make_mask","repo":"ThanThoai/Visual-Question-Answering_Vietnamese","repo_kind":"listed","path":"model.py","file_url":"https://github.com/ThanThoai/Visual-Question-Answering_Vietnamese/blob/HEAD/model.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e766a0a3063c15d0"}},{"code_sha256_prefix":"06b045032beeb885","entry":"optimal_transport_dist","repo":"milvlg/rosita","repo_kind":"listed","path":"rosita/modeling/pretrain_tasks/rosita.py","file_url":"https://github.com/milvlg/rosita/blob/HEAD/rosita/modeling/pretrain_tasks/rosita.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"06b045032beeb885"}},{"code_sha256_prefix":"412cdf6ed5178c2c","entry":"trace","repo":"milvlg/rosita","repo_kind":"listed","path":"rosita/modeling/pretrain_tasks/rosita.py","file_url":"https://github.com/milvlg/rosita/blob/HEAD/rosita/modeling/pretrain_tasks/rosita.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"412cdf6ed5178c2c"}},{"code_sha256_prefix":"01007aebf0b1b84a","entry":"ImgfeatHead","repo":"milvlg/rosita","repo_kind":"listed","path":"rosita/modeling/pretrain_tasks/rosita.py","file_url":"https://github.com/milvlg/rosita/blob/HEAD/rosita/modeling/pretrain_tasks/rosita.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"01007aebf0b1b84a"}},{"code_sha256_prefix":"b97a0ae0d5f606f5","entry":"MHAtt","repo":"milvlg/rosita","repo_kind":"listed","path":"rosita/modeling/pretrain_tasks/rosita.py","file_url":"https://github.com/milvlg/rosita/blob/HEAD/rosita/modeling/pretrain_tasks/rosita.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b97a0ae0d5f606f5"}},{"code_sha256_prefix":"8bc6b6ae220fc251","entry":"MmITMHead","repo":"milvlg/rosita","repo_kind":"listed","path":"rosita/modeling/pretrain_tasks/rosita.py","file_url":"https://github.com/milvlg/rosita/blob/HEAD/rosita/modeling/pretrain_tasks/rosita.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8bc6b6ae220fc251"}},{"code_sha256_prefix":"b90f9bcd35cb01ad","entry":"MmQAHead","repo":"milvlg/rosita","repo_kind":"listed","path":"rosita/modeling/pretrain_tasks/rosita.py","file_url":"https://github.com/milvlg/rosita/blob/HEAD/rosita/modeling/pretrain_tasks/rosita.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b90f9bcd35cb01ad"}},{"code_sha256_prefix":"4fde87aac2867c6f","entry":"Net","repo":"milvlg/rosita","repo_kind":"listed","path":"rosita/modeling/pretrain_tasks/rosita.py","file_url":"https://github.com/milvlg/rosita/blob/HEAD/rosita/modeling/pretrain_tasks/rosita.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4fde87aac2867c6f"}},{"code_sha256_prefix":"00f7c9f9cc6a5223","entry":"SelfAtt","repo":"milvlg/rosita","repo_kind":"listed","path":"rosita/modeling/pretrain_tasks/rosita.py","file_url":"https://github.com/milvlg/rosita/blob/HEAD/rosita/modeling/pretrain_tasks/rosita.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"00f7c9f9cc6a5223"}},{"code_sha256_prefix":"5b65287416e63add","entry":"TextMLMHead","repo":"milvlg/rosita","repo_kind":"listed","path":"rosita/modeling/pretrain_tasks/rosita.py","file_url":"https://github.com/milvlg/rosita/blob/HEAD/rosita/modeling/pretrain_tasks/rosita.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5b65287416e63add"}},{"code_sha256_prefix":"5d4575f5eacfe1f9","entry":"get_data_en","repo":"ThanThoai/Visual-Question-Answering_Vietnamese","repo_kind":"listed","path":"merge_raw_data.py","file_url":"https://github.com/ThanThoai/Visual-Question-Answering_Vietnamese/blob/HEAD/merge_raw_data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5d4575f5eacfe1f9"}},{"code_sha256_prefix":"b9f7aa42c0afe2be","entry":"get_list_text","repo":"ThanThoai/Visual-Question-Answering_Vietnamese","repo_kind":"listed","path":"merge_raw_data.py","file_url":"https://github.com/ThanThoai/Visual-Question-Answering_Vietnamese/blob/HEAD/merge_raw_data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b9f7aa42c0afe2be"}},{"code_sha256_prefix":"e6bf616333cbcc3b","entry":"prepare_data","repo":"ThanThoai/Visual-Question-Answering_Vietnamese","repo_kind":"listed","path":"extract_image.py","file_url":"https://github.com/ThanThoai/Visual-Question-Answering_Vietnamese/blob/HEAD/extract_image.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e6bf616333cbcc3b"}},{"code_sha256_prefix":"ecf115f3542eefd4","entry":"read_json","repo":"ThanThoai/Visual-Question-Answering_Vietnamese","repo_kind":"listed","path":"merge_raw_data.py","file_url":"https://github.com/ThanThoai/Visual-Question-Answering_Vietnamese/blob/HEAD/merge_raw_data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ecf115f3542eefd4"}},{"code_sha256_prefix":"bdc984dffbfed9a3","entry":"train_model","repo":"ThanThoai/Visual-Question-Answering_Vietnamese","repo_kind":"listed","path":"extract_image.py","file_url":"https://github.com/ThanThoai/Visual-Question-Answering_Vietnamese/blob/HEAD/extract_image.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bdc984dffbfed9a3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}