{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/dont-take-the-easy-way-out-ensemble-based","title":"Don't Take the Easy Way Out: Ensemble Based Methods for Avoiding Known Dataset Biases","arxiv_id":"1909.03683","date":"2019-09-09","proceeding":"IJCNLP 2019 11","authors":["Christopher Clark","Mark Yatskar","Luke Zettlemoyer"],"abstract":"State-of-the-art models often make use of superficial patterns in the data that do not generalize well to out-of-domain or adversarial settings. For example, textual entailment models often learn that particular key words imply entailment, irrespective of context, and visual question answering models learn to predict prototypical answers, without considering evidence in the image. In this paper, we show that if we have prior knowledge of such biases, we can train a model to be more robust to domain shift. Our method has two stages: we (1) train a naive model that makes predictions exclusively based on dataset biases, and (2) train a robust model as part of an ensemble with the naive one in order to encourage it to focus on other patterns in the data that are more likely to generalize. Experiments on five datasets with out-of-domain test sets show significantly improved robustness in all settings, including a 12 point gain on a changing priors visual question answering dataset and a 9 point gain on an adversarial question answering test set.","url_abs":"https://arxiv.org/abs/1909.03683v1","url_pdf":"https://arxiv.org/pdf/1909.03683v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"dont-take-the-easy-way-out-ensemble-based","repo_url":"https://github.com/chrisc36/debias","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"dont-take-the-easy-way-out-ensemble-based","repo_url":"https://github.com/chrisc36/bottom-up-attention-vqa","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"dont-take-the-easy-way-out-ensemble-based","repo_url":"https://github.com/itaigat/removing-bias-in-multi-modal-classifiers","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"natural-language-inference","task_name":"Natural Language Inference"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/visual-question-answering-on-vqa-cp","task":"Visual Question Answering (VQA)","dataset":"VQA-CP","model":"Learned-Mixin +H","rank_in_archive_order":5,"of":10,"metrics":{"Score":"52.05"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/1909.03683","atlas_url":"https://app.syntology.ai/?focus=1909.03683","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1909.03683"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/itaigat/removing-bias-in-multi-modal-classifiers","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/chrisc36/bottom-up-attention-vqa","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/chrisc36/debias","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_draft_wrong":3,"unverified":3},"by_repo_kind":{"official":{"samples":3,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":3,"samples":[{"code_sha256_prefix":"4bd23f3562cfbd1c","entry":"elementwise_logsumexp","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_draft_wrong","verification_level":2,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"4bd23f3562cfbd1c"}},{"code_sha256_prefix":"0e2c50fe59debfb7","entry":"convert_sigmoid_logits_to_binary_logprobs","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"0e2c50fe59debfb7"}},{"code_sha256_prefix":"21690a7e909f0f41","entry":"renormalize_binary_logits","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"21690a7e909f0f41"}},{"code_sha256_prefix":"3a05eec27da874b5","entry":"compute_attention_mask","repo":"chrisc36/debias","repo_kind":"official","path":"debias/modules/attention_layers.py","file_url":"https://github.com/chrisc36/debias/blob/HEAD/debias/modules/attention_layers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3a05eec27da874b5"}},{"code_sha256_prefix":"c8984e69674b1a0b","entry":"word_to_char_ids","repo":"chrisc36/debias","repo_kind":"official","path":"debias/modules/char_encoder.py","file_url":"https://github.com/chrisc36/debias/blob/HEAD/debias/modules/char_encoder.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c8984e69674b1a0b"}},{"code_sha256_prefix":"5defc61880c73860","entry":"words_to_char_ids_py","repo":"chrisc36/debias","repo_kind":"official","path":"debias/modules/char_encoder.py","file_url":"https://github.com/chrisc36/debias/blob/HEAD/debias/modules/char_encoder.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5defc61880c73860"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}