{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/dopanim-a-dataset-of-doppelganger-animals","title":"dopanim: A Dataset of Doppelganger Animals with Noisy Annotations from Multiple Humans","arxiv_id":"2407.20950","date":"2024-07-30","proceeding":null,"authors":["Marek Herde","Denis Huseljic","Lukas Rauch","Bernhard Sick"],"abstract":"Human annotators typically provide annotated data for training machine learning models, such as neural networks. Yet, human annotations are subject to noise, impairing generalization performances. Methodological research on approaches counteracting noisy annotations requires corresponding datasets for a meaningful empirical evaluation. Consequently, we introduce a novel benchmark dataset, dopanim, consisting of about 15,750 animal images of 15 classes with ground truth labels. For approximately 10,500 of these images, 20 humans provided over 52,000 annotations with an accuracy of circa 67%. Its key attributes include (1) the challenging task of classifying doppelganger animals, (2) human-estimated likelihoods as annotations, and (3) annotator metadata. We benchmark well-known multi-annotator learning approaches using seven variants of this dataset and outline further evaluation use cases such as learning beyond hard class labels and active learning. Our dataset and a comprehensive codebase are publicly available to emulate the data collection process and to reproduce all empirical results.","url_abs":"https://arxiv.org/abs/2407.20950v1","url_pdf":"https://arxiv.org/pdf/2407.20950v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"dopanim-a-dataset-of-doppelganger-animals","repo_url":"https://github.com/ies-research/multi-annotator-machine-learning","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"BSD-3-Clause"}}],"tasks":[{"task_slug":"active-learning","task_name":"Active Learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2407.20950","atlas_url":"https://app.syntology.ai/?focus=2407.20950","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.20950"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ies-research/multi-annotator-machine-learning","reach":{"status":"ok","spdx":"BSD-3-Clause"}}],"summary":{"ran":4,"unverified":1},"by_repo_kind":{"official":{"samples":5,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"03ad90af5feea8a9","entry":"gt_resnet","repo":"ies-research/multi-annotator-machine-learning","repo_kind":"official","path":"maml/architectures/_resnet.py","file_url":"https://github.com/ies-research/multi-annotator-machine-learning/blob/HEAD/maml/architectures/_resnet.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"03ad90af5feea8a9"}},{"code_sha256_prefix":"96bac99626b81250","entry":"gt_tabnet","repo":"ies-research/multi-annotator-machine-learning","repo_kind":"official","path":"maml/architectures/_tabnet.py","file_url":"https://github.com/ies-research/multi-annotator-machine-learning/blob/HEAD/maml/architectures/_tabnet.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"96bac99626b81250"}},{"code_sha256_prefix":"c109e7d9b73598a7","entry":"insert_missing_annotations","repo":"ies-research/multi-annotator-machine-learning","repo_kind":"official","path":"maml/utils/_annotator_simulation.py","file_url":"https://github.com/ies-research/multi-annotator-machine-learning/blob/HEAD/maml/utils/_annotator_simulation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"c109e7d9b73598a7"}},{"code_sha256_prefix":"b1f376b8e90d6a45","entry":"multisample_from_probs","repo":"ies-research/multi-annotator-machine-learning","repo_kind":"official","path":"maml/utils/_annotator_simulation.py","file_url":"https://github.com/ies-research/multi-annotator-machine-learning/blob/HEAD/maml/utils/_annotator_simulation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"b1f376b8e90d6a45"}},{"code_sha256_prefix":"fb243cfdb634f32d","entry":"simulate_annotator_classifiers","repo":"ies-research/multi-annotator-machine-learning","repo_kind":"official","path":"maml/utils/_annotator_simulation.py","file_url":"https://github.com/ies-research/multi-annotator-machine-learning/blob/HEAD/maml/utils/_annotator_simulation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"fb243cfdb634f32d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}