{"url":"/dataset/carb","name":"CaRB","full_name":"Crowdsourced automatic open Relation extraction Benchmark","description_markdown":"CaRB [Bhardwaj et al., 2019] is developed by re-annotating the dev and test splits of OIE2016 via crowd-sourcing. Besides improving annotation quality, CaRB also provides a new matching scorer. CaRB scorer uses token level match and it matches relation with relation, arguments with arguments.\r\n\r\nSource: https://arxiv.org/pdf/2205.11725.pdf (section 3.1)","description_withheld":null,"homepage":"https://github.com/dair-iitd/CaRB","introduced_date":"2019-11-01","introduced_date_note":null,"introduced_by":{"paper":"/paper/carb-a-crowdsourced-benchmark-for-open-ie","title":"CaRB: A Crowdsourced Benchmark for Open IE","first_author":"Sangnie Bhardwaj","url":null},"license":null,"modalities":[],"tasks":[{"name":"Open Information Extraction","url":"/task/open-information-extraction","datasets_with_task":"/datasets/task/open-information-extraction"}],"languages":[],"variants":["CaRB"],"data_loaders":[],"num_papers_in_archive":35,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[{"leaderboard":"/sota/open-information-extraction-on-carb","task":"Open Information Extraction","dataset_variant":"CaRB","rows":29,"metrics":["F1"],"first_row_in_archive_order":{"model":"MacroIE","paper":"/paper/a-survey-on-neural-open-information","metrics":{"F1":"54.8"},"code_links":[]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"}],"papers_with_a_benchmark_row":[{"paper":"/paper/improving-open-information-extraction-with","title":"Improving Open Information Extraction with Large Language Models: A Study on Demonstration Uncertainty","date":"2023-09-07","rows_on_this_dataset":3,"code_links":0,"syntology":null},{"paper":"/paper/a-survey-on-neural-open-information","title":"A Survey on Neural Open Information Extraction: Current Status and Future Directions","date":"2022-05-24","rows_on_this_dataset":10,"code_links":0,"syntology":null},{"paper":"/paper/openie6-iterative-grid-labeling-and","title":"OpenIE6: Iterative Grid Labeling and Coordination Analysis for Open Information Extraction","date":"2020-10-07","rows_on_this_dataset":10,"code_links":1,"syntology":null},{"paper":"/paper/multi-2oie-multilingual-open-information","title":"Multi$^2$OIE: Multilingual Open Information Extraction Based on Multi-Head Attention with BERT","date":"2020-09-17","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":7,"samples_ran":6,"samples_unverified":1,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/open-information-extraction-from-conjunctive","title":"Open Information Extraction from Conjunctive Sentences","date":"2018-08-01","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/demonyms-and-compound-relational-nouns-in","title":"Demonyms and Compound Relational Nouns in Nominal Open IE","date":"2016-06-01","rows_on_this_dataset":1,"code_links":2,"syntology":null},{"paper":"/paper/getting-more-out-of-syntax-with-props","title":"Getting More Out Of Syntax with PropS","date":"2016-03-04","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/open-language-learning-for-information","title":"Open Language Learning for Information Extraction","date":"2012-07-01","rows_on_this_dataset":1,"code_links":0,"syntology":null}],"syntology_totals":{"read_at":"2026-09-24T18:15:14+00:00","papers_with_samples":1,"samples_harvested":7,"samples_ran":6,"samples_unverified":1,"pointer_only_for_licence":0,"papers_with_no_sample_that_ran":0,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}