{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/supernotes-driving-consensus-in-crowd-sourced","title":"Supernotes: Driving Consensus in Crowd-Sourced Fact-Checking","arxiv_id":"2411.06116","date":"2024-11-09","proceeding":null,"authors":["Soham De","Michiel A. Bakker","Jay Baxter","Martin Saveski"],"abstract":"X's Community Notes, a crowd-sourced fact-checking system, allows users to annotate potentially misleading posts. Notes rated as helpful by a diverse set of users are prominently displayed below the original post. While demonstrably effective at reducing misinformation's impact when notes are displayed, there is an opportunity for notes to appear on many more posts: for 91% of posts where at least one note is proposed, no notes ultimately achieve sufficient support from diverse users to be shown on the platform. This motivates the development of Supernotes: AI-generated notes that synthesize information from several existing community notes and are written to foster consensus among a diverse set of users. Our framework uses an LLM to generate many diverse Supernote candidates from existing proposed notes. These candidates are then evaluated by a novel scoring model, trained on millions of historical Community Notes ratings, selecting candidates that are most likely to be rated helpful by a diverse set of users. To test our framework, we ran a human subjects experiment in which we asked participants to compare the Supernotes generated by our framework to the best existing community notes for 100 sample posts. We found that participants rated the Supernotes as significantly more helpful, and when asked to choose between the two, preferred the Supernotes 75.2% of the time. Participants also rated the Supernotes more favorably than the best existing notes on quality, clarity, coverage, context, and argumentativeness. Finally, in a follow-up experiment, we asked participants to compare the Supernotes against LLM-generated summaries and found that the participants rated the Supernotes significantly more helpful, demonstrating that both the LLM-based candidate generation and the consensus-driven scoring play crucial roles in creating notes that effectively build consensus among diverse users.","url_abs":"https://arxiv.org/abs/2411.06116v1","url_pdf":"https://arxiv.org/pdf/2411.06116v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"links_only","authors_date_abstract":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license), from the Kaggle arXiv metadata snapshot of 2026-09-12"},"code_links":[{"paper_slug":"supernotes-driving-consensus-in-crowd-sourced","repo_url":"https://github.com/saveski-lab/supernotes","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok","spdx":"CC-BY-4.0"}}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2411.06116","atlas_url":"https://app.syntology.ai/?focus=2411.06116","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.06116"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/saveski-lab/supernotes","reach":{"status":"ok","spdx":"CC-BY-4.0"}}],"summary":{"unverified":9},"by_repo_kind":{"official":{"samples":9,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":9,"samples":[{"code_sha256_prefix":"aeb810e59fc176ec","entry":"aggregate","repo":"saveski-lab/supernotes","repo_kind":"official","path":"supernotes/aggregator.py","file_url":"https://github.com/saveski-lab/supernotes/blob/HEAD/supernotes/aggregator.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"CC-BY-4.0","inline_ok":false,"mcp_get_code":{"code_sha256":"aeb810e59fc176ec"}},{"code_sha256_prefix":"3d4e78f506db8e4e","entry":"lenCheck","repo":"saveski-lab/supernotes","repo_kind":"official","path":"supernotes/evaluator.py","file_url":"https://github.com/saveski-lab/supernotes/blob/HEAD/supernotes/evaluator.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"CC-BY-4.0","inline_ok":false,"mcp_get_code":{"code_sha256":"3d4e78f506db8e4e"}},{"code_sha256_prefix":"bdc80dcc3209d3e5","entry":"linkCheck","repo":"saveski-lab/supernotes","repo_kind":"official","path":"supernotes/evaluator.py","file_url":"https://github.com/saveski-lab/supernotes/blob/HEAD/supernotes/evaluator.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"CC-BY-4.0","inline_ok":false,"mcp_get_code":{"code_sha256":"bdc80dcc3209d3e5"}},{"code_sha256_prefix":"c2ffbd1a1e08f626","entry":"predict_rating","repo":"saveski-lab/supernotes","repo_kind":"official","path":"supernotes/phm.py","file_url":"https://github.com/saveski-lab/supernotes/blob/HEAD/supernotes/phm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"CC-BY-4.0","inline_ok":false,"mcp_get_code":{"code_sha256":"c2ffbd1a1e08f626"}},{"code_sha256_prefix":"d97066a82ef137e9","entry":"predict_rating_sample","repo":"saveski-lab/supernotes","repo_kind":"official","path":"supernotes/phm.py","file_url":"https://github.com/saveski-lab/supernotes/blob/HEAD/supernotes/phm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"CC-BY-4.0","inline_ok":false,"mcp_get_code":{"code_sha256":"d97066a82ef137e9"}},{"code_sha256_prefix":"e1c7311abdf1e558","entry":"predict_rating_sample_argmax","repo":"saveski-lab/supernotes","repo_kind":"official","path":"supernotes/phm.py","file_url":"https://github.com/saveski-lab/supernotes/blob/HEAD/supernotes/phm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"CC-BY-4.0","inline_ok":false,"mcp_get_code":{"code_sha256":"e1c7311abdf1e558"}},{"code_sha256_prefix":"2f4e81bcf059f275","entry":"prompt_generator","repo":"saveski-lab/supernotes","repo_kind":"official","path":"supernotes/summarizer.py","file_url":"https://github.com/saveski-lab/supernotes/blob/HEAD/supernotes/summarizer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"CC-BY-4.0","inline_ok":false,"mcp_get_code":{"code_sha256":"2f4e81bcf059f275"}},{"code_sha256_prefix":"17b27b7f71eb882f","entry":"prompt_generator_notweet","repo":"saveski-lab/supernotes","repo_kind":"official","path":"supernotes/summarizer.py","file_url":"https://github.com/saveski-lab/supernotes/blob/HEAD/supernotes/summarizer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"CC-BY-4.0","inline_ok":false,"mcp_get_code":{"code_sha256":"17b27b7f71eb882f"}},{"code_sha256_prefix":"01eb020ff2518d8d","entry":"sample_users","repo":"saveski-lab/supernotes","repo_kind":"official","path":"supernotes/aggregator.py","file_url":"https://github.com/saveski-lab/supernotes/blob/HEAD/supernotes/aggregator.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"CC-BY-4.0","inline_ok":false,"mcp_get_code":{"code_sha256":"01eb020ff2518d8d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}