{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2603-26791","title":"Crystal: Characterizing Relative Impact of Scholarly Publications","arxiv_id":"2603.26791","date":"2026-03-25","proceeding":null,"authors":["Hannah Collison","Benjamin Van Durme","Daniel Khashabi"],"abstract":"Assessing a cited paper's impact is typically done by analyzing its citation context in isolation within the citing paper. While this focuses on the most directly relevant text, it prevents relative comparisons across all the works a paper cites. We propose Crystal, which instead jointly ranks all cited papers within a citing paper using large language models (LLMs). To mitigate LLMs' positional bias, we rank each list three times in a randomized order and aggregate the impact labels through majority voting. This joint approach leverages the full citation context, rather than evaluating citations independently, to more reliably distinguish impactful references. Crystal outperforms a prior state-of-the-art impact classifier by +9.5% accuracy and +8.3% F1 on a dataset of human-annotated citations. Crystal further gains efficiency through fewer LLM calls and outperforms prior baselines using an open-weight model, enabling scalable, cost-effective citation impact analysis. In a case study of ACL Test-of-Time award-winning papers, we find that Crystal's impact characterizations align closely with long-term scientific recognition. We release Crystal-Bank, a 46.8k-paper dataset with rankings and impact labels, along with code.","url_abs":"https://arxiv.org/abs/2603.26791","url_pdf":"https://arxiv.org/pdf/2603.26791","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":null,"mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2603.26791"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/allenai/multicite","reach":null}],"summary":{"ran_draft_wrong":1,"ran_honours":1,"unverified":10},"by_repo_kind":{"found_in_text":{"samples":12,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":12,"samples":[{"code_sha256_prefix":"d6eb7fc276a790c2","entry":"normalize_answer","repo":"allenai/multicite","repo_kind":"found_in_text","path":"qa/qasper_evaluator.py","file_url":"https://github.com/allenai/multicite/blob/HEAD/qa/qasper_evaluator.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d6eb7fc276a790c2"}},{"code_sha256_prefix":"5f8f36a4b206aba8","entry":"token_f1_score","repo":"allenai/multicite","repo_kind":"found_in_text","path":"qa/qasper_evaluator.py","file_url":"https://github.com/allenai/multicite/blob/HEAD/qa/qasper_evaluator.py","link_basis":"plan_row","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5f8f36a4b206aba8"}},{"code_sha256_prefix":"7588d2b9e8ea0f11","entry":"compute_metrics","repo":"allenai/multicite","repo_kind":"found_in_text","path":"classification/run_citation_classification.py","file_url":"https://github.com/allenai/multicite/blob/HEAD/classification/run_citation_classification.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7588d2b9e8ea0f11"}},{"code_sha256_prefix":"72d1c6cc5d8438bf","entry":"evaluate_per_class","repo":"allenai/multicite","repo_kind":"found_in_text","path":"classification/multilabel_evaluation.py","file_url":"https://github.com/allenai/multicite/blob/HEAD/classification/multilabel_evaluation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"72d1c6cc5d8438bf"}},{"code_sha256_prefix":"0386a811be881625","entry":"evaluate_strict","repo":"allenai/multicite","repo_kind":"found_in_text","path":"classification/multilabel_evaluation.py","file_url":"https://github.com/allenai/multicite/blob/HEAD/classification/multilabel_evaluation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0386a811be881625"}},{"code_sha256_prefix":"5b166e3041050322","entry":"evaluate_weak","repo":"allenai/multicite","repo_kind":"found_in_text","path":"classification/multilabel_evaluation.py","file_url":"https://github.com/allenai/multicite/blob/HEAD/classification/multilabel_evaluation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5b166e3041050322"}},{"code_sha256_prefix":"020201c0401fb88e","entry":"load_context_file","repo":"allenai/multicite","repo_kind":"found_in_text","path":"qa/convert_ours_to_qa.py","file_url":"https://github.com/allenai/multicite/blob/HEAD/qa/convert_ours_to_qa.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"020201c0401fb88e"}},{"code_sha256_prefix":"3bb85cd6650872f2","entry":"load_data","repo":"allenai/multicite","repo_kind":"found_in_text","path":"qa/convert_ours_to_qa.py","file_url":"https://github.com/allenai/multicite/blob/HEAD/qa/convert_ours_to_qa.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3bb85cd6650872f2"}},{"code_sha256_prefix":"dbae70d77c504ac2","entry":"majority","repo":"allenai/multicite","repo_kind":"found_in_text","path":"qa/eval_qa.py","file_url":"https://github.com/allenai/multicite/blob/HEAD/qa/eval_qa.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"dbae70d77c504ac2"}},{"code_sha256_prefix":"d612d80a2f1ae3e5","entry":"majority_sl","repo":"allenai/multicite","repo_kind":"found_in_text","path":"qa/eval_qa.py","file_url":"https://github.com/allenai/multicite/blob/HEAD/qa/eval_qa.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d612d80a2f1ae3e5"}},{"code_sha256_prefix":"d40d49e176b4ed43","entry":"paragraph_f1_score_wo_ground_truth","repo":"allenai/multicite","repo_kind":"found_in_text","path":"qa/qasper_evaluator.py","file_url":"https://github.com/allenai/multicite/blob/HEAD/qa/qasper_evaluator.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d40d49e176b4ed43"}},{"code_sha256_prefix":"f6314153726dce6b","entry":"replace_markup","repo":"allenai/multicite","repo_kind":"found_in_text","path":"qa/convert_ours_to_qa.py","file_url":"https://github.com/allenai/multicite/blob/HEAD/qa/convert_ours_to_qa.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f6314153726dce6b"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.CL","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}