{"url":"/sota/logical-reasoning-on-lingoly","task":{"name":"Logical Reasoning","url":"/task/logical-reasoning","note":null},"dataset":{"name":"LingOly","url":"/dataset/lingoly"},"category":"Miscellaneous","categories":["Methodology","Miscellaneous","Reasoning"],"category_note":null,"description":null,"description_from":null,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","rank":"the archive's row order at snapshot; not re-ranked","rows_end_at":"2025-07-28","rows_withheld_as_spam":0,"metric_values":"the archive's strings, untouched"},"metrics":["Delta_NoContext","Exact Match Accuracy"],"metric_direction":{"note":"inferred from the metric name only (the archive records no direction); null = not inferred, chart draws points only","by_metric":{"Delta_NoContext":null,"Exact Match Accuracy":"higher"}},"counts":{"rows":11,"rows_with_code":11,"rows_with_paper_page":11,"rows_dated":11,"rows_using_additional_data":0},"rows":[{"rank_in_archive_order":1,"model":"Claude Opus","metrics":{"Delta_NoContext":"28.8%","Exact Match Accuracy":"46.3%"},"uses_additional_data":false,"paper_date":"2024-06-10","paper":"/paper/lingoly-a-benchmark-of-olympiad-level","paper_url":"https://arxiv.org/abs/2406.06196v3","paper_title":"LINGOLY: A Benchmark of Olympiad-Level Linguistic Reasoning Puzzles in Low-Resource and Extinct Languages","code":"https://github.com/am-bean/lingOly","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":2}},{"rank_in_archive_order":2,"model":"GPT-4o","metrics":{"Delta_NoContext":"25.1%","Exact Match Accuracy":"37.6%"},"uses_additional_data":false,"paper_date":"2024-06-10","paper":"/paper/lingoly-a-benchmark-of-olympiad-level","paper_url":"https://arxiv.org/abs/2406.06196v3","paper_title":"LINGOLY: A Benchmark of Olympiad-Level Linguistic Reasoning Puzzles in Low-Resource and Extinct Languages","code":"https://github.com/am-bean/lingOly","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":2}},{"rank_in_archive_order":3,"model":"Gemini 1.5 Pro","metrics":{"Delta_NoContext":"23.4%","Exact Match Accuracy":"32.1%"},"uses_additional_data":false,"paper_date":"2024-06-10","paper":"/paper/lingoly-a-benchmark-of-olympiad-level","paper_url":"https://arxiv.org/abs/2406.06196v3","paper_title":"LINGOLY: A Benchmark of Olympiad-Level Linguistic Reasoning Puzzles in Low-Resource and Extinct Languages","code":"https://github.com/am-bean/lingOly","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":2}},{"rank_in_archive_order":4,"model":"GPT-4","metrics":{"Delta_NoContext":"21.5%","Exact Match Accuracy":"33.4%"},"uses_additional_data":false,"paper_date":"2024-06-10","paper":"/paper/lingoly-a-benchmark-of-olympiad-level","paper_url":"https://arxiv.org/abs/2406.06196v3","paper_title":"LINGOLY: A Benchmark of Olympiad-Level Linguistic Reasoning Puzzles in Low-Resource and Extinct Languages","code":"https://github.com/am-bean/lingOly","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":2}},{"rank_in_archive_order":5,"model":"Command R+","metrics":{"Delta_NoContext":"11.6%","Exact Match Accuracy":"21.5%"},"uses_additional_data":false,"paper_date":"2024-06-10","paper":"/paper/lingoly-a-benchmark-of-olympiad-level","paper_url":"https://arxiv.org/abs/2406.06196v3","paper_title":"LINGOLY: A Benchmark of Olympiad-Level Linguistic Reasoning Puzzles in Low-Resource and Extinct Languages","code":"https://github.com/am-bean/lingOly","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":2}},{"rank_in_archive_order":6,"model":"GPT-3.5","metrics":{"Delta_NoContext":"11.2%","Exact Match Accuracy":"21.2%"},"uses_additional_data":false,"paper_date":"2024-06-10","paper":"/paper/lingoly-a-benchmark-of-olympiad-level","paper_url":"https://arxiv.org/abs/2406.06196v3","paper_title":"LINGOLY: A Benchmark of Olympiad-Level Linguistic Reasoning Puzzles in Low-Resource and Extinct Languages","code":"https://github.com/am-bean/lingOly","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":2}},{"rank_in_archive_order":7,"model":"Mixtral 8x7B","metrics":{"Delta_NoContext":"6.4%","Exact Match Accuracy":"14.2%"},"uses_additional_data":false,"paper_date":"2024-06-10","paper":"/paper/lingoly-a-benchmark-of-olympiad-level","paper_url":"https://arxiv.org/abs/2406.06196v3","paper_title":"LINGOLY: A Benchmark of Olympiad-Level Linguistic Reasoning Puzzles in Low-Resource and Extinct Languages","code":"https://github.com/am-bean/lingOly","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":2}},{"rank_in_archive_order":8,"model":"Llama 3 8B","metrics":{"Delta_NoContext":"4.9%","Exact Match Accuracy":"11.4%"},"uses_additional_data":false,"paper_date":"2024-06-10","paper":"/paper/lingoly-a-benchmark-of-olympiad-level","paper_url":"https://arxiv.org/abs/2406.06196v3","paper_title":"LINGOLY: A Benchmark of Olympiad-Level Linguistic Reasoning Puzzles in Low-Resource and Extinct Languages","code":"https://github.com/am-bean/lingOly","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":2}},{"rank_in_archive_order":9,"model":"Llama 3 70B","metrics":{"Delta_NoContext":"2.9%","Exact Match Accuracy":"10.3%"},"uses_additional_data":false,"paper_date":"2024-06-10","paper":"/paper/lingoly-a-benchmark-of-olympiad-level","paper_url":"https://arxiv.org/abs/2406.06196v3","paper_title":"LINGOLY: A Benchmark of Olympiad-Level Linguistic Reasoning Puzzles in Low-Resource and Extinct Languages","code":"https://github.com/am-bean/lingOly","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":2}},{"rank_in_archive_order":10,"model":"Gemma 7B","metrics":{"Delta_NoContext":"2.2%","Exact Match Accuracy":"4.9%"},"uses_additional_data":false,"paper_date":"2024-06-10","paper":"/paper/lingoly-a-benchmark-of-olympiad-level","paper_url":"https://arxiv.org/abs/2406.06196v3","paper_title":"LINGOLY: A Benchmark of Olympiad-Level Linguistic Reasoning Puzzles in Low-Resource and Extinct Languages","code":"https://github.com/am-bean/lingOly","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":2}},{"rank_in_archive_order":11,"model":"Llama 2 70B","metrics":{"Delta_NoContext":"1.1%","Exact Match Accuracy":"6.4%"},"uses_additional_data":false,"paper_date":"2024-06-10","paper":"/paper/lingoly-a-benchmark-of-olympiad-level","paper_url":"https://arxiv.org/abs/2406.06196v3","paper_title":"LINGOLY: A Benchmark of Olympiad-Level Linguistic Reasoning Puzzles in Low-Resource and Extinct Languages","code":"https://github.com/am-bean/lingOly","n_code_links":1,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":2}}],"since_archive":{"present":false,"note":"No Syntology-extracted rows are published in this build."},"syntology":{"read_at":"2026-09-24T18:15:14+00:00","claim":"Per row: N of M harvested code samples from that row's paper executed on a synthesized fixture; the other M-N are unverified. Not a reproduction of the row's number; not a correctness claim. n_pointer_only_licence counts samples the site points at rather than redistributes (a licence axis, independent of ran/unverified).","rows_with_graph_line":11,"rows_with_any_sample_ran":11,"distinct_papers_with_graph_line":1,"distinct_papers_with_any_sample_ran":1,"samples_over_distinct_papers":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":2,"note":"each paper (arXiv id) counted once, however many rows it is behind; this is the page-level figure"},"samples_row_weighted":{"n_ran":22,"n_unverified":0,"n_samples":22,"n_pointer_only_licence":22,"note":"row-weighted: a paper behind several rows is counted once per row; inflated relative to samples_over_distinct_papers by design, kept for readers summing the per-row syntology blocks"}}}