{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/sonnet-or-not-bot-poetry-evaluation-for-large","title":"Sonnet or Not, Bot? Poetry Evaluation for Large Models and Datasets","arxiv_id":"2406.18906","date":"2024-06-27","proceeding":null,"authors":["Melanie Walsh","Anna Preus","Maria Antoniak"],"abstract":"Large language models (LLMs) can now generate and recognize poetry. But what do LLMs really know about poetry? We develop a task to evaluate how well LLMs recognize one aspect of English-language poetry--poetic form--which captures many different poetic features, including rhyme scheme, meter, and word or line repetition. By using a benchmark dataset of over 4.1k human expert-annotated poems, we show that state-of-the-art LLMs can successfully identify both common and uncommon fixed poetic forms--such as sonnets, sestinas, and pantoums--with surprisingly high accuracy. However, performance varies significantly by poetic form; the models struggle to identify unfixed poetic forms, especially those based on topic or visual features. We additionally measure how many poems from our benchmark dataset are present in popular pretraining datasets or memorized by GPT-4, finding that pretraining presence and memorization may improve performance on this task, but results are inconclusive. We release a benchmark evaluation dataset with 1.4k public domain poems and form annotations, results of memorization experiments and data audits, and code.","url_abs":"https://arxiv.org/abs/2406.18906v3","url_pdf":"https://arxiv.org/pdf/2406.18906v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"sonnet-or-not-bot-poetry-evaluation-for-large","repo_url":"https://github.com/maria-antoniak/poetry-eval","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":null}],"tasks":[{"task_slug":"form","task_name":"Form"},{"task_slug":"memorization","task_name":"Memorization"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt-4","method_name":"GPT-4"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2406.18906","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.18906"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/maria-antoniak/poetry-eval","reach":null}],"summary":{"ran_violates":2,"ran_draft_wrong":1},"by_repo_kind":{"official":{"samples":3,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":3,"samples":[{"code_sha256_prefix":"b7f18205079d0a15","entry":"contains_word_longer_than_two_chars","repo":"maria-antoniak/poetry-eval","repo_kind":"official","path":"llm-poetic-form-tagging-scripts/llama3-poetry-tagger.py","file_url":"https://github.com/maria-antoniak/poetry-eval/blob/HEAD/llm-poetic-form-tagging-scripts/llama3-poetry-tagger.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b7f18205079d0a15"}},{"code_sha256_prefix":"da18e1df51732444","entry":"find_first_valid_line","repo":"maria-antoniak/poetry-eval","repo_kind":"official","path":"llm-poetic-form-tagging-scripts/llama3-poetry-tagger.py","file_url":"https://github.com/maria-antoniak/poetry-eval/blob/HEAD/llm-poetic-form-tagging-scripts/llama3-poetry-tagger.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"da18e1df51732444"}},{"code_sha256_prefix":"0e488be335f2909d","entry":"is_valid_first_line","repo":"maria-antoniak/poetry-eval","repo_kind":"official","path":"llm-poetic-form-tagging-scripts/llama3-poetry-tagger.py","file_url":"https://github.com/maria-antoniak/poetry-eval/blob/HEAD/llm-poetic-form-tagging-scripts/llama3-poetry-tagger.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0e488be335f2909d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}