{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/compute-bleu","entry":"compute_bleu","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":30,"n_papers_ran":14,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":18,"n_samples_ran":6,"n_samples_fingerprinted":2,"n_places":30,"n_places_pointer_only":5,"by_status":{"ran_honours":3,"ran_violates":0,"ran_draft_wrong":0,"ran_fixture":0,"ran":3,"unverified":12},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2606.12789","paper":"/paper/arxiv-2606-12789","title":"How Fine-Grained Should a RAG Benchmark Be? A Hierarchical Framework for Synthetic Question Generation","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"fensorechase/rag-diverse-benchmarks-synthetic-qa","path":"rag_system/generate/complete_analysis.py","file_url":"https://github.com/fensorechase/rag-diverse-benchmarks-synthetic-qa/blob/HEAD/rag_system/generate/complete_analysis.py","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"52f277ebffee2f1f","mcp_get_code":{"code_sha256":"52f277ebffee2f1f"}},{"arxiv_id":"2601.19871","paper":"/paper/arxiv-2601-19871","title":"Reflective Translation for Low-Resource Machine Translation via Structured Self-Reflection","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"Nickcheng123/reflective-translation-mt","path":"src/reflective_translation/pipeline.py","file_url":"https://github.com/Nickcheng123/reflective-translation-mt/blob/HEAD/src/reflective_translation/pipeline.py","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"5ce1568daa37b696","mcp_get_code":{"code_sha256":"5ce1568daa37b696"}},{"arxiv_id":"2409.19804","paper":"/paper/does-rag-introduce-unfairness-in-llms","title":"Does RAG Introduce Unfairness in LLMs? Evaluating Fairness in Retrieval-Augmented Generation Systems","date":"2024-09-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"elviswxy/rag_fairness","path":"flashrag/evaluator/_bleu.py","file_url":"https://github.com/elviswxy/rag_fairness/blob/HEAD/flashrag/evaluator/_bleu.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"2dd8e40379d7bbb9","mcp_get_code":{"code_sha256":"2dd8e40379d7bbb9"}},{"arxiv_id":"2409.13191","paper":"/paper/an-adapted-large-language-model-facilitates","title":"Diabetica: Adapting Large Language Model to Enhance Multiple Medical Tasks in Diabetes Care and Management","date":"2024-09-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"waltonfuture/Diabetica","path":"evaluation/fill-in-the-blank_eval.py","file_url":"https://github.com/waltonfuture/Diabetica/blob/HEAD/evaluation/fill-in-the-blank_eval.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"fc8f0da4acefd1b7","mcp_get_code":{"code_sha256":"fc8f0da4acefd1b7"}},{"arxiv_id":"2405.10587","paper":"/paper/rdrec-rationale-distillation-for-llm-based","title":"RDRec: Rationale Distillation for LLM-based Recommendation","date":"2024-05-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"WangXFng/RDRec","path":"utils/bleu.py","file_url":"https://github.com/WangXFng/RDRec/blob/HEAD/utils/bleu.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"49289a7f94c227bd","mcp_get_code":{"code_sha256":"49289a7f94c227bd"}},{"arxiv_id":"2402.07945","paper":"/paper/screenagent-a-vision-language-model-driven","title":"ScreenAgent: A Vision Language Model-driven Computer Control Agent","date":"2024-02-09","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"niuzaisheng/ScreenAgent","path":"client/bleu.py","file_url":"https://github.com/niuzaisheng/ScreenAgent/blob/HEAD/client/bleu.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"49289a7f94c227bd","mcp_get_code":{"code_sha256":"49289a7f94c227bd"}},{"arxiv_id":"2401.16745","paper":"/paper/mt-eval-a-multi-turn-capabilities-evaluation","title":"MT-Eval: A Multi-Turn Capabilities Evaluation Benchmark for Large Language Models","date":"2024-01-30","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"kwanwaichung/mt-eval","path":"utils/bleu.py","file_url":"https://github.com/kwanwaichung/mt-eval/blob/HEAD/utils/bleu.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"2dd8e40379d7bbb9","mcp_get_code":{"code_sha256":"2dd8e40379d7bbb9"}},{"arxiv_id":"2401.15545","paper":"/paper/ppm-automated-generation-of-diverse","title":"PPM: Automated Generation of Diverse Programming Problems for Benchmarking Code Generation Models","date":"2024-01-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"seekingdream/ppm","path":"diversity.py","file_url":"https://github.com/seekingdream/ppm/blob/HEAD/diversity.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"b059913a28aa8d55","mcp_get_code":{"code_sha256":"b059913a28aa8d55"}},{"arxiv_id":"2311.07468","paper":"/paper/are-we-falling-in-a-middle-intelligence-trap","title":"An Analysis and Mitigation of the Reversal Curse","date":"2023-11-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"trestad/mitigating-reversal-curse","path":"generate.py","file_url":"https://github.com/trestad/mitigating-reversal-curse/blob/HEAD/generate.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f0d20ebd827390d6","mcp_get_code":{"code_sha256":"f0d20ebd827390d6"}},{"arxiv_id":"2308.12241","paper":"/paper/llmrec-benchmarking-large-language-models-on","title":"LLMRec: Benchmarking Large Language Models on Recommendation Task","date":"2023-08-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"williamliujl/llmrec","path":"utils/evaluate/bleu.py","file_url":"https://github.com/williamliujl/llmrec/blob/HEAD/utils/evaluate/bleu.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"49289a7f94c227bd","mcp_get_code":{"code_sha256":"49289a7f94c227bd"}},{"arxiv_id":"2211.09374","paper":"/paper/execution-based-evaluation-for-data-science","title":"Execution-based Evaluation for Data Science Code Generation Models","date":"2022-11-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jun-jie-huang/exeds","path":"codegpt/bleu.py","file_url":"https://github.com/jun-jie-huang/exeds/blob/HEAD/codegpt/bleu.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"2dd8e40379d7bbb9","mcp_get_code":{"code_sha256":"2dd8e40379d7bbb9"}},{"arxiv_id":"2206.08474","paper":"/paper/xlcost-a-benchmark-dataset-for-cross-lingual","title":"XLCoST: A Benchmark Dataset for Cross-lingual Code Intelligence","date":"2022-06-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"reddy-lab-code-research/xlcost","path":"code/translation/code/bleu.py","file_url":"https://github.com/reddy-lab-code-research/xlcost/blob/HEAD/code/translation/code/bleu.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"49289a7f94c227bd","mcp_get_code":{"code_sha256":"49289a7f94c227bd"}},{"arxiv_id":"2206.05239","paper":"/paper/structcoder-structure-aware-transformer-for","title":"StructCoder: Structure-Aware Transformer for Code Generation","date":"2022-06-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"reddy-lab-code-research/structcoder","path":"bleu.py","file_url":"https://github.com/reddy-lab-code-research/structcoder/blob/HEAD/bleu.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"49289a7f94c227bd","mcp_get_code":{"code_sha256":"49289a7f94c227bd"}},{"arxiv_id":"2203.15685","paper":"/paper/envedit-environment-editing-for-vision-and","title":"EnvEdit: Environment Editing for Vision-and-Language Navigation","date":"2022-03-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jialuli-luka/envedit","path":"r2r_src/bleu.py","file_url":"https://github.com/jialuli-luka/envedit/blob/HEAD/r2r_src/bleu.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"7b5b4685db18fbca","mcp_get_code":{"code_sha256":"7b5b4685db18fbca"}},{"arxiv_id":"2109.08365","paper":"/paper/codeqa-a-question-answering-dataset-for","title":"CodeQA: A Question Answering Dataset for Source Code Comprehension","date":"2021-09-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jadecxliu/codeqa","path":"codeBERT/code/eval/bleu/google_bleu.py","file_url":"https://github.com/jadecxliu/codeqa/blob/HEAD/codeBERT/code/eval/bleu/google_bleu.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"225b0921f040310c","mcp_get_code":{"code_sha256":"225b0921f040310c"}},{"arxiv_id":"2107.07201","paper":"/paper/neighbor-view-enhanced-model-for-vision-and","title":"Neighbor-view Enhanced Model for Vision and Language Navigation","date":"2021-07-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"MarSaKi/NvEM","path":"r2r_src/bleu.py","file_url":"https://github.com/MarSaKi/NvEM/blob/HEAD/r2r_src/bleu.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"7b5b4685db18fbca","mcp_get_code":{"code_sha256":"7b5b4685db18fbca"}},{"arxiv_id":"2010.07954","paper":"/paper/room-across-room-multilingual-vision-and","title":"Room-Across-Room: Multilingual Vision-and-Language Navigation with Dense Spatiotemporal Grounding","date":"2020-10-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"VegB/Diagnose_VLN","path":"rxr/model/CLIP-ViL-VLN/r2r_src/bleu.py","file_url":"https://github.com/VegB/Diagnose_VLN/blob/HEAD/rxr/model/CLIP-ViL-VLN/r2r_src/bleu.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"7b5b4685db18fbca","mcp_get_code":{"code_sha256":"7b5b4685db18fbca"}},{"arxiv_id":"2006.09286","paper":"/paper/on-the-computational-power-of-transformers","title":"On the Computational Power of Transformers and its Implications in Sequence Modeling","date":"2020-06-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"satwik77/Transformer-Computation-Analysis","path":"Transformer/src/utils/bleu.py","file_url":"https://github.com/satwik77/Transformer-Computation-Analysis/blob/HEAD/Transformer/src/utils/bleu.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"ae3b0a421287c5c9","mcp_get_code":{"code_sha256":"ae3b0a421287c5c9"}},{"arxiv_id":"2006.04771","paper":"/paper/copy-that-editing-sequences-by-copying-spans-1","title":"Copy that! Editing Sequences by Copying Spans","date":"2020-06-08","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"microsoft/aaai21-copy-that","path":"model/testencdec.py","file_url":"https://github.com/microsoft/aaai21-copy-that/blob/HEAD/model/testencdec.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"912f05720e2aef3a","mcp_get_code":{"code_sha256":"912f05720e2aef3a"}},{"arxiv_id":"1909.09436","paper":"/paper/codesearchnet-challenge-evaluating-the-state","title":"CodeSearchNet Challenge: Evaluating the State of Semantic Code Search","date":"2019-09-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"nchen909/codeattention","path":"evaluator/bleu.py","file_url":"https://github.com/nchen909/codeattention/blob/HEAD/evaluator/bleu.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"49289a7f94c227bd","mcp_get_code":{"code_sha256":"49289a7f94c227bd"}},{"arxiv_id":"1909.06356","paper":"/paper/addressing-semantic-drift-in-question","title":"Addressing Semantic Drift in Question Generation for Semi-Supervised Question Answering","date":"2019-09-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ZhangShiyue/QGforQA","path":"LIB/EVAL/bleu.py","file_url":"https://github.com/ZhangShiyue/QGforQA/blob/HEAD/LIB/EVAL/bleu.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"9212e9aaf973fc1a","mcp_get_code":{"code_sha256":"9212e9aaf973fc1a"}},{"arxiv_id":"1904.04195","paper":"/paper/learning-to-navigate-unseen-environments-back","title":"Learning to Navigate Unseen Environments: Back Translation with Environmental Dropout","date":"2019-04-08","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"airsplay/R2R-EnvDrop","path":"r2r_src/bleu.py","file_url":"https://github.com/airsplay/R2R-EnvDrop/blob/HEAD/r2r_src/bleu.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"7b5b4685db18fbca","mcp_get_code":{"code_sha256":"7b5b4685db18fbca"}},{"arxiv_id":"1901.00158","paper":"/paper/text-infilling","title":"Text Infilling","date":"2019-01-01","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"VegB/Text_Infilling","path":"text_infilling/bleu_tool.py","file_url":"https://github.com/VegB/Text_Infilling/blob/HEAD/text_infilling/bleu_tool.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"dda226c5d7902119","mcp_get_code":{"code_sha256":"dda226c5d7902119"}},{"arxiv_id":"1808.08866","paper":"/paper/a-study-of-reinforcement-learning-for-neural","title":"A Study of Reinforcement Learning for Neural Machine Translation","date":"2018-08-27","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"apeterswu/RL4NMT","path":"tensor2tensor/utils/bleu_hook.py","file_url":"https://github.com/apeterswu/RL4NMT/blob/HEAD/tensor2tensor/utils/bleu_hook.py","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"0479f465eb03e39f","mcp_get_code":{"code_sha256":"0479f465eb03e39f"}},{"arxiv_id":"1711.04090","paper":"/paper/mojitalk-generating-emotional-responses-at","title":"MojiTalk: Generating Emotional Responses at Scale","date":"2017-11-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Claude-Zhou/MojiTalk","path":"bleu.py","file_url":"https://github.com/Claude-Zhou/MojiTalk/blob/HEAD/bleu.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"2c08a727cace5066","mcp_get_code":{"code_sha256":"2c08a727cace5066"}},{"arxiv_id":"aaai_6453","paper":null,"title":"arXiv:aaai_6453","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"siat-nlp/TransDG","path":"src/utils/bleu.py","file_url":"https://github.com/siat-nlp/TransDG/blob/HEAD/src/utils/bleu.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"c2167c0487876139","mcp_get_code":{"code_sha256":"c2167c0487876139"}},{"arxiv_id":"2025.acl-long.1142","paper":null,"title":"arXiv:2025.acl-long.1142","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"GenIRAG/PER-PSE","path":"flashrag/evaluator/_bleu.py","file_url":"https://github.com/GenIRAG/PER-PSE/blob/HEAD/flashrag/evaluator/_bleu.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"2dd8e40379d7bbb9","mcp_get_code":{"code_sha256":"2dd8e40379d7bbb9"}},{"arxiv_id":"2024.findings-naacl.46","paper":null,"title":"arXiv:2024.findings-naacl.46","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"knalin55/LEEETs-Dial","path":"evaluation_utils.py","file_url":"https://github.com/knalin55/LEEETs-Dial/blob/HEAD/evaluation_utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"9fc45e6d9ca90f15","mcp_get_code":{"code_sha256":"9fc45e6d9ca90f15"}},{"arxiv_id":"2022.findings-naacl.160","paper":null,"title":"arXiv:2022.findings-naacl.160","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"malllabiisc/SGCP","path":"src/bleu.py","file_url":"https://github.com/malllabiisc/SGCP/blob/HEAD/src/bleu.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"ae3b0a421287c5c9","mcp_get_code":{"code_sha256":"ae3b0a421287c5c9"}},{"arxiv_id":"2020.findings-emnlp.289","paper":null,"title":"arXiv:2020.findings-emnlp.289","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"h4ste/mtft_zsl","path":"fslks/eval/bleu.py","file_url":"https://github.com/h4ste/mtft_zsl/blob/HEAD/fslks/eval/bleu.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"f23551c5f115973e","mcp_get_code":{"code_sha256":"f23551c5f115973e"}}]}