{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2510-10681","title":"RePro: Training Language Models to Faithfully Recycle the Web for Pretraining","arxiv_id":"2510.10681","date":"2025-10-12","proceeding":"ICML","authors":["Zichun Yu","Chenyan Xiong"],"abstract":"High-quality data is a cornerstone of large language model (LLM) pretraining, yet its growth has not kept pace with the needs of frontier models. In this paper, we introduce RePro, a novel web recycling method that trains a relatively small LM with reinforcement learning to generate effective and faithful rephrasings of pretraining data. Specifically, we design one quality reward and three faithfulness rewards, optimizing the LM rephraser to convert organic data into high-quality rephrasings while maintaining its core semantics and structure. In our experiment, we train a rephraser as small as 1B parameters to recycle 72B tokens sampled from DCLM-RefinedWeb. Pretraining results on 400M, 1.4B, and 2.8B models demonstrate that RePro delivers 3.7%-14.5% relative accuracy gains over organic-only baseline on 22 downstream tasks, doubling the performance gains achieved by the state-of-the-art web recycling method that prompts a 70B rephraser. Experiments with different amounts of recycled data highlight that RePro improves organic data efficiency by 2-3x. Individual and distributional analyses validate that RePro preserves more critical information and faithfully reflects the characteristics of organic data compared to prompting-based methods. Together, these results show that RePro provides an efficient and controllable path to effectively recycle organic data for pretraining. Our code is available at https://github.com/cxcscmu/RePro.","url_abs":"https://arxiv.org/abs/2510.10681","url_pdf":"https://arxiv.org/pdf/2510.10681","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2510.10681","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2510.10681"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/cxcscmu/RePro","reach":{"status":"ok"}}],"summary":{"ran_draft_wrong":1,"unverified":16},"by_repo_kind":{"found_in_text":{"samples":17,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":17,"samples":[{"code_sha256_prefix":"e67c82963b22e823","entry":"set_args_for_val","repo":"cxcscmu/RePro","repo_kind":"found_in_text","path":"pretrain/eval/eval_openlm_ckpt.py","file_url":"https://github.com/cxcscmu/RePro/blob/HEAD/pretrain/eval/eval_openlm_ckpt.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e67c82963b22e823"}},{"code_sha256_prefix":"f81781e970831f36","entry":"do_once","repo":"cxcscmu/RePro","repo_kind":"found_in_text","path":"pretrain/baselines/mappers/core_utils.py","file_url":"https://github.com/cxcscmu/RePro/blob/HEAD/pretrain/baselines/mappers/core_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f81781e970831f36"}},{"code_sha256_prefix":"63466964df9945fb","entry":"factory_function","repo":"cxcscmu/RePro","repo_kind":"found_in_text","path":"pretrain/baselines/core/factory_utils.py","file_url":"https://github.com/cxcscmu/RePro/blob/HEAD/pretrain/baselines/core/factory_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"63466964df9945fb"}},{"code_sha256_prefix":"8f658ea2014771b8","entry":"get_aggregated_results","repo":"cxcscmu/RePro","repo_kind":"found_in_text","path":"pretrain/eval/aggregated_metrics.py","file_url":"https://github.com/cxcscmu/RePro/blob/HEAD/pretrain/eval/aggregated_metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8f658ea2014771b8"}},{"code_sha256_prefix":"5a857aa173b66cad","entry":"get_aggregator","repo":"cxcscmu/RePro","repo_kind":"found_in_text","path":"pretrain/baselines/core/factories.py","file_url":"https://github.com/cxcscmu/RePro/blob/HEAD/pretrain/baselines/core/factories.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5a857aa173b66cad"}},{"code_sha256_prefix":"ca51ecf816ab03c9","entry":"get_mapper","repo":"cxcscmu/RePro","repo_kind":"found_in_text","path":"pretrain/baselines/core/factories.py","file_url":"https://github.com/cxcscmu/RePro/blob/HEAD/pretrain/baselines/core/factories.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ca51ecf816ab03c9"}},{"code_sha256_prefix":"c22f387135336fa2","entry":"get_transform","repo":"cxcscmu/RePro","repo_kind":"found_in_text","path":"pretrain/baselines/core/factories.py","file_url":"https://github.com/cxcscmu/RePro/blob/HEAD/pretrain/baselines/core/factories.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c22f387135336fa2"}},{"code_sha256_prefix":"830a150f8d23852f","entry":"histogram","repo":"cxcscmu/RePro","repo_kind":"found_in_text","path":"pretrain/baselines/aggregators.py","file_url":"https://github.com/cxcscmu/RePro/blob/HEAD/pretrain/baselines/aggregators.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"830a150f8d23852f"}},{"code_sha256_prefix":"c259a6af6d26a457","entry":"initialize_mapper","repo":"cxcscmu/RePro","repo_kind":"found_in_text","path":"pretrain/baselines/core/factory_utils.py","file_url":"https://github.com/cxcscmu/RePro/blob/HEAD/pretrain/baselines/core/factory_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c259a6af6d26a457"}},{"code_sha256_prefix":"9e0723c9952cbf52","entry":"is_compressed","repo":"cxcscmu/RePro","repo_kind":"found_in_text","path":"pretrain/baselines/core/file_utils.py","file_url":"https://github.com/cxcscmu/RePro/blob/HEAD/pretrain/baselines/core/file_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9e0723c9952cbf52"}},{"code_sha256_prefix":"e378c50a1a63c85f","entry":"is_factory","repo":"cxcscmu/RePro","repo_kind":"found_in_text","path":"pretrain/baselines/core/factory_utils.py","file_url":"https://github.com/cxcscmu/RePro/blob/HEAD/pretrain/baselines/core/factory_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e378c50a1a63c85f"}},{"code_sha256_prefix":"eabedfcf5a503e17","entry":"is_s3","repo":"cxcscmu/RePro","repo_kind":"found_in_text","path":"pretrain/baselines/core/file_utils.py","file_url":"https://github.com/cxcscmu/RePro/blob/HEAD/pretrain/baselines/core/file_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"eabedfcf5a503e17"}},{"code_sha256_prefix":"77cf05a2c9609525","entry":"percentiles","repo":"cxcscmu/RePro","repo_kind":"found_in_text","path":"pretrain/baselines/aggregators.py","file_url":"https://github.com/cxcscmu/RePro/blob/HEAD/pretrain/baselines/aggregators.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"77cf05a2c9609525"}},{"code_sha256_prefix":"ef65c4b7a32dccc8","entry":"split_paragraphs","repo":"cxcscmu/RePro","repo_kind":"found_in_text","path":"pretrain/baselines/mappers/core_utils.py","file_url":"https://github.com/cxcscmu/RePro/blob/HEAD/pretrain/baselines/mappers/core_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ef65c4b7a32dccc8"}},{"code_sha256_prefix":"c547a9e449d2b01b","entry":"split_sentences","repo":"cxcscmu/RePro","repo_kind":"found_in_text","path":"pretrain/baselines/mappers/core_utils.py","file_url":"https://github.com/cxcscmu/RePro/blob/HEAD/pretrain/baselines/mappers/core_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c547a9e449d2b01b"}},{"code_sha256_prefix":"7aecd73638e7fe41","entry":"split_train_val","repo":"cxcscmu/RePro","repo_kind":"found_in_text","path":"pretrain/baselines/train_fasttext_classifier.py","file_url":"https://github.com/cxcscmu/RePro/blob/HEAD/pretrain/baselines/train_fasttext_classifier.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7aecd73638e7fe41"}},{"code_sha256_prefix":"97cc94f573da6893","entry":"threshold_transform","repo":"cxcscmu/RePro","repo_kind":"found_in_text","path":"pretrain/baselines/aggregators.py","file_url":"https://github.com/cxcscmu/RePro/blob/HEAD/pretrain/baselines/aggregators.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"97cc94f573da6893"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.CL","source":"arxiv_api"},"syntology_extracted_results":null}