{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/a-survey-on-reproducibility-by-evaluating","title":"A Survey on Reproducibility by Evaluating Deep Reinforcement Learning Algorithms on Real-World Robots","arxiv_id":"1909.03772","date":"2019-09-09","proceeding":null,"authors":["Nicolai A. Lynnerup","Laura Nolling","Rasmus Hasle","John Hallam"],"abstract":"As reinforcement learning (RL) achieves more success in solving complex tasks, more care is needed to ensure that RL research is reproducible and that algorithms herein can be compared easily and fairly with minimal bias. RL results are, however, notoriously hard to reproduce due to the algorithms' intrinsic variance, the environments' stochasticity, and numerous (potentially unreported) hyper-parameters. In this work we investigate the many issues leading to irreproducible research and how to manage those. We further show how to utilise a rigorous and standardised evaluation approach for easing the process of documentation, evaluation and fair comparison of different algorithms, where we emphasise the importance of choosing the right measurement metrics and conducting proper statistics on the results, for unbiased reporting of the results.","url_abs":"https://arxiv.org/abs/1909.03772v2","url_pdf":"https://arxiv.org/pdf/1909.03772v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"a-survey-on-reproducibility-by-evaluating","repo_url":"https://github.com/dti-research/SenseActExperiments","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"BSD-3-Clause"}}],"tasks":[{"task_slug":"deep-reinforcement-learning","task_name":"Deep Reinforcement Learning"},{"task_slug":"reinforcement-learning","task_name":"Reinforcement Learning"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"},{"task_slug":"reinforcement-learning-2","task_name":"reinforcement-learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1909.03772","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1909.03772"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/dti-research/SenseActExperiments","reach":{"status":"ok","spdx":"BSD-3-Clause"}}],"summary":{"unverified":8},"by_repo_kind":{"official":{"samples":8,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"72d7383d8fc116f0","entry":"bootstrap","repo":"dti-research/SenseActExperiments","repo_kind":"official","path":"evaluate/evaluate.py","file_url":"https://github.com/dti-research/SenseActExperiments/blob/HEAD/evaluate/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"72d7383d8fc116f0"}},{"code_sha256_prefix":"0aa883b0e9f732ae","entry":"compute_mean_ci","repo":"dti-research/SenseActExperiments","repo_kind":"official","path":"evaluate/evaluate.py","file_url":"https://github.com/dti-research/SenseActExperiments/blob/HEAD/evaluate/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"0aa883b0e9f732ae"}},{"code_sha256_prefix":"c95c349d9b6d63e8","entry":"create_callback","repo":"dti-research/SenseActExperiments","repo_kind":"official","path":"code/helper.py","file_url":"https://github.com/dti-research/SenseActExperiments/blob/HEAD/code/helper.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"c95c349d9b6d63e8"}},{"code_sha256_prefix":"a30533080afa3611","entry":"fit_distributions","repo":"dti-research/SenseActExperiments","repo_kind":"official","path":"evaluate/dist_fitter.py","file_url":"https://github.com/dti-research/SenseActExperiments/blob/HEAD/evaluate/dist_fitter.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"a30533080afa3611"}},{"code_sha256_prefix":"80f105a970d9f936","entry":"get_xticks","repo":"dti-research/SenseActExperiments","repo_kind":"official","path":"evaluate/dist_fitter.py","file_url":"https://github.com/dti-research/SenseActExperiments/blob/HEAD/evaluate/dist_fitter.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"80f105a970d9f936"}},{"code_sha256_prefix":"45b1de17ab516b93","entry":"read_csv_files","repo":"dti-research/SenseActExperiments","repo_kind":"official","path":"evaluate/dist_fitter.py","file_url":"https://github.com/dti-research/SenseActExperiments/blob/HEAD/evaluate/dist_fitter.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"45b1de17ab516b93"}},{"code_sha256_prefix":"16373edbff04e2aa","entry":"read_csv_files","repo":"dti-research/SenseActExperiments","repo_kind":"official","path":"evaluate/evaluate.py","file_url":"https://github.com/dti-research/SenseActExperiments/blob/HEAD/evaluate/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"16373edbff04e2aa"}},{"code_sha256_prefix":"61a80260f7ad4bf9","entry":"read_csv_files","repo":"dti-research/SenseActExperiments","repo_kind":"official","path":"evaluate/plot_w_std_dev.py","file_url":"https://github.com/dti-research/SenseActExperiments/blob/HEAD/evaluate/plot_w_std_dev.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"61a80260f7ad4bf9"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}