{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/a-new-burrows-wheeler-transform-markov","title":"A New Burrows Wheeler Transform Markov Distance","arxiv_id":"1912.13046","date":"2019-12-30","proceeding":null,"authors":["Edward Raff","Charles Nicholas","Mark McLean"],"abstract":"Prior work inspired by compression algorithms has described how the Burrows Wheeler Transform can be used to create a distance measure for bioinformatics problems. We describe issues with this approach that were not widely known, and introduce our new Burrows Wheeler Markov Distance (BWMD) as an alternative. The BWMD avoids the shortcomings of earlier efforts, and allows us to tackle problems in variable length DNA sequence clustering. BWMD is also more adaptable to other domains, which we demonstrate on malware classification tasks. Unlike other compression-based distance metrics known to us, BWMD works by embedding sequences into a fixed-length feature vector. This allows us to provide significantly improved clustering performance on larger malware corpora, a weakness of prior methods.","url_abs":"https://arxiv.org/abs/1912.13046v1","url_pdf":"https://arxiv.org/pdf/1912.13046v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"a-new-burrows-wheeler-transform-markov","repo_url":"https://github.com/EdwardRaff/pyBWMD","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":null},{"paper_slug":"a-new-burrows-wheeler-transform-markov","repo_url":"https://github.com/MaksimEkin/COVID19-Literature-Clustering","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"a-new-burrows-wheeler-transform-markov","repo_url":"https://github.com/MaksimEkin/spam-message-classification","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"a-new-burrows-wheeler-transform-markov","repo_url":"https://github.com/prasanna4567/COVID-19_MLDM","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"clustering","task_name":"Clustering"},{"task_slug":"malware-classification","task_name":"Malware Classification"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1912.13046","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1912.13046"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/prasanna4567/COVID-19_MLDM","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/MaksimEkin/COVID19-Literature-Clustering","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/EdwardRaff/pyBWMD","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/MaksimEkin/spam-message-classification","reach":{"status":"ok"}}],"summary":{"ran_violates":1,"ran_honours":1,"unverified":1},"by_repo_kind":{"listed":{"samples":3,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":3,"samples":[{"code_sha256_prefix":"56f087476b48503e","entry":"isFile","repo":"EdwardRaff/pyBWMD","repo_kind":"listed","path":"pyBWMD/bwmd.py","file_url":"https://github.com/EdwardRaff/pyBWMD/blob/HEAD/pyBWMD/bwmd.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"56f087476b48503e"}},{"code_sha256_prefix":"d0409c8f5c3b680b","entry":"makeMatrix","repo":"EdwardRaff/pyBWMD","repo_kind":"listed","path":"pyBWMD/bwmd.py","file_url":"https://github.com/EdwardRaff/pyBWMD/blob/HEAD/pyBWMD/bwmd.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d0409c8f5c3b680b"}},{"code_sha256_prefix":"4c16c93dfc6b693c","entry":"vectorize","repo":"EdwardRaff/pyBWMD","repo_kind":"listed","path":"pyBWMD/bwmd.py","file_url":"https://github.com/EdwardRaff/pyBWMD/blob/HEAD/pyBWMD/bwmd.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4c16c93dfc6b693c"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}