{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/lightweight-correlation-aware-table","title":"Lightweight Correlation-Aware Table Compression","arxiv_id":"2410.14066","date":"2024-10-17","proceeding":null,"authors":["Mihail Stoian","Alexander van Renen","Jan Kobiolka","Ping-Lin Kuo","Josif Grabocka","Andreas Kipf"],"abstract":"The growing adoption of data lakes for managing relational data necessitates efficient, open storage formats that provide high scan performance and competitive compression ratios. While existing formats achieve fast scans through lightweight encoding techniques, they have reached a plateau in terms of minimizing storage footprint. Recently, correlation-aware compression schemes have been shown to reduce file sizes further. Yet, current approaches either incur significant scan overheads or require manual specification of correlations, limiting their practicability. We present $\\texttt{Virtual}$, a framework that integrates seamlessly with existing open formats to automatically leverage data correlations, achieving substantial compression gains while having minimal scan performance overhead. Experiments on data-gov datasets show that $\\texttt{Virtual}$ reduces file sizes by up to 40% compared to Apache Parquet.","url_abs":"https://arxiv.org/abs/2410.14066v3","url_pdf":"https://arxiv.org/pdf/2410.14066v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"lightweight-correlation-aware-table","repo_url":"https://github.com/utndatasystems/virtual","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2410.14066","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.14066"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/utndatasystems/virtual","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":9,"unverified":2},"by_repo_kind":{"official":{"samples":11,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"8d1469ccb137db5b","entry":"compute_error","repo":"utndatasystems/virtual","repo_kind":"official","path":"virtual/models/k_regression.py","file_url":"https://github.com/utndatasystems/virtual/blob/HEAD/virtual/models/k_regression.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8d1469ccb137db5b"}},{"code_sha256_prefix":"568f06b3f91e0b1c","entry":"conflict","repo":"utndatasystems/virtual","repo_kind":"official","path":"virtual/v_optimizer.py","file_url":"https://github.com/utndatasystems/virtual/blob/HEAD/virtual/v_optimizer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"568f06b3f91e0b1c"}},{"code_sha256_prefix":"7b5dcb4a12862adc","entry":"debug","repo":"utndatasystems/virtual","repo_kind":"official","path":"virtual/expr_parser.py","file_url":"https://github.com/utndatasystems/virtual/blob/HEAD/virtual/expr_parser.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7b5dcb4a12862adc"}},{"code_sha256_prefix":"9a4bf3c885bc2941","entry":"get_csv_header","repo":"utndatasystems/virtual","repo_kind":"official","path":"virtual/utils.py","file_url":"https://github.com/utndatasystems/virtual/blob/HEAD/virtual/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9a4bf3c885bc2941"}},{"code_sha256_prefix":"5797870b3880d5dd","entry":"get_status","repo":"utndatasystems/virtual","repo_kind":"official","path":"virtual/btrblocks_utils.py","file_url":"https://github.com/utndatasystems/virtual/blob/HEAD/virtual/btrblocks_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5797870b3880d5dd"}},{"code_sha256_prefix":"b43d7a3657167b8a","entry":"optimize","repo":"utndatasystems/virtual","repo_kind":"official","path":"virtual/expr_optimizer.py","file_url":"https://github.com/utndatasystems/virtual/blob/HEAD/virtual/expr_optimizer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b43d7a3657167b8a"}},{"code_sha256_prefix":"b6367791f0672728","entry":"schema_to_yaml","repo":"utndatasystems/virtual","repo_kind":"official","path":"virtual/btrblocks_utils.py","file_url":"https://github.com/utndatasystems/virtual/blob/HEAD/virtual/btrblocks_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b6367791f0672728"}},{"code_sha256_prefix":"f092a01a9bb9f053","entry":"solve_custom_cols","repo":"utndatasystems/virtual","repo_kind":"official","path":"virtual/v_driller.py","file_url":"https://github.com/utndatasystems/virtual/blob/HEAD/virtual/v_driller.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f092a01a9bb9f053"}},{"code_sha256_prefix":"1334ba2a4187a7f7","entry":"to_csv","repo":"utndatasystems/virtual","repo_kind":"official","path":"virtual/utils.py","file_url":"https://github.com/utndatasystems/virtual/blob/HEAD/virtual/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1334ba2a4187a7f7"}},{"code_sha256_prefix":"72429b37f32034a1","entry":"print_expr","repo":"utndatasystems/virtual","repo_kind":"official","path":"virtual/expr_parser.py","file_url":"https://github.com/utndatasystems/virtual/blob/HEAD/virtual/expr_parser.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"72429b37f32034a1"}},{"code_sha256_prefix":"c0f051172ca97276","entry":"storage_cost","repo":"utndatasystems/virtual","repo_kind":"official","path":"virtual/models/k_regression.py","file_url":"https://github.com/utndatasystems/virtual/blob/HEAD/virtual/models/k_regression.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c0f051172ca97276"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}