Fast and Efficient Compression of Floating-Point Data Export

@article{citeulike:3450179, abstract = {Large scale scientific simulation codes typically run on a cluster of CPUs that write/read time steps to/from a single file system. As data sets are constantly growing in size, this increasingly leads to I/O bottlenecks. When the rate at which data is produced exceeds the available I/O bandwidth, the simulation stalls and the CPUs are idle. Data compression can alleviate this problem by using some CPU cycles to reduce the amount of data needed to be transfered. Most compression schemes, however, are designed to operate offline and seek to maximize compression, notthroughput.Furthermore, they often require quantizing floating-point values onto a uniform integer grid, which disqualifies their use in applications where exact values must be retained. We propose a simple scheme for lossless, online compression of floating-point data that transparently integrates into the I/O ofmany applications.A plug-in scheme for data-dependent prediction makes our scheme applicable to a wide variety of data used invisualization, such as unstructured meshes, point sets, images, and voxel grids. We achieve state-of-the-art compression rates and speeds, the latter in part due to an improved entropy coder. We demonstrate that this significantly accelerates I/O throughput in real simulation runs. Unlike previous schemes, our method also adapts well to variable-precision floating-point and integer data.}, address = {Piscataway, NJ, USA}, author = {Isenburg, Martin}, citeulike-article-id = {3450179}, citeulike-linkout-0 = {http://portal.acm.org/citation.cfm?id=1187627.1187859}, citeulike-linkout-1 = {http://dx.doi.org/10.1109/TVCG.2006.143}, doi = {10.1109/TVCG.2006.143}, issn = {1077-2626}, journal = {IEEE Transactions on Visualization and Computer Graphics}, keywords = {compression, storage}, number = {5}, pages = {1245--1250}, posted-at = {2008-10-25 16:20:34}, priority = {2}, publisher = {IEEE Educational Activities Department}, title = {Fast and Efficient Compression of Floating-Point Data}, url = {http://dx.doi.org/10.1109/TVCG.2006.143}, volume = {12}, year = {2006} }

TY - JOUR ID - citeulike:3450179 L3 - citeulike-article-id:3450179 N2 - Large scale scientific simulation codes typically run on a cluster of CPUs that write/read time steps to/from a single file system. As data sets are constantly growing in size, this increasingly leads to I/O bottlenecks. When the rate at which data is produced exceeds the available I/O bandwidth, the simulation stalls and the CPUs are idle. Data compression can alleviate this problem by using some CPU cycles to reduce the amount of data needed to be transfered. Most compression schemes, however, are designed to operate offline and seek to maximize compression, notthroughput.Furthermore, they often require quantizing floating-point values onto a uniform integer grid, which disqualifies their use in applications where exact values must be retained. We propose a simple scheme for lossless, online compression of floating-point data that transparently integrates into the I/O ofmany applications.A plug-in scheme for data-dependent prediction makes our scheme applicable to a wide variety of data used invisualization, such as unstructured meshes, point sets, images, and voxel grids. We achieve state-of-the-art compression rates and speeds, the latter in part due to an improved entropy coder. We demonstrate that this significantly accelerates I/O throughput in real simulation runs. Unlike previous schemes, our method also adapts well to variable-precision floating-point and integer data. IS - 5 JF - IEEE Transactions on Visualization and Computer Graphics SN - 1077-2626 AD - Piscataway, NJ, USA EP - 1250 TI - Fast and Efficient Compression of Floating-Point Data PB - IEEE Educational Activities Department VL - 12 SP - 1245 KW - compression KW - storage AU - Isenburg, Martin PY - 2006/// UR - http://dx.doi.org/10.1109/TVCG.2006.143 ER -