Decoupling computation and data scheduling in distributed data-intensive applications Export

@inproceedings{citeulike:2728289, abstract = {In high-energy physics, bioinformatics, and other disciplines, we encounter applications involving numerous, loosely coupled jobs that both access and generate large data sets. So-called Data Grids seek to harness geographically distributed resources for such large-scale data-intensive problems. Yet effective scheduling in such environments is challenging, due to a need to address a variety of metrics and constraints while dealing with multiple, potentially independent sources of jobs and a large number of storage, compute, and network resources. We describe a scheduling framework that addresses these problems. Within this framework, data movement operations may be either tightly bound to job scheduling decisions or, alternatively, performed by a decoupled, asynchronous process on the basis of observed data access patterns and load. We develop a family of algorithms and use simulation studies to evaluate various combinations. Our results suggest that while it is necessary to consider the impact of replication, it is not always necessary to couple data movement and computation scheduling. Instead, these two activities can be addressed separately, thus significantly simplifying the design and implementation.}, author = {Ranganathan, K. and Foster, I.}, booktitle = {High Performance Distributed Computing, 2002. HPDC-11 2002. Proceedings. 11th IEEE International Symposium on}, citeulike-article-id = {2728289}, citeulike-linkout-0 = {http://dx.doi.org/10.1109/HPDC.2002.1029935}, citeulike-linkout-1 = {http://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=1029935}, doi = {10.1109/HPDC.2002.1029935}, journal = {High Performance Distributed Computing, 2002. HPDC-11 2002. Proceedings. 11th IEEE International Symposium on}, keywords = {scheduling, storage}, pages = {352--358}, posted-at = {2009-11-08 12:07:32}, priority = {2}, title = {Decoupling computation and data scheduling in distributed data-intensive applications}, url = {http://dx.doi.org/10.1109/HPDC.2002.1029935}, year = {2002} }

TY - CONF ID - citeulike:2728289 L3 - citeulike-article-id:2728289 N2 - In high-energy physics, bioinformatics, and other disciplines, we encounter applications involving numerous, loosely coupled jobs that both access and generate large data sets. So-called Data Grids seek to harness geographically distributed resources for such large-scale data-intensive problems. Yet effective scheduling in such environments is challenging, due to a need to address a variety of metrics and constraints while dealing with multiple, potentially independent sources of jobs and a large number of storage, compute, and network resources. We describe a scheduling framework that addresses these problems. Within this framework, data movement operations may be either tightly bound to job scheduling decisions or, alternatively, performed by a decoupled, asynchronous process on the basis of observed data access patterns and load. We develop a family of algorithms and use simulation studies to evaluate various combinations. Our results suggest that while it is necessary to consider the impact of replication, it is not always necessary to couple data movement and computation scheduling. Instead, these two activities can be addressed separately, thus significantly simplifying the design and implementation. JF - High Performance Distributed Computing, 2002. HPDC-11 2002. Proceedings. 11th IEEE International Symposium on EP - 358 TI - Decoupling computation and data scheduling in distributed data-intensive applications T2 - High Performance Distributed Computing, 2002. HPDC-11 2002. Proceedings. 11th IEEE International Symposium on SP - 352 KW - scheduling KW - storage AU - Ranganathan, K AU - Foster, I PY - 2002/// UR - http://dx.doi.org/10.1109/HPDC.2002.1029935 ER -