%0 Conference Paper %1 tantisiriroj2011duality %A Tantisiriroj, Wittawat %A Son, Seung Woo %A Patil, Swapnil %A Lang, Samuel J. %A Gibson, Garth %A Ross, Robert B. %B Proceedings of 2011 International Conference for High Performance Computing, Networking, Storage and Analysis %C New York, NY, USA %D 2011 %I ACM %K cloud distributed file hdfs hpc master performance pvfs seminar seminar:dfs system uni ws1213 %P 67:1--67:12 %R 10.1145/2063384.2063474 %T On the duality of data-intensive file system design: reconciling HDFS and PVFS %U http://doi.acm.org/10.1145/2063384.2063474 %X Data-intensive applications fall into two computing styles: Internet services (cloud computing) or high-performance computing (HPC). In both categories, the underlying file system is a key component for scalable application performance. In this paper, we explore the similarities and differences between PVFS, a parallel file system used in HPC at large scale, and HDFS, the primary storage system used in cloud computing with Hadoop. We integrate PVFS into Hadoop and compare its performance to HDFS using a set of data-intensive computing benchmarks. We study how HDFS-specific optimizations can be matched using PVFS and how consistency, durability, and persistence tradeoffs made by these file systems affect application performance. We show how to embed multiple replicas into a PVFS file, including a mapping with a complete copy local to the writing client, to emulate HDFS's file layout policies. We also highlight implementation issues with HDFS's dependence on disk bandwidth and benefits from pipelined replication. %@ 978-1-4503-0771-0

@inproceedings{tantisiriroj2011duality, abstract = {Data-intensive applications fall into two computing styles: Internet services (cloud computing) or high-performance computing (HPC). In both categories, the underlying file system is a key component for scalable application performance. In this paper, we explore the similarities and differences between PVFS, a parallel file system used in HPC at large scale, and HDFS, the primary storage system used in cloud computing with Hadoop. We integrate PVFS into Hadoop and compare its performance to HDFS using a set of data-intensive computing benchmarks. We study how HDFS-specific optimizations can be matched using PVFS and how consistency, durability, and persistence tradeoffs made by these file systems affect application performance. We show how to embed multiple replicas into a PVFS file, including a mapping with a complete copy local to the writing client, to emulate HDFS's file layout policies. We also highlight implementation issues with HDFS's dependence on disk bandwidth and benefits from pipelined replication.}, acmid = {2063474}, added-at = {2012-10-30T16:39:40.000+0100}, address = {New York, NY, USA}, articleno = {67}, author = {Tantisiriroj, Wittawat and Son, Seung Woo and Patil, Swapnil and Lang, Samuel J. and Gibson, Garth and Ross, Robert B.}, biburl = {https://www.bibsonomy.org/bibtex/21102ab3cc19f7e1c4e025bd0af2a0842/telekoma}, booktitle = {Proceedings of 2011 International Conference for High Performance Computing, Networking, Storage and Analysis}, description = {On the duality of data-intensive file system design}, doi = {10.1145/2063384.2063474}, interhash = {9a174f889754f9bc295fa46b75eeb6e3}, intrahash = {1102ab3cc19f7e1c4e025bd0af2a0842}, isbn = {978-1-4503-0771-0}, keywords = {cloud distributed file hdfs hpc master performance pvfs seminar seminar:dfs system uni ws1213}, location = {Seattle, Washington}, numpages = {12}, pages = {67:1--67:12}, publisher = {ACM}, series = {SC '11}, timestamp = {2012-10-30T16:39:40.000+0100}, title = {On the duality of data-intensive file system design: reconciling HDFS and PVFS}, url = {http://doi.acm.org/10.1145/2063384.2063474}, year = 2011 }