@inproceedings{fbce02285def45e8b2038de7708109b8,
title = "Libra: Improved partitioning strategies for massive comparative metagenomics analysis",
abstract = "Big-data analytics platforms, such as Hadoop, are appealing for scientific computation because they are ubiquitous, well-supported, and well-understood. Unfortunately, load-balancing is a common challenge of implementing large-scale scientific computing applications on these platforms. In this paper we present the design and implementation of Libra, a Hadoop-based tool for comparative metagenomics (comparing samples of genetic material collected from the environment). We describe the computation that Libra performs and how that computation is implemented using Hadoop tasks, including the techniques used by Libra to ensure that the task workloads are balanced despite nonuniform sample sizes and skewed distributions of genetic material in the samples. On a 10-machine Hadoop cluster Libra can analyze the entire Tara Ocean Viromes of ~4.2 billion reads in fewer than 20 hours.",
keywords = "Comparative genomics, Genome distance, K-mer, Metagenomics, Parallel, Task-partitioning",
author = "Illyoung Choi and Mahew Bomhoff and Ponsero, {Alise J.} and Hurwitz, {Bonnie L.} and Ken Youens-Clark and Hartman, {John H.}",
note = "Funding Information: We thank Binil Benjamin, Russell Lewis and Hurwitz Lab for comments on the implementation of the algorithm and feedback on the manuscript. We thank staff at the University of Arizona Information Technology Services for access and system support for the Hadoop cluster. This research was funded in part by NSF grant OAR-1640775. Publisher Copyright: {\textcopyright} 2018 Association for Computing Machinery.; 9th Workshop on Scientific Cloud Computing, ScienceCloud 2018 ; Conference date: 11-06-2018",
year = "2018",
month = jun,
day = "11",
doi = "10.1145/3217880.3217882",
language = "English (US)",
series = "Proceedings of the 9th Workshop on Scientific Cloud Computing, ScienceCloud 2018 - Co-located with HPDC 2018",
publisher = "Association for Computing Machinery, Inc",
booktitle = "Proceedings of the 9th Workshop on Scientific Cloud Computing, ScienceCloud 2018 - Co-located with HPDC 2018",
}