@article {Sun2020.11.14.382994, author = {Zheng Sun and Shi Huang and Meng Zhang and Qi-Yun Zhu and Niina Haiminen and Anna-Paola Carrieri and Yoshiki V{\'a}zquez-Baeza and Laxmi Parida and Ho-Cheol Kim and Rob Knight and Yang-Yu Liu}, title = {Challenges in Benchmarking Metagenomic Profilers}, elocation-id = {2020.11.14.382994}, year = {2020}, doi = {10.1101/2020.11.14.382994}, publisher = {Cold Spring Harbor Laboratory}, abstract = {Accurate microbial identification and abundance estimation are crucial for metagenomics analysis. Various methods for classifying metagenomic data and estimating taxonomic profiles, broadly referred to as metagenomic profilers, have been developed. Yet, benchmarking metagenomic profilers remains challenging because some tools are designed to report relative sequence abundance while others report relative taxonomic abundance. Here, we show how misleading conclusions can be drawn by neglecting this distinction between relative abundance types when benchmarking metagenomic profilers. Moreover, we show compelling evidence that interchanging sequence abundance and taxonomic abundance will influence both per-sample summary statistics and cross-sample comparisons. We suggest that the microbiome research community should pay attention to potentially misleading biological conclusions arising from this issue when benchmarking metagenomic profilers, by carefully considering the type of abundance data that was analyzed and interpreted, and clearly stating the strategy used for metagenomic profiling.Competing Interest StatementThe authors have declared no competing interest.}, URL = {https://www.biorxiv.org/content/early/2020/11/18/2020.11.14.382994}, eprint = {https://www.biorxiv.org/content/early/2020/11/18/2020.11.14.382994.full.pdf}, journal = {bioRxiv} }