output abundances per sample in one file, optimise scatter plot

Author: d4straub

bin/compare_performance.r

@@ -179,12 +179,20 @@ if (length(SAMPLES) == 0) {
 
 # (B) ANALYSE
 
-# Initialize dataframe
+# Initialize dataframes
 df <- data.frame(
 	sample=character(),
 	type=character(),
 	value=character(),
 	stringsAsFactors=FALSE)
+abundances <- data.frame(
+	sample=character(),
+	ID=character(),
+	query=character(),
+	target=character(),
+	observed_abund=numeric(),
+	expected_abund=numeric(),
+	stringsAsFactors=FALSE)
 
 # for each sample
 for (sample in SAMPLES) {
@@ -238,10 +246,10 @@ for (sample in SAMPLES) {
 	data$observed_abund <- data$observed_abund/sum(data$observed_abund,na.rm=TRUE)
 	# Sort by expected abundance
 	data <- data[order(data$expected_abund, decreasing = TRUE), ]
-	# write table
-	outfile <- paste0(sample,"_abundances.tsv")
-	print(paste("write",outfile))
-	write.table(data, file = outfile, row.names = FALSE, col.names = TRUE, quote = FALSE, na = '', sep="\t")
+	# append sample data to table
+	data$sample <- rep(sample, nrow(data))
+	data <- data[, c("sample", "ID", "query", "target", "observed_abund", "expected_abund")]
+	abundances <- rbind( abundances, data )
 
 	# (3) STATISTICS
 
@@ -274,12 +282,13 @@ for (sample in SAMPLES) {
 	invisible(dev.off())
 
 	# Scatter plot: Observed vs. Expected Abundance (log-log)
+	data_matches <- data[data$observed_abund >0 & data$expected_abund >0,] # only on matched observed to expected
 	outfile <- paste0(sample,"_scatter_loglog")
 	print(paste("write",outfile))
 	svg(paste0(outfile,".svg"), width = 8, height = 6)
 	plot(
-		log10(data$expected_abund),
-		log10(data$observed_abund),
+		log10(data_matches$expected_abund),
+		log10(data_matches$observed_abund),
 		xlab = "log10(Expected Abundance)",
 		ylab = "log10(Observed Abundance)",
 		main = "Scatter Plot: Observed vs. Expected Abundance (log-log)",
@@ -321,6 +330,12 @@ for (sample in SAMPLES) {
 
 # (5) TABLES - metrics overall
 
+# write table
+outfile <- "abundances_per-sample.tsv"
+abundances$tag <- rep(tag, nrow(abundances))
+print(paste("write",outfile))
+write.table(abundances, file = outfile, row.names = FALSE, col.names = TRUE, quote = FALSE, na = '', sep="\t")
+
 # Write detailed output
 outfile <- "performance_per-sample.tsv"
 df$tag <- rep(tag, nrow(df))

conf/modules.config

@@ -1248,12 +1248,12 @@ process {
             [
                 path: { "${params.outdir}/comparison/${meta.id}" },
                 mode: params.publish_dir_mode,
-                pattern: "*{performance_summary.tsv,.log,performance_boxplot.svg,performance_boxplot.png,performance_per-sample.tsv}"
+                pattern: "*{performance_summary.tsv,.log,performance_boxplot.svg,performance_boxplot.png,performance_per-sample.tsv,abundances_per-sample.tsv}"
             ],
             [
                 path: { "${params.outdir}/comparison/${meta.id}/per-sample" },
                 mode: params.publish_dir_mode,
-                pattern: "*{rank_abundance_curve.svg,_scatter_loglog.svg,_abundance_barplot.svg,_abundances.tsv}"
+                pattern: "*{rank_abundance_curve.svg,_scatter_loglog.svg,_abundance_barplot.svg}"
             ]
         ]
     }

docs/output.md

@@ -721,14 +721,14 @@ The following additional files will be produced:
 <summary>Output files</summary>
 
 - `comparison/<parameter md5sum>_<pipeline version>`
+  - `abundances_per-sample.tsv`: Tab-separated table with abundance of expected and observed sequences per sample, in long format.
   - `performance_summary.tsv`: Tab-separated table with aggregated performance metrics.
   - `performance_per-sample.tsv`: Tab-separated table with performance metrics per sample, in long format.
   - `performance.log`: Log file, complementary to `nucleotide-differences.log`.
   - `performance_boxplot.png`: Boxplot of number of aggregated performance metrics in png format.
   - `performance_boxplot.svg`: Boxplot of number of aggregated performance metrics in svg format.
 - `comparison/<parameter md5sum>_<pipeline version>/per-sample`
   - `<sample>_abundance_barplot.svg`: Side-by-Side bar plots in svg format.
-  - `<sample>_abundances.tsv`: Tab-separated table with expected and observed abundances.
   - `<sample>_rank_abundance_curve.svg`: Rank Abundance Curves in svg format.
   - `<sample>_scatter_loglog.svg`: Scatter plot: Observed vs. Expected Abundance (log-log) in svg format.
 

modules/local/compare_performance.nf

@@ -17,7 +17,7 @@ process COMPARE_PERFORMANCE {
     path("*.png")                            , emit: png
     path("performance_summary.tsv")          , emit: performance
     path("performance_per-sample.tsv")       , emit: performance_per_sample
-    path("*_abundances.tsv")                 , emit: abundance_table_per_sample
+    path("abundances_per-sample.tsv")        , emit: abundance_table_per_sample
     path("performance.log")                  , emit: log
     path("Warnings.txt")                     , emit: warnings
     path "versions.yml"                      , emit: versions, topic: versions