Skip to content

Commit b4e4693

Browse files
rujinlongrujinlong
committed
test
1 parent bd03afa commit b4e4693

172 files changed

Lines changed: 9957 additions & 28 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/stub_test.yml‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
name: Stub pipeline test
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
- dev_ru
8+
pull_request:
9+
branches:
10+
- main
11+
workflow_dispatch:
12+
13+
jobs:
14+
stub_viroprofiler:
15+
name: Stub test - VIROPROFILER workflow
16+
runs-on: ubuntu-latest
17+
timeout-minutes: 30
18+
19+
steps:
20+
- name: Check out repository
21+
uses: actions/checkout@v4
22+
23+
- name: Install Nextflow
24+
uses: nf-core/setup-nextflow@v2
25+
with:
26+
version: ">=23.04.0"
27+
28+
- name: Resolve samplesheet paths
29+
run: |
30+
export STUB_R1_PATH="${{ github.workspace }}/tests/data/stub_R1.fastq.gz"
31+
export STUB_R2_PATH="${{ github.workspace }}/tests/data/stub_R2.fastq.gz"
32+
envsubst '$STUB_R1_PATH $STUB_R2_PATH' < tests/data/samplesheet_stub.csv > /tmp/samplesheet_stub.csv
33+
34+
- name: Run stub test
35+
run: |
36+
nextflow run main.nf \
37+
-stub \
38+
-profile test_stub \
39+
--input /tmp/samplesheet_stub.csv \
40+
--outdir output_stub \
41+
-with-trace \
42+
2>&1 | tee nextflow_stub.log
43+
44+
- name: Verify success
45+
run: |
46+
if grep -q "ERROR ~" nextflow_stub.log; then
47+
echo "=== Errors found ==="
48+
grep "ERROR ~" nextflow_stub.log
49+
exit 1
50+
fi
51+
grep -q "Succeeded\|Pipeline completed successfully" nextflow_stub.log || \
52+
{ echo "Pipeline did not report success"; tail -20 nextflow_stub.log; exit 1; }
53+
54+
- name: Upload logs on failure
55+
if: failure()
56+
uses: actions/upload-artifact@v4
57+
with:
58+
name: stub-viroprofiler-logs
59+
path: |
60+
nextflow_stub.log
61+
.nextflow.log
62+
output_stub/pipeline_info/
63+
64+
stub_contiganno:
65+
name: Stub test - CONTIGANNO workflow
66+
runs-on: ubuntu-latest
67+
timeout-minutes: 30
68+
69+
steps:
70+
- name: Check out repository
71+
uses: actions/checkout@v4
72+
73+
- name: Install Nextflow
74+
uses: nf-core/setup-nextflow@v2
75+
with:
76+
version: ">=23.04.0"
77+
78+
- name: Run stub test
79+
run: |
80+
nextflow run main.nf \
81+
-stub \
82+
-profile test_stub \
83+
--input_contigs "${{ github.workspace }}/tests/data/stub_contigs.fasta" \
84+
--outdir output_stub_contiganno \
85+
2>&1 | tee nextflow_stub_contiganno.log
86+
87+
- name: Verify success
88+
run: |
89+
if grep -q "ERROR ~" nextflow_stub_contiganno.log; then
90+
echo "=== Errors found ==="
91+
grep "ERROR ~" nextflow_stub_contiganno.log
92+
exit 1
93+
fi
94+
grep -q "Succeeded\|Pipeline completed successfully" nextflow_stub_contiganno.log || \
95+
{ echo "Pipeline did not report success"; tail -20 nextflow_stub_contiganno.log; exit 1; }
96+
97+
- name: Upload logs on failure
98+
if: failure()
99+
uses: actions/upload-artifact@v4
100+
with:
101+
name: stub-contiganno-logs
102+
path: |
103+
nextflow_stub_contiganno.log
104+
.nextflow.log
105+
output_stub_contiganno/pipeline_info/

‎bin/normalize_abundance.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ def main(fin_abundance, fin_covfrac, fout, covfrac_cutoff, reads_length, inflati
2323
df_covfrac = read_coverm(fin_covfrac)
2424

2525
# normalize coverage fraction: set value to 0 if they are less than the cutoff, otherwise set to 1
26-
df_covfrac = df_covfrac.map(lambda x: 0 if x < covfrac_cutoff else 1)
26+
df_covfrac = (df_covfrac >= covfrac_cutoff).astype(int)
2727

2828
# multiply the abundance by the normalized coverage fraction
2929
df_abundance_normalized = df_abundance * df_covfrac

‎bin/parse_mmseqsTaxa.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,11 @@ def main(fin, fout_prefix, unclassified, dbsource):
6565
df = pd.read_csv(fin, sep="\t", names=cols)
6666

6767
# Remove unclassified contigs
68+
n_total = len(df)
6869
df = df[~df.lineage.isna()]
70+
n_unclassified = n_total - len(df)
71+
if n_unclassified > 0:
72+
print(f"[parse_mmseqsTaxa] {n_unclassified}/{n_total} contigs have no taxonomy lineage and were excluded.", flush=True)
6973

7074
# Format taxonomy ranks
7175
formatted_taxa = df.apply(lambda x: format_mmseqs_taxa(x, unclassified, col_lineage="lineage", dbsource=dbsource), axis=1)
@@ -76,7 +80,7 @@ def main(fin, fout_prefix, unclassified, dbsource):
7680

7781
# save selected columns for merging with vConTACT2 clusters
7882
if dbsource == "ICTV":
79-
df_sel = df_formatted[['contig_id', 'Realm', 'Kingdom', 'Phylum', 'Class', 'Order', 'Family', 'Genus', 'Species', 'Strain']]
83+
df_sel = df_formatted[['contig_id', 'Realm', 'Kingdom', 'Phylum', 'Class', 'Order', 'Family', 'Genus', 'Species']]
8084
elif dbsource == "NCBI":
8185
df_sel = df_formatted[['contig_id', 'Kingdom', 'Phylum', 'Class', 'Order', 'Family', 'Genus', 'Species']]
8286
df_sel.to_csv('{}.tsv'.format(fout_prefix), sep="\t", index=False)

‎conf/test_stub.config‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
3+
Nextflow config file for stub tests (pipeline topology validation)
4+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5+
Usage:
6+
nextflow run main.nf -stub -profile test_stub --input /path/to/samplesheet_stub.csv
7+
----------------------------------------------------------------------------------------
8+
*/
9+
10+
params {
11+
config_profile_name = 'Stub test profile'
12+
config_profile_description = 'Validates pipeline topology without running any tools'
13+
14+
// Minimal resources (stub blocks are near-instant)
15+
max_cpus = 2
16+
max_memory = '4.GB'
17+
max_time = '1.h'
18+
19+
// Override in CI via --input with resolved absolute paths
20+
input = "${projectDir}/tests/data/samplesheet_stub.csv"
21+
db = "/tmp/stub_db"
22+
outdir = "output_stub"
23+
tracedir = "output_stub/pipeline_info"
24+
25+
// Disable heavyweight optional modules
26+
use_abricate = false
27+
use_decontam = false
28+
use_eggnog = false
29+
use_dram = true
30+
use_iphop = true
31+
use_kraken2 = false
32+
use_phamb = false
33+
binning = false
34+
replicyc = "bacphlip"
35+
reads_type = "raw"
36+
assemblies = "scaffolds"
37+
mode = "all"
38+
}
39+
40+
// Disable ALL container engines for stub runs
41+
docker.enabled = false
42+
singularity.enabled = false
43+
podman.enabled = false
44+
shifter.enabled = false
45+
charliecloud.enabled = false
46+
apptainer.enabled = false
47+
params.enable_conda = false

‎docker/viroprofiler-geneannot/Dockerfile‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@ COPY ./docker/viroprofiler-geneannot/env_*.yml /tmp
1313

1414
# install DRAM to `base` env
1515
RUN --mount=type=cache,target=/opt/conda/pkgs micromamba install -n base -f /tmp/env_dram.yml -y
16-
COPY ./docker/viroprofiler-geneannot/database_handler.py /opt/conda/lib/python3.10/site-packages/mag_annotator
16+
COPY ./docker/viroprofiler-geneannot/database_handler.py /tmp/database_handler.py
17+
RUN SITE_PKGS=$(python3 -c "import site; print(site.getsitepackages()[0])") && \
18+
cp /tmp/database_handler.py $SITE_PKGS/mag_annotator/
1719

1820
# Install emapper to `viroprofiler_emapper` env
1921
RUN --mount=type=cache,target=/opt/conda/pkgs micromamba create -f /tmp/env_emapper.yml -y

‎docker/viroprofiler-replicyc/Dockerfile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ COPY ./docker/viroprofiler-replicyc/env_*.yml /tmp
1212

1313
# Install packages into `base` env
1414
RUN --mount=type=cache,target=/opt/conda/pkgs micromamba install -n base -f /tmp/env_replidec.yml -y
15-
RUN cd /opt/conda/lib/python3.10/site-packages/Replidec && \
15+
RUN REPLIDEC_DIR=$(python3 -c "import site; print(site.getsitepackages()[0])")/Replidec && \
16+
cd $REPLIDEC_DIR && \
1617
wget https://zenodo.org/record/6975142/files/db_v0.2.3.tar.gz && \
1718
tar xzf db_v0.2.3.tar.gz && \
1819
rm db_v0.2.3.tar.gz

‎docker/viroprofiler-taxa/Dockerfile‎

Lines changed: 7 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -12,30 +12,18 @@ RUN --mount=type=cache,target=/opt/conda/pkgs micromamba install -n base -f /tmp
1212
ARG MAMBA_DOCKERFILE_ACTIVATE=1
1313

1414

15-
RUN cd /opt/conda/lib/python3.8/site-packages/vcontact2/data && \
15+
RUN VCONTACT2_DATA=$(python3 -c "import site; print(site.getsitepackages()[0])")/vcontact2/data && \
16+
cd $VCONTACT2_DATA && \
1617
/opt/conda/bin/wget https://bitbucket.org/MAVERICLab/vcontact2/raw/6089ed83fd03a844de3140c442863c7b29881d43/vcontact2/data/ViralRefSeq-prokaryotes-v211.Merged-reference.csv && \
1718
/opt/conda/bin/wget https://bitbucket.org/MAVERICLab/vcontact2/raw/6089ed83fd03a844de3140c442863c7b29881d43/vcontact2/data/ViralRefSeq-prokaryotes-v211.faa.gz && \
1819
/opt/conda/bin/wget https://bitbucket.org/MAVERICLab/vcontact2/raw/6089ed83fd03a844de3140c442863c7b29881d43/vcontact2/data/ViralRefSeq-prokaryotes-v211.protein2contig.csv
1920

20-
# Set PATH manually, because nextflow doesn't activate base env by default.
21-
# ENV PATH=/opt/MMseqs2/build/bin:/opt/conda/bin:$PATH
22-
ENV PATH=/opt/mmseqs/bin:/opt/MMseqs2/build/bin:/opt/conda/bin:$PATH
23-
24-
RUN cd /opt && \
25-
git clone https://github.com/soedinglab/MMseqs2.git && \
26-
cd /opt/MMseqs2 && \
27-
git checkout 3b9cf88 && \
28-
mkdir build && \
29-
cd build && \
30-
cmake -DHAVE_SANITIZER=1 -DCMAKE_BUILD_TYPE=ASan -DCMAKE_INSTALL_PREFIX=. .. && \
31-
make -j 1 && \
32-
make install
33-
3421
RUN micromamba clean --all --yes
3522

36-
37-
# COPY ./docker/viroprofiler-taxa/mmseqs /opt/conda/bin
38-
# RUN chmod +x /opt/conda/bin/mmseqs
23+
# Set PATH manually, because nextflow doesn't activate base env by default.
24+
ENV PATH=/opt/mmseqs/bin:/opt/conda/bin:$PATH
3925

4026
RUN cd /opt && \
41-
wget https://mmseqs.com/latest/mmseqs-linux-avx2.tar.gz; tar xvfz mmseqs-linux-avx2.tar.gz; rm mmseqs-linux-avx2.tar.gz
27+
wget https://mmseqs.com/latest/mmseqs-linux-avx2.tar.gz && \
28+
tar xvfz mmseqs-linux-avx2.tar.gz && \
29+
rm mmseqs-linux-avx2.tar.gz

‎modules/local/abricate.nf‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,4 +30,15 @@ process ABRICATE {
3030
abricate: \$(echo \$(abricate -v) | sed 's/^abricate //' ))
3131
END_VERSIONS
3232
"""
33+
34+
stub:
35+
"""
36+
printf '#FILE\tSEQUENCE\tSTART\tEND\tSTRAND\tGENE\tCOVERAGE\tCOVERAGE_MAP\tGAPS\t%COVERAGE\t%IDENTITY\tDATABASE\tACCESSION\tPRODUCT\tRESISTANCE\n' > ARG_argannot.tsv
37+
printf 'SEQUENCE\tSTART\tEND\tSTRAND\tGENE\tCOVERAGE\tCOVERAGE_MAP\tGAPS\t%COVERAGE\t%IDENTITY\tDATABASE\tACCESSION\tPRODUCT\tRESISTANCE\n' > anno_abricate.tsv
38+
39+
cat <<-END_VERSIONS > versions.yml
40+
"${task.process}":
41+
abricate: 1.0.1
42+
END_VERSIONS
43+
"""
3344
}

‎modules/local/abundance.nf‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,17 @@ process MAPPING2CONTIGS {
3030
samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//')
3131
END_VERSIONS
3232
"""
33+
34+
stub:
35+
"""
36+
touch ${meta.id}.bam
37+
38+
cat <<-END_VERSIONS > versions.yml
39+
"${task.process}":
40+
minimap2: 2.24
41+
samtools: 1.15.1
42+
END_VERSIONS
43+
"""
3344
}
3445

3546
process CONTIGINDEX {
@@ -49,13 +60,25 @@ process CONTIGINDEX {
4960
def args = task.ext.args ?: ''
5061
"""
5162
mkdir -p bowtie2
52-
bowtie2-build $args --threads $task.cpus $contigs bowtie2/bowtie2idx
63+
bowtie2-build $args --threads $task.cpus $contigs bowtie2/bowtie2idx
5364
5465
cat <<-END_VERSIONS > versions.yml
5566
"${task.process}":
5667
bowtie2: \$(echo \$(bowtie2 --version 2>&1) | sed 's/^.*bowtie2-align-s version //; s/ .*\$//')
5768
END_VERSIONS
5869
"""
70+
71+
stub:
72+
"""
73+
mkdir -p bowtie2
74+
touch bowtie2/idx.1.bt2
75+
touch bowtie2/idx.rev.1.bt2
76+
77+
cat <<-END_VERSIONS > versions.yml
78+
"${task.process}":
79+
bowtie2: 2.4.4
80+
END_VERSIONS
81+
"""
5982
}
6083

6184

@@ -93,6 +116,18 @@ process MAPPING2CONTIGS2 {
93116
coverm: \$(echo \$(coverm --version 2>&1) | sed 's/^.*coverm //; s/ .*\$//')
94117
END_VERSIONS
95118
"""
119+
120+
stub:
121+
"""
122+
touch ${meta.id}.bam
123+
124+
cat <<-END_VERSIONS > versions.yml
125+
"${task.process}":
126+
bowtie2: 2.4.4
127+
samtools: 1.15.1
128+
coverm: 0.6.1
129+
END_VERSIONS
130+
"""
96131
}
97132

98133

@@ -131,4 +166,14 @@ process ABUNDANCE {
131166
# compresss count table
132167
pigz -p $task.cpus abundance_contigs_*.tsv
133168
"""
169+
170+
stub:
171+
"""
172+
printf 'Contig\tsample1\n' | gzip > abundance_contigs_count.tsv.gz
173+
printf 'Contig\tsample1\n' | gzip > abundance_contigs_covered_fraction.tsv.gz
174+
printf 'Contig\tsample1\n' | gzip > abundance_contigs_tpm.tsv.gz
175+
printf 'Contig\tsample1\n' | gzip > abundance_contigs_rpkm.tsv.gz
176+
printf 'Contig\tsample1\n' | gzip > abundance_contigs_trimmed_mean.tsv.gz
177+
printf 'Contig\tsample1\n' | gzip > abundance_contigs_reads_per_base.tsv.gz
178+
"""
134179
}

0 commit comments

Comments
 (0)