Dev fpo

meteor/counter.py

@@ -401,7 +401,7 @@ def launch_counting(
         cramfile_strain: Path,
         count_file: Path,
         ref_json: dict,
-        census_json: dict,
+        stage1_json_data: dict,
         stage1_json: Path,
     ):
         """Function that count reads from a cram file, using the given methods in count:
@@ -453,8 +453,8 @@ def launch_counting(
         self.write_stat(count_file, abundance, database)
         counted_reads = len(reads)
         config = self.set_counter_config(counted_reads, count_file)
-        census_json.update(config)
-        self.save_config(census_json, stage1_json)
+        stage1_json_data.update(config)
+        self.save_config(stage1_json_data, stage1_json)
         if self.keep_filtered_alignments:
             cramfile_strain_unsorted = Path(mkstemp(dir=self.meteor.tmp_dir)[1])
             self.save_cram_strain(
@@ -485,114 +485,99 @@ def launch_counting(
     def execute(self) -> None:
         """Compute the mapping"""
         mapping_done = True
-        try:
-            # Get the ini ref
-            ref_json = self.get_reference_info(self.meteor.ref_dir)
-            Component.check_catalogue(ref_json)
-            self.meteor.ref_name = ref_json["reference_info"]["reference_name"]
-            if not self.identity_user:
-                if ref_json["reference_info"]["database_type"] == "complete":
-                    self.identity_threshold = self.DEFAULT_IDENTITY_THRESHOLD_COMPLETE
-                else:
-                    self.identity_threshold = self.DEFAULT_IDENTITY_THRESHOLD_TAXO
-            else:
-                self.identity_threshold = self.identity_user
-        except AssertionError:
+        ref_json = self.get_reference_info(self.meteor.ref_dir)
+        Component.check_catalogue(ref_json)
+        self.meteor.ref_name = ref_json["reference_info"]["reference_name"]
+        if self.identity_user:
+            self.identity_threshold = self.identity_user
+        elif ref_json["reference_info"]["database_type"] == "complete":
+            self.identity_threshold = self.DEFAULT_IDENTITY_THRESHOLD_COMPLETE
+        else:
+             self.identity_threshold = self.DEFAULT_IDENTITY_THRESHOLD_TAXO
+
+        census_json_files = list(
+            self.meteor.fastq_dir.glob("*_census_stage_0.json")
+        )
+        if len(census_json_files) == 0:
             logging.error(
-                "No *_reference.json file found in %s. "
-                "One *_reference.json is expected",
-                self.meteor.ref_dir,
+                "No *_census_stage_0.json file found in %s",
+                self.meteor.fastq_dir,
             )
             sys.exit(1)
-        try:
-            census_json_files = list(
-                self.meteor.fastq_dir.glob("*_census_stage_0.json")
-            )
-            assert len(census_json_files) > 0
-
-            #  mapping of each sample against reference
-            for library in census_json_files:
-                census_json = self.read_json(library)
-                sample_info = census_json["sample_info"]
-                stage1_dir = self.meteor.mapping_dir / sample_info["sample_name"]
-                stage1_dir.mkdir(exist_ok=True, parents=True)
-                # if self.pysam_test:
-                self.json_data[library] = {
-                    "census": census_json,
-                    "directory": stage1_dir,
-                    "Stage1FileName": stage1_dir
-                    / f"{sample_info['sample_name']}_census_stage_1.json",
-                    "reference": ref_json,
-                }
-                if not self.json_data[library]["Stage1FileName"].exists():
-                    mapping_done = False
-            # mapping already done and no overwriting
-            if mapping_done:
-                logging.info(
-                    "Mapping already done for sample: %s",
-                    sample_info["sample_name"],
-                )
-                logging.info("Skipped !")
-            else:
-                logging.info("Launch mapping")
-                self.launch_mapping()
-            # running counter
-            raw_cram_file = (
-                self.json_data[library]["directory"]
-                / f"{sample_info['sample_name']}_raw.cram"
-            )
-            cram_file = (
-                self.json_data[library]["directory"]
-                / f"{sample_info['sample_name']}.cram"
-            )
-            count_file = (
-                self.json_data[library]["directory"]
-                / f"{sample_info['sample_name']}.tsv.xz"
-            )
-            start = perf_counter()
+
+        #  mapping of each sample against reference
+        for library in census_json_files:
+            census_json = self.read_json(library)
+            sample_name = census_json["sample_info"]["sample_name"]
+            stage1_dir = self.meteor.mapping_dir / sample_name
+            stage1_dir.mkdir(exist_ok=True, parents=True)
             stage1_json = (
                 self.meteor.mapping_dir
-                / sample_info["sample_name"]
-                / f"{sample_info['sample_name']}_census_stage_1.json"
+                / sample_name
+                / f"{sample_name}_census_stage_1.json"
             )
-            census_json = self.read_json(stage1_json)
-            self.launch_counting(
-                raw_cram_file,
-                cram_file,
-                count_file,
-                ref_json,
-                census_json,
-                stage1_json,
+            self.json_data[library] = {
+                "census": census_json,
+                "directory": stage1_dir,
+                "Stage1FileName": stage1_json,
+                "reference": ref_json,
+            }
+            if not stage1_json.exists():
+                mapping_done = False
+        # mapping already done and no overwriting
+        if mapping_done:
+            logging.info(
+                "Mapping already done for sample: %s",
+                sample_name,
             )
-            # Add final mapping rate
-            census_json = self.read_json(stage1_json)
-            census_json["counting"]["final_mapping_rate"] = (
-                round(
-                    census_json["counting"]["counted_reads"]
-                    / census_json["mapping"]["total_read_count"]
-                    * 100,
-                    2
-                )
+            logging.info("Skipped !")
+        else:
+            logging.info("Launch mapping")
+            self.launch_mapping()
+        # running counter
+        stage1_json_data = self.read_json(stage1_json)
+        raw_cram_file = (stage1_dir /
+            stage1_json_data["mapping"]["mapping_file"]
+        )
+        cram_file = (
+            stage1_dir
+            / f"{sample_name}.cram"
+        )
+        count_file = (
+            stage1_dir
+            / f"{sample_name}.tsv.xz"
+        )
+        start = perf_counter()
+        self.launch_counting(
+            raw_cram_file,
+            cram_file,
+            count_file,
+            ref_json,
+            stage1_json_data,
+            stage1_json,
+        )
+        # Add final mapping rate
+        stage1_json_data = self.read_json(stage1_json)
+        stage1_json_data["counting"]["final_mapping_rate"] = (
+            round(
+                stage1_json_data["counting"]["counted_reads"]
+                / stage1_json_data["mapping"]["total_read_count"]
+                * 100,
+                2
             )
-            self.save_config(
-                census_json,
-                stage1_json
+        )
+        self.save_config(
+            stage1_json_data,
+            stage1_json
+        )
+        logging.info("Completed counting in %f seconds", perf_counter() - start)
+        if not self.keep_all_alignments:
+            logging.info(
+                "Raw cram file is not kept (--ka). "
+                "Re-counting operation will need to be performed from scratch."
             )
+            raw_cram_file.unlink(missing_ok=True)
+            raw_cram_file.with_suffix(".cram.crai").unlink(missing_ok=True)
 
-            logging.info("Completed counting in %f seconds", perf_counter() - start)
-            if not self.keep_all_alignments:
-                logging.info(
-                    "Raw cram file is not kept (--ka). "
-                    "Re-counting operation will need to be performed from scratch."
-                )
-                raw_cram_file.unlink(missing_ok=True)
-                raw_cram_file.with_suffix(".cram.crai").unlink(missing_ok=True)
-        except AssertionError:
-            logging.error(
-                "No *_census_stage_0.json file found in %s",
-                self.meteor.fastq_dir,
-            )
-            sys.exit(1)
-        else:
-            # Not sure if it's a good idea to delete a temporary file
-            rmtree(self.meteor.tmp_dir, ignore_errors=True)
+        # Not sure if it's a good idea to delete a temporary file
+        rmtree(self.meteor.tmp_dir, ignore_errors=True)

meteor/downloader.py

@@ -67,22 +67,22 @@ def show_progress(self, block_num: int, block_size: int, total_size: int):
             )
         self.progress_bar.update(block_size)
 
-    def extract_tar(self, catalogue: Path) -> None:
+    def extract_tar(self, tar_archive: Path) -> None:
         """Extract tar file
 
-        :param catalogue: (Path) A path object to the given catalog
+        :param tar_archive: (Path) A path object to the tar archive file
         """
-        logging.info("Extracting %s catalogue", self.choice)
-        with tarfile.open(catalogue) as tar:
+        logging.info("Extracting archive %s", tar_archive.name)
+        with tarfile.open(tar_archive) as tar:
             tar.extractall(path=self.meteor.ref_dir, filter='data')
-        catalogue.unlink(missing_ok=True)
+        tar_archive.unlink(missing_ok=True)
 
     def execute(self) -> None:
         try:
-            # for choice in self.user_choice:
             logging.info(
-                "Download %s catalogue",
-                self.catalogues_config[self.choice][self.data_type]["filename"],
+                "Downloading %s catalogue (%s version)",
+                self.choice,
+                "fast" if self.taxonomy else "full"
             )
             url = self.catalogues_config[self.choice][self.data_type]["catalogue"]
             md5_expect = self.catalogues_config[self.choice][self.data_type]["md5"]
@@ -94,7 +94,7 @@ def execute(self) -> None:
             self.progress_bar.close()
             if self.choice == Component.TEST_CATALOGUE:
                 for sample in self.catalogues_config[self.choice]["samples"]:
-                    logging.info("Download %s fastq file", sample)
+                    logging.info("Downloading %s fastq file", sample)
                     url_fastq = self.catalogues_config[self.choice]["samples"][sample][
                         "catalogue"
                     ]
@@ -116,8 +116,8 @@ def execute(self) -> None:
                 assert md5_expect == self.getmd5(catalogue)
             self.extract_tar(catalogue)
             logging.info(
-                "The catalogue is now ready to be used in the folder: %s",
-                str(catalogue.with_suffix("").stem),
+                "The catalogue is now available in the directory %s",
+                str(catalogue).removesuffix('.tar.xz'),
             )
         except AssertionError:
             logging.error("MD5sum of %s has a different value than expected", catalogue)

meteor/merging.py

@@ -223,7 +223,10 @@ def execute(self) -> None:
             for filename in fnmatch.filter(files, "*census_stage_2.json")
         ]
         if len(all_census) == 0:
-            logging.error("No census stage 2 found in the specified repository.")
+            logging.error(
+                "No *_census_stage_2.json files found in %s",
+                self.meteor.profile_dir,
+            )
             sys.exit(1)
         else:
             logging.info("%d census files have been detected.", len(all_census))

meteor/meteor.py

@@ -43,6 +43,8 @@ def get_logging() -> logging.Logger:  # pragma: no cover
     :return: (logging.logger) A logger object
     """
     logger = logging.getLogger()
+    if logger.hasHandlers():
+        logger.handlers.clear()
     logger.setLevel(logging.INFO)
     formatter = logging.Formatter("%(asctime)s :: %(levelname)s :: %(message)s")
     # Stream in the the console

meteor/phylogeny.py

@@ -14,8 +14,7 @@
 import re
 import logging
 import pandas as pd
-
-# import sys
+import sys
 # from subprocess import run, Popen, PIPE
 # from packaging.version import parse
 
@@ -444,6 +443,7 @@ def execute(self) -> None:
                     logging.error(
                         "MSP %s generated an exception: %s", msp_file.name, exc
                     )
+                    sys.exit(1)
 
         logging.info("Completed phylogeny in %f seconds", perf_counter() - start)
         logging.info(

meteor/treebuilder.py

@@ -124,7 +124,10 @@ def execute(self) -> None:
         all_census = [Path(root) / f for root, _, files in os.walk(self.meteor.strain_dir, followlinks=True)
                       for f in fnmatch.filter(files, "*census_stage_3.json")]
         if len(all_census) == 0:
-            logging.error("No census stage found in the specified repository.")
+            logging.error(
+                "No *_census_stage_3.json files found in %s",
+                self.meteor.strain_dir,
+            )
             sys.exit(1)
         else:
             logging.info("%d samples have been detected.", len(all_census))