feat: write per-step processing logs to separate files

Each submission processing step now logs to both the main parsing.log
and its own file under a new parsing_logs/ folder in the submission's
data folder:

  parsed.log    - RDF generation / parsing
  metadata.log  - metadata extraction
  labels.log    - missing labels generation
  obsolete.log  - obsolete classes generation
  indexed.log   - index all data / terms / properties (shared)
  metrics.log   - metrics calculation
  diff.log      - submission version diff

A memoized step_logger helper in OntologyProcessor wraps the main
logger and a per-file logger in a MultiLogger so every step broadcasts
to both destinations. MultiLogger's broadcast methods are fixed to
splat args (they previously logged messages as arrays). The
parsing_logs/ folder is removed on archive.

Author: maboukerfa

lib/ontologies_linked_data/models/ontology_submission.rb

@@ -454,6 +454,10 @@ def parsing_log_path
         return File.join(self.data_folder, 'parsing.log')
       end
 
+      def parsing_logs_folder
+        return File.join(self.data_folder, 'parsing_logs')
+      end
+
       def triples_file_path
         self.bring(:uploadFilePath) if self.bring?(:uploadFilePath)
         self.bring(:masterFileName) if self.bring?(:masterFileName)

lib/ontologies_linked_data/services/submission_process/operations/submission_archiver.rb

@@ -3,7 +3,7 @@ module Services
     class OntologySubmissionArchiver < OntologySubmissionProcess
 
       FILES_TO_DELETE = ['labels.ttl', 'mappings.ttl', 'obsolete.ttl', 'owlapi.xrdf', 'errors.log']
-      FOLDERS_TO_DELETE = ['unzipped']
+      FOLDERS_TO_DELETE = ['unzipped', 'parsing_logs']
       FILE_SIZE_ZIPPING_THRESHOLD = 100 * 1024 * 1024 # 100MB
 
       def process(force: false)

lib/ontologies_linked_data/services/submission_process/submission_processor.rb

@@ -34,30 +34,30 @@ def process_submission(logger, options = {})
             @submission.archive
           else
 
-            @submission.generate_rdf(logger, reasoning: process_reasoning?(options)) if process_rdf?(options)
+            @submission.generate_rdf(step_logger(logger, 'parsed.log'), reasoning: process_reasoning?(options)) if process_rdf?(options)
 
             parsed = @submission.ready?(status: %i[rdf])
 
-            @submission = @submission.extract_metadata(logger, user_params: options[:params], heavy_extraction: extract_metadata?(options))
+            @submission = @submission.extract_metadata(step_logger(logger, 'metadata.log'), user_params: options[:params], heavy_extraction: extract_metadata?(options))
 
-            @submission.generate_missing_labels(logger) if generate_missing_labels?(options)
+            @submission.generate_missing_labels(step_logger(logger, 'labels.log')) if generate_missing_labels?(options)
 
-            @submission.generate_obsolete_classes(logger) if generate_obsolete_classes?(options)
+            @submission.generate_obsolete_classes(step_logger(logger, 'obsolete.log')) if generate_obsolete_classes?(options)
 
             if !parsed && (index_search?(options) || index_properties?(options) || index_all_data?(options))
               raise StandardError, "The submission #{@submission.ontology.acronym}/submissions/#{@submission.submissionId}
                                 cannot be indexed because it has not been successfully parsed"
             end
 
-            @submission.index_all(logger, commit: process_index_commit?(options)) if index_all_data?(options)
+            @submission.index_all(step_logger(logger, 'indexed.log'), commit: process_index_commit?(options)) if index_all_data?(options)
 
-            @submission.index_terms(logger, commit: process_index_commit?(options)) if index_search?(options)
+            @submission.index_terms(step_logger(logger, 'indexed.log'), commit: process_index_commit?(options)) if index_search?(options)
 
-            @submission.index_properties(logger, commit: process_index_commit?(options)) if index_properties?(options)
+            @submission.index_properties(step_logger(logger, 'indexed.log'), commit: process_index_commit?(options)) if index_properties?(options)
 
-            @submission.generate_metrics(logger) if process_metrics?(options)
+            @submission.generate_metrics(step_logger(logger, 'metrics.log')) if process_metrics?(options)
 
-            @submission.generate_diff(logger) if process_diff?(options)
+            @submission.generate_diff(step_logger(logger, 'diff.log')) if process_diff?(options)
           end
           @submission.save
           logger.info("Submission processing of #{@submission.id} completed successfully")
@@ -75,6 +75,19 @@ def notify_submission_processed(logger)
         logger.error("Email sending failed: #{e.message}\n#{e.backtrace.join("\n\t")}"); logger.flush
       end
 
+      # Build a logger for a processing step that writes both to the main log
+      # (the one passed into #process) and to a step-specific file located in the
+      # submission's parsing_logs folder. Loggers are memoized per file name so
+      # steps sharing a file (e.g. the indexers) append to the same handle.
+      def step_logger(main_logger, filename)
+        @step_loggers ||= {}
+        @step_loggers[filename] ||= begin
+          FileUtils.mkdir_p(@submission.parsing_logs_folder)
+          file_logger = Logger.new(File.join(@submission.parsing_logs_folder, filename))
+          LinkedData::Utils::MultiLogger.new(loggers: [main_logger, file_logger])
+        end
+      end
+
       def process_archive?(options)
         options[:archive].eql?(true)
       end

lib/ontologies_linked_data/utils/multi_logger.rb

@@ -2,8 +2,18 @@
 
 module LinkedData::Utils
   class MultiLogger < OmniLogger
-    def flush()
-      @loggers.each { |logger| logger.flush }
+    def flush
+      @loggers.each(&:flush)
+    end
+
+    # OmniLogger's generated level methods broadcast with `logger.send(level, args)`,
+    # passing the args as a single array (so `info("msg")` logs `["msg"]`). Redefine
+    # them here to splat the args and forward the block to each underlying logger.
+    Logger::Severity.constants.each do |level|
+      name = level.downcase
+      define_method(name) do |*args, &block|
+        @loggers.each { |logger| logger.send(name, *args, &block) }
+      end
     end
   end
-end
+end