Improve scoop

.rubocop.yml

@@ -105,6 +105,7 @@ Metrics/ClassLength:
     - 'lib/cypress/demographics_randomizer.rb'
     - 'lib/cypress/expected_results_calculator.rb'
     - 'lib/cypress/population_clone_job.rb'
+    - 'lib/cypress/scoop_and_filter.rb'
     - 'lib/validators/cat3_population_validator.rb'
 Metrics/CyclomaticComplexity:
   Max: 12

Gemfile

@@ -126,7 +126,6 @@ group :development, :test do
   # remove scss_lint, incompatible with sass dependency upgrades
   # gem 'scss_lint', require: false
   gem 'selenium-webdriver'
-  gem 'webrick'
 end
 
 group :development do

Gemfile.lock

@@ -265,7 +265,6 @@ GEM
       railties (>= 6.1.0)
     faker (3.8.0)
       i18n (>= 1.8.11, < 2)
-    ffi (1.17.4)
     ffi (1.17.4-aarch64-linux-gnu)
     ffi (1.17.4-aarch64-linux-musl)
     ffi (1.17.4-arm-linux-gnu)
@@ -350,7 +349,7 @@ GEM
     local_time (3.0.3)
     log4r (1.1.10)
     logger (1.7.0)
-    loofah (2.25.1)
+    loofah (2.25.2)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
     macaddr (1.7.2)
@@ -373,7 +372,6 @@ GEM
     mini_magick (5.3.1)
       logger
     mini_mime (1.1.5)
-    mini_portile2 (2.8.9)
     minitest (5.27.0)
     minitest-rails (8.1.0)
       minitest (~> 5.20)
@@ -415,9 +413,6 @@ GEM
     newrelic_rpm (10.4.0)
       logger
     nio4r (2.7.5)
-    nokogiri (1.19.4)
-      mini_portile2 (~> 2.8.2)
-      racc (~> 1.4)
     nokogiri (1.19.4-aarch64-linux-gnu)
       racc (~> 1.4)
     nokogiri (1.19.4-aarch64-linux-musl)
@@ -499,8 +494,8 @@ GEM
       activesupport (>= 5.0.0)
       minitest
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.7.0)
-      loofah (~> 2.25)
+    rails-html-sanitizer (1.7.1)
+      loofah (~> 2.25, >= 2.25.2)
       nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0)
     rails-perftest (0.0.7)
     rails_best_practices (1.23.4)
@@ -578,9 +573,6 @@ GEM
       logger
     ruby2_keywords (0.0.5)
     rubyzip (1.3.0)
-    sass-embedded (1.99.0)
-      google-protobuf (~> 4.31)
-      rake (>= 13)
     sass-embedded (1.99.0-aarch64-linux-gnu)
       google-protobuf (~> 4.31)
     sass-embedded (1.99.0-aarch64-linux-musl)
@@ -669,7 +661,6 @@ GEM
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    webrick (1.9.2)
     websocket (1.2.11)
     websocket-driver (0.8.2)
       base64
@@ -689,7 +680,6 @@ PLATFORMS
   arm-linux-musl
   arm-linux-musleabihf
   arm64-darwin
-  ruby
   x86_64-darwin
   x86_64-linux-gnu
   x86_64-linux-musl
@@ -771,7 +761,6 @@ DEPENDENCIES
   vmstat
   web-console (~> 4.2.0)
   webmock
-  webrick
 
 RUBY VERSION
    ruby 3.4.9p82

app/models/product_test.rb

@@ -92,11 +92,13 @@ def generate_patients(job_id = nil)
     if product.randomize_patients
       # If we're using a "slim test deck", don't pass in any random IDs
       random_ids = slim_test_deck? ? [] : gather_patient_ids
-      Cypress::PopulationCloneJob.new('test_id' => id, 'patient_ids' => master_patient_ids, 'randomization_ids' => random_ids,
+      Cypress::PopulationCloneJob.new('test_id' => id, 'patient_ids' => master_patient_ids,
+                                      'scoop_and_filter' => true, 'randomization_ids' => random_ids,
                                       'include_virtual' => (measure_ids & APP_CONSTANTS['telehealth_ineligible_measures']),
                                       'randomize_demographics' => true, 'generate_provider' => product.c4_test, 'job_id' => job_id).perform
     else
-      Cypress::PopulationCloneJob.new('test_id' => id, 'patient_ids' => master_patient_ids, 'disable_randomization' => true).perform
+      Cypress::PopulationCloneJob.new('test_id' => id, 'patient_ids' => master_patient_ids,
+                                      'scoop_and_filter' => true, 'disable_randomization' => true).perform
     end
   end
 

features/support/env.rb

@@ -86,7 +86,7 @@ def default_drivers
 
 default_drivers
 
-Capybara.server = :webrick
+Capybara.server = :puma, { Silent: true }
 Capybara.default_max_wait_time = 15
 # Capybara.ignore_hidden_elements = false
 

lib/cypress/patient_zipper.rb

@@ -79,7 +79,7 @@ def self.zip(file, patients, format)
           sf_patient.id = patient.id
           # CMS529 requires that entries point to the ecounters that they are related to
           sf_patient.add_encounter_ids_to_events if measures.map(&:hqmf_id).intersect?(APP_CONSTANTS['result_measures'].map(&:hqmf_id))
-          patient_scoop_and_filter.scoop_and_filter(sf_patient)
+          patient_scoop_and_filter.replace_negated_codes_with_valueset(sf_patient)
           z.put_next_entry("#{next_entry_path(patient, i)}.#{FORMAT_EXTENSIONS[format.to_sym]}")
           z << formatter.export(sf_patient)
         end

lib/cypress/population_clone_job.rb

@@ -47,6 +47,8 @@ def perform
         end
       end
 
+      @patient_scoop_and_filter = Cypress::ScoopAndFilter.new(@test.measures) if @options['scoop_and_filter']
+
       # get single provider if @test is a measure test. measure tests have a single provider for each patient while filtering tests can have different
       # providers for each patient
       provider = @test.provider if @test.instance_of?(MeasureTest)
@@ -96,6 +98,7 @@ def clone_and_save_patient(patient, prng, provider = nil, allow_dups: false)
         original_patient_id: patient.id,
         correlation_id: options['test_id']
       }
+      @patient_scoop_and_filter&.scoop_and_filter(cloned_patient, replace_negations: false, persist_scoop: true)
       unnumerify cloned_patient if patient.givenNames.map { |n| n =~ /\d/ }.any? || patient.familyName =~ /\d/
       cloned_patient.medical_record_number = next_medical_record_number unless options['disable_randomization']
       @test.reload

lib/cypress/scoop_and_filter.rb

@@ -20,47 +20,92 @@ def get_non_demographic_category_statuses(measures)
       end.flatten.uniq.compact
     end
 
-    def scoop_and_filter(patient)
-      demographic_criteria = patient.qdmPatient.dataElements.collect { |de| de if de.qdmCategory == 'patient_characteristic' }.compact
+    def scoop_and_filter(patient, replace_negations: true, persist_scoop: false)
+      de_to_delete = []
+      de_to_delete += patient.qdmPatient.dataElements.map { |de| de unless data_element_used_by_measure(de) }
+      patient.qdmPatient.dataElements.each do |data_element|
+        scoop_and_filter_data_element_codes(data_element, persist_scoop)
+      end
+      # keep data element if codes is not empty
+      de_to_delete += patient.qdmPatient.dataElements.map { |de| de unless de.dataElementCodes.present? }
+      if persist_scoop
+        de_to_delete.compact.each(&:destroy)
+      else
+        ids_to_delete = de_to_delete.compact.map(&:id)
+        patient.qdmPatient.dataElements.keep_if { |data_element| !ids_to_delete.include?(data_element.id) }
+      end
+      replace_negated_codes_with_valueset(patient) if replace_negations
+      patient
+    end
+
+    def replace_negated_codes_with_valueset(patient)
       # If a negated code belongs to multiple valuesets, we need to add a cloned entry for each valueset.
       # This array stores the cloned entries to be added
       multi_vs_negation_elements = []
-      patient.qdmPatient.dataElements.keep_if { |de| data_element_used_by_measure(de) }
+      ids_to_delete = []
       patient.qdmPatient.dataElements.each do |data_element|
-        scoop_and_filter_data_element_codes(data_element, multi_vs_negation_elements, patient)
-        data_element.dataElementCodes.first
+        next unless data_element.respond_to?('negationRationale') && data_element.negationRationale
+
+        replace_negated_code_with_valueset(data_element, multi_vs_negation_elements)
+        if data_element.dataElementCodes.blank?
+          ids_to_delete << data_element.id
+          next
+        end
+
+        # add data element valueset and other potentially relevant valueset descriptions
+        codes = (multi_vs_negation_elements + [data_element]).map { |de| "#{de.dataElementCodes.first.code}:#{de.dataElementCodes.first.system}" }
+        Cypress::QrdaPostProcessor.build_code_descriptions(codes, patient, patient.bundle)
       end
-      # keep data element if codes is not empty
-      patient.qdmPatient.dataElements.keep_if { |data_element| data_element.dataElementCodes.present? }
-      patient.qdmPatient.dataElements.concat(demographic_criteria)
       patient.qdmPatient.dataElements.concat(multi_vs_negation_elements)
+      patient.qdmPatient.dataElements.keep_if { |data_element| !ids_to_delete.include?(data_element.id) }
       patient
     end
 
     private
 
     # Method to remove codes from a data element that are not relevant to measure.
     # Multi_vs_negation_elements is an array of cloned elements to add to patient record to capture all of the negated valuesets
-    def scoop_and_filter_data_element_codes(data_element, multi_vs_negation_elements, patient)
+    def scoop_and_filter_data_element_codes(data_element, persist_scoop)
+      return if data_element.qdmCategory == 'patient_characteristic'
+
       # keep if data_element code and codesystem is in one of the relevant_codes
       # Also keep all negated valuesets, we'll deal with those later
       data_element.dataElementCodes.keep_if do |de_code|
-        @relevant_codes.include?(code: de_code.code, system: de_code.system) || de_code.system == '1.2.3.4.5.6.7.8.9.10'
+        relevant_code?(de_code) || de_code.system == '1.2.3.4.5.6.7.8.9.10'
       end
       # Return if all codes have been removed
       return if data_element.dataElementCodes.blank?
 
-      remove_irrelevant_valuesets_and_add_description_to_data_element(data_element)
-      # Return if all codes and valuesets have been removed
-      return if data_element.dataElementCodes.blank?
-      return unless data_element.respond_to?('negationRationale') && data_element.negationRationale
+      scoop_and_filter_data_element_fields(data_element)
+      # For repeatability, don't do this if you are saving the record
+      remove_irrelevant_valuesets_and_add_description_to_data_element(data_element) unless persist_scoop
+    end
 
-      replace_negated_code_with_valueset(data_element, multi_vs_negation_elements)
-      return if data_element.dataElementCodes.blank?
+    def scoop_and_filter_data_element_fields(data_element)
+      # Iterate through each field to see coded fields include relevant codes
+      data_element.fields.keys.each do |field_name|
+        next if data_element[field_name].nil?
+
+        # Diagnoses and Facility Locations are unique because they are arrays
+        if field_name == 'diagnoses'
+          data_element.diagnoses.keep_if do |diagnosis|
+            relevant_code?(diagnosis.code)
+          end
+          data_element.diagnoses = nil if data_element.diagnoses.blank?
+        elsif field_name == 'facilityLocations'
+          data_element.facilityLocations.keep_if do |facility_location|
+            relevant_code?(facility_location.code)
+          end
+          data_element.facilityLocations = nil if data_element.facilityLocations.blank?
+        end
+        next unless data_element.fields[field_name].type == QDM::Code
+
+        data_element[field_name] = nil unless relevant_code?(data_element[field_name])
+      end
+    end
 
-      # add data element valueset and other potentially relevant valueset descriptions
-      codes = (multi_vs_negation_elements + [data_element]).map { |de| "#{de.dataElementCodes.first.code}:#{de.dataElementCodes.first.system}" }
-      Cypress::QrdaPostProcessor.build_code_descriptions(codes, patient, patient.bundle)
+    def relevant_code?(code)
+      @relevant_codes.include?(code: code.code, system: code.system)
     end
 
     def data_element_category_and_status(data_element)
@@ -69,6 +114,8 @@ def data_element_category_and_status(data_element)
 
     # returns true if a patients data element is used by a measure
     def data_element_used_by_measure(data_element)
+      return true if data_element.qdmCategory == 'patient_characteristic'
+
       !@de_category_statuses_for_measures.index do |dcs|
         dcs[:category] == data_element['qdmCategory'] && dcs[:status] == data_element['qdmStatus']
       end.nil?

test/models/cat1_filter_task_test.rb

@@ -22,7 +22,7 @@ def test_task_good_results_should_pass
     perform_enqueued_jobs do
       te = task.execute(testfile, user)
       te.reload
-      assert_empty te.execution_errors, 'test execution with known good results should have no errors'
+      assert_empty te.execution_errors.only_errors, 'test execution with known good results should have no errors'
     end
   end
 end

test/unit/lib/patient_zipper_test.rb

@@ -72,8 +72,7 @@ class PatientZipperTest < ActiveSupport::TestCase
         doc = Nokogiri::XML(zip_entry.get_input_stream, &:strict)
         doc.root.add_namespace_definition('cda', 'urn:hl7-org:v3')
         doc.root.add_namespace_definition('sdtc', 'urn:hl7-org:sdtc')
-        # There should be 1 entry with encounter ids. The fixture measures do not have PhysicalExamPerformed criteria, so that entry will be filtered out
-        assert_equal 1, doc.xpath('//sdtc:templateId[@root="2.16.840.1.113883.10.20.24.3.150"]').size, 'There should be 1 entry with encounter ids'
+        assert_equal 2, doc.xpath('//sdtc:templateId[@root="2.16.840.1.113883.10.20.24.3.150"]').size, 'There should be 2 entry with encounter ids'
         count += 1
       end
     end