🚧 Validate files comparable to importing

.gitignore

@@ -18,3 +18,4 @@ tmp/**
 .tool-versions
 .ruby-version
 .claude/*
+*.code-workspace

app/models/bulkrax/csv_entry.rb

@@ -307,7 +307,11 @@ def build_value(property_name, mapping_config)
     # metadata that does not have a specific Bulkrax entry is mapped to the key name, as matching keys coming in are mapped by the csv parser automatically
     def key_for_export(key)
       clean_key = key_without_numbers(key)
-      unnumbered_key = mapping[clean_key] ? mapping[clean_key]['from'].first : clean_key
+      unnumbered_key = if mapping[clean_key]
+                         Bulkrax::FieldResolver.headers_for_field(mapping, clean_key).first
+                       else
+                         clean_key
+                       end
       # Bring the number back if there is one
       "#{unnumbered_key}#{key.sub(clean_key, '')}"
     end

app/models/bulkrax/xml_entry.rb

@@ -106,9 +106,10 @@ def each_candidate_metadata_node_name_and_content(elements: field_mapping_from_v
     #       than other parsers, in that they will make assumptions about each encountered column (in
     #       the case of CSV) or node (in the case of OAI).  tl;dr - Here there be dragons.
     def field_mapping_from_values_for_xml_element_names
-      Bulkrax.field_mappings[self.importerexporter.parser_klass].map do |_k, v|
-        v[:from]
-      end.flatten.compact.uniq
+      (Bulkrax.field_mappings[self.importerexporter.parser_klass] || {}).flat_map do |_k, v|
+        next [] unless v.is_a?(Hash)
+        Array(v['from'] || v[:from])
+      end.compact.uniq
     end
 
     # Included for potential downstream adopters

app/models/concerns/bulkrax/has_matchers.rb

@@ -181,18 +181,7 @@ def schema_form_definitions
     # @param field [String] the importer field name
     # @return [Array] hyrax fields
     def field_to(field)
-      fields = mapping&.map do |key, value|
-        return unless value
-
-        if value['from'].instance_of?(Array)
-          key if value['from'].include?(field) || key == field
-        elsif (value['from'] == field) || key == field
-          key
-        end
-      end&.compact
-
-      return [field] if fields.blank?
-      return fields
+      Bulkrax::FieldResolver.fields_for_header(mapping, field)
     end
 
     # Check whether a field is explicitly excluded in the mapping

app/parsers/bulkrax/application_parser.rb

@@ -135,10 +135,7 @@ def get_field_mapping_hash_for(key)
 
     # @return [Array<String>]
     def model_field_mappings
-      model_mappings = Bulkrax.field_mappings[self.class.to_s]&.dig('model', :from) || []
-      model_mappings |= ['model']
-
-      model_mappings
+      Bulkrax::FieldResolver.headers_for_field(Bulkrax.field_mappings[self.class.to_s], 'model')
     end
 
     # @return [String]

app/parsers/bulkrax/csv_parser.rb

@@ -109,14 +109,8 @@ def required_elements?(record)
 
     def missing_elements(record)
       keys_from_record = keys_without_numbers(record.reject { |_, v| v.blank? }.keys.compact.uniq.map(&:to_s))
-      keys = []
       mapping_values = importerexporter.mapping.stringify_keys
-      mapping_values.each do |k, v|
-        from_values = Array.wrap(v.is_a?(Hash) ? (v['from'] || v[:from]) : nil)
-        from_values.each do |vf|
-          keys << k if vf.present? && keys_from_record.include?(vf.to_s.strip)
-        end
-      end
+      keys = keys_from_record.flat_map { |header| Bulkrax::FieldResolver.fields_for_header(mapping_values, header) }
       required_elements.map(&:to_s) - keys.uniq.map(&:to_s)
     end
 
@@ -363,15 +357,16 @@ def file_paths
       return [] if importerexporter.metadata_only?
 
       # Compute once — these don't vary per record.
-      file_mapping  = Bulkrax.field_mappings.dig(self.class.to_s, 'file', :from)&.first&.to_sym || :file
+      file_keys     = Bulkrax::FieldResolver.headers_for_field(Bulkrax.field_mappings[self.class.to_s], 'file').map(&:to_sym)
       split_pattern = self.class.file_split_pattern
       files_dir     = path_to_files
 
       @file_paths ||= records.map do |r|
-        next if r[file_mapping].blank?
+        raw_values = file_keys.filter_map { |k| r[k] }.reject(&:blank?)
+        next if raw_values.empty?
         raise StandardError, "Record references local files but no files directory could be resolved from the import path" if files_dir.nil?
 
-        r[file_mapping].split(split_pattern).map do |f|
+        raw_values.flat_map { |v| v.split(split_pattern) }.map do |f|
           file = File.join(files_dir, f.strip.tr(' ', '_'))
           if File.exist?(file) # rubocop:disable Style/GuardClause
             file
@@ -422,16 +417,8 @@ def unzip_with_primary_csv(file_to_unzip)
       dest_dir = importer_unzip_path(mkdir: true)
       Zip::File.open(file_to_unzip) do |zip_file|
         entries = real_zip_entries(zip_file)
-        primary = select_primary_csv!(entries)
-        primary_dir = File.dirname(primary.name)
-
-        entries.each do |entry|
-          if entry == primary
-            extract_to(zip_file, entry, dest_dir, File.basename(entry.name))
-          else
-            extract_to(zip_file, entry, dest_dir, File.join('files', relative_to(primary_dir, entry.name)))
-          end
-        end
+        plan = Bulkrax::ZipPlacementPlanner.plan(entries, mode: :primary_csv)
+        plan.placements.each { |entry, dest| extract_to(zip_file, entry, dest_dir, dest) }
       end
     end
 
@@ -447,13 +434,8 @@ def unzip_attachments_only(file_to_unzip)
       dest_dir = importer_unzip_path(mkdir: true)
       Zip::File.open(file_to_unzip) do |zip_file|
         entries = real_zip_entries(zip_file)
-        wrapper = single_top_level_wrapper(entries)
-
-        entries.each do |entry|
-          relative = wrapper ? entry.name.delete_prefix("#{wrapper}/") : entry.name
-          next if relative.empty?
-          extract_to(zip_file, entry, dest_dir, File.join('files', relative))
-        end
+        plan = Bulkrax::ZipPlacementPlanner.plan(entries, mode: :attachments_only)
+        plan.placements.each { |entry, dest| extract_to(zip_file, entry, dest_dir, dest) }
       end
     end
 
@@ -493,39 +475,6 @@ def real_zip_entries(zip_file)
       entries
     end
 
-    # Picks the single primary CSV from zip entries, enforcing the
-    # shallowest-level rule. Raises {Bulkrax::UnzipError} on failure.
-    def select_primary_csv!(entries)
-      csvs = entries.select { |e| e.name.end_with?('.csv') }
-      raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.no_csv') if csvs.empty?
-
-      by_depth = csvs.group_by { |e| e.name.count('/') }
-      shallowest = by_depth[by_depth.keys.min]
-
-      raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.multiple_csv') if shallowest.size > 1
-
-      shallowest.first
-    end
-
-    # If every entry shares a single top-level directory, returns that
-    # directory name; otherwise nil.
-    def single_top_level_wrapper(entries)
-      tops = entries.map { |e| e.name.split('/').first }.uniq
-      return nil unless tops.size == 1
-      # If the single top segment is a file (no slashes in the entry), not a dir,
-      # there's no wrapper to strip.
-      return nil if entries.any? { |e| e.name == tops.first }
-      tops.first
-    end
-
-    # Returns `path` with `prefix/` removed from the front, if present, and
-    # a leading `files/` segment also stripped so callers can join under
-    # `files/` without doubling when the zip already uses that convention.
-    def relative_to(prefix, path)
-      remaining = prefix == '.' || prefix.empty? ? path : path.delete_prefix("#{prefix}/")
-      remaining.delete_prefix('files/')
-    end
-
     # Extracts a zip entry to `dest_dir/relative_dest`. Creates intermediate
     # directories and honors the rubyzip 2/3 extract-method signature.
     # The destination path is validated by {#safe_extract_path} — an unsafe

app/parsers/concerns/bulkrax/csv_parser/csv_validation.rb

@@ -41,6 +41,33 @@ def validate_csv(csv_file:, zip_file: nil, admin_set_id: nil)
 
         private
 
+        # Builds a ZipPlacementPlanner::Plan for the uploaded zip so row
+        # validators can check referenced file paths against the set of
+        # paths that will actually exist under files/ after extraction.
+        # Returns nil if no zip was uploaded or the zip can't be opened.
+        def build_zip_plan(zip_file)
+          return nil if zip_file.nil?
+
+          zip_path = zip_file.respond_to?(:path) ? zip_file.path : zip_file
+          Zip::File.open(zip_path) do |zf|
+            entries = zf.entries.select { |e| e.file? && !macos_junk_entry?(e.name) }
+            next if entries.empty?
+
+            mode = entries.any? { |e| e.name.end_with?('.csv') } ? :primary_csv : :attachments_only
+            Bulkrax::ZipPlacementPlanner.plan(entries, mode: mode)
+          end
+        rescue StandardError => e
+          Rails.logger.warn("CsvParser.validate_csv: could not build zip plan — #{e.class}: #{e.message}")
+          nil
+        end
+
+        # The macOS junk filter lives on ApplicationParser as an instance
+        # method. From class methods we re-implement the predicate inline
+        # to avoid reaching into that instance surface from here.
+        def macos_junk_entry?(name)
+          name.start_with?('__MACOSX/') || File.basename(name) == '.DS_Store' || File.basename(name).start_with?('._')
+        end
+
         # Builds notices, runs row validators, file validator, and hierarchy extraction.
         # Returns [notices, row_errors, file_validator, collections, works, file_sets].
         def run_validations(csv_data, all_ids, headers, source_id_key, mappings, field_metadata, missing_required, zip_file, admin_set_id, mapping_manager: nil) # rubocop:disable Metrics/ParameterLists
@@ -49,8 +76,9 @@ def run_validations(csv_data, all_ids, headers, source_id_key, mappings, field_m
           append_missing_source_id!(missing_required, headers, source_id_key, csv_data.map { |r| r[:model] }.compact.uniq)
           append_missing_model_notice!(notices, headers, csv_data)
 
-          row_errors                       = run_row_validators(csv_data, all_ids, source_id_key, mappings, field_metadata, find_record, notices, mapping_manager: mapping_manager)
-          file_validator                   = CsvTemplate::FileValidator.new(csv_data, zip_file, admin_set_id)
+          zip_plan                         = build_zip_plan(zip_file)
+          row_errors                       = run_row_validators(csv_data, all_ids, source_id_key, mappings, field_metadata, find_record, notices, mapping_manager: mapping_manager, zip_plan: zip_plan)
+          file_validator                   = Bulkrax::FileValidator.new(csv_data, zip_file, admin_set_id)
           collections, works, file_sets    = extract_hierarchy_items(csv_data, all_ids, find_record, mappings)
           [notices, row_errors, file_validator, collections, works, file_sets]
         end
@@ -65,12 +93,12 @@ def parse_csv_inputs(csv_file, admin_set_id)
           mapping_manager = CsvTemplate::MappingManager.new
           mappings        = mapping_manager.mappings
 
-          source_id_key = resolve_validation_key(mapping_manager, flag: 'source_identifier', default: :source_identifier)
-          parent_key    = resolve_validation_key(mapping_manager, flag: 'related_parents_field_mapping',  default: :parents)
-          children_key  = resolve_validation_key(mapping_manager, flag: 'related_children_field_mapping', default: :children)
-          file_key      = resolve_validation_key(mapping_manager, key: 'file',                            default: :file)
+          source_id_key = resolve_validation_key(mapping_manager, flag: 'source_identifier', headers: headers, default: :source_identifier)
+          parent_key    = resolve_validation_key(mapping_manager, flag: 'related_parents_field_mapping',  headers: headers, default: :parents)
+          children_key  = resolve_validation_key(mapping_manager, flag: 'related_children_field_mapping', headers: headers, default: :children)
+          file_headers  = Bulkrax::FieldResolver.headers_for_field(mappings, 'file')
 
-          csv_data       = parse_validation_rows(raw_csv, source_id_key, parent_key, children_key, file_key)
+          csv_data       = parse_validation_rows(raw_csv, source_id_key, parent_key, children_key, file_headers)
           all_models     = csv_data.map { |r| r[:model].to_s }.reject(&:blank?).uniq
           all_models    |= [Bulkrax.default_work_type] if Bulkrax.default_work_type.present?
           field_analyzer = CsvTemplate::FieldAnalyzer.new(mappings, admin_set_id)
@@ -105,7 +133,7 @@ def extract_hierarchy_items(csv_data, all_ids, find_record, mappings)
         end
 
         # Runs all registered row validators and returns the collected errors.
-        def run_row_validators(csv_data, all_ids, source_id_key, mappings, field_metadata, find_record, notices = [], mapping_manager: nil) # rubocop:disable Metrics/ParameterLists
+        def run_row_validators(csv_data, all_ids, source_id_key, mappings, field_metadata, find_record, notices = [], mapping_manager: nil, zip_plan: nil) # rubocop:disable Metrics/ParameterLists
           context = {
             errors: [],
             warnings: [],
@@ -121,7 +149,8 @@ def run_row_validators(csv_data, all_ids, source_id_key, mappings, field_metadat
             field_metadata: field_metadata,
             find_record_by_source_identifier: find_record,
             relationship_graph: build_relationship_graph(csv_data, mappings),
-            notices: notices
+            notices: notices,
+            zip_plan: zip_plan
           }
           csv_data.each_with_index do |record, index|
             row_number = index + 2 # 1-indexed, plus header row

app/parsers/concerns/bulkrax/csv_parser/csv_validation_helpers.rb

@@ -8,15 +8,27 @@ module CsvValidationHelpers # rubocop:disable Metrics/ModuleLength
 
       # Resolve a symbol key from mappings for use as a record hash key.
       # Returns a Symbol matching the parser's symbol-keyed record hash.
-      def resolve_validation_key(mapping_manager, key: nil, flag: nil, default:)
+      #
+      # When `flag:` is provided along with `headers:`, picks the alias
+      # that actually appears in the CSV's headers — falling back to the
+      # first `from:` alias when none match. This keeps the validator's
+      # column resolution in lockstep with the import pipeline (which
+      # aliases the matching column upstream in CsvEntry.data_for_entry).
+      def resolve_validation_key(mapping_manager, key: nil, flag: nil, headers: nil, default:)
+        if flag && headers
+          chosen = Bulkrax::FieldResolver.present_header_for_flag(mapping_manager.mappings, flag, headers)
+          return chosen.to_sym if chosen
+        end
+
         options = mapping_manager.resolve_column_name(key: key, flag: flag, default: default.to_s)
         options.first&.to_sym || default
       end
 
       # Parse rows from a CsvEntry.read_data result into the canonical record shape.
       # CsvEntry.read_data returns CSV::Row objects with symbol headers; blank rows
       # are already filtered by CsvWrapper.
-      def parse_validation_rows(raw_csv, source_id_key, parent_key, children_key, file_key)
+      def parse_validation_rows(raw_csv, source_id_key, parent_key, children_key, file_headers)
+        file_syms = Array(file_headers).map(&:to_sym)
         raw_csv.map do |row|
           # CSV::Row#to_h converts symbol headers → string-keyed hash
           row_hash = row.to_h.transform_keys(&:to_s)
@@ -25,7 +37,7 @@ def parse_validation_rows(raw_csv, source_id_key, parent_key, children_key, file
             model: row[:model],
             parent: row[parent_key],
             children: row[children_key],
-            file: row[file_key],
+            file: file_syms.map { |sym| row[sym] }.reject(&:blank?),
             raw_row: row_hash
           }
         end
@@ -64,7 +76,7 @@ def build_valid_validation_headers(mapping_manager, field_analyzer, all_models,
         Rails.logger.error("CsvParser.validate_csv: error building valid headers – #{e.message}")
         standard = %w[model source_identifier parents children file]
         model_fields = field_metadata.values.flat_map { |m| m[:properties] }
-                                            .map { |prop| mapping_manager.key_to_mapped_column(prop) }
+                                            .map { |prop| Bulkrax::FieldResolver.headers_for_field(mapping_manager.mappings, prop).first }
         (standard + model_fields).uniq
       end
 
@@ -77,8 +89,7 @@ def non_property_mapping_aliases(mappings)
                             CsvTemplate::ColumnDescriptor::COLUMN_DESCRIPTIONS[:files].flat_map(&:keys) +
                             CsvTemplate::ColumnDescriptor::COLUMN_DESCRIPTIONS[:relationships].flat_map(&:keys)
         non_property_keys.flat_map do |key|
-          entry = mappings[key]
-          entry.is_a?(Hash) ? Array(entry["from"]) : []
+          Bulkrax::FieldResolver.headers_for_field(mappings, key)
         end
       end
 
@@ -237,11 +248,13 @@ def find_record_by_source_identifier(identifier, work_identifier, work_identifie
       end
 
       # Returns the raw CSV column name (String) for a relationship field.
-      # Looks for the mapping entry flagged with +flag+ and returns its first +from+ value,
-      # falling back to +default+ when none is found.
+      # Looks for the mapping entry flagged with +flag+ and returns its first
+      # alias, falling back to +default+ when none is found.
       def resolve_relationship_column(mappings, flag, default)
-        entry = mappings.find { |_k, v| v.is_a?(Hash) && v[flag] }
-        entry&.last&.dig('from')&.first || default
+        entry = mappings.find { |_k, v| v.is_a?(Hash) && (v[flag] || v[flag.to_sym]) }
+        return default unless entry
+
+        Bulkrax::FieldResolver.headers_for_field(mappings, entry.first).first || default
       end
 
       def resolve_parent_split_pattern(mappings)

app/services/bulkrax/csv_template/column_builder.rb

@@ -22,9 +22,7 @@ def required_columns
       private
 
       def mapped_core_columns
-        @descriptor.core_columns.map do |column|
-          @service.mapping_manager.key_to_mapped_column(column)
-        end
+        @descriptor.core_columns.map { |column| header_for(column) }
       end
 
       def property_columns
@@ -35,7 +33,7 @@ def property_columns
         properties = field_lists
                      .flat_map { |item| item.values.flat_map { |config| config["properties"] || [] } }
                      .uniq
-                     .map { |property| @service.mapping_manager.key_to_mapped_column(property) }
+                     .map { |property| header_for(property) }
                      .uniq
 
         (properties - required_columns).sort
@@ -50,11 +48,16 @@ def relationship_columns
 
       def file_columns
         CsvTemplate::ColumnDescriptor::COLUMN_DESCRIPTIONS[:files].flat_map do |property_hash|
-          property_hash.keys.map do |key|
-            @service.mapping_manager.key_to_mapped_column(key)
-          end
+          property_hash.keys.map { |key| header_for(key) }
         end
       end
+
+      # Picks one CSV column header to emit for a canonical mapping key.
+      # Any of the key's `from:` aliases (or the key itself) is a valid
+      # header — see Bulkrax::FieldResolver.headers_for_field.
+      def header_for(key)
+        Bulkrax::FieldResolver.headers_for_field(@service.mappings, key).first
+      end
     end
   end
 end