Merge to master: Release 2.3.2 - Submissions endpoint pagination and fixes (#52)

* add get submission all including all properties test

* extract and use submission_include_params where we use submission.bring

* use retrieve_submissions helper in the :acronym/submissions endpoint

Author: syphax-bouazzouni

Gemfile.lock

@@ -53,7 +53,7 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git
-  revision: 0d6632bb39c1d24a930fe088bc464c3a51d68c9b
+  revision: f44f7baa96eb3ee10dfab4a8aca154161ba7dd89
   branch: master
   specs:
     ontologies_linked_data (0.0.1)
@@ -112,7 +112,7 @@ GEM
     bcrypt_pbkdf (1.1.0)
     bigdecimal (1.4.2)
     builder (3.2.4)
-    capistrano (3.17.3)
+    capistrano (3.18.0)
       airbrussh (>= 1.0.0)
       i18n
       rake (>= 10.0.0)

controllers/admin_controller.rb

@@ -68,7 +68,7 @@ class AdminController < ApplicationController
       latest = ont.latest_submission(status: :any)
       error 404, "Ontology #{params["acronym"]} contains no submissions" if latest.nil?
       check_last_modified(latest)
-      latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param))
+      latest.bring(*submission_include_params)
       NcboCron::Models::OntologySubmissionParser.new.queue_submission(latest, actions)
       halt 204
     end
@@ -84,7 +84,7 @@ class AdminController < ApplicationController
         latest = ont.latest_submission(status: :any)
       end
       check_last_modified(latest) if latest
-      latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) if latest
+      latest.bring(*submission_include_params) if latest
       reply(latest || {})
     end
 

controllers/ontologies_controller.rb

@@ -38,21 +38,12 @@ class OntologiesController < ApplicationController
       else
         latest = ont.latest_submission(status: :any)
       end
-      check_last_modified(latest) if latest
-      # When asking to display all metadata, we are using bring_remaining which is more performant than including all metadata (remove this when the query to get metadata will be fixed)
+
       if latest
-        if includes_param.first == :all
-          # Bring what we need to display all attr of the submission
-          latest.bring_remaining
-          latest.bring({:contact=>[:name, :email],
-                      :ontology=>[:acronym, :name, :administeredBy, :group, :viewingRestriction, :doNotUpdate, :flat,
-                                  :hasDomain, :summaryOnly, :acl, :viewOf, :ontologyType],
-                      :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]})
-        else
-          latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param))
-        end
+        check_last_modified(latest)
+        latest.bring(*submission_include_params)
       end
-      #remove the whole previous if block and replace by it: latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) if latest
+
       reply(latest || {})
     end
 
@@ -62,7 +53,7 @@ class OntologiesController < ApplicationController
     patch '/:acronym/latest_submission' do
       ont = Ontology.find(params["acronym"]).first
       error 422, "You must provide an existing `acronym` to patch" if ont.nil?
-      
+
       submission = ont.latest_submission(status: :any)
 
       submission.bring(*OntologySubmission.attributes)

controllers/ontology_submissions_controller.rb

@@ -1,9 +1,15 @@
 class OntologySubmissionsController < ApplicationController
   get "/submissions" do
     check_last_modified_collection(LinkedData::Models::OntologySubmission)
-    #using appplication_helper method
-    options = {also_include_views: params["also_include_views"], status: (params["include_status"] || "ANY")}
-    reply retrieve_latest_submissions(options).values
+    options = {
+               also_include_views: params["also_include_views"],
+               status: (params["include_status"] || "ANY")
+    }
+    subs = retrieve_latest_submissions(options)
+    subs = subs.values unless page?
+    # Force to show ontology reviews, notes and projects by default only for this request
+    LinkedData::Models::Ontology.serialize_default(*(LinkedData::Models::Ontology.hypermedia_settings[:serialize_default] + [:reviews, :notes, :projects]))
+    reply subs
   end
 
   ##
@@ -19,22 +25,18 @@ class OntologySubmissionsController < ApplicationController
     ##
     # Display all submissions of an ontology
     get do
-      ont = Ontology.find(params["acronym"]).include(:acronym).first
+      ont = Ontology.find(params["acronym"]).include(:acronym, :administeredBy, :acl, :viewingRestriction).first
       error 422, "Ontology #{params["acronym"]} does not exist" unless ont
       check_last_modified_segment(LinkedData::Models::OntologySubmission, [ont.acronym])
-      if includes_param.first == :all
-        # When asking to display all metadata, we are using bring_remaining which is more performant than including all metadata (remove this when the query to get metadata will be fixed)
-        ont.bring(submissions: [:released, :creationDate, :status, :submissionId,
-                                {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, :group, :hasDomain, :views, :viewOf, :flat],
-                                 :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus])
-
-        ont.submissions.each do |sub|
-          sub.bring_remaining
-        end
-      else
-        ont.bring(submissions: OntologySubmission.goo_attrs_to_load(includes_param))
-      end
-      reply ont.submissions.sort {|a,b| b.submissionId.to_i <=> a.submissionId.to_i }  # descending order of submissionId
+      check_access(ont)
+      options = {
+        also_include_views: true,
+        status: (params["include_status"] || "ANY"),
+        ontology: params["acronym"]
+      }
+      subs = retrieve_submissions(options)
+
+      reply subs.sort {|a,b| b.submissionId.to_i <=> a.submissionId.to_i }  # descending order of submissionId
     end
 
     ##
@@ -53,7 +55,7 @@ class OntologySubmissionsController < ApplicationController
       ont.bring(:submissions)
       ont_submission = ont.submission(params["ontology_submission_id"])
       error 404, "`submissionId` not found" if ont_submission.nil?
-      ont_submission.bring(*OntologySubmission.goo_attrs_to_load(includes_param))
+      ont_submission.bring(*submission_include_params)
       reply ont_submission
     end
 

helpers/access_control_helper.rb

@@ -10,11 +10,7 @@ module AccessControlHelper
       def check_access(obj)
         return obj unless LinkedData.settings.enable_security
         if obj.is_a?(Enumerable)
-          if obj.first.is_a?(LinkedData::Models::Base) && obj.first.access_based_on?
-            check_access(obj.first)
-          else
             filter_access(obj)
-          end
         else
           if obj.respond_to?(:read_restricted?) && obj.read_restricted?
             readable = obj.readable?(env["REMOTE_USER"])

helpers/application_helper.rb

@@ -52,6 +52,10 @@ def populate_from_params(obj, params)
             value = is_arr ? value : [value]
             new_value = []
             value.each do |cls|
+              if uri_as_needed(cls["ontology"]).nil?
+                new_value << cls
+                next
+              end
               sub = LinkedData::Models::Ontology.find(uri_as_needed(cls["ontology"])).first.latest_submission
               new_value << LinkedData::Models::Class.find(cls["class"]).in(sub).first
             end
@@ -356,40 +360,16 @@ def replace_url_prefix(id)
       end
 
       def retrieve_latest_submissions(options = {})
-        status = (options[:status] || "RDF").to_s.upcase
-        include_ready = status.eql?("READY") ? true : false
-        status = "RDF" if status.eql?("READY")
-        any = true if status.eql?("ANY")
-        include_views = options[:also_include_views] || false
-        includes = OntologySubmission.goo_attrs_to_load(includes_param)
-
-        includes << :submissionStatus unless includes.include?(:submissionStatus)
-        if any
-          submissions_query = OntologySubmission.where
-        else
-          submissions_query = OntologySubmission.where(submissionStatus: [ code: status])
-        end
+        submissions = retrieve_submissions(options)
 
-        submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views
-        submissions_query = submissions_query.filter(filter) if filter?
-        # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs
-        if includes_param.first == :all
-          includes = [:submissionId, {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl,
-                                       :group, :hasDomain, :views, :viewOf, :flat], :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus]
-        end
-        submissions = submissions_query.include(includes).to_a
-        
-        # Figure out latest parsed submissions using all submissions
-        latest_submissions = {}
+        latest_submissions = page? ? submissions : {} # latest_submission doest not work with pagination
         submissions.each do |sub|
-          # To retrieve all metadata, but slow when a lot of ontologies
-          if includes_param.first == :all
-            sub.bring_remaining
+          unless page?
+            next if include_ready?(options) && !sub.ready?
+            next if sub.ontology.nil?
+            latest_submissions[sub.ontology.acronym] ||= sub
+            latest_submissions[sub.ontology.acronym] = sub if sub.submissionId.to_i > latest_submissions[sub.ontology.acronym].submissionId.to_i
           end
-          next if include_ready && !sub.ready?
-          next if sub.ontology.nil?
-          latest_submissions[sub.ontology.acronym] ||= sub
-          latest_submissions[sub.ontology.acronym] = sub if sub.submissionId.to_i > latest_submissions[sub.ontology.acronym].submissionId.to_i
         end
         latest_submissions
       end

helpers/request_params_helper.rb

@@ -13,6 +13,10 @@ def settings_params(klass)
         [attributes, page, size, order_by, bring_unmapped]
       end
 
+      def page?
+        !params[:page].nil?
+      end
+
       def is_set?(param)
         !param.nil? && param != ""
       end
@@ -25,6 +29,38 @@ def filter
         build_filter
       end
 
+      def apply_filters(object, query)
+        attributes_to_filter = object.attributes(:all).select{|x| params.keys.include?(x.to_s)}
+        filters = attributes_to_filter.map {|key| [key, params[key]&.split(',')]}.to_h
+        add_direct_filters(filters, query)
+      end
+
+      def apply_submission_filters(query)
+
+        filters = {
+          naturalLanguage: params[:naturalLanguage]&.split(',') , #%w[http://lexvo.org/id/iso639-3/fra http://lexvo.org/id/iso639-3/eng],
+          hasOntologyLanguage_acronym: params[:hasOntologyLanguage]&.split(',') , #%w[OWL SKOS],
+          ontology_hasDomain_acronym:  params[:hasDomain]&.split(',') , #%w[Crop Vue_francais],
+          ontology_group_acronym: params[:group]&.split(','), #%w[RICE CROP],
+          isOfType: params[:isOfType]&.split(','), #["http://omv.ontoware.org/2005/05/ontology#Vocabulary"],
+          hasFormalityLevel: params[:hasFormalityLevel]&.split(','), #["http://w3id.org/nkos/nkostype#thesaurus"],
+          ontology_viewingRestriction: params[:viewingRestriction]&.split(','), #["private"]
+        }
+        inverse_filters = {
+          status: params[:status], #"retired",
+          submissionStatus: params[:submissionStatus] #"RDF",
+        }
+
+        query = add_direct_filters(filters, query)
+
+        query = add_inverse_filters(inverse_filters, query)
+
+        query = add_acronym_name_filters(query)
+
+        add_order_by_patterns(query)
+      end
+
+
       def get_order_by_from(params, default_order = :asc)
         if is_set?(params['sortby'])
           orders = (params["order"] || default_order.to_s).split(',')
@@ -50,6 +86,67 @@ def bring_unmapped_to(page_data, sub, klass)
       end
 
       private
+      def extract_attr(key)
+        attr, sub_attr, sub_sub_attr = key.to_s.split('_')
+
+        return attr.to_sym unless sub_attr
+
+        return {attr.to_sym => [sub_attr.to_sym]} unless  sub_sub_attr
+
+        {attr.to_sym => [sub_attr.to_sym => sub_sub_attr.to_sym]}
+      end
+
+      def add_direct_filters(filters, query)
+        filters.each do |key, values|
+          attr = extract_attr(key)
+          next if Array(values).empty?
+
+          filter = Goo::Filter.new(attr).regex(values.first)
+          values.drop(1).each do |v|
+            filter = filter.or(Goo::Filter.new(attr).regex(v))
+          end
+          query = query.filter(filter)
+        end
+        query
+      end
+
+      def add_inverse_filters(inverse_filters, query)
+        inverse_filters.each do |key, value|
+          attr = extract_attr(key)
+          next unless value
+
+          filter = Goo::Filter.new(attr).regex("^(?:(?!#{value}).)*$")
+          query = query.filter(filter)
+        end
+        query
+      end
+
+      def add_acronym_name_filters(query)
+        if params[:acronym]
+          filter = Goo::Filter.new(extract_attr(:ontology_acronym)).regex(params[:acronym])
+          if params[:name]
+            filter.or(Goo::Filter.new(extract_attr(:ontology_name)).regex(params[:name]))
+          end
+          query = query.filter(filter)
+        elsif params[:name]
+          filter = Goo::Filter.new(extract_attr(:ontology_name)).regex(params[:name])
+          query = query.filter(filter)
+        end
+        query
+      end
+
+      def add_order_by_patterns(query)
+        if params[:order_by]
+          attr, sub_attr = params[:order_by].to_s.split('_')
+          if sub_attr
+            order_pattern = { attr.to_sym => { sub_attr.to_sym => (sub_attr.eql?("name") ? :asc : :desc) } }
+          else
+            order_pattern = { attr.to_sym => :desc }
+          end
+          query = query.order_by(order_pattern)
+        end
+        query
+      end
 
       def sort_order_item(param, order)
         [param.to_sym, order.to_sym]

helpers/submission_helper.rb

@@ -0,0 +1,67 @@
+require 'sinatra/base'
+
+module Sinatra
+  module Helpers
+    module SubmissionHelper
+      def submission_include_params
+        # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs
+        includes = OntologySubmission.goo_attrs_to_load(includes_param)
+        if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:ontology)}
+          includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects,:acl, :viewOf]}
+        end
+
+        if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:contact)}
+          includes << {:contact=>[:name, :email]}
+        end
+        includes
+      end
+
+      def submission_attributes_all
+        out = [LinkedData::Models::OntologySubmission.embed_values_hash]
+        out << {:contact=>[:name, :email]}
+        out << {:ontology=>[:acronym, :name, :administeredBy, :group, :viewingRestriction, :doNotUpdate, :flat,
+                            :hasDomain, :summaryOnly, :acl, :viewOf, :ontologyType]}
+
+        out
+      end
+
+      def retrieve_submissions(options)
+        status = (options[:status] || "RDF").to_s.upcase
+        status = "RDF" if status.eql?("READY")
+        ontology_acronym = options[:ontology]
+        any = status.eql?("ANY")
+        include_views = options[:also_include_views] || false
+        includes, page, size, order_by, _ = settings_params(LinkedData::Models::OntologySubmission)
+        includes << :submissionStatus unless includes.include?(:submissionStatus)
+
+        submissions_query = LinkedData::Models::OntologySubmission
+        submissions_query = submissions_query.where(ontology: [acronym: ontology_acronym]) if ontology_acronym
+
+        if any
+          submissions_query = submissions_query.where unless ontology_acronym
+        else
+          submissions_query = submissions_query.where({ submissionStatus: [code: status] })
+        end
+
+        submissions_query = apply_submission_filters(submissions_query)
+        submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views
+        submissions_query = submissions_query.filter(filter) if filter?
+
+
+        submissions = submissions_query.include(submission_include_params)
+        if page?
+          submissions.page(page, size).all
+        else
+          submissions.to_a
+        end
+      end
+
+      def include_ready?(options)
+        options[:status] && options[:status].to_s.upcase.eql?("READY")
+      end
+
+    end
+  end
+end
+
+helpers Sinatra::Helpers::SubmissionHelper