feat: integrate Sidekiq for background job processing and update configurations

Author: maboukerfa

Gemfile

@@ -10,6 +10,7 @@ gem 'rake'
 gem 'rexml' # Investigate why unicorn fails to start under ruby 3 without adding rexml gem to the Gemfile
 gem 'sinatra'
 gem 'rackup'
+gem 'sidekiq', '7.3.9'
 
 github 'sinatra/sinatra' do
   gem 'sinatra-contrib'
@@ -54,7 +55,7 @@ gem 'pandoc-ruby'
 gem 'ncbo_annotator', git: 'https://github.com/agroportal/ncbo_annotator.git', branch: 'development'
 gem 'ncbo_cron', git: 'https://github.com/agroportal/ncbo_cron.git', branch: 'development'
 gem 'ncbo_ontology_recommender', git: 'https://github.com/agroportal/ncbo_ontology_recommender.git', branch: 'development'
-gem 'ontologies_linked_data', github: 'agroportal/ontologies_linked_data', branch: 'development'
+gem 'ontologies_linked_data', github: 'agroportal/ontologies_linked_data', branch: 'feat/sidekiq-integration'
 gem 'goo', github: 'agroportal/goo', branch: 'development'
 gem 'sparql-client', github: 'agroportal/sparql-client', branch: 'development'
 

Gemfile.lock

@@ -29,7 +29,7 @@ GIT
 
 GIT
   remote: https://github.com/agroportal/ncbo_cron.git
-  revision: 2eb5cfd12e6e8fd9d3b0a6610d836a358cdfc409
+  revision: 1447bf247caca0858cf4d454ec44fec6f3a44b55
   branch: development
   specs:
     ncbo_cron (0.0.1)
@@ -57,8 +57,8 @@ GIT
 
 GIT
   remote: https://github.com/agroportal/ontologies_linked_data.git
-  revision: e34ef6d84858c7235a6a28be77aca4a528e84b32
-  branch: development
+  revision: 3c3ff6749e7332521de5f9e43a22d1c47050d485
+  branch: feat/sidekiq-integration
   specs:
     ontologies_linked_data (0.0.1)
       activesupport
@@ -128,7 +128,7 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.2.3)
+    activesupport (7.2.3.1)
       base64
       benchmark (>= 0.3)
       bigdecimal
@@ -137,7 +137,7 @@ GEM
       drb
       i18n (>= 1.6, < 2)
       logger (>= 1.4.2)
-      minitest (>= 5.1)
+      minitest (>= 5.1, < 6)
       securerandom (>= 0.3)
       tzinfo (~> 2.0, >= 2.0.5)
     addressable (2.8.9)
@@ -148,12 +148,12 @@ GEM
     ast (2.4.3)
     base64 (0.3.0)
     bcp47_spec (0.2.1)
-    bcrypt (3.1.21)
+    bcrypt (3.1.22)
     bcrypt_pbkdf (1.1.2)
     bcrypt_pbkdf (1.1.2-arm64-darwin)
     bcrypt_pbkdf (1.1.2-x86_64-darwin)
     benchmark (0.5.0)
-    bigdecimal (4.0.1)
+    bigdecimal (4.1.0)
     builder (3.3.0)
     capistrano (3.20.0)
       airbrussh (>= 1.0.0)
@@ -229,10 +229,12 @@ GEM
     google-cloud-env (2.3.1)
       base64 (~> 0.2)
       faraday (>= 1.0, < 3.a)
-    google-cloud-errors (1.5.0)
+    google-cloud-errors (1.6.0)
     google-logging-utils (0.2.0)
     google-protobuf (3.25.3)
+    google-protobuf (3.25.3-aarch64-linux)
     google-protobuf (3.25.3-arm64-darwin)
+    google-protobuf (3.25.3-x86-linux)
     google-protobuf (3.25.3-x86_64-darwin)
     google-protobuf (3.25.3-x86_64-linux)
     googleapis-common-protos (1.8.0)
@@ -252,9 +254,15 @@ GEM
     grpc (1.70.1)
       google-protobuf (>= 3.25, < 5.0)
       googleapis-common-protos-types (~> 1.0)
+    grpc (1.70.1-aarch64-linux)
+      google-protobuf (>= 3.25, < 5.0)
+      googleapis-common-protos-types (~> 1.0)
     grpc (1.70.1-arm64-darwin)
       google-protobuf (>= 3.25, < 5.0)
       googleapis-common-protos-types (~> 1.0)
+    grpc (1.70.1-x86-linux)
+      google-protobuf (>= 3.25, < 5.0)
+      googleapis-common-protos-types (~> 1.0)
     grpc (1.70.1-x86_64-darwin)
       google-protobuf (>= 3.25, < 5.0)
       googleapis-common-protos-types (~> 1.0)
@@ -272,7 +280,7 @@ GEM
     i18n (1.14.8)
       concurrent-ruby (~> 1.0)
     io-console (0.8.2)
-    json (2.19.1)
+    json (2.19.3)
     json-canonicalization (0.4.0)
     json-ld (3.2.5)
       htmlentities (~> 4.3)
@@ -300,13 +308,11 @@ GEM
       net-imap
       net-pop
       net-smtp
-    mcp (0.8.0)
-      json-schema (>= 4.1)
     method_source (1.1.0)
     mime-types (3.7.0)
       logger
       mime-types-data (~> 3.2025, >= 3.2025.0507)
-    mime-types-data (3.2026.0303)
+    mime-types-data (3.2026.0317)
     mini_mime (1.1.5)
     minitest (5.27.0)
     minitest-fail-fast (0.2.0)
@@ -344,7 +350,7 @@ GEM
       net-ssh (>= 5.0.0, < 8.0.0)
     net-smtp (0.5.1)
       net-protocol
-    net-ssh (7.3.0)
+    net-ssh (7.3.2)
     netrc (0.11.0)
     oj (3.16.16)
       bigdecimal (>= 3.0)
@@ -356,7 +362,7 @@ GEM
     pandoc-ruby (2.1.10)
     parallel (1.27.0)
     parseconfig (1.1.2)
-    parser (3.3.10.2)
+    parser (3.3.11.1)
       ast (~> 2.4.1)
       racc
     pony (1.13.1)
@@ -409,7 +415,7 @@ GEM
       rexml (~> 3.2)
     redis (5.4.1)
       redis-client (>= 0.22.0)
-    redis-client (0.27.0)
+    redis-client (0.28.0)
       connection_pool
     redis-rack-cache (2.2.1)
       rack-cache (>= 1.10, < 2)
@@ -435,19 +441,18 @@ GEM
     rsolr (2.6.0)
       builder (>= 2.1.2)
       faraday (>= 0.9, < 3, != 2.0.0)
-    rubocop (1.85.1)
+    rubocop (1.86.0)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
-      mcp (~> 0.6)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 2.9.3, < 3.0)
       rubocop-ast (>= 1.49.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.49.0)
+    rubocop-ast (1.49.1)
       parser (>= 3.3.7.2)
       prism (~> 1.7)
     ruby-progressbar (1.13.0)
@@ -460,6 +465,12 @@ GEM
     sentry-ruby (5.28.1)
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
+    sidekiq (7.3.9)
+      base64
+      connection_pool (>= 2.3.0)
+      logger
+      rack (>= 2.2.4)
+      redis-client (>= 0.22.2)
     signet (0.21.0)
       addressable (~> 2.8)
       faraday (>= 0.17.5, < 3.a)
@@ -510,15 +521,17 @@ GEM
     uri (1.1.1)
     uuid (2.3.9)
       macaddr (~> 1.0)
-    webmock (3.26.1)
+    webmock (3.26.2)
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
     webrick (1.9.2)
 
 PLATFORMS
+  aarch64-linux
   arm64-darwin
   ruby
+  x86-linux
   x86_64-darwin
   x86_64-linux
 
@@ -575,6 +588,7 @@ DEPENDENCIES
   rubocop
   sentry-ruby (~> 5.24)
   shotgun!
+  sidekiq (= 7.3.9)
   simplecov
   simplecov-cobertura
   sinatra
@@ -586,4 +600,4 @@ DEPENDENCIES
   webrick
 
 BUNDLED WITH
-   2.6.7
+   2.6.9

app.rb

@@ -29,6 +29,10 @@
 require_relative 'lib/rack/slice_detection'
 require_relative 'lib/rack/request_lang'
 
+# sidekiq requirements
+require 'sidekiq/web'
+require 'sidekiq'
+
 # Logging setup
 require_relative 'config/logging'
 

bin/ontoportal

@@ -96,7 +96,9 @@ build_docker_run_cmd() {
   local ncbo_path="$5"
 
   local docker_run_cmd="docker compose --profile vo -p ontoportal_docker run --rm -it --name api-service"
+  local sidekiq_run_cmd="docker compose --profile vo -p ontoportal_docker run --rm -d --name sidekiq-service"
   local bash_cmd=""
+  local sidekiq_bash_cmd=""
 
   # Conditionally add bind mounts only if the paths are not empty
   for path_var in "linked_data_path:ontologies_linked_data" "goo_path:goo" "sparql_client_path:sparql-client" "ncbo_path:ncbo_cron" ; do
@@ -107,15 +109,30 @@ build_docker_run_cmd() {
       echo "[+] Run: bundle config local.$value ${!path}"
       container_path="/srv/ontoportal/$value"
       docker_run_cmd+=" -v $host_path:$container_path"
+      sidekiq_run_cmd+=" -v $host_path:$container_path"
       bash_cmd+="(git config --global --add safe.directory $container_path && bundle config local.$value $container_path) &&"
+      sidekiq_bash_cmd+="(git config --global --add safe.directory $container_path && bundle config local.$value $container_path) &&"
     else
       bash_cmd+=" (bundle config unset local.$value) &&"
+      sidekiq_bash_cmd+=" (bundle config unset local.$value) &&"
     fi
   done
 
+  # Build sidekiq command with same bundle setup
+  local sidekiq_custom_command="bundle exec sidekiq -C /srv/ontoportal/ontologies_api/config/sidekiq.yml -r /srv/ontoportal/ontologies_api/app.rb --env=development"
+  sidekiq_bash_cmd+=" (bundle check || bundle install || bundle update) && $sidekiq_custom_command"
+  sidekiq_run_cmd+=" sidekiq bash -c \"$sidekiq_bash_cmd\""
+
+  # Start sidekiq detached first
+  echo "[+] Starting Sidekiq in detached mode..."
+  echo "[+] Run docker command $sidekiq_run_cmd"
+  eval "$sidekiq_run_cmd"
+
+  # Build and run API command
   bash_cmd+=" (bundle check || bundle install || bundle update) && $custom_command"
   docker_run_cmd+=" --service-ports api bash -c \"$bash_cmd\""
 
+  echo "[+] Starting API server..."
   echo "[+] Run docker command $docker_run_cmd"
   eval "$docker_run_cmd"
 }

config.ru

@@ -1,2 +1,6 @@
 require './app.rb'
 run Sinatra::Application
+
+map '/sidekiq' do
+  run Sidekiq::Web
+end

config/sidekiq.yml

@@ -0,0 +1,14 @@
+:concurrency: 5
+
+:queues:
+  - default
+  - mailers
+  - critical
+
+#:timeout: 25
+
+:verbose: true
+
+:logfile: ./log/sidekiq.log
+
+:pidfile: ./tmp/pids/sidekiq.pid

docker-compose.yml

@@ -1,9 +1,11 @@
 x-app: &app
   image: agroportal/ontologies_api:development
-  environment: &env
+  environment:
     # default bundle config resolves to /usr/local/bundle/config inside of the container
     # we are setting it to local app directory if we need to use 'bundle config local'
-    BUNDLE_PATH: /srv/ontoportal/bundle
+
+    # BUNDLE_PATH: /srv/ontoportal/bundle
+    &env
     COVERAGE: 'true' # enable simplecov code coverage
     REDIS_HOST: redis-ut
     REDIS_PORT: 6379
@@ -27,13 +29,10 @@ x-app: &app
   tty: true
   command: /bin/bash
 
-
-
 services:
   api:
     <<: *app
-    env_file:
-      .env
+    env_file: .env
     environment:
       <<: *env
       BUNDLE_APP_CONFIG: /srv/ontoportal/ontologies_api/.bundle
@@ -53,15 +52,28 @@ services:
       - "9393:9393"
     volumes:
       # bundle volume for hosting gems installed by bundle; it speeds up gem install in local development
+      - bundle:/srv/ontoportal/bundle
       - app_api:/srv/ontoportal/ontologies_api
       - repository:/srv/ontoportal/data/repository
-      - bundle:/srv/ontoportal/bundle
+
+  sidekiq:
+    <<: *app
+    env_file: .env
+    environment:
+      <<: *env
+      BUNDLE_APP_CONFIG: /srv/ontoportal/ontologies_api/.bundle
+
+    volumes:
+      - app_api:/srv/ontoportal/ontologies_api
+      - repository:/srv/ontoportal/data/repository
+    depends_on:
+      redis-ut:
+        condition: service_healthy
 
   ncbo_cron:
     <<: *app
     image: agroportal/ncbo_cron:development
-    env_file:
-      .env
+    env_file: .env
     environment:
       <<: *env
       BUNDLE_APP_CONFIG: /srv/ontoportal/ncbo_cron/.bundle
@@ -84,7 +96,6 @@ services:
       virtuoso-ut:
         condition: service_started
 
-
   mgrep-ut:
     image: ontoportal/mgrep-ncbo:0.1
     ports:
@@ -141,11 +152,7 @@ services:
       - agdata:/agraph/data
       #      - ./agraph/etc:/agraph/etc
     command: >
-      bash -c "/agraph/bin/agraph-control --config /agraph/etc/agraph.cfg start
-      ; agtool repos create ontoportal_test --supersede
-      ; agtool users add anonymous
-      ; agtool users grant anonymous root:ontoportal_test:rw
-      ; tail -f /agraph/data/agraph.log"
+      bash -c "/agraph/bin/agraph-control --config /agraph/etc/agraph.cfg start ; agtool repos create ontoportal_test --supersede ; agtool users add anonymous ; agtool users grant anonymous root:ontoportal_test:rw ; tail -f /agraph/data/agraph.log"    
     # healthcheck:
     #   test: ["CMD-SHELL", "curl -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1"]
     #   start_period: 10s
@@ -191,6 +198,7 @@ services:
       - gb
 
 volumes:
+  bundle:
   app_api:
   app_cron:
   agdata:
@@ -201,4 +209,3 @@ volumes:
   mgrep:
   logs:
   history:
-  bundle: