Merge branch 'main' into feature/anonymous-users

Author: ccoulombe

ansible/all.yml

@@ -30,6 +30,9 @@
     - galaxyproject.gxadmin
     - usegalaxy_ca.backup_pg
 
+- name: tusd
+  import_playbook: tusd.yml
+
 - name: galaxy
   import_playbook: galaxy.yml
 

ansible/files/galaxy/config/tpv_rules_local.yml

@@ -71,6 +71,32 @@ tools:
   toolshed.g2.bx.psu.edu/repos/devteam/fastqc/fastqc/.*:
     cores: min(max(int(input_size * 4), 1), 16)
     mem: min(max(int(input_size * 7), 5.6), 120)
+  toolshed.g2.bx.psu.edu/repos/bgruening/sylph_profile/sylph_profile/.*:
+    cores: 4
+    mem: 16
+  toolshed.g2.bx.psu.edu/repos/iuc/bwa_mem2/bwa_mem2/.*:
+    cores: 32
+    mem: 240
+    rules:
+      - id: bwa_mem2_small_input_rule
+        if: input_size < 0.1
+        cores: 2
+        mem: 5.6
+      - id: bwa_mem2_medium_input_rule
+        if: 0.1 <= input_size < 1
+        cores: 4
+        mem: 14
+      - id: bwa_mem2_large_input_rule
+        if: 1 <= input_size < 20
+        cores: 8
+        mem: 28
+      - id: bwa_mem2_pretty_large_input_rule
+        if: 20 <= input_size < 100
+        cores: 16
+        mem: 120
+  toolshed.g2.bx.psu.edu/repos/iuc/seqkit_sort/seqkit_sort/.*:
+    cores: 1
+    mem: 7
 
 roles:
   training.*:
@@ -87,7 +113,7 @@ destinations:
       - name: SINGULARITYENV__JAVA_OPTIONS
         value: $_JAVA_OPTIONS
       - execute: ulimit -f 1048576000  # 500GB, our default quota
-      - execute: ulimit -u 10000
+      - execute: ulimit -u 4096
 
   slurm-training:
     runner: slurm
@@ -97,7 +123,7 @@ destinations:
     params:
       native_specification: --nodes=1 --ntasks=1 --cpus-per-task={cores} --mem={round(mem*1024)} --time=48:00:00 --partition=training
       singularity_enabled: true
-      singularity_volumes: "$galaxy_root:ro,$tool_directory:ro,$working_directory:rw,$job_directory:rw,/mnt/nfs/data/datasets:ro,/mnt/nfs1/data/datasets:ro,/mnt/nfs2/data/datasets:ro,/cvmfs/data.galaxyproject.org:ro"
+      singularity_volumes: "$galaxy_root:ro,$tool_directory:ro,$working_directory:rw,$job_directory:rw,/mnt/nfs/data/datasets:ro,/mnt/nfs1/data/datasets:ro,/mnt/nfs2/data/datasets:ro,/cvmfs/data.galaxyproject.org:ro,/mnt/nfs/galaxy/var/tool_data:ro"
       tmp_dir: true
     env:
       LC_ALL: C
@@ -117,7 +143,7 @@ destinations:
     params:
       native_specification: --nodes=1 --ntasks=1 --cpus-per-task={cores} --mem={round(mem*1024)} --time=168:00:00 --partition=main #set to main for autumn season
       singularity_enabled: true
-      singularity_volumes: "$galaxy_root:ro,$tool_directory:ro,$working_directory:rw,$job_directory:rw,/mnt/nfs/data/datasets:ro,/mnt/nfs1/data/datasets:ro,/mnt/nfs2/data/datasets:ro,/cvmfs/data.galaxyproject.org:ro"
+      singularity_volumes: "$galaxy_root:ro,$tool_directory:ro,$working_directory:rw,$job_directory:rw,/mnt/nfs/data/datasets:ro,/mnt/nfs1/data/datasets:ro,/mnt/nfs2/data/datasets:ro,/cvmfs/data.galaxyproject.org:ro,/mnt/nfs/galaxy/var/tool_data:ro"
       tmp_dir: true
     env:
       # Ensuring a consistent collation environment is good for reproducibility.

ansible/galaxy.yml

@@ -23,7 +23,6 @@
       tags:
         - always
   roles:
-    - galaxyproject.tusd
     - role: galaxyproject.galaxy
       tags:
         - never
@@ -49,6 +48,8 @@
     - role: usegalaxy_ca.pgcleanup
       become: true
       become_user: "{{ galaxy_user_name }}"
+      tags:
+        - cleanup
 
   post_tasks:
     - name: fetch edugain idps the first time

ansible/group_vars/galaxyservers.yml

@@ -48,11 +48,11 @@ galaxy_job_config:
       value: -1
       # Limit on the number of jobs a user with a registered Galaxy account can have active across all environments.
     - type: registered_user_concurrent_jobs
-      value: 6
+      value: 10
       # Limit total jobs running on the galaxy server (upload, etc)
     - type: environment_total_concurrent_jobs
       id: local_env
-      value: 10
+      value: 30
       # Limit total jobs running on the computes
     - type: environment_total_concurrent_jobs
       id: tpv_dispatcher
@@ -196,18 +196,18 @@ galaxy_config:
     galaxy_user: "{{ galaxy_user_name }}"
     virtualenv: "{{ galaxy_venv_dir }}"
     gunicorn:
-      # listening options
-      # bind: "unix:{{ galaxy_mutable_config_dir }}/gunicorn.sock"
-      bind: "unix:{{ galaxy_root }}/gunicorn.sock"
-      # performance options
-      workers: 2
-      # Other options that will be passed to gunicorn
-      # This permits setting of 'secure' headers like REMOTE_USER (and friends)
-      # https://docs.gunicorn.org/en/stable/settings.html#forwarded-allow-ips
-      extra_args: '--forwarded-allow-ips="*"'
-      # This lets Gunicorn start Galaxy completely before forking which is faster.
-      # https://docs.gunicorn.org/en/stable/settings.html#preload-app
-      preload: true
+      - bind: "unix:{{ galaxy_root }}/gunicorn0.sock"
+        workers: 2
+        timeout: 600
+        restart_timeout: 7200
+        extra_args: '--forwarded-allow-ips="*"'
+        preload: true
+      - bind: "unix:{{ galaxy_root }}/gunicorn1.sock"
+        workers: 2
+        timeout: 600
+        restart_timeout: 7200
+        extra_args: '--forwarded-allow-ips="*"'
+        preload: true
     celery:
       concurrency: 2
       loglevel: DEBUG
@@ -312,9 +312,10 @@ nginx_conf_ssl_certificate_key: /etc/ssl/user/privkey-www-data.pem
 
 # TUS
 tusd_version: v1.13.0
-galaxy_tusd_host: localhost
+galaxy_tusd_host: "{{ groups['tusdservers'][0] }}"
 galaxy_tusd_port: 1080
 galaxy_tus_upload_store: /mnt/nfs/data/tus
+tusd_base_path: /api/upload/resumable_upload # Setting the tusd base path as the Galaxy upload location prevents tusd from returning the wrong upload URL.
 
 #NFS
 nfs_host: "{{ groups['nfsservers'][0] }}"

ansible/roles/usegalaxy_ca.logrotate/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+journal_rotate: "4"
+journal_size: 10G
+journal_size_staging: 2G

ansible/roles/usegalaxy_ca.logrotate/tasks/main.yml

@@ -0,0 +1,28 @@
+---
+- name: Set lower log size limit on staging
+  ansible.builtin.set_fact:
+    journal_size: "{{ journal_size_staging }}"
+  when: inventory_hostname is match('staging-')
+
+- name: Create journald directory
+  ansible.builtin.file:
+    path: /etc/systemd/journald.conf.d
+    state: directory
+
+- name: Copy logrotate config
+  ansible.builtin.template:
+    src: rsyslog.j2
+    dest: /etc/logrotate.d/rsyslog
+
+- name: Copy journald config
+  ansible.builtin.template:
+    src: 00-journal-size.conf.j2
+    dest: /etc/systemd/journald.conf.d/00-journal-size.conf.j2
+
+- name: Restart logrotate and journald
+  ansible.builtin.service:
+    name: "{{ item }}"
+    state: restarted
+  loop:
+    - logrotate.service
+    - systemd-journald.service

ansible/roles/usegalaxy_ca.logrotate/templates/00-journal-size.conf.j2

@@ -0,0 +1,3 @@
+[Journal]
+SystemMaxUse={{ journal_size }}
+Compress=yes

ansible/roles/usegalaxy_ca.logrotate/templates/rsyslog.j2

@@ -0,0 +1,26 @@
+/var/log/syslog
+/var/log/mail.info
+/var/log/mail.warn
+/var/log/mail.err
+/var/log/mail.log
+/var/log/daemon.log
+/var/log/kern.log
+/var/log/auth.log
+/var/log/user.log
+/var/log/lpr.log
+/var/log/cron.log
+/var/log/debug
+/var/log/messages
+{
+	rotate {{ journal_rotate }}
+	weekly
+	size {{ journal_size }}
+	missingok
+	notifempty
+	compress
+	delaycompress
+	sharedscripts
+	postrotate
+		/usr/lib/rsyslog/rsyslog-rotate
+	endscript
+}

ansible/roles/usegalaxy_ca.pgcleanup/defaults/main.yml

@@ -1,9 +1,9 @@
 # This cron job runs every Monday at 06:00
-# It cleans up datasets that have been deleted for more than 14 days
+# It cleans up datasets that have been deleted for more than 7 days
 galaxy_pgcleanup_cron_weekday: "1"
 galaxy_pgcleanup_cron_hour: "06"
 galaxy_pgcleanup_cron_min: "0"
-galaxy_pgcleanup_days: 30
+galaxy_pgcleanup_days: 7
 # order of action is important!
 galaxy_pgcleanup_actions:
   - delete_userless_histories

ansible/roles/usegalaxy_ca.pgcleanup/tasks/main.yml

@@ -8,4 +8,4 @@
     job: >
       source {{ galaxy_venv_dir }}/bin/activate;
       cd {{ galaxy_server_dir }};
-      {{ galaxy_venv_dir }}/bin/python ./scripts/cleanup_datasets/pgcleanup.py -w 128MB --config-file {{ galaxy_config_file }} --log-dir {{ galaxy_log_dir }} --older-than {{ galaxy_pgcleanup_days }} {{ galaxy_pgcleanup_actions | join(' ') }} 2>&1 | tee -a {{ galaxy_log_dir }}/cleanup_datasets.log;
+      {{ galaxy_venv_dir }}/bin/python ./scripts/cleanup_datasets/pgcleanup.py --no-update-time -w 128MB --config-file {{ galaxy_config_file }} --log-dir {{ galaxy_log_dir }} --older-than {{ galaxy_pgcleanup_days }} {{ galaxy_pgcleanup_actions | join(' ') }} 2>&1 | tee -a {{ galaxy_log_dir }}/cleanup_datasets.log;