Use stack env file templating

config/dump_2_stack_db_migration/example_inventory/group_vars/all

@@ -74,22 +74,25 @@ db_dumps:
 
 # Key/value pairs to set in the stack env file.  These will replace existing
 # values in the env file or add a new line if the name doesn't already exist.
-stack_env_vars:
 
-  STROOM_CONTENT_PACK_IMPORT_ENABLED: "false"
+
+# All stack_env_ variables are used for substitution in the .env.j2 file that
+# comes bundled with the stack
 
   # Uncomment this if you have a DNS server, else the containers can't resolve it
-  HOST_IP: "{{ hostvars['stroom-dump-2-stack-db-migration-host']['public_ip_address'] 
+stack_env_host_ip: "{{ hostvars['stroom-dump-2-stack-db-migration-host']['public_ip_address'] 
     | default(hostvars['stroom-dump-2-stack-db-migration-host']['inventory_hostname'])}}"
 
-  # MYSQL_DOCKER_REPO: "mysql"
-  # STROOM_AUTH_SERVICE_DOCKER_REPO: "gchq/stroom-auth-service"
-  # STROOM_AUTH_UI_DOCKER_REPO: "gchq/stroom-auth-ui"
-  # STROOM_DOCKER_REPO: "gchq/stroom"
-  # STROOM_LOG_SENDER_DOCKER_REPO: "gchq/stroom-log-sender"
-  # STROOM_NGINX_DOCKER_REPO: "gchq/stroom-nginx"
-  # STROOM_PROXY_DOCKER_REPO: "gchq/stroom-proxy"
-  # STROOM_UI_DOCKER_REPO: "gchq/stroom-ui"
+stack_env_stroom_content_pack_import_enabled: "false"
+
+# MYSQL_DOCKER_REPO: "mysql"
+# STROOM_AUTH_SERVICE_DOCKER_REPO: "gchq/stroom-auth-service"
+# STROOM_AUTH_UI_DOCKER_REPO: "gchq/stroom-auth-ui"
+# STROOM_DOCKER_REPO: "gchq/stroom"
+# STROOM_LOG_SENDER_DOCKER_REPO: "gchq/stroom-log-sender"
+# STROOM_NGINX_DOCKER_REPO: "gchq/stroom-nginx"
+# STROOM_PROXY_DOCKER_REPO: "gchq/stroom-proxy"
+# STROOM_UI_DOCKER_REPO: "gchq/stroom-ui"
 
 # If you have no internet connection then you may want to set some of the following
 

config/remote_proxy/example_inventory/group_vars/all

@@ -13,9 +13,6 @@ stack_name: "stroom_proxy"
 # The version of the stack to deploy
 stack_version: "v7.0-.beta.26-2"
 
-# The path on a file server where the stack archive file can be found
-stroom_stack_archive_url_base: "https://github.com/gchq/stroom-resources/releases/download/stroom-stacks-{{ stack_version }}"
-
 # The local directory where the stack configuration is kept. This configuration
 # will be pushed out to the proxy hosts. This directory should be under source
 # control.
@@ -31,52 +28,85 @@ stack_install_root_dir: "/home/{{ stroom_user }}"
 # Set to true to delete all the self signed development certificates
 remove_development_certificates: false
 
-# Key/value pairs to set in the stack env file.  These will replace existing
-# values in the env file or add a new line if the name doesn't already exist.
-# It is debatable if we should hold a jinja2 template for this file or use this
-# apporach
-stack_env_vars:
-
-  # Uncomment this if you have a DNS server, else the containers can't resolve it
-  #HOST_IP: "{{ inventory_hostname }}"
-
-  # The key/cert/ca files used by nginx
-  # Must exist in files_and_templates/volumes/nginx/certs
-  #NGINX_SSL_CERTIFICATE_KEY: "example.server.key"
-  #NGINX_SSL_CERTIFICATE: "example.server.cert"
-  #NGINX_SSL_CA_CERTIFICATE: "example.ca.cert"
-
-  #STROOM_NGINX_DOCKER_REPO: "gchq/stroom-nginx"
-  #STROOM_PROXY_DOCKER_REPO: "gchq/stroom-proxy"
-
-  # The java key/truststores to use when checking feed status
-  # Must exist in files_and_templates/volumes/stroom-proxy-remote/certs
-  #STROOM_PROXY_REMOTE_CLIENT_KEYSTORE_PASSWORD: "a new password"
-  #STROOM_PROXY_REMOTE_CLIENT_KEYSTORE_PATH: "/stroom-proxy/certs/example.client.jks"
-  #STROOM_PROXY_REMOTE_CLIENT_TRUSTSTORE_PASSWORD: "a new password"
-  #STROOM_PROXY_REMOTE_CLIENT_TRUSTSTORE_PATH: "/stroom-proxy/certs/example.ca.jks"
-
-  # The API token used for checking feed status as provided by stroom admins
-  #STROOM_PROXY_REMOTE_FEED_STATUS_API_KEY: "eyJhbG......ciOwd7Q"
-
-  # Set this with the dns/hostname/IP of the downstream proxy/stroom
-  # DNS/hostname will only work if there is a DNS server.
-  STROOM_PROXY_REMOTE_FEED_STATUS_URL: "https://192.168.102.11/api/proxy/feedStatus/v1"
-
-  # Set this with the dns/hostname/IP of the downstream proxy/stroom
-  # DNS/hostname will only work if there is a DNS server.
-  STROOM_PROXY_REMOTE_FORWARD_URL: "https://192.168.102.11/stroom/datafeed"
-
-  # The java key/truststores to use when forwarding data downstream
-  # Must exist in files_and_templates/volumes/stroom-proxy-remote/certs
-  #STROOM_PROXY_REMOTE_FORWARDING_KEYSTORE_PASSWORD: "a new password"
-  #STROOM_PROXY_REMOTE_FORWARDING_KEYSTORE_PATH: "/stroom-proxy/certs/example.client.jks"
-  #STROOM_PROXY_REMOTE_FORWARDING_TRUSTSTORE_PASSWORD: "a new password"
-  #STROOM_PROXY_REMOTE_FORWARDING_TRUSTSTORE_PATH: "/stroom-proxy/certs/example.ca.jks"
-
-  # These need to be set for dev testing when using IPs and self signed certs
-  STROOM_PROXY_REMOTE_FORWARDING_HOST_VERIFICATION_ENABLED: "false"
-  STROOM_PROXY_REMOTE_JERSEY_VERIFY_HOSTNAME: "false"
-  STROOM_PROXY_REMOTE_JERSEY_TRUST_SELF_SIGNED: "true"
+# Set this to the path of a templated env file on the controller
+# if you don't want to use the templated env file supplied in the stack
+# release archive.
+# local_templated_env_file_path:
+
+# All stack_env_ variables are used for substitution in the .env.j2 file that
+# comes bundled with the stack
+
+# Uncomment this if you have a DNS server, else the containers can't resolve it
+stack_env_host_ip: "{{ hostvars[groups['stroom_remote_proxy_stack'][0]]['public_ip_address'] | default(inventory_hostname) }}"
+
+# The key/cert/ca files used by nginx
+# Must exist in files_and_templates/volumes/nginx/certs
+#stack_env_nginx_ssl_certificate_key: "example.server.key"
+#stack_env_nginx_ssl_certificate: "example.server.cert"
+#stack_env_nginx_ssl_ca_certificate: "example.ca.cert"
+
+#stack_env_stroom_nginx_docker_repo: "gchq/stroom-nginx"
+#stack_env_stroom_proxy_docker_repo: "gchq/stroom-proxy"
+#stack_env_stroom_log_sender_docker_repo: "gchq/stroom-log-sender"
+
+# The java key/truststores to use when checking feed status
+# Must exist in files_and_templates/volumes/stroom-proxy-remote/certs
+# The absolute path inside the container
+#stack_env_stroom_proxy_remote_client_keystore_password: "a new password"
+#stack_env_stroom_proxy_remote_client_keystore_path: "/stroom-proxy/certs/example.client.jks"
+#stack_env_stroom_proxy_remote_client_truststore_password: "a new password"
+#stack_env_stroom_proxy_remote_client_truststore_path: "/stroom-proxy/certs/example.ca.jks"
+
+# The API token used for checking feed status as provided by stroom admins
+#stack_env_stroom_proxy_remote_feed_status_api_key: "eyJhbG......ciOwd7Q"
+
+# Set this with the dns/hostname/IP of the downstream proxy/stroom
+# DNS/hostname will only work if there is a DNS server.
+stack_env_stroom_proxy_remote_feed_status_url: "https://192.168.102.11/api/proxy/feedStatus/v1"
+
+# Set this with the dns/hostname/IP of the downstream proxy/stroom
+# DNS/hostname will only work if there is a DNS server.
+stack_env_stroom_proxy_remote_forward_url: "https://192.168.102.11/stroom/datafeed"
+
+# The java key/truststores to use when forwarding data downstream
+# Must exist in files_and_templates/volumes/stroom-proxy-remote/certs
+# The absolute path inside the container
+#stack_env_stroom_proxy_remote_forwarding_keystore_password: "a new password"
+#stack_env_stroom_proxy_remote_forwarding_keystore_path: "/stroom-proxy/certs/example.client.jks"
+#stack_env_stroom_proxy_remote_forwarding_truststore_password: "a new password"
+#stack_env_stroom_proxy_remote_forwarding_truststore_path: "/stroom-proxy/certs/example.ca.jks"
+
+# These need to be set for dev testing when using IPs and self signed certs
+stack_env_stroom_proxy_remote_forwarding_host_verification_enabled: "false"
+stack_env_stroom_proxy_remote_jersey_verify_hostname: "false"
+stack_env_stroom_proxy_remote_jersey_trust_self_signed: "true"
+
+# Configure the sending of proxy/nginx logs to a downstream stroom/proxy
+#stack_env_stroom_log_sender_datafeed_url: "https://a-downstream-stroom-or-proxy/stroom/datafeeddirect"
+#stack_env_stroom_log_sender_default_environment: "DEV"
+# Must exist in files_and_templates/volumes/stroom-proxy-remote/certs
+# The absolute path inside the container
+#stack_env_stroom_log_sender_ca_cert_file: "/stroom-log-sender/certs/ca.pem.crt"
+#stack_env_stroom_log_sender_cert_file: "/stroom-log-sender/certs/client.pem.crt"
+#stack_env_stroom_log_sender_private_key_file: "/stroom-log-sender/certs/client.unencrypted.key"
+
+# If you have no internet connection then you will want to set the following
+# to locations on your network
+
+# The path on a file server where the stack archive file can be found
+# stroom_stack_archive_url_base: "https://github.com/gchq/stroom-resources/releases/download/stroom-stacks-{{ stack_version }}"
+
+# The URL to download the docker-ce yum repo file from
+# docker_yum_repo_file_url: https://download.docker.com/linux/centos/docker-ce.repo
+
+# The baseurl value to use in the docker-ce yum repo file if you are not downloading
+# the repo file from docker_yum_repo_file_url
+# docker_yum_repo_base_url: https://download.docker.com/linux/centos/7/$basearch/stable
+
+# The URL to download the docker-compose binary from
+# docker_compose_url: "https://github.com/docker/compose/releases/download/{{ docker_compose_version }}/docker-compose-Linux-x86_64"
+
+# epel_repo_url:
+# epel_repo_gpg_key_url:
 
 # vim: set filetype=yaml shiftwidth=2 tabstop=2 expandtab:

config/single_node_stroom_core_stack/example_inventory/group_vars/all

@@ -5,7 +5,7 @@ ansible_connection: ssh
 # The user should have passwordless sudo and an ssh key pair that has already
 # been set up on the remote host(s). The ssh key to use can be set on the command line
 # using '--private-key=~/.ssh/my_private_key_id_rsa'
-ansible_user: vagrant
+ansible_user: centos
 
 # The name of the stack to deploy
 stack_name: "stroom_core_test"
@@ -34,41 +34,43 @@ remove_development_certificates: false
 # If you have DNS in place then either hard code it here or get it from the inventory file
 # in a similar way to this. If you have no DNS then use an IP that is resolvable from
 # outside the host and from inside the containers.
-nginx_advertised_host: "{{ hostvars[groups['stroom_core_stack'][0]]['public_ip_address'] }}"
+# public_ip_address can be added to the inventory to allow this to work with vagrant locally when there is no DNS
+nginx_advertised_host: "{{ hostvars[groups['stroom_core_stack'][0]]['public_ip_address'] | default(inventory_hostname) }}"
 
 # Key/value pairs to set in the stack env file.  These will replace existing
 # values in the env file or add a new line if the name doesn't already exist.
 # It is debatable if we should hold a jinja2 template for this file or use this
 # apporach
-stack_env_vars:
 
-  HOST_IP: "{{ nginx_advertised_host }}"
+stack_env_host_ip: "{{ nginx_advertised_host }}"
+stack_env_db_host_ip: "{{ nginx_advertised_host }}"
+stack_env_nginx_advertised_host: "{{ nginx_advertised_host }}"
+stack_env_docker_host_hostname: "{{ nginx_advertised_host }}"
+stack_env_docker_host_ip: "{{ nginx_advertised_host }}"
 
-  NGINX_ADVERTISED_HOST: "{{ nginx_advertised_host }}"
+#stack_env_nginx_ssl_certificate_key: "example.server.key"
+#stack_env_nginx_ssl_certificate: "example.server.cert"
+#stack_env_nginx_ssl_ca_certificate: "example.ca.cert"
 
-  #NGINX_SSL_CERTIFICATE_KEY: "example.server.key"
-  #NGINX_SSL_CERTIFICATE: "example.server.cert"
-  #NGINX_SSL_CA_CERTIFICATE: "example.ca.cert"
+#stack_env_stroom_nginx_docker_repo: "gchq/stroom-nginx"
+#stack_env_stroom_proxy_docker_repo: "gchq/stroom-proxy"
 
-  #STROOM_NGINX_DOCKER_REPO: "gchq/stroom-nginx"
-  #STROOM_PROXY_DOCKER_REPO: "gchq/stroom-proxy"
+#stack_env_stroom_proxy_remote_client_keystore_password: "a new password"
+#stack_env_stroom_proxy_remote_client_keystore_path: "/stroom-proxy/certs/example.client.jks"
 
-  #STROOM_PROXY_REMOTE_CLIENT_KEYSTORE_PASSWORD: "a new password"
-  #STROOM_PROXY_REMOTE_CLIENT_KEYSTORE_PATH: "/stroom-proxy/certs/example.client.jks"
+#stack_env_stroom_proxy_remote_client_truststore_password: "a new password"
+#stack_env_stroom_proxy_remote_client_truststore_path: "/stroom-proxy/certs/example.ca.jks"
 
-  #STROOM_PROXY_REMOTE_CLIENT_TRUSTSTORE_PASSWORD: "a new password"
-  #STROOM_PROXY_REMOTE_CLIENT_TRUSTSTORE_PATH: "/stroom-proxy/certs/example.ca.jks"
+#stack_env_stroom_proxy_remote_feed_status_api_key: "eyJhbG......ciOwd7Q"
+#stack_env_stroom_proxy_remote_feed_status_url: "https://downstream-stroom-or-proxy.some.domain/api/proxy/feedStatus/v1"
 
-  #STROOM_PROXY_REMOTE_FEED_STATUS_API_KEY: "eyJhbG......ciOwd7Q"
-  #STROOM_PROXY_REMOTE_FEED_STATUS_URL: "https://downstream-stroom-or-proxy.some.domain/api/proxy/feedStatus/v1"
-
-  #STROOM_PROXY_REMOTE_FORWARD_URL: "https://downstream-stroom-or-proxy.some.domain/stroom/datafeed"
+#stack_env_stroom_proxy_remote_forward_url: "https://downstream-stroom-or-proxy.some.domain/stroom/datafeed"
 
   ## Use same (key|trust)store for client and forwarding
-  #STROOM_PROXY_REMOTE_FORWARDING_KEYSTORE_PASSWORD: "a new password"
-  #STROOM_PROXY_REMOTE_FORWARDING_KEYSTORE_PATH: "/stroom-proxy/certs/example.client.jks"
+#stack_env_stroom_proxy_remote_forwarding_keystore_password: "a new password"
+#stack_env_stroom_proxy_remote_forwarding_keystore_path: "/stroom-proxy/certs/example.client.jks"
 
-  #STROOM_PROXY_REMOTE_FORWARDING_TRUSTSTORE_PASSWORD: "a new password"
-  #STROOM_PROXY_REMOTE_FORWARDING_TRUSTSTORE_PATH: "/stroom-proxy/certs/example.ca.jks"
+#stack_env_stroom_proxy_remote_forwarding_truststore_password: "a new password"
+#stack_env_stroom_proxy_remote_forwarding_truststore_path: "/stroom-proxy/certs/example.ca.jks"
 
 # vim: set filetype=yaml shiftwidth=2 tabstop=2 expandtab:

stroom/roles/setup/common/tasks/main.yml

@@ -37,13 +37,13 @@
     state: present
   vars:
     packages:
-      - bash
+      - bash # All stroom shell scripts expect bash or bash 4+
       - bind-utils # for nslookup
       - coreutils  # realpath is used by some stroom scripts
       - curl
       - git # Needed for migration testing, but useful to have
       - htop # Useful process monitoring
-      - httpie  # Used by some stroom scripts
+      - httpie  # Used by some stroom scripts, handy for inspecting REST endpoints
       - iproute  # needed for the ip command used in our scripts
       - jq  # Used by some stroom scripts, handy for REST json munging
       - net-tools

stroom/roles/stack/deploy/defaults/main.yml

@@ -1,6 +1,8 @@
 ---
-stack_bin_dir: "{{ stack_install_root_dir }}/{{ stack_name }}/{{ stack_name }}-{{ stack_version }}"
-stack_config_dir: "{{ stack_install_root_dir }}/{{ stack_name }}/{{ stack_name }}-{{ stack_version }}/config"
-stack_volumes_dir: "{{ stack_install_root_dir }}/{{ stack_name }}/volumes"
+stack_version_base_dir: "{{ stack_install_root_dir }}/{{ stack_name }}/{{ stack_version }}"
+stack_bin_dir: "{{ stack_version_base_dir }}/bin"
+stack_config_dir: "{{ stack_bin_dir }}/config"
+stack_volumes_dir: "{{ stack_version_base_dir }}/volumes"
+
 stack_env_var_file: "{{ stack_config_dir }}/{{ stack_name }}.env"
 stack_env_vars: {}

stroom/roles/stack/deploy/tasks/main.yml

@@ -10,7 +10,7 @@
 
 - debug:
     msg:
-      - "Deploying/configuring stroom-proxy in {{ stack_install_root_dir }} as user {{ stroom_user }}"
+      - "Deploying/configuring stack in {{ stack_install_root_dir }} as user {{ stroom_user }}"
   tags:
     - deploy
     - configure
@@ -20,7 +20,7 @@
   become_user: "{{ stroom_user }}"
   block:
 
-########### DEPLOY VANILLA REMOTE PROXY STACK  ###########
+########### DEPLOY VANILLA STACK  ###########
 
     - import_role:
         name: stack/download
@@ -44,3 +44,10 @@
         - deploy
         - configure
 
+    - name: Pull docker images
+      command: "./pull_images.sh -m"
+      args:
+        chdir: "{{ stack_bin_dir }}"
+      tags:
+        - deploy
+        - pull_images