feat: [PPT-2260] Add support for ES client authentication

.gitignore

@@ -8,3 +8,4 @@ app
 bin
 .vscode
 .DS_*
+.env*

docker-compose.yml

@@ -9,8 +9,13 @@ x-deployment-env: &deployment-env
   PG_DATABASE_URL: ${PG_DATABASE_URL:-postgresql://postgres:password@postgres:5432/ingest_spec}
 
 x-elastic-client-env: &elastic-client-env
-  ELASTIC_HOST: ${ELASTIC_HOST:-elastic}
-  ELASTIC_PORT: ${ELASTIC_PORT:-9200}
+  # ELASTIC_HOST: ${ELASTIC_HOST:-elastic}
+  # ELASTIC_PORT: ${ELASTIC_PORT:-9200}
+  # Basic auth credentials (alternative to ELASTIC_URI)
+  # ELASTIC_USER: ${ELASTIC_USER:-elastic}
+  # ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-testpass123}
+  # Alternative: use ELASTIC_URI with embedded credentials
+  ELASTIC_URI: ${ELASTIC_URI:-http://elastic:testpass123@elastic:9200}
 
 x-search-ingest-client-env: &search-ingest-client-env
   PLACE_SEARCH_INGEST_URI: ${PLACE_SEARCH_INGEST_URI:-http://search-ingest:3000}
@@ -32,10 +37,7 @@ services:
       - postgres
     environment:
       # Service Hosts
-      <<: [
-        *elastic-client-env,
-        *deployment-env,
-      ]
+      <<: [*elastic-client-env, *deployment-env]
       # Environment
       GITHUB_ACTION: ${GITHUB_ACTION:-}
 
@@ -44,6 +46,13 @@ services:
     restart: always
     environment:
       discovery.type: single-node
+      # Enable security features
+      xpack.security.enabled: "true"
+      # Set built-in user passwords
+      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-testpass123}
+      # Disable SSL for testing (not recommended for production)
+      xpack.security.http.ssl.enabled: "false"
+      xpack.security.transport.ssl.enabled: "false"
 
   postgres:
     hostname: postgres
@@ -59,4 +68,4 @@ services:
       POSTGRES_PASSWORD: password
       POSTGRES_DB: ingest_spec
     ports:
-       - "5432:5432"
+      - "5432:5432"

shard.lock

@@ -114,7 +114,7 @@ shards:
 
   openssl_ext:
     git: https://github.com/spider-gazelle/openssl_ext.git
-    version: 2.4.4
+    version: 2.5.2
 
   pars:
     git: https://github.com/spider-gazelle/pars.git
@@ -130,7 +130,7 @@ shards:
 
   pg-orm:
     git: https://github.com/spider-gazelle/pg-orm.git
-    version: 2.0.4
+    version: 2.0.5
 
   place_calendar:
     git: https://github.com/placeos/calendar.git

shard.yml

@@ -1,5 +1,5 @@
 name: search-ingest
-version: 2.11.2
+version: 2.11.3
 crystal: ">= 1.0.0"
 license: MIT
 authors:

spec/spec_models.cr

@@ -19,7 +19,7 @@ class RayGun < AbstractBase
   attribute barrel_length : Float32 = 23.42
   attribute rounds : Int32 = 32
   attribute ip : String = "127.0.0.1", es_type: "ip"
-  attribute last_shot : Time = ->{ Time.utc }
+  attribute last_shot : Time = -> { Time.utc }
 end
 
 class Programmer < AbstractBase
@@ -34,14 +34,14 @@ end
 
 class Beverage::Coffee < AbstractBase
   attribute temperature : Int32 = 54
-  attribute created_at : Time = ->{ Time.utc }
+  attribute created_at : Time = -> { Time.utc }
 
   belongs_to Programmer, dependent: :destroy
 end
 
 class Migraine < AbstractBase
   table :ouch
-  attribute duration : Time = ->{ Time.utc + 50.years }
+  attribute duration : Time = -> { Time.utc + 50.years }
 
   belongs_to Programmer
 end

spec/table_manager_spec.cr

@@ -10,7 +10,7 @@ module SearchIngest
       it "applies new mapping to an index" do
         delete_test_indices
         index = Broke.table_name
-        get_schema = ->{
+        get_schema = -> {
           response = JSON.parse(Elastic.client &.get("/#{index}").body)
           # Pluck the fields of interest
           mappings_field = response.dig(index, "mappings")

src/constants.cr

@@ -29,6 +29,8 @@ module SearchIngest
   ES_URI               = self.env_with_deprecation("ELASTIC_URI", "ES_URI").try(&->URI.parse(String))
   ES_HOST              = self.env_with_deprecation("ELASTIC_HOST", "ES_HOST") || "localhost"
   ES_PORT              = self.env_with_deprecation("ELASTIC_PORT", "ES_PORT").try(&.to_i) || 9200
+  ES_USER              = self.env_with_deprecation("ELASTIC_USER", "ES_USER")
+  ES_PASSWORD          = self.env_with_deprecation("ELASTIC_PASSWORD", "ES_PASSWORD")
   ES_CONN_POOL         = self.env_with_deprecation("ELASTIC_CONN_POOL", "ES_CONN_POOL").try(&.to_i)
   ES_IDLE_POOL         = self.env_with_deprecation("ELASTIC_IDLE_POOL", "ES_IDLE_POOL").try(&.to_i)
   ES_CONN_POOL_TIMEOUT = self.env_with_deprecation("ELASTIC_CONN_POOL_TIMEOUT", "ES_CONN_POOL_TIMEOUT").try(&.to_f64) || 5.0

src/search-ingest/client.cr

@@ -63,7 +63,7 @@ module SearchIngest
     def initialize(
       @uri : URI = DEFAULT_URI,
       @request_id : String? = nil,
-      @api_version : String = API_VERSION
+      @api_version : String = API_VERSION,
     )
       @connection = HTTP::Client.new(@uri)
     end

src/search-ingest/elastic.cr

@@ -20,6 +20,8 @@ module SearchIngest
       setting host : String = ES_HOST
       setting port : Int32 = ES_PORT
       setting tls : Bool = ES_TLS
+      setting user : String? = ES_USER
+      setting password : String? = ES_PASSWORD
       setting pool_size : Int32 = ES_CONN_POOL || SearchIngest::MANAGED_TABLES.size
       setting idle_pool_size : Int32 = ES_IDLE_POOL || (SearchIngest::MANAGED_TABLES.size // 4)
       setting pool_timeout : Float64 = ES_CONN_POOL_TIMEOUT
@@ -31,16 +33,29 @@ module SearchIngest
       host : String = settings.host,
       port : Int32 = settings.port,
       tls : Bool = settings.tls,
-      uri : URI? = settings.uri
+      uri : URI? = settings.uri,
+      user : String? = settings.user,
+      password : String? = settings.password,
     )
       if uri.nil?
         @client = HTTP::Client.new(host: host, port: port, tls: tls)
+        # Set basic auth if user and password are provided
+        if user && password
+          @client.basic_auth(user, password)
+        end
       else
         if ES_TLS
           context = OpenSSL::SSL::Context::Client.new
           context.verify_mode = OpenSSL::SSL::VerifyMode::NONE
         end
         @client = HTTP::Client.new(uri: uri, tls: context)
+
+        # Extract auth from URI if present, otherwise use separate user/password
+        if (usr = uri.user) && (pwd = uri.password)
+          @client.basic_auth(usr, pwd)
+        elsif user && password
+          @client.basic_auth(user, password)
+        end
       end
     end
 
@@ -168,7 +183,7 @@ module SearchIngest
       document_type : String,
       document_any : Hash(String, JSON::Any)? = nil,
       parent_id : String? = nil,
-      no_children : Bool = true
+      no_children : Bool = true,
     )
       case action
       in .update?, .create?
@@ -328,7 +343,7 @@ module SearchIngest
       document_type : String,
       document_any : Hash(String, JSON::Any)? = nil,
       parent_id : String? = nil,
-      no_children : Bool = true
+      no_children : Bool = true,
     )
       case action
       in .update?

src/search-ingest/schemas.cr

@@ -170,7 +170,7 @@ module SearchIngest
     # Collects all properties relevant to an index and collapse them into a schema
     def collect_index_properties(
       model : String | Class,
-      children : Array(String)? = nil
+      children : Array(String)? = nil,
     ) : Array(Field)
       name = self.class.document_name(model)
       if !children || children.empty?

src/search-ingest/table_manager.cr

@@ -31,7 +31,7 @@ module SearchIngest
     def initialize(
       @tables : Array(Table::Interface),
       backfill : Bool = false,
-      watch : Bool = false
+      watch : Bool = false,
     )
       Log.debug { {bulk_api: Elastic.bulk?, backfill: backfill, watch: watch, message: "starting TableManager"} }