implement startup readiness health checks

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/health/GenericHealthCheck.java

@@ -0,0 +1,36 @@
+package tech.ebp.oqm.core.api.health;
+
+import jakarta.enterprise.inject.Instance;
+import org.eclipse.microprofile.health.HealthCheck;
+import org.eclipse.microprofile.health.HealthCheckResponse;
+import org.eclipse.microprofile.health.HealthCheckResponseBuilder;
+
+import java.util.function.Function;
+
+public class GenericHealthCheck<T> implements HealthCheck {
+    private final String healthCheckName;
+    private final Instance<T> providers;
+    private final Function<T, HealthStatus> statusGetter;
+
+    public GenericHealthCheck(String healthCheckName, Instance<T> providers, Function<T, HealthStatus> statusGetter) {
+        this.healthCheckName = healthCheckName;
+        this.providers = providers;
+        this.statusGetter = statusGetter;
+    }
+
+    @Override
+    public HealthCheckResponse call() {
+        HealthCheckResponseBuilder builder = HealthCheckResponse.named(this.healthCheckName);
+        boolean allUp = true;
+
+        for (T provider : this.providers) {
+            HealthStatus status = this.statusGetter.apply(provider);
+            boolean up = status.isUp();
+            allUp &= up;
+            builder.withData(status.getName() + ".up", up);
+            builder.withData(status.getName() + ".status", status.getStatusMessage());
+        }
+
+        return (allUp ? builder.up() : builder.down()).build();
+    }
+}

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/health/HealthStatus.java

@@ -0,0 +1,24 @@
+package tech.ebp.oqm.core.api.health;
+
+import lombok.Getter;
+
+@Getter
+public class HealthStatus {
+    private final String name;
+    private volatile boolean up = false;
+    private volatile String statusMessage = "Status not set";
+
+    public HealthStatus(String name) {
+        this.name = name;
+    }
+
+    public void markUp(String message) {
+        up = true;
+        statusMessage = message;
+    }
+
+    public void markDown(String message) {
+        up = false;
+        statusMessage = message;
+    }
+}

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/health/SchemaUpgradeHealthCheck.java

@@ -10,7 +10,7 @@
 public class SchemaUpgradeHealthCheck implements HealthCheck {
 
 	@Inject
-	ObjectSchemaUpgradeService objectSchemaUpgradeService;
+    ObjectSchemaUpgradeService objectSchemaUpgradeService;
 
 	@Override
 	public HealthCheckResponse call() {

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/health/service/LivenessHealthCheck.java

@@ -0,0 +1,18 @@
+package tech.ebp.oqm.core.api.health.service;
+
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.inject.Instance;
+import jakarta.inject.Inject;
+import org.eclipse.microprofile.health.Liveness;
+import tech.ebp.oqm.core.api.health.GenericHealthCheck;
+import tech.ebp.oqm.core.api.health.utils.HasLivenessCheck;
+
+@Liveness
+@ApplicationScoped
+public class LivenessHealthCheck extends GenericHealthCheck<HasLivenessCheck> {
+
+    @Inject
+    LivenessHealthCheck(Instance<HasLivenessCheck> providers) {
+        super("Service Health - Liveness", providers, HasLivenessCheck::getLivenessStatus);
+    }
+}

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/health/service/ReadinessHealthCheck.java

@@ -0,0 +1,18 @@
+package tech.ebp.oqm.core.api.health.service;
+
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.inject.Instance;
+import jakarta.inject.Inject;
+import org.eclipse.microprofile.health.Readiness;
+import tech.ebp.oqm.core.api.health.GenericHealthCheck;
+import tech.ebp.oqm.core.api.health.utils.HasReadinessCheck;
+
+@Readiness
+@ApplicationScoped
+public class ReadinessHealthCheck extends GenericHealthCheck<HasReadinessCheck> {
+
+    @Inject
+    ReadinessHealthCheck(Instance<HasReadinessCheck> providers) {
+        super("Service Health - Readiness", providers, HasReadinessCheck::getReadinessStatus);
+    }
+}

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/health/service/StartupHealthCheck.java

@@ -0,0 +1,18 @@
+package tech.ebp.oqm.core.api.health.service;
+
+import io.quarkus.runtime.Startup;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.inject.Instance;
+import jakarta.inject.Inject;
+import tech.ebp.oqm.core.api.health.GenericHealthCheck;
+import tech.ebp.oqm.core.api.health.utils.HasStartupCheck;
+
+@Startup
+@ApplicationScoped
+public class StartupHealthCheck extends GenericHealthCheck<HasStartupCheck> {
+
+    @Inject
+    StartupHealthCheck(Instance<HasStartupCheck> providers) {
+        super("Service Health - Startup", providers, HasStartupCheck::getStartupStatus);
+    }
+}

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/health/utils/HasLivenessCheck.java

@@ -0,0 +1,7 @@
+package tech.ebp.oqm.core.api.health.utils;
+
+import tech.ebp.oqm.core.api.health.HealthStatus;
+
+public interface HasLivenessCheck {
+    HealthStatus getLivenessStatus();
+}

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/health/utils/HasReadinessCheck.java

@@ -0,0 +1,7 @@
+package tech.ebp.oqm.core.api.health.utils;
+
+import tech.ebp.oqm.core.api.health.HealthStatus;
+
+public interface HasReadinessCheck {
+    HealthStatus getReadinessStatus();
+}

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/health/utils/HasStartupCheck.java

@@ -0,0 +1,7 @@
+package tech.ebp.oqm.core.api.health.utils;
+
+import tech.ebp.oqm.core.api.health.HealthStatus;
+
+public interface HasStartupCheck {
+    HealthStatus getStartupStatus();
+}

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/scheduled/LifecycleBean.java

@@ -8,10 +8,8 @@
 import lombok.extern.slf4j.Slf4j;
 import org.eclipse.microprofile.config.ConfigProvider;
 import org.eclipse.microprofile.config.inject.ConfigProperty;
-import tech.ebp.oqm.core.api.model.object.upgrade.TotalUpgradeResult;
 import tech.ebp.oqm.core.api.service.TempFileService;
 import tech.ebp.oqm.core.api.service.mongo.CustomUnitService;
-import tech.ebp.oqm.core.api.service.schemaVersioning.ObjectSchemaUpgradeService;
 import tech.ebp.oqm.core.api.service.serviceState.db.OqmDatabaseService;
 
 import java.nio.file.Paths;
@@ -38,7 +36,7 @@ public class LifecycleBean {
 	TempFileService tempFileService;
 
 	@Inject
-	OqmDatabaseService dbService;
+    OqmDatabaseService dbService;
 
 	private ZonedDateTime startDateTime;
 

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/scheduled/MongoDbInit.java

@@ -6,21 +6,23 @@
 import jakarta.enterprise.inject.Instance;
 import jakarta.inject.Inject;
 import jakarta.inject.Singleton;
+import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import tech.ebp.oqm.core.api.model.object.upgrade.TotalUpgradeResult;
 import tech.ebp.oqm.core.api.service.mongo.InventoryItemService;
-import tech.ebp.oqm.core.api.service.mongo.MongoDbAwareService;
 import tech.ebp.oqm.core.api.service.mongo.MongoService;
 import tech.ebp.oqm.core.api.service.schemaVersioning.ObjectSchemaUpgradeService;
 import tech.ebp.oqm.core.api.service.serviceState.InstanceMutexService;
 import tech.ebp.oqm.core.api.service.serviceState.db.DbCacheEntry;
 import tech.ebp.oqm.core.api.service.serviceState.db.OqmDatabaseService;
+import tech.ebp.oqm.core.api.health.HealthStatus;
+import tech.ebp.oqm.core.api.health.utils.HasReadinessCheck;
 
 import java.util.Optional;
 
 @Singleton
 @Slf4j
-public class MongoDbInit {
+public class MongoDbInit implements HasReadinessCheck {
 
     @Inject
     InventoryItemService inventoryItemService;
@@ -51,9 +53,7 @@ private void ensureItemMutexesExist() {
         for (DbCacheEntry curDb : this.oqmDatabaseService.getDatabases()) {
             log.info("Ensuring inventory item mutexes exist for database: {}", curDb.getDbName());
             this.inventoryItemService.iterator(curDb.getDbId().toHexString()).forEachRemaining((item) -> {
-                this.instanceMutexService.register(
-                    this.instanceMutexService.getMutexIdFor(curDb.getDbId().toHexString(), item)
-                );
+                this.instanceMutexService.register(this.instanceMutexService.getMutexIdFor(curDb.getDbId().toHexString(), item));
             });
             log.info("DONE Ensuring inventory item mutexes exist for database: {}", curDb.getDbName());
         }
@@ -82,16 +82,29 @@ private void initDbs(){
 	}
 
 
-    void onStart(
-        @Observes
-        StartupEvent ev
-    ) {
-		log.info("Starting initial db initialization tasks.");
-
-		this.upgradeDbs();
-		this.initDbs();
-		this.ensureItemMutexesExist();
-
-		log.info("FINISHED initial db initialization tasks.");
+    @Getter
+    private final HealthStatus readinessStatus = new HealthStatus("Mongo DB Init");
+
+    void onStart(@Observes StartupEvent ev) {
+        readinessStatus.markDown("Startup initialization in progress");
+        try {
+            //ensures the db service bean is initialized, and the extension has had time to init
+            try {
+                this.oqmDatabaseService.collectionStats();
+                this.oqmDatabaseService.getReadinessStatus().markUp("Database service initialized");
+            } catch (RuntimeException e) {
+                this.oqmDatabaseService.getReadinessStatus().markDown("Database service init failed: " + e.getMessage());
+                throw e;
+            }
+            this.upgradeDbs();
+            this.initDbs();
+            this.ensureItemMutexesExist();
+
+            readinessStatus.markUp("Initial db initialization tasks finished");
+            log.info("FINISHED initial db initialization tasks.");
+        } catch (RuntimeException e) {
+            readinessStatus.markDown("Initial db initialization failed: " + e.getMessage());
+            throw e;
+        }
     }
 }