Feature/runtime lineage

.circleci/api-load-test.sh

@@ -14,7 +14,7 @@
 set -e
 
 # Build version of Marquez
-readonly MARQUEZ_VERSION=0.52.0-SNAPSHOT
+readonly MARQUEZ_VERSION=0.51.2-SNAPSHOT
 # Fully qualified path to marquez.jar
 readonly MARQUEZ_JAR="api/build/libs/marquez-api-${MARQUEZ_VERSION}.jar"
 

.circleci/db-migration.sh

@@ -13,7 +13,7 @@
 # Version of PostgreSQL
 readonly POSTGRES_VERSION="14"
 # Version of Marquez
-readonly MARQUEZ_VERSION=0.51.0
+readonly MARQUEZ_VERSION=0.51.1
 # Build version of Marquez
 readonly MARQUEZ_BUILD_VERSION="$(git log --pretty=format:'%h' -n 1)" # SHA1
 readonly POSTGRES_PORT=5432

.env.example

@@ -3,4 +3,4 @@ API_ADMIN_PORT=5001
 WEB_PORT=3000
 POSTGRES_PORT=5432
 SEARCH_PORT=9200
-TAG=0.51.0
+TAG=0.51.1

Dockerfile

@@ -1,23 +1,43 @@
+# Copyright 2018-2023 contributors to the Marquez project
+# SPDX-License-Identifier: Apache-2.0
+
 FROM eclipse-temurin:17 AS base
 WORKDIR /usr/src/app
 COPY gradle gradle
+RUN ls -la gradle
 COPY gradle.properties gradle.properties
+RUN ls -la gradle.properties
 COPY gradlew gradlew
+RUN ls -la gradlew
 COPY settings.gradle settings.gradle
-RUN ./gradlew --version
+RUN ls -la settings.gradle
+
+# Make wrapper executable and fix line endings
+RUN chmod +x ./gradlew
+RUN sed -i 's/\r$//' ./gradlew
 
 FROM base AS build
 WORKDIR /usr/src/app
 COPY build.gradle build.gradle
+RUN ls -la build.gradle
 COPY api ./api
+RUN ls -la api
 COPY clients/java ./clients/java
-RUN ./gradlew --no-daemon clean :api:shadowJar
+RUN ls -la clients/java
+RUN ./gradlew clean :api:shadowJar --no-daemon --refresh-dependencies
 
 FROM eclipse-temurin:17
-RUN apt-get update && apt-get install -y postgresql-client bash coreutils
+RUN apt-get update && apt-get install -y postgresql-client bash coreutils dos2unix
 WORKDIR /usr/src/app
 COPY --from=build /usr/src/app/api/build/libs/marquez-*.jar /usr/src/app
+RUN ls -la /usr/src/app/marquez-*.jar
 COPY marquez.dev.yml marquez.dev.yml
+RUN ls -la marquez.dev.yml
 COPY docker/entrypoint.sh entrypoint.sh
+RUN dos2unix entrypoint.sh && \
+    chmod +x entrypoint.sh && \
+    ls -la entrypoint.sh && \
+    cat entrypoint.sh
+
 EXPOSE 5000 5001
-ENTRYPOINT ["/usr/src/app/entrypoint.sh"]
+CMD ["/usr/src/app/entrypoint.sh"]

api/build.gradle

@@ -18,6 +18,8 @@ import org.apache.tools.ant.filters.ReplaceTokens
 plugins {
     id 'maven-publish'
     id 'signing'
+    id 'application'
+    id 'com.github.johnrengelman.shadow'
 }
 
 ext {
@@ -26,41 +28,90 @@ ext {
     testcontainersVersion = '1.18.3'
     sentryVersion = '6.34.0'
 }
-
+configurations.all {
+    resolutionStrategy {
+        force 'com.google.guava:guava:32.1.3-jre'
+    }
+}
 dependencies {
     implementation project(':clients:java')
     implementation "io.dropwizard:dropwizard-core:${dropwizardVersion}"
     implementation "io.dropwizard:dropwizard-jdbi3:${dropwizardVersion}"
     implementation "io.dropwizard:dropwizard-json-logging:${dropwizardVersion}"
     implementation "io.dropwizard:dropwizard-http2:${dropwizardVersion}"
     implementation "io.dropwizard:dropwizard-assets:${dropwizardVersion}"
-    implementation "io.prometheus:simpleclient:${prometheusVersion}"
-    implementation "io.openlineage:openlineage-java:${openlineageVersion}"
-    implementation "io.prometheus:simpleclient_dropwizard:${prometheusVersion}"
-    implementation "io.prometheus:simpleclient_hotspot:${prometheusVersion}"
-    implementation "io.prometheus:simpleclient_servlet:${prometheusVersion}"
+
+    // Lombok
+    compileOnly 'org.projectlombok:lombok:1.18.30'
+    annotationProcessor 'org.projectlombok:lombok:1.18.30'
+
+    // Jakarta EE dependencies
+    implementation platform("jakarta.platform:jakarta.jakartaee-bom:${jakartaVersion}")
+    implementation "jakarta.platform:jakarta.jakartaee-api:${jakartaVersion}"
+    implementation "jakarta.validation:jakarta.validation-api:${jakartaValidationVersion}"
+    implementation 'jakarta.annotation:jakarta.annotation-api:2.1.1'
+    implementation 'jakarta.transaction:jakarta.transaction-api:2.0.1'
+    implementation 'jakarta.servlet:jakarta.servlet-api:5.0.0'
+    implementation 'jakarta.ws.rs:jakarta.ws.rs-api:3.1.0'
+    implementation 'jakarta.validation:jakarta.validation-api:3.0.2'
+    implementation 'org.hibernate.validator:hibernate-validator:8.0.1.Final'
+    implementation 'org.glassfish:jakarta.el:4.0.2'
+
+    // GraphQL dependencies with Jakarta EE 9 support
+    implementation ("com.graphql-java:graphql-java:${graphqlJavaVersion}") {
+        exclude group: 'com.google.guava', module: 'guava'
+    }
+    implementation ("com.graphql-java-kickstart:graphql-java-servlet:${graphqlServletVersion}") {
+        exclude group: 'com.google.guava', module: 'guava'
+    }
+    implementation ("com.graphql-java-kickstart:graphql-java-kickstart:${graphqlServletVersion}") {
+        exclude group: 'com.google.guava', module: 'guava'
+    }
+
+    implementation "io.prometheus:simpleclient_servlet_jakarta:0.16.0"
+    implementation "io.prometheus:simpleclient_common:0.16.0"
+    implementation "io.prometheus:simpleclient_dropwizard:0.16.0"
+    implementation "io.prometheus:simpleclient_hotspot:0.16.0"
+
     implementation "org.jdbi:jdbi3-core:${jdbi3Version}"
     implementation "org.jdbi:jdbi3-jackson2:${jdbi3Version}"
     implementation "org.jdbi:jdbi3-postgres:${jdbi3Version}"
     implementation "org.jdbi:jdbi3-sqlobject:${jdbi3Version}"
+    implementation "io.dropwizard.metrics:metrics-jdbi3:4.2.25"
     implementation 'com.google.guava:guava:32.1.3-jre'
     implementation 'org.dhatim:dropwizard-sentry:2.1.6'
     implementation "io.sentry:sentry:${sentryVersion}"
     implementation 'org.flywaydb:flyway-core:8.5.13'
     implementation "org.postgresql:postgresql:${postgresqlVersion}"
-    implementation 'com.graphql-java:graphql-java:20.9'
-    implementation 'com.graphql-java-kickstart:graphql-java-servlet:12.0.0'
+    implementation "io.openlineage:openlineage-java:0.30.1"
+    implementation 'org.apache.httpcomponents:httpclient:4.5.14'
 
-    implementation 'org.opensearch.client:opensearch-rest-client:2.17.1'
-    implementation 'org.opensearch.client:opensearch-java:2.16.0'
+    implementation 'org.opensearch.client:opensearch-rest-client:2.19.1'
+    implementation 'org.opensearch.client:opensearch-java:2.22.0'
 
+    testImplementation "io.dropwizard:dropwizard-core:${dropwizardVersion}"
+    testImplementation "io.dropwizard:dropwizard-jdbi3:${dropwizardVersion}"
     testImplementation "io.dropwizard:dropwizard-testing:${dropwizardVersion}"
     testImplementation "org.jdbi:jdbi3-testing:${jdbi3Version}"
     testImplementation "org.jdbi:jdbi3-testcontainers:${jdbi3Version}"
     testImplementation "org.junit.vintage:junit-vintage-engine:${junit5Version}"
     testImplementation "org.testcontainers:postgresql:${testcontainersVersion}"
     testImplementation "org.testcontainers:junit-jupiter:${testcontainersVersion}"
-    testImplementation 'org.apache.httpcomponents:httpclient:4.5.14'
+    testImplementation "org.junit.jupiter:junit-jupiter-api:${junit5Version}"
+    testImplementation "org.junit.jupiter:junit-jupiter-engine:${junit5Version}"
+    testImplementation "org.junit.jupiter:junit-jupiter-params:${junit5Version}"
+    testImplementation "org.testcontainers:testcontainers:${testcontainersVersion}"
+
+    // Add Jakarta EE dependencies for tests
+    testImplementation platform("jakarta.platform:jakarta.jakartaee-bom:${jakartaVersion}")
+    testImplementation "jakarta.platform:jakarta.jakartaee-api:${jakartaVersion}"
+    testImplementation "jakarta.validation:jakarta.validation-api:${jakartaValidationVersion}"
+    testImplementation 'jakarta.annotation:jakarta.annotation-api:2.1.1'
+    testImplementation 'jakarta.transaction:jakarta.transaction-api:2.0.1'
+    testImplementation 'jakarta.servlet:jakarta.servlet-api:5.0.0'
+    testImplementation 'jakarta.ws.rs:jakarta.ws.rs-api:3.1.0'
+    testImplementation 'jakarta.validation:jakarta.validation-api:3.0.2'
+    testImplementation 'org.hibernate.validator:hibernate-validator:8.0.1.Final'
 }
 
 task testUnit(type: Test) {
@@ -83,6 +134,11 @@ task testDataAccess(type: Test) {
 
 test {
     useJUnitPlatform()
+    testLogging {
+        events "passed", "skipped", "failed"
+        showStandardStreams = true
+        exceptionFormat = 'full'
+    }
 }
 
 publishing {
@@ -158,6 +214,24 @@ shadowJar {
     from(projectDir) {
         include 'LICENSE'
     }
+    mergeServiceFiles()
+    // Include all dependencies by default
+    exclude 'io/dropwizard/logback/shaded/guava/**'
+    exclude 'META-INF/maven/com.google.guava/**' // Optional: only if you want zero guava metadata
+
+    dependencies {
+          exclude { dep ->
+            dep.moduleGroup == 'com.google.guava' &&
+            (dep.moduleName == 'guava' && dep.moduleVersion == '31.0.1-jre')
+        }
+        //exclude(dependency('com.google.guava:guava'))
+        // Exclude test dependencies
+        exclude(dependency('org.junit:.*'))
+        exclude(dependency('org.testcontainers:.*'))
+        exclude(dependency('org.junit.jupiter:.*'))
+        exclude(dependency('org.junit.vintage:.*'))
+        exclude(dependency('com.google.guava:guava:31.0.1-jre'))
+    }
     manifest {
         attributes(
                 'Created-By': "Gradle ${gradle.gradleVersion}",

api/src/main/java/marquez/MarquezApp.java

@@ -7,22 +7,22 @@
 
 import com.codahale.metrics.jdbi3.InstrumentedSqlLogger;
 import com.fasterxml.jackson.databind.SerializationFeature;
-import io.dropwizard.Application;
 import io.dropwizard.assets.AssetsBundle;
 import io.dropwizard.configuration.EnvironmentVariableSubstitutor;
 import io.dropwizard.configuration.SubstitutingSourceProvider;
+import io.dropwizard.core.Application;
+import io.dropwizard.core.setup.Bootstrap;
+import io.dropwizard.core.setup.Environment;
 import io.dropwizard.db.DataSourceFactory;
 import io.dropwizard.db.ManagedDataSource;
 import io.dropwizard.jdbi3.JdbiFactory;
-import io.dropwizard.setup.Bootstrap;
-import io.dropwizard.setup.Environment;
 import io.prometheus.client.CollectorRegistry;
 import io.prometheus.client.dropwizard.DropwizardExports;
-import io.prometheus.client.exporter.MetricsServlet;
 import io.prometheus.client.hotspot.DefaultExports;
+import io.prometheus.client.servlet.jakarta.exporter.MetricsServlet;
 import io.sentry.Sentry;
+import jakarta.servlet.DispatcherType;
 import java.util.EnumSet;
-import javax.servlet.DispatcherType;
 import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 import marquez.api.filter.JobRedirectFilter;
@@ -65,6 +65,12 @@ public final class MarquezApp extends Application<MarquezConfig> {
   private static final String PROMETHEUS_ENDPOINT = "/metrics";
   private static final String PROMETHEUS_ENDPOINT_V2 = "/v2beta/metrics";
 
+  private static Jdbi jdbiInstance; // Static reference for testing
+
+  public static Jdbi getJdbiInstanceForTesting() { // Static getter for testing
+    return jdbiInstance;
+  }
+
   public static void main(final String[] args) throws Exception {
     new MarquezApp().run(args);
   }
@@ -76,20 +82,18 @@ public String getName() {
 
   @Override
   public void initialize(@NonNull Bootstrap<MarquezConfig> bootstrap) {
-    // Enable metric collection for prometheus.
+    // Enable Prometheus metrics
     CollectorRegistry.defaultRegistry.register(
         new DropwizardExports(bootstrap.getMetricRegistry()));
     DatabaseMetrics.registry.register(new DropwizardExports(bootstrap.getMetricRegistry()));
-    DefaultExports.initialize(); // Add metrics for CPU, JVM memory, etc.
+    DefaultExports.initialize();
     DefaultExports.register(DatabaseMetrics.registry);
 
-    // Enable variable substitution with environment variables.
     bootstrap.setConfigurationSourceProvider(
         new SubstitutingSourceProvider(
             bootstrap.getConfigurationSourceProvider(),
             new EnvironmentVariableSubstitutor(ERROR_ON_UNDEFINED)));
 
-    // Add CLI commands
     bootstrap.addCommand(new DbMigrateCommand());
     bootstrap.addCommand(new DbRetentionCommand());
     bootstrap.addCommand(new MetadataCommand());
@@ -98,7 +102,6 @@ public void initialize(@NonNull Bootstrap<MarquezConfig> bootstrap) {
     bootstrap.getObjectMapper().disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
     Utils.addZonedDateTimeMixin(bootstrap.getObjectMapper());
 
-    // Add graphql playground
     bootstrap.addBundle(
         new AssetsBundle(
             "/assets",
@@ -118,8 +121,7 @@ public void run(@NonNull MarquezConfig config, @NonNull Environment env) {
       DbMigration.migrateDbOrError(config.getFlywayFactory(), source, config.isMigrateOnStartup());
     } catch (FlywayException errorOnDbMigrate) {
       log.info("Stopping app...");
-      // Propagate throwable up the stack.
-      onFatalError(errorOnDbMigrate); // Signal app termination.
+      onFatalError(errorOnDbMigrate);
     }
 
     if (isSentryEnabled(config)) {
@@ -138,6 +140,8 @@ public void run(@NonNull MarquezConfig config, @NonNull Environment env) {
     }
 
     final Jdbi jdbi = newJdbi(config, env, source);
+    jdbiInstance = jdbi; // Assign to static field
+
     final MarquezContext marquezContext =
         MarquezContext.builder()
             .jdbi(jdbi)
@@ -149,16 +153,12 @@ public void run(@NonNull MarquezConfig config, @NonNull Environment env) {
     registerServlets(env);
     registerFilters(env, marquezContext);
 
-    // Add scheduled jobs to lifecycle.
     if (config.hasDbRetentionPolicy()) {
-      // Add job to apply retention policy to database.
       env.lifecycle().manage(new DbRetentionJob(jdbi, config.getDbRetention()));
     }
 
-    // Add job to refresh materialized views.
     env.lifecycle().manage(new MaterializeViewRefresherJob(jdbi));
 
-    // set namespaceFilter
     ExclusionsConfig exclusions = config.getExclude();
     Exclusions.use(exclusions);
   }
@@ -168,7 +168,6 @@ private boolean isSentryEnabled(MarquezConfig config) {
         && !config.getSentry().getDsn().equals(SentryConfig.DEFAULT_DSN);
   }
 
-  /** Returns a new {@link Jdbi} object. */
   private Jdbi newJdbi(
       @NonNull MarquezConfig config, @NonNull Environment env, @NonNull ManagedDataSource source) {
     final JdbiFactory factory = new JdbiFactory();
@@ -197,6 +196,9 @@ public void registerResources(
           .addMapping("/api/v1-beta/graphql", "/api/v1/schema.json");
     }
 
+    // Prometheus metrics endpoint
+    env.servlets().addServlet(PROMETHEUS, new MetricsServlet()).addMapping(PROMETHEUS_ENDPOINT);
+
     log.debug("Registering resources...");
     for (final Object resource : context.getResources()) {
       env.jersey().register(resource);
@@ -205,9 +207,6 @@ public void registerResources(
 
   private void registerServlets(@NonNull Environment env) {
     log.debug("Registering servlets...");
-
-    // Expose metrics for monitoring.
-    env.servlets().addServlet(PROMETHEUS, new MetricsServlet()).addMapping(PROMETHEUS_ENDPOINT);
     env.servlets()
         .addServlet(PROMETHEUS_V2, new MetricsServlet(DatabaseMetrics.registry))
         .addMapping(PROMETHEUS_ENDPOINT_V2);

api/src/main/java/marquez/MarquezConfig.java

@@ -7,7 +7,7 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.collect.ImmutableSet;
-import io.dropwizard.Configuration;
+import io.dropwizard.core.Configuration;
 import io.dropwizard.db.DataSourceFactory;
 import lombok.Getter;
 import lombok.NoArgsConstructor;

api/src/main/java/marquez/MarquezContext.java

@@ -8,7 +8,6 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
-import graphql.kickstart.servlet.GraphQLHttpServlet;
 import java.util.ArrayList;
 import java.util.List;
 import lombok.Getter;
@@ -110,7 +109,7 @@ public final class MarquezContext {
   @Getter private final ImmutableList<Object> resources;
   @Getter private final JdbiExceptionExceptionMapper jdbiException;
   @Getter private final JsonProcessingExceptionMapper jsonException;
-  @Getter private final GraphQLHttpServlet graphqlServlet;
+  @Getter private final jakarta.servlet.Servlet graphqlServlet;
   @Getter private final SearchConfig searchConfig;
 
   private MarquezContext(

api/src/main/java/marquez/api/BaseResource.java

@@ -6,10 +6,10 @@
 package marquez.api;
 
 import com.google.common.collect.ImmutableSet;
+import jakarta.annotation.Nullable;
+import jakarta.ws.rs.core.UriInfo;
 import java.net.URI;
 import java.util.Optional;
-import javax.annotation.Nullable;
-import javax.ws.rs.core.UriInfo;
 import lombok.NonNull;
 import marquez.api.exceptions.DatasetNotFoundException;
 import marquez.api.exceptions.FieldNotFoundException;