feat(docker): add Dockerfile.local for local development and fix migrations (#8866)

* fix(migrations): add new fields and improve migration scripts for various plugins

* feat(docker): add Dockerfile.local for local development and update docker-compose configuration

* chore: add license to dockerfile

---------

Co-authored-by: Fábio Luciano <fabio.gois@sicoob.com.br>

Author: fabioluciano

backend/Dockerfile

@@ -63,8 +63,8 @@ COPY --from=debian-arm64 /lib/aarch64-linux-gnu /rootfs-arm64/lib/aarch64-linux-
 
 RUN for arch in aarch64 x86_64 ; do \
         mkdir -p /tmp/build/${arch} && cd /tmp/build/${arch} && \
-        wget https://github.com/libgit2/libgit2/archive/refs/tags/v1.3.2.tar.gz -O - | tar -xz && \
-        cd libgit2-1.3.2 && \
+        wget https://github.com/libgit2/libgit2/archive/refs/tags/v1.3.0.tar.gz -O - | tar -xz && \
+        cd libgit2-1.3.0 && \
         mkdir build && cd build && \
         if [ "$arch" = "aarch64" ] ; then \
             cmake .. -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc \

backend/Dockerfile.local

@@ -0,0 +1,92 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#Apache DevLake is an effort undergoing incubation at The Apache Software
+#Foundation (ASF), sponsored by the Apache Incubator PMC.
+#
+#Incubation is required of all newly accepted projects until a further review
+#indicates that the infrastructure, communications, and decision making process
+#have stabilized in a manner consistent with other successful ASF projects.
+#
+#While incubation status is not necessarily a reflection of the completeness or stability of the code,
+#it does indicate that the project has yet to be fully endorsed by the ASF.
+
+FROM golang:1.20.5-bookworm as builder
+
+ARG GOPROXY=
+ARG HTTP_PROXY=
+ARG HTTPS_PROXY=
+
+RUN apt-get update && apt-get install -y \
+    gcc \
+    binutils \
+    libfindbin-libs-perl \
+    cmake \
+    libssh2-1-dev \
+    libssl-dev \
+    zlib1g-dev
+
+# Install libgit2 for native platform only
+RUN mkdir -p /tmp/build && cd /tmp/build && \
+    wget https://github.com/libgit2/libgit2/archive/refs/tags/v1.3.0.tar.gz -O - | tar -xz && \
+    cd libgit2-1.3.0 && \
+    mkdir build && cd build && \
+    cmake .. -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local && \
+    make -j$(nproc) install && \
+    ldconfig
+
+RUN go install github.com/vektra/mockery/v2@v2.20.0
+RUN go install github.com/swaggo/swag/cmd/swag@v1.16.1
+
+WORKDIR /app
+COPY . /app
+ENV GOBIN=/app/bin
+
+ARG TAG=
+ARG SHA=
+
+# Generate mocks and swagger
+RUN make mock
+RUN make swag
+
+# Build plugins
+RUN make build-plugin
+
+# Build server
+RUN VERSION=${TAG}@${SHA} make build-server
+
+FROM debian:bookworm-slim
+
+RUN apt-get update && apt-get install -y \
+    libssh2-1 \
+    libssl3 \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy libgit2
+COPY --from=builder /usr/local/lib/libgit2.so* /usr/local/lib/
+RUN ldconfig
+
+WORKDIR /app
+
+COPY --from=builder /app/bin ./bin
+
+# Create empty python plugins dir to avoid startup error
+RUN mkdir -p python/plugins
+
+ENV PORT=8080
+EXPOSE 8080
+
+CMD ["./bin/lake"]

backend/plugins/github/models/migrationscripts/20240821_add_index_to_tool_github_jobs.go

@@ -43,6 +43,11 @@ func (script *addIndexToGithubJobs) Up(basicRes context.BasicRes) errors.Error {
 		if err := db.Exec(sql); err != nil {
 			return err
 		}
+	} else if u.Scheme == "postgres" || u.Scheme == "postgresql" {
+		sql := "CREATE INDEX IF NOT EXISTS idx_repo_id_connection_id ON _tool_github_jobs (repo_id, connection_id)"
+		if err := db.Exec(sql); err != nil {
+			return err
+		}
 	}
 	return nil
 }

backend/plugins/q_dev/models/migrationscripts/20250623_add_display_name_fields.go

@@ -21,29 +21,44 @@ import (
 	"github.com/apache/incubator-devlake/core/context"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/helpers/migrationhelper"
 )
 
 var _ plugin.MigrationScript = (*addDisplayNameFields)(nil)
 
 type addDisplayNameFields struct{}
 
-func (*addDisplayNameFields) Up(basicRes context.BasicRes) errors.Error {
-	db := basicRes.GetDal()
+type QDevConnection20250623 struct {
+	IdentityStoreId     string `gorm:"type:VARCHAR(255)"`
+	IdentityStoreRegion string `gorm:"type:VARCHAR(255)"`
+}
+
+func (QDevConnection20250623) TableName() string {
+	return "_tool_q_dev_connections"
+}
+
+type QDevUserData20250623 struct {
+	DisplayName string `gorm:"type:VARCHAR(255)"`
+}
 
-	// Add Identity Center fields to connections table
-	// Ignore error if column already exists (MySQL error 1060)
-	_ = db.Exec("ALTER TABLE _tool_q_dev_connections ADD COLUMN identity_store_id VARCHAR(255)")
-	_ = db.Exec("ALTER TABLE _tool_q_dev_connections ADD COLUMN identity_store_region VARCHAR(255)")
+func (QDevUserData20250623) TableName() string {
+	return "_tool_q_dev_user_data"
+}
 
-	// Add display_name column to user_data table
-	// Ignore error if column already exists (MySQL error 1060)
-	_ = db.Exec("ALTER TABLE _tool_q_dev_user_data ADD COLUMN display_name VARCHAR(255)")
+type QDevUserMetrics20250623 struct {
+	DisplayName string `gorm:"type:VARCHAR(255)"`
+}
 
-	// Add display_name column to user_metrics table
-	// Ignore error if column already exists (MySQL error 1060)
-	_ = db.Exec("ALTER TABLE _tool_q_dev_user_metrics ADD COLUMN display_name VARCHAR(255)")
+func (QDevUserMetrics20250623) TableName() string {
+	return "_tool_q_dev_user_metrics"
+}
 
-	return nil
+func (*addDisplayNameFields) Up(basicRes context.BasicRes) errors.Error {
+	return migrationhelper.AutoMigrateTables(basicRes,
+		&QDevConnection20250623{},
+		&QDevUserData20250623{},
+		&QDevUserMetrics20250623{},
+	)
 }
 
 func (*addDisplayNameFields) Version() uint64 {

backend/plugins/q_dev/models/migrationscripts/20251123_add_scope_config_id_to_s3_slice.go

@@ -20,23 +20,21 @@ package migrationscripts
 import (
 	"github.com/apache/incubator-devlake/core/context"
 	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/helpers/migrationhelper"
 )
 
 type addScopeConfigIdToS3Slice struct{}
 
+type QDevS3Slice20251123 struct {
+	ScopeConfigId uint64 `gorm:"type:BIGINT DEFAULT 0"`
+}
+
+func (QDevS3Slice20251123) TableName() string {
+	return "_tool_q_dev_s3_slices"
+}
+
 func (*addScopeConfigIdToS3Slice) Up(basicRes context.BasicRes) errors.Error {
-	db := basicRes.GetDal()
-
-	// Add scope_config_id column to _tool_q_dev_s3_slices table
-	err := db.Exec(`
-		ALTER TABLE _tool_q_dev_s3_slices
-		ADD COLUMN scope_config_id BIGINT UNSIGNED DEFAULT 0
-	`)
-	if err != nil {
-		return errors.Convert(err)
-	}
-
-	return nil
+	return migrationhelper.AutoMigrateTables(basicRes, &QDevS3Slice20251123{})
 }
 
 func (*addScopeConfigIdToS3Slice) Version() uint64 {

backend/plugins/sonarqube/models/migrationscripts/20230316_modify_character_set.go

@@ -18,61 +18,53 @@ limitations under the License.
 package migrationscripts
 
 import (
-	"net/url"
-
 	"github.com/apache/incubator-devlake/core/context"
 	"github.com/apache/incubator-devlake/core/errors"
 )
 
 type modifyCharacterSet struct{}
 
 func (*modifyCharacterSet) Up(basicRes context.BasicRes) errors.Error {
-	dbUrl := basicRes.GetConfig("DB_URL")
-	if dbUrl == "" {
-		return errors.BadInput.New("DB_URL is required")
-	}
-	u, err1 := url.Parse(dbUrl)
-	if err1 != nil {
-		return errors.Convert(err1)
-	}
-	if u.Scheme == "mysql" {
-		err := basicRes.GetDal().Exec(`ALTER TABLE _tool_sonarqube_projects CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
+	db := basicRes.GetDal()
+	// Character set conversion is MySQL-specific; PostgreSQL uses UTF-8 by default
+	if db.Dialect() == "mysql" {
+		err := db.Exec(`ALTER TABLE _tool_sonarqube_projects CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
 		if err != nil {
 			return err
 		}
-		err = basicRes.GetDal().Exec(`ALTER TABLE _tool_sonarqube_issues CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
+		err = db.Exec(`ALTER TABLE _tool_sonarqube_issues CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
 		if err != nil {
 			return err
 		}
-		err = basicRes.GetDal().Exec(`ALTER TABLE _tool_sonarqube_issue_code_blocks CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
+		err = db.Exec(`ALTER TABLE _tool_sonarqube_issue_code_blocks CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
 		if err != nil {
 			return err
 		}
-		err = basicRes.GetDal().Exec(`ALTER TABLE _tool_sonarqube_hotspots CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
+		err = db.Exec(`ALTER TABLE _tool_sonarqube_hotspots CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
 		if err != nil {
 			return err
 		}
-		err = basicRes.GetDal().Exec(`ALTER TABLE _tool_sonarqube_file_metrics CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
+		err = db.Exec(`ALTER TABLE _tool_sonarqube_file_metrics CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
 		if err != nil {
 			return err
 		}
-		err = basicRes.GetDal().Exec(`ALTER TABLE _tool_sonarqube_accounts CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
+		err = db.Exec(`ALTER TABLE _tool_sonarqube_accounts CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
 		if err != nil {
 			return err
 		}
-		err = basicRes.GetDal().Exec(`ALTER TABLE cq_projects CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
+		err = db.Exec(`ALTER TABLE cq_projects CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
 		if err != nil {
 			return err
 		}
-		err = basicRes.GetDal().Exec(`ALTER TABLE cq_issues CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
+		err = db.Exec(`ALTER TABLE cq_issues CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
 		if err != nil {
 			return err
 		}
-		err = basicRes.GetDal().Exec(`ALTER TABLE cq_issue_code_blocks CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
+		err = db.Exec(`ALTER TABLE cq_issue_code_blocks CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
 		if err != nil {
 			return err
 		}
-		err = basicRes.GetDal().Exec(`ALTER TABLE cq_file_metrics CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
+		err = db.Exec(`ALTER TABLE cq_file_metrics CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
 		if err != nil {
 			return err
 		}

backend/plugins/sonarqube/models/migrationscripts/20240325_modify_commit_character_type.go

@@ -18,29 +18,21 @@ limitations under the License.
 package migrationscripts
 
 import (
-	"net/url"
-
 	"github.com/apache/incubator-devlake/core/context"
 	"github.com/apache/incubator-devlake/core/errors"
 )
 
 type modifyCommitCharacterType struct{}
 
 func (*modifyCommitCharacterType) Up(basicRes context.BasicRes) errors.Error {
-	dbUrl := basicRes.GetConfig("DB_URL")
-	if dbUrl == "" {
-		return errors.BadInput.New("DB_URL is required")
-	}
-	u, err1 := url.Parse(dbUrl)
-	if err1 != nil {
-		return errors.Convert(err1)
-	}
-	if u.Scheme == "mysql" {
-		err := basicRes.GetDal().Exec(`ALTER TABLE commits MODIFY COLUMN message LONGTEXT CHARACTER SET binary;`)
+	db := basicRes.GetDal()
+	// Binary column types are MySQL-specific; PostgreSQL uses bytea natively
+	if db.Dialect() == "mysql" {
+		err := db.Exec(`ALTER TABLE commits MODIFY COLUMN message LONGTEXT CHARACTER SET binary;`)
 		if err != nil {
 			return err
 		}
-		err = basicRes.GetDal().Exec(`ALTER TABLE commit_files MODIFY COLUMN file_path VARBINARY(255);`)
+		err = db.Exec(`ALTER TABLE commit_files MODIFY COLUMN file_path VARBINARY(255);`)
 		if err != nil {
 			return err
 		}

backend/plugins/sonarqube/models/migrationscripts/20240508_modify_commit_character_type.go

@@ -18,25 +18,17 @@ limitations under the License.
 package migrationscripts
 
 import (
-	"net/url"
-
 	"github.com/apache/incubator-devlake/core/context"
 	"github.com/apache/incubator-devlake/core/errors"
 )
 
 type modifyCommitCharacterType0508 struct{}
 
 func (*modifyCommitCharacterType0508) Up(basicRes context.BasicRes) errors.Error {
-	dbUrl := basicRes.GetConfig("DB_URL")
-	if dbUrl == "" {
-		return errors.BadInput.New("DB_URL is required")
-	}
-	u, err1 := url.Parse(dbUrl)
-	if err1 != nil {
-		return errors.Convert(err1)
-	}
-	if u.Scheme == "mysql" {
-		err := basicRes.GetDal().Exec(`ALTER TABLE commit_files MODIFY COLUMN file_path VARBINARY(1024);`)
+	db := basicRes.GetDal()
+	// VARBINARY is MySQL-specific; PostgreSQL uses bytea natively
+	if db.Dialect() == "mysql" {
+		err := db.Exec(`ALTER TABLE commit_files MODIFY COLUMN file_path VARBINARY(1024);`)
 		if err != nil {
 			return err
 		}

backend/plugins/testmo/models/migrationscripts/20250629_add_scope_config_id_to_projects.go

@@ -21,13 +21,21 @@ import (
 	"github.com/apache/incubator-devlake/core/context"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/helpers/migrationhelper"
 )
 
 type addScopeConfigIdToProjects struct{}
 
+type TestmoProject20250629 struct {
+	ScopeConfigId uint64 `gorm:"type:BIGINT NOT NULL DEFAULT 0"`
+}
+
+func (TestmoProject20250629) TableName() string {
+	return "_tool_testmo_projects"
+}
+
 func (*addScopeConfigIdToProjects) Up(basicRes context.BasicRes) errors.Error {
-	db := basicRes.GetDal()
-	return db.Exec("ALTER TABLE `_tool_testmo_projects` ADD COLUMN `scope_config_id` bigint unsigned NOT NULL DEFAULT 0")
+	return migrationhelper.AutoMigrateTables(basicRes, &TestmoProject20250629{})
 }
 
 func (*addScopeConfigIdToProjects) Version() uint64 {

docker-compose-dev.yml

@@ -79,9 +79,10 @@ services:
       - mysql
 
   devlake:
-    image: devlake.docker.scarf.sh/apache/devlake:latest
+    image: devlake-local:latest
     build:
       context: backend
+      dockerfile: Dockerfile.local
       args:
         HTTPS_PROXY: "${HTTPS_PROXY}"
         GOPROXY: "${GOPROXY}"
@@ -93,6 +94,10 @@ services:
     env_file:
       - ./.env
     environment:
+      DB_URL: postgres://merico:merico@postgres:5432/lake?sslmode=disable
+      E2E_DB_URL: postgres://merico:merico@postgres:5432/lake_test?sslmode=disable
+      REMOTE_PLUGIN_DIR: ""
+      FORCE_MIGRATION: "true"
       LOGGING_DIR: /app/logs
       TZ: UTC
       # LOGOUT_URI: https://xxx.amazoncognito.com/logout?client_id=yyy&logout_uri=http%3A%2F%2Flocalhost%3A4180%2Foauth2%2Fsign_out