Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/cfg/integration-test-core.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -147,3 +147,6 @@ jobs:

- name: Run 1-1-restore tests
run: make pkg-integration-test TABLETS=${{ env.tablets }} PKG=./pkg/service/one2onerestore

- name: Verify SM server startup
run: make run-server SSL_ENABLED=${{ env.ssl-enabled }} IP_FAMILY=${{ env.ip-family }}
2 changes: 2 additions & 0 deletions .github/workflows/integration-tests-2024.1.21-IPV4.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,8 @@ jobs:
run: make pkg-integration-test PKG=./pkg/schema/migrate
- name: Run 1-1-restore tests
run: make pkg-integration-test TABLETS=${{ env.tablets }} PKG=./pkg/service/one2onerestore
- name: Verify SM server startup
run: make run-server SSL_ENABLED=${{ env.ssl-enabled }} IP_FAMILY=${{ env.ip-family }}
name: integration-tests-2024.1.21-IPV4
"on":
pull_request:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,8 @@ jobs:
run: make pkg-integration-test PKG=./pkg/schema/migrate
- name: Run 1-1-restore tests
run: make pkg-integration-test TABLETS=${{ env.tablets }} PKG=./pkg/service/one2onerestore
- name: Verify SM server startup
run: make run-server SSL_ENABLED=${{ env.ssl-enabled }} IP_FAMILY=${{ env.ip-family }}
name: integration-tests-2024.1.21-IPV6-raftschema-nossl
"on":
pull_request:
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/integration-tests-2025.1.11-IPV4.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,8 @@ jobs:
run: make pkg-integration-test PKG=./pkg/schema/migrate
- name: Run 1-1-restore tests
run: make pkg-integration-test TABLETS=${{ env.tablets }} PKG=./pkg/service/one2onerestore
- name: Verify SM server startup
run: make run-server SSL_ENABLED=${{ env.ssl-enabled }} IP_FAMILY=${{ env.ip-family }}
name: integration-tests-2025.1.11-IPV4
"on":
pull_request:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,8 @@ jobs:
run: make pkg-integration-test PKG=./pkg/schema/migrate
- name: Run 1-1-restore tests
run: make pkg-integration-test TABLETS=${{ env.tablets }} PKG=./pkg/service/one2onerestore
- name: Verify SM server startup
run: make run-server SSL_ENABLED=${{ env.ssl-enabled }} IP_FAMILY=${{ env.ip-family }}
name: integration-tests-2025.1.11-IPV6-tablets-nossl
"on":
pull_request:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ jobs:
run: make pkg-integration-test PKG=./pkg/schema/migrate
- name: Run 1-1-restore tests
run: make pkg-integration-test TABLETS=${{ env.tablets }} PKG=./pkg/service/one2onerestore
- name: Verify SM server startup
run: make run-server SSL_ENABLED=${{ env.ssl-enabled }} IP_FAMILY=${{ env.ip-family }}
name: integration-tests-2025.3.7-IPV4-tablets-rclone-native
"on":
pull_request:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ jobs:
run: make pkg-integration-test PKG=./pkg/schema/migrate
- name: Run 1-1-restore tests
run: make pkg-integration-test TABLETS=${{ env.tablets }} PKG=./pkg/service/one2onerestore
- name: Verify SM server startup
run: make run-server SSL_ENABLED=${{ env.ssl-enabled }} IP_FAMILY=${{ env.ip-family }}
name: integration-tests-2025.3.7-IPV6-nossl-rclone-rclone
"on":
pull_request:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ jobs:
run: make pkg-integration-test PKG=./pkg/schema/migrate
- name: Run 1-1-restore tests
run: make pkg-integration-test TABLETS=${{ env.tablets }} PKG=./pkg/service/one2onerestore
- name: Verify SM server startup
run: make run-server SSL_ENABLED=${{ env.ssl-enabled }} IP_FAMILY=${{ env.ip-family }}
name: integration-tests-2025.4.3-IPV4-native-rclone
"on":
pull_request:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ jobs:
run: make pkg-integration-test PKG=./pkg/schema/migrate
- name: Run 1-1-restore tests
run: make pkg-integration-test TABLETS=${{ env.tablets }} PKG=./pkg/service/one2onerestore
- name: Verify SM server startup
run: make run-server SSL_ENABLED=${{ env.ssl-enabled }} IP_FAMILY=${{ env.ip-family }}
name: integration-tests-2025.4.3-IPV6-tablets-nossl-rclone-rclone
"on":
pull_request:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ jobs:
run: make pkg-integration-test PKG=./pkg/schema/migrate
- name: Run 1-1-restore tests
run: make pkg-integration-test TABLETS=${{ env.tablets }} PKG=./pkg/service/one2onerestore
- name: Verify SM server startup
run: make run-server SSL_ENABLED=${{ env.ssl-enabled }} IP_FAMILY=${{ env.ip-family }}
name: integration-tests-2026.1.0-rc2-IPV4-tablets-native-native
"on":
pull_request:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ jobs:
run: make pkg-integration-test PKG=./pkg/schema/migrate
- name: Run 1-1-restore tests
run: make pkg-integration-test TABLETS=${{ env.tablets }} PKG=./pkg/service/one2onerestore
- name: Verify SM server startup
run: make run-server SSL_ENABLED=${{ env.ssl-enabled }} IP_FAMILY=${{ env.ip-family }}
name: integration-tests-2026.1.0-rc2-IPV6-nossl-rclone-rclone
"on":
pull_request:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ jobs:
run: make pkg-integration-test PKG=./pkg/schema/migrate
- name: Run 1-1-restore tests
run: make pkg-integration-test TABLETS=${{ env.tablets }} PKG=./pkg/service/one2onerestore
- name: Verify SM server startup
run: make run-server SSL_ENABLED=${{ env.ssl-enabled }} IP_FAMILY=${{ env.ip-family }}
name: integration-tests-latest-IPV4-native-native
"on":
pull_request:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ jobs:
run: make pkg-integration-test PKG=./pkg/schema/migrate
- name: Run 1-1-restore tests
run: make pkg-integration-test TABLETS=${{ env.tablets }} PKG=./pkg/service/one2onerestore
- name: Verify SM server startup
run: make run-server SSL_ENABLED=${{ env.ssl-enabled }} IP_FAMILY=${{ env.ip-family }}
name: integration-tests-latest-IPV6-tablets-nossl-rclone-rclone
"on":
pull_request:
Expand Down
15 changes: 15 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,13 @@ ifeq ($(IP_FAMILY), IPV6)
MINIO_ENDPOINT := https://[2001:0DB9:200::99]:9000
endif

ifeq ($(SSL_ENABLED),true)
MANAGER_CONFIG := testing/scylla-manager/scylla-manager-ssl.yaml
ifeq ($(IP_FAMILY), IPV6)
MANAGER_CONFIG := testing/scylla-manager/scylla-manager-ssl-ipv6.yaml
endif
endif

.PHONY: fmt
fmt: ## Format source code
@$(GOBIN)/golangci-lint run -c .golangci-fmt.yml --fix $(PKG)
Expand Down Expand Up @@ -222,6 +229,7 @@ clean-server: ## Remove development server container

.PHONY: run-server
run-server: build-server ## Build and run development server
@$(MAKE) clean-server
@docker run --name "scylla_manager_server" \
--network scylla_manager_second \
-p "5080:5080" \
Expand All @@ -230,9 +238,16 @@ run-server: build-server ## Build and run development server
-v "$(PWD)/scylla-manager.dev:/usr/bin/scylla-manager:ro" \
-v "$(PWD)/sctool.dev:/usr/bin/sctool:ro" \
-v "$(PWD)/$(MANAGER_CONFIG):/etc/scylla-manager/scylla-manager.yaml:ro" \
-v "$(PWD)/testing/scylla/certs:/etc/scylla-manager/certs:ro" \
-v "/tmp:/tmp" \
-d --read-only --rm scylladb/scylla-manager-dev scylla-manager
@docker network connect scylla_manager_public scylla_manager_server
@echo "==> Waiting for SM server to start"
@for i in $$(seq 15 -1 0); do \
[ $$i -eq 0 ] && echo -e "\nERROR: SM server failed to start" && exit 1; \
docker exec scylla_manager_server sctool status &> /dev/null && break; \
echo -n "." && sleep 1; \
done; echo ""

.PHONY: build
build: build-cli build-agent build-server ## Build all project binaries
Expand Down
5 changes: 3 additions & 2 deletions pkg/cmd/scylla-manager/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"net"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
Expand Down Expand Up @@ -100,15 +101,15 @@ var rootCmd = &cobra.Command{
netwait.DefaultWaiter.Logger = logger.Named("wait")
// Wait for database
logger.Info(ctx, "Checking database connectivity...")
initHost, err := netwait.AnyHostPort(ctx, c.Database.Hosts, "9042")
initHost, err := netwait.AnyHostPort(ctx, c.Database.Hosts, strconv.Itoa(c.Database.Port))
if err != nil {
return errors.Wrapf(
err,
"no connection to database, make sure Scylla server is running and that database section in c file(s) %s is set correctly",
strings.Join(rootArgs.configFiles, ", "),
)
}
c.Database.InitAddr = net.JoinHostPort(initHost, "9042")
c.Database.InitAddr = net.JoinHostPort(initHost, strconv.Itoa(c.Database.Port))

// Create keyspace if needed
ok, err := keyspaceExists(ctx, c, logger)
Expand Down
12 changes: 7 additions & 5 deletions pkg/testutils/db/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"crypto/tls"
"fmt"
"net/http"
"os"
"strings"
"sync"
"testing"
Expand Down Expand Up @@ -104,10 +103,6 @@ func CreateManagedClusterSession(tb testing.TB, empty bool, client *scyllaclient
Username: user,
Password: pass,
}
if os.Getenv("SSL_ENABLED") == "true" {
cluster.SslOpts = testconfig.CQLSSLOptions()
cluster.Port = testconfig.CQLPort()
}

session, err := gocqlx.WrapSession(cluster.CreateSession())
if err != nil {
Expand All @@ -124,6 +119,13 @@ func createCluster(hosts ...string) *gocql.ClusterConfig {
cluster.Timeout = 30 * time.Second
cluster.Consistency = gocql.Quorum
cluster.MaxWaitSchemaAgreement = 2 * time.Minute // travis might be slow
// SSL_ENABLED env var configures TLS setup for
// both managed clusters and SM DB cluster with
// the same TLS certs.
if testconfig.IsSSLEnabled() {
cluster.SslOpts = testconfig.CQLSSLOptions()
cluster.Port = testconfig.CQLPort()
}
return cluster
}

Expand Down
9 changes: 9 additions & 0 deletions testing/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ up:
@cd scylla/certs && ./generate.sh
@echo "==> Generating Scylla configuration"
@cp scylla/config/scylla.yaml scylla/scylla.yaml
@cp scylla/config/scylla-sm.yaml scylla/scylla-sm.yaml
@$(YQ) write -i scylla/scylla.yaml 'object_storage_endpoints[0].(name)' $(MINIO_HOST)

ifeq ($(SSL_ENABLED),true)
Expand All @@ -77,6 +78,10 @@ ifeq ($(SSL_ENABLED),true)
# merge into scylla.yaml values from config/scylla-ssl.yaml with overwrite option (-x)
@$(YQ) merge -i -x scylla/scylla.yaml scylla/config/scylla-ssl.yaml
@cp scylla/config/cqlshrc-ssl scylla/cqlshrc
# The same for SM DB
@$(YQ) delete -i scylla/scylla-sm.yaml 'native_transport_port'
@$(YQ) delete -i scylla/scylla-sm.yaml 'alternator_port'
@$(YQ) merge -i -x scylla/scylla-sm.yaml scylla/config/scylla-ssl.yaml
else
@cp scylla/config/cqlshrc scylla/cqlshrc
endif
Expand Down Expand Up @@ -131,6 +136,10 @@ endif
@./nodes_exec "rm /root/.cqlshrc || true"
@./nodes_exec "mkdir -p /root/.cassandra"
@./nodes_cp "scylla/cqlshrc" "/root/.cassandra/cqlshrc"
# Also handle cqlshrc for SM DB - even though it doesn't use password authentication, it's needed for SSL_ENABLED setup
@$(COMPOSE) exec -T scylla-manager-db bash -c "rm /root/.cqlshrc || true"
@$(COMPOSE) exec -T scylla-manager-db bash -c "mkdir -p /root/.cassandra"
@docker cp scylla/cqlshrc $$(docker ps -qf name=scylla-manager-db):/root/.cassandra/cqlshrc

@echo "==> Adding Minio user"
./minio/add_user.sh || true
Expand Down
7 changes: 6 additions & 1 deletion testing/docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -160,9 +160,11 @@ services:
second:

scylla-manager-db:
image: scylladb/${SCYLLA_VERSION}
image: scylladb/agent-${SCYLLA_VERSION}
privileged: true
ports:
- "9042:9042"
- "9142:9142"
- "10000:10000"
volumes:
- type: bind
Expand All @@ -171,6 +173,9 @@ services:
- type: bind
source: ./scylla/scylla-sm.yaml
target: /etc/scylla/scylla.yaml
- type: bind
source: ./scylla/certs/
target: /etc/scylla/certs
networks:
second:

Expand Down
6 changes: 3 additions & 3 deletions testing/scylla-manager/scylla-manager-ipv6.yaml
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
http: [::]:5080
https: [::]:5443
http: "[::]:5080"
https: "[::]:5443"

logger:
level: info
development: true

database:
hosts:
- [2001:0DB9:200::100]
- "2001:0DB9:200::100"
timeout: 1s
20 changes: 20 additions & 0 deletions testing/scylla-manager/scylla-manager-ssl-ipv6.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
http: "[::]:5080"
https: "[::]:5443"

logger:
level: info
development: true

database:
hosts:
- "2001:0DB9:200::100"
ssl: true
port: 9142
timeout: 1s

ssl:
cert_file: /etc/scylla-manager/certs/ca.crt
validate: true
user_cert_file: /etc/scylla-manager/certs/cl.crt
user_key_file: /etc/scylla-manager/certs/cl.key
Comment thread
Michal-Leszczynski marked this conversation as resolved.

20 changes: 20 additions & 0 deletions testing/scylla-manager/scylla-manager-ssl.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
http: "0.0.0.0:5080"
https: "0.0.0.0:5443"

logger:
level: info
development: true

database:
hosts:
- "192.168.200.100"
ssl: true

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

WDYT about removing "ssl: true" completely ?

It would make sense if we skip cert validation, but this PR is opposite - it forces certificate validation

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think I follow. Removing ssl: true results in SM not using TLS when connecting to SM DB. This PR made it so SM DB is set up with TLS when SSL_ENABLED=true. So just removing it would result in errors. I guess that you just wanted to remove some cert validation. We can remove it from this config, as then we don't validate SM DB TLS cert which is required in default secure TLS. We could remove it from SM DB scylla.yaml, but since scylla allows for easy setup of mTLS and we already validate SM certs when connecting to first/second cluster nodes with TLS, I kept the mTLS setup. It's just that by default we have one-way TLS (scylla validates SM, SM doesn't validate scylla), so adding regular TLS results in mTLS out of the box.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From UX perspective - I see ssl: true as an unnecessary boilerplate when you include certificates into the config.

Comment is about this UX exactly. But it's nit.

If we skip cert validation - then ssl: true without certs could mean - encrypt transmission but do not verify certs.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually, there are small differences in behavior controlled by the ssl field.
It's true that if someone adds TLS certs to config, then we can infer that ssl: true.
On the other hand, if we are missing TLS certs, we might still want to use TLS against scylla that does not verify client certs.
Similar thing happens when we think about custom CAs. Setup with ssl: true, no TLS certs, no custom CA still uses TLS and validates TLS certs against system CAs.

We might want to think about making it so we always validate TLS certs against both system and custom CA (if specified), so that the CA case is more comprehensive, but for now we can't just remove the ssl: bool field.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Or did you mean to just not require ss: true when any other TLS config field is specified, but keep this config option in general?

port: 9142
timeout: 1s

ssl:
cert_file: /etc/scylla-manager/certs/ca.crt
validate: true
user_cert_file: /etc/scylla-manager/certs/cl.crt
user_key_file: /etc/scylla-manager/certs/cl.key
Comment thread
Michal-Leszczynski marked this conversation as resolved.

6 changes: 3 additions & 3 deletions testing/scylla-manager/scylla-manager.yaml
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
http: 0.0.0.0:5080
https: 0.0.0.0:5443
http: "0.0.0.0:5080"
https: "0.0.0.0:5443"

logger:
level: info
development: true

database:
hosts:
- 192.168.200.100
- "192.168.200.100"
timeout: 1s
3 changes: 2 additions & 1 deletion testing/scylla/.gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/cqlshrc
/scylla.yaml
/scylla-second-cluster.yaml
/scylla-second-cluster.yaml
/scylla-sm.yaml
25 changes: 25 additions & 0 deletions testing/scylla/certs/db.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,28 @@ basicConstraints = CA:true
# Extensions to add to a certificate request
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names

[alt_names]
DNS.1 = db.scylla.com
IP.1 = 127.0.0.1
IP.2 = 192.168.200.11
IP.3 = 192.168.200.12
IP.4 = 192.168.200.13
IP.5 = 192.168.200.21
IP.6 = 192.168.200.22
IP.7 = 192.168.200.23
IP.8 = 192.168.200.31
IP.9 = 192.168.200.32
IP.10 = 192.168.100.99
Comment thread
Michal-Leszczynski marked this conversation as resolved.
IP.11 = 192.168.200.100
IP.12 = 2001:0DB9:200::11
IP.13 = 2001:0DB9:200::12
IP.14 = 2001:0DB9:200::13
IP.15 = 2001:0DB9:200::21
IP.16 = 2001:0DB9:200::22
IP.17 = 2001:0DB9:200::23
IP.18 = 2001:0DB9:200::31
IP.19 = 2001:0DB9:200::32
IP.20 = 2001:0DB9:100::99
IP.21 = 2001:0DB9:200::100
4 changes: 2 additions & 2 deletions testing/scylla/certs/generate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@ openssl genrsa -out db.key 4096 &> /dev/null
openssl genrsa -out cl.key 4096 &> /dev/null

# Generate certificate for Certificate Authority (ca)
openssl req -x509 -new -nodes -key ca.key -days 3650 -config ca.cfg -out ca.crt
openssl req -x509 -new -nodes -key ca.key -days 3650 -config ca.cfg -extensions v3_ca -out ca.crt
# Generate certificate for Database node (db)
openssl req -new -key db.key -out db.csr -config db.cfg
openssl x509 -req -in db.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out db.crt -days 365
openssl x509 -req -in db.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out db.crt -days 365 -extfile db.cfg -extensions v3_req
# Generate certificate for Client (cl)
openssl req -new -key cl.key -out cl.csr -config cl.cfg
openssl x509 -req -in cl.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out cl.crt -days 365
File renamed without changes.
Loading