Skip to content

test: isolate kafka tests from host env #4561

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
pellared wants to merge 6 commits into
base: main
Choose a base branch
from
Draft

test: isolate kafka tests from host env #4561

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a02609d
test: avoid fixed container names
pellared Apr 30, 2026
a0efb67
Merge branch 'main' into fix-4537
pellared Apr 30, 2026
0fb5d60
avoid fixed hostnames and custom Docker network setup
pellared Apr 30, 2026
a7ede07
Remove fixed port bindings from Kafka tests
pellared Apr 30, 2026
612328e
go mod tidy
pellared Apr 30, 2026
c57f519
apply Copilot suggestions
pellared Apr 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion instrumentation/github.com/confluentinc/confluent-kafka-go/kafka/splunkkafka/test/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ go 1.25.5

require (
github.com/confluentinc/confluent-kafka-go v1.9.2
github.com/moby/moby/api v1.54.2
github.com/ory/dockertest/v4 v4.0.0
github.com/signalfx/splunk-otel-go/instrumentation/github.com/confluentinc/confluent-kafka-go/kafka/splunkkafka v1.32.0
github.com/stretchr/testify v1.11.1
Expand All @@ -29,6 +28,7 @@ require (
github.com/go-logr/stdr v1.2.2 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/moby/api v1.54.2 // indirect
github.com/moby/moby/client v0.4.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
Expand Down
66 changes: 31 additions & 35 deletions ...mentation/github.com/confluentinc/confluent-kafka-go/kafka/splunkkafka/test/kafka_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,11 @@ import (
"fmt"
"log"
"net"
"net/netip"
"os"
"testing"
"time"

"github.com/confluentinc/confluent-kafka-go/kafka"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/network"
"github.com/ory/dockertest/v4"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand All @@ -53,9 +50,10 @@ var (

const (
bootstrapServersConfigKey = "bootstrap.servers"
kafkaBootstrapServers = "127.0.0.1:9092"
)

var kafkaBootstrapServers string

func TestMain(m *testing.M) {
deprecated := flag.Bool("deprecated", false, "run integration tests of a deprecated module")
flag.Parse()
Expand All @@ -74,69 +72,59 @@ func TestMain(m *testing.M) {
log.Fatalf("Could not connect to docker: %v", err)
}

confNet, err := pool.CreateNetwork(ctx, "confluent", nil)
if err != nil {
log.Fatalf("Could not create docker network: %v", err)
}

_, err = pool.Run(ctx, "confluentinc/cp-zookeeper",
zkRes, err := pool.Run(ctx, "confluentinc/cp-zookeeper",
dockertest.WithTag("6.2.0"),
dockertest.WithHostname("zookeeper"),
dockertest.WithName("zookeeper"),
dockertest.WithPortBindings(network.PortMap{
network.MustParsePort("2181/tcp"): {
{HostIP: netip.MustParseAddr("127.0.0.1"), HostPort: "2181"},
},
}),
dockertest.WithEnv([]string{
"ZOOKEEPER_CLIENT_PORT=2181",
"ZOOKEEPER_TICK_TIME=2000",
}),
dockertest.WithHostConfig(func(hostConfig *container.HostConfig) {
hostConfig.NetworkMode = container.NetworkMode(confNet.ID())
}),
dockertest.WithoutReuse(),
)
if err != nil {
log.Fatalf("Could not create zookeeper: %v", err)
}
zookeeperIP := bridgeIP(zkRes)
if zookeeperIP == "" {
log.Fatal("Could not determine zookeeper container IP")
}
zookeeperHostPort := zkRes.GetHostPort("2181/tcp")
if zookeeperHostPort == "" {
log.Fatal("Could not determine zookeeper host port")
}

// Wait for the Kafka to come up using dockertest retry.
if err = pool.Retry(ctx, 10*time.Minute, func() error {
_, dialErr := net.Dial("tcp", "127.0.0.1:2181")
_, dialErr := net.Dial("tcp", zookeeperHostPort)
return dialErr
}); err != nil {
log.Fatalf("Could not connect to Kafka broker: %v", err)
}

_, err = pool.Run(ctx, "confluentinc/cp-kafka",
kafkaRes, err := pool.Run(ctx, "confluentinc/cp-kafka",
dockertest.WithTag("6.2.0"),
dockertest.WithHostname("broker"),
dockertest.WithName("broker"),
dockertest.WithPortBindings(network.PortMap{
network.MustParsePort("9092/tcp"): {
{HostIP: netip.MustParseAddr("127.0.0.1"), HostPort: "9092"},
},
}),
dockertest.WithEnv([]string{
"KAFKA_BROKER_ID=1",
"KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181",
"KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT",
"KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://127.0.0.1:9092",
fmt.Sprintf("KAFKA_ZOOKEEPER_CONNECT=%s:2181", zookeeperIP),
"KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092",
"KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT",
"KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1",
"KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1",
"KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1",
"KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0",
fmt.Sprintf("KAFKA_CREATE_TOPICS=%s:1:1", testTopic),
}),
dockertest.WithHostConfig(func(hostConfig *container.HostConfig) {
hostConfig.NetworkMode = container.NetworkMode(confNet.ID())
}),
dockertest.WithEntrypoint([]string{"/bin/bash", "-c"}),
dockertest.WithCmd([]string{`export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://$(hostname -i):9092"; exec /etc/confluent/docker/run`}),
dockertest.WithoutReuse(),
)
if err != nil {
log.Fatalf("Could not create kakfa container: %v", err)
Comment thread
pellared marked this conversation as resolved.
Outdated
}
kafkaIP := bridgeIP(kafkaRes)
if kafkaIP == "" {
log.Fatal("Could not determine Kafka container IP")
}
kafkaBootstrapServers = net.JoinHostPort(kafkaIP, "9092")
Comment thread
pellared marked this conversation as resolved.
Outdated

// Wait for the Kafka to come up using dockertest retry.
if err = pool.Retry(ctx, 10*time.Minute, verifyCanProduceToKafka); err != nil {
Expand All @@ -157,6 +145,14 @@ func TestMain(m *testing.M) {
os.Exit(code)
}

func bridgeIP(res dockertest.Resource) string {
endpoint := res.Container().NetworkSettings.Networks["bridge"]
if endpoint == nil || !endpoint.IPAddress.IsValid() {
return ""
}
return endpoint.IPAddress.String()
Comment thread
pellared marked this conversation as resolved.
Outdated
}

func verifyCanProduceToKafka() error {
p, err := kafka.NewProducer(&kafka.ConfigMap{
bootstrapServersConfigKey: kafkaBootstrapServers,
Expand Down
2 changes: 1 addition & 1 deletion instrumentation/github.com/confluentinc/confluent-kafka-go/v2/kafka/splunkkafka/test/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ go 1.25.5

require (
github.com/confluentinc/confluent-kafka-go/v2 v2.14.1
github.com/moby/moby/api v1.54.2
github.com/ory/dockertest/v4 v4.0.0
github.com/signalfx/splunk-otel-go/instrumentation/github.com/confluentinc/confluent-kafka-go/v2/kafka/splunkkafka v1.32.0
github.com/stretchr/testify v1.11.1
Expand Down Expand Up @@ -36,6 +35,7 @@ require (
github.com/google/uuid v1.6.0 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/go-archive v0.2.0 // indirect
github.com/moby/moby/api v1.54.2 // indirect
github.com/moby/moby/client v0.4.1 // indirect
github.com/moby/sys/atomicwriter v0.1.0 // indirect
github.com/moby/sys/capability v0.4.0 // indirect
Expand Down
66 changes: 31 additions & 35 deletions ...tation/github.com/confluentinc/confluent-kafka-go/v2/kafka/splunkkafka/test/kafka_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,11 @@ import (
"fmt"
"log"
"net"
"net/netip"
"os"
"testing"
"time"

"github.com/confluentinc/confluent-kafka-go/v2/kafka"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/network"
"github.com/ory/dockertest/v4"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand All @@ -52,9 +49,10 @@ var (

const (
bootstrapServersConfigKey = "bootstrap.servers"
kafkaBootstrapServers = "127.0.0.1:9092"
)

var kafkaBootstrapServers string

func TestMain(m *testing.M) {
flag.Parse()
if testing.Short() {
Expand All @@ -68,69 +66,59 @@ func TestMain(m *testing.M) {
log.Fatalf("Could not connect to docker: %v", err)
}

confNet, err := pool.CreateNetwork(ctx, "confluent", nil)
if err != nil {
log.Fatalf("Could not create docker network: %v", err)
}

_, err = pool.Run(ctx, "confluentinc/cp-zookeeper",
zkRes, err := pool.Run(ctx, "confluentinc/cp-zookeeper",
dockertest.WithTag("6.2.0"),
dockertest.WithHostname("zookeeper"),
dockertest.WithName("zookeeper"),
dockertest.WithPortBindings(network.PortMap{
network.MustParsePort("2181/tcp"): {
{HostIP: netip.MustParseAddr("127.0.0.1"), HostPort: "2181"},
},
}),
dockertest.WithEnv([]string{
"ZOOKEEPER_CLIENT_PORT=2181",
"ZOOKEEPER_TICK_TIME=2000",
}),
dockertest.WithHostConfig(func(hostConfig *container.HostConfig) {
hostConfig.NetworkMode = container.NetworkMode(confNet.ID())
}),
dockertest.WithoutReuse(),
)
if err != nil {
log.Fatalf("Could not create zookeeper: %v", err)
}
zookeeperIP := bridgeIP(zkRes)
if zookeeperIP == "" {
log.Fatal("Could not determine zookeeper container IP")
}
zookeeperHostPort := zkRes.GetHostPort("2181/tcp")
if zookeeperHostPort == "" {
log.Fatal("Could not determine zookeeper host port")
}

// Wait for the Kafka to come up using dockertest retry.
if err = pool.Retry(ctx, 10*time.Minute, func() error {
_, dialErr := net.Dial("tcp", "127.0.0.1:2181")
_, dialErr := net.Dial("tcp", zookeeperHostPort)
return dialErr
}); err != nil {
log.Fatalf("Could not connect to Kafka broker: %v", err)
}

_, err = pool.Run(ctx, "confluentinc/cp-kafka",
kafkaRes, err := pool.Run(ctx, "confluentinc/cp-kafka",
dockertest.WithTag("6.2.0"),
dockertest.WithHostname("broker"),
dockertest.WithName("broker"),
dockertest.WithPortBindings(network.PortMap{
network.MustParsePort("9092/tcp"): {
{HostIP: netip.MustParseAddr("127.0.0.1"), HostPort: "9092"},
},
}),
dockertest.WithEnv([]string{
"KAFKA_BROKER_ID=1",
"KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181",
"KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT",
"KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://127.0.0.1:9092",
fmt.Sprintf("KAFKA_ZOOKEEPER_CONNECT=%s:2181", zookeeperIP),
"KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092",
"KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT",
"KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1",
"KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1",
"KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1",
"KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0",
fmt.Sprintf("KAFKA_CREATE_TOPICS=%s:1:1", testTopic),
}),
dockertest.WithHostConfig(func(hostConfig *container.HostConfig) {
hostConfig.NetworkMode = container.NetworkMode(confNet.ID())
}),
dockertest.WithEntrypoint([]string{"/bin/bash", "-c"}),
dockertest.WithCmd([]string{`export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://$(hostname -i):9092"; exec /etc/confluent/docker/run`}),
dockertest.WithoutReuse(),
)
if err != nil {
log.Fatalf("Could not create kakfa container: %v", err)
Comment thread
pellared marked this conversation as resolved.
Outdated
}
kafkaIP := bridgeIP(kafkaRes)
if kafkaIP == "" {
log.Fatal("Could not determine Kafka container IP")
}
kafkaBootstrapServers = net.JoinHostPort(kafkaIP, "9092")
Comment thread
pellared marked this conversation as resolved.
Outdated

// Wait for the Kafka to come up using dockertest retry.
if err = pool.Retry(ctx, 10*time.Minute, verifyCanProduceToKafka); err != nil {
Expand All @@ -151,6 +139,14 @@ func TestMain(m *testing.M) {
os.Exit(code)
}

func bridgeIP(res dockertest.Resource) string {
endpoint := res.Container().NetworkSettings.Networks["bridge"]
if endpoint == nil || !endpoint.IPAddress.IsValid() {
return ""
}
return endpoint.IPAddress.String()
Comment thread
pellared marked this conversation as resolved.
Outdated
}

func verifyCanProduceToKafka() error {
p, err := kafka.NewProducer(&kafka.ConfigMap{
bootstrapServersConfigKey: kafkaBootstrapServers,
Expand Down
Loading