ci: add ipv6 support to storage network pipeline and fix multus misconfiguration

Signed-off-by: Yang Chiu <yang.chiu@suse.com>

Author: yangchiu

pipelines/storage_network/Jenkinsfile

@@ -18,6 +18,8 @@ def REGISTRY_URL = ""
 // parameter for hdd test
 def USE_HDD = params.USE_HDD ? params.USE_HDD : false
 
+def NETWORK_STACK = params.NETWORK_STACK ? params.NETWORK_STACK : "ipv4"
+
 node {
 
     withCredentials([
@@ -82,6 +84,7 @@ node {
                                        --env TF_VAR_mtu=${MTU_SIZE} \
                                        --env TF_VAR_multus_version=${MULTUS_VERSION} \
                                        --env TF_VAR_thick_plugin=${THICK_PLUGIN} \
+                                       --env TF_VAR_network_stack="${NETWORK_STACK}" \
                                        ${imageName}
                 """
             }
@@ -97,7 +100,7 @@ node {
             }
 
             stage ('download support bundle') {
-                sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/storage_network/scripts/download-support-bundle.sh  ${JOB_BASE_NAME}-${BUILD_NUMBER}-bundle.zip"
+                sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/storage_network/scripts/download-support-bundle.sh ${JOB_BASE_NAME}-${BUILD_NUMBER}-bundle.zip"
                 sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/${JOB_BASE_NAME}-${BUILD_NUMBER}-bundle.zip ."
 				archiveArtifacts allowEmptyArchive: true, artifacts: '**/*.zip', followSymlinks: false
 			}

pipelines/storage_network/scripts/download-support-bundle.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+source pipelines/utilities/longhorn_ui.sh
+
 set -e
 
 SUPPORT_BUNDLE_FILE_NAME=${1:-"lh-support-bundle.zip"}
@@ -10,10 +12,6 @@ set_kubeconfig_envvar(){
     export KUBECONFIG="${PWD}/pipelines/storage_network/terraform/k3s.yaml"
 }
 
-export_longhorn_ui_url(){
-  export LONGHORN_CLIENT_URL="http://$(cat /tmp/controlplane_public_ip):30000"
-}
-
 set_kubeconfig_envvar
 export_longhorn_ui_url
 
@@ -27,7 +25,7 @@ SUPPORT_BUNDLE_READY=false
 while [[ ${SUPPORT_BUNDLE_READY} == false ]]; do
     PERCENT=`kubectl exec -n longhorn-system svc/longhorn-frontend -- curl -H 'Accept: application/json' ${SUPPORT_BUNDLE_URL} | jq -r '.progressPercentage' || true`
     echo ${PERCENT}
-    
+
     if [[ ${PERCENT} == 100 ]]; then SUPPORT_BUNDLE_READY=true; fi
 done
 

pipelines/storage_network/scripts/longhorn-setup.sh

@@ -8,8 +8,11 @@ source pipelines/utilities/create_aws_secret.sh
 source pipelines/utilities/create_registry_secret.sh
 source pipelines/utilities/install_backupstores.sh
 source pipelines/utilities/storage_network.sh
+source pipelines/utilities/coredns.sh
 source pipelines/utilities/create_longhorn_namespace.sh
 source pipelines/utilities/longhorn_manifest.sh
+source pipelines/utilities/longhorn_status.sh
+source pipelines/utilities/longhorn_ui.sh
 source pipelines/utilities/run_longhorn_test.sh
 
 # create and clean tmpdir
@@ -44,6 +47,9 @@ main(){
   fi
   deploy_network_attachment_definition
 
+  patch_coredns_ipv6_name_servers
+  scale_up_coredns
+
   create_longhorn_namespace
   install_backupstores
   setup_azurite_backup_store
@@ -55,7 +61,13 @@ main(){
   customize_longhorn_manifest_registry
   install_longhorn
 
+  setup_longhorn_ui_nodeport
+  export_longhorn_ui_url
+
   update_storage_network_setting
+  # instance manager pods should restart after the storage network setting applied
+  wait_longhorn_status_running
+  validate_storage_network_setting_taking_effect
 
   run_longhorn_test
 }

pipelines/storage_network/scripts/terraform-setup.sh

@@ -10,4 +10,6 @@ for id in ${NETWORK_INTERFACE_IDS}; do
   aws ec2 modify-network-interface-attribute --network-interface-id "${id}" --no-source-dest-check
 done
 
+terraform -chdir=pipelines/storage_network/terraform output -raw controlplane_public_ip > /tmp/controlplane_public_ip
+
 exit $?

pipelines/storage_network/terraform/data.tf

@@ -12,8 +12,9 @@ data "aws_ami" "aws_ami_sles" {
 data "template_file" "provision_k3s_server" {
   template = var.k8s_distro_name == "k3s" ? file("${path.module}/user-data-scripts/provision_k3s_server.sh.tpl") : null
   vars = {
+    network_stack = var.network_stack
+    control_plane_ipv4 = aws_eip.aws_eip[0].public_ip
     k3s_cluster_secret = random_password.cluster_secret.result
-    k3s_server_public_ip = aws_eip.aws_eip[0].public_ip
     k3s_version =  var.k8s_distro_version
     thick_plugin = var.thick_plugin
   }
@@ -23,6 +24,7 @@ data "template_file" "provision_k3s_server" {
 data "template_file" "provision_k3s_agent" {
   template = var.k8s_distro_name == "k3s" ? file("${path.module}/user-data-scripts/provision_k3s_agent.sh.tpl") : null
   vars = {
+    network_stack = var.network_stack
     k3s_server_url = "https://${aws_eip.aws_eip[0].public_ip}:6443"
     k3s_cluster_secret = random_password.cluster_secret.result
     k3s_version =  var.k8s_distro_version
@@ -48,5 +50,8 @@ data "template_file" "routes" {
     N1 = aws_network_interface.instance_eth1[0].private_ip
     N2 = aws_network_interface.instance_eth1[1].private_ip
     N3 = aws_network_interface.instance_eth1[2].private_ip
+    N1_v6 = aws_network_interface.instance_eth1[0].ipv6_address_list[0]
+    N2_v6 = aws_network_interface.instance_eth1[1].ipv6_address_list[0]
+    N3_v6 = aws_network_interface.instance_eth1[2].ipv6_address_list[0]
   }
 }

pipelines/storage_network/terraform/main.tf

@@ -28,6 +28,10 @@ resource "random_password" "cluster_secret" {
 resource "aws_vpc" "aws_vpc" {
   cidr_block = "10.0.0.0/16"
 
+  assign_generated_ipv6_cidr_block  = true
+  enable_dns_support                = true
+  enable_dns_hostnames              = true
+
   tags = {
     Name = "${var.aws_vpc_name}-${random_string.random_suffix.id}"
     Owner = var.resources_owner
@@ -55,16 +59,23 @@ resource "aws_route_table" "aws_public_rt" {
     gateway_id = aws_internet_gateway.aws_igw.id
   }
 
+  route {
+    ipv6_cidr_block = "::/0"
+    gateway_id      = aws_internet_gateway.aws_igw.id
+  }
+
   tags = {
     Name = "lh_aws_public_rt-${random_string.random_suffix.id}"
     Owner = var.resources_owner
   }
 }
 
 resource "aws_subnet" "aws_subnet_1" {
-  vpc_id     = aws_vpc.aws_vpc.id
-  availability_zone = "us-east-1c"
-  cidr_block = "10.0.1.0/24"
+  vpc_id                          = aws_vpc.aws_vpc.id
+  availability_zone               = "us-east-1c"
+  cidr_block                      = "10.0.1.0/24"
+  ipv6_cidr_block                 = cidrsubnet(aws_vpc.aws_vpc.ipv6_cidr_block, 8, 0)
+  assign_ipv6_address_on_creation = true
 
   tags = {
     Name = "lh_subnet_1-${random_string.random_suffix.id}"
@@ -73,9 +84,11 @@ resource "aws_subnet" "aws_subnet_1" {
 }
 
 resource "aws_subnet" "aws_subnet_2" {
-  vpc_id     = aws_vpc.aws_vpc.id
-  availability_zone = "us-east-1c"
-  cidr_block = "10.0.2.0/24"
+  vpc_id                          = aws_vpc.aws_vpc.id
+  availability_zone               = "us-east-1c"
+  cidr_block                      = "10.0.2.0/24"
+  ipv6_cidr_block                 = cidrsubnet(aws_vpc.aws_vpc.ipv6_cidr_block, 8, 1)
+  assign_ipv6_address_on_creation = true
 
   tags = {
     Name = "lh_subnet_2-${random_string.random_suffix.id}"
@@ -109,19 +122,19 @@ resource "aws_security_group" "aws_secgrp" {
   vpc_id      = aws_vpc.aws_vpc.id
 
   ingress {
-    description = "Allow SSH"
-    from_port   = 22
-    to_port     = 22
-    protocol    = "tcp"
+    description = "Allow all ports"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
     cidr_blocks = ["0.0.0.0/0"]
   }
 
   ingress {
-    description = "Allow all ports"
+    description = "Allow all ports over IPv6"
     from_port   = 0
     to_port     = 0
     protocol    = "-1"
-    cidr_blocks = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
   }
 
   egress {
@@ -131,6 +144,13 @@ resource "aws_security_group" "aws_secgrp" {
     cidr_blocks = ["0.0.0.0/0"]
   }
 
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
   tags = {
     Name = "lh_aws_secgrp-${random_string.random_suffix.id}"
     Owner = var.resources_owner
@@ -145,6 +165,7 @@ resource "aws_key_pair" "aws_pair_key" {
 resource "aws_network_interface" "instance_eth0" {
   subnet_id   = aws_subnet.aws_subnet_1.id
   security_groups = [aws_security_group.aws_secgrp.id]
+  ipv6_address_count = 1
 
   count = var.aws_instance_count
 
@@ -195,6 +216,7 @@ resource "aws_network_interface" "instance_eth1" {
 
   subnet_id   = aws_subnet.aws_subnet_2.id
   security_groups = [aws_security_group.aws_secgrp.id]
+  ipv6_address_count = 1
 
   count = var.aws_instance_count
 
@@ -257,7 +279,16 @@ resource "null_resource" "rsync_kubeconfig_file" {
   }
 
   provisioner "local-exec" {
-    command = "rsync -aPvz --rsync-path=\"sudo rsync\" -e \"ssh -o StrictHostKeyChecking=no -l ec2-user -i ${var.aws_ssh_private_key_file_path}\" ${aws_eip.aws_eip[0].public_ip}:/etc/rancher/k3s/k3s.yaml .  && sed -i 's#https://127.0.0.1:6443#https://${aws_eip.aws_eip[0].public_ip}:6443#' k3s.yaml"
+    command = <<EOT
+    export K3S_SERVER_IP=$(
+        [ "${var.network_stack}" = "ipv6" ] && echo "[${aws_instance.aws_instance[0].ipv6_addresses[0]}]" || echo ${aws_eip.aws_eip[0].public_ip}
+    )
+    export LOCAL_IP=$(
+        [ "${var.network_stack}" = "ipv6" ] && echo "\[::1\]" || echo "127.0.0.1"
+    )
+    rsync -aPvz --rsync-path="sudo rsync" -e "ssh -o StrictHostKeyChecking=no -l ec2-user -i ${var.aws_ssh_private_key_file_path}" "${aws_eip.aws_eip[0].public_ip}:/etc/rancher/k3s/k3s.yaml" . && \
+    sed -i "s#https://$LOCAL_IP:6443#https://$K3S_SERVER_IP:6443#" k3s.yaml
+EOT
   }
 }
 

pipelines/storage_network/terraform/output.tf

@@ -4,4 +4,12 @@ output "network_interface_ids" {
     aws_network_interface.instance_eth1
   ]
   value = join(" ", concat(aws_network_interface.instance_eth0[*].id, aws_network_interface.instance_eth1[*].id))
-}
+}
+
+output "controlplane_public_ip" {
+  depends_on = [
+    aws_eip.aws_eip,
+    aws_instance.aws_instance
+  ]
+  value = var.network_stack == "ipv6" ? "[${aws_instance.aws_instance[0].ipv6_addresses[0]}]" : aws_eip.aws_eip[0].public_ip
+}

pipelines/storage_network/terraform/user-data-scripts/flannel.sh.tpl

@@ -1,5 +1,6 @@
 #!/bin/bash
 STORAGE_NETWORK_PREFIX="192.168"
+STORAGE_NETWORK_PREFIX_V6="fd00:168"
 
 ETH1_IP=`ip a | grep eth1 | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*'  | awk '{print $2}'`
 
@@ -16,4 +17,11 @@ FLANNEL_NETWORK=$STORAGE_NETWORK_PREFIX.0.0/16
 FLANNEL_SUBNET=$STORAGE_NETWORK_PREFIX.$NET.0/24
 FLANNEL_MTU=${mtu}
 FLANNEL_IPMASQ=true
-EOF
+EOF
+
+cat << EOF | sudo tee -a /run/flannel/multus-subnet-$STORAGE_NETWORK_PREFIX_V6.0.0.env
+FLANNEL_NETWORK=$STORAGE_NETWORK_PREFIX_V6::/48
+FLANNEL_SUBNET=$STORAGE_NETWORK_PREFIX_V6:$NET::/64
+FLANNEL_MTU=${mtu}
+FLANNEL_IPMASQ=false
+EOF

pipelines/storage_network/terraform/user-data-scripts/provision_k3s_agent.sh.tpl

@@ -5,7 +5,7 @@ set -e
 sudo systemctl restart guestregister # Sometimes registration fails on first boot.
 sudo zypper ref
 sudo zypper install -y -t pattern devel_basis
-sudo zypper install -y open-iscsi nfs-client cryptsetup
+sudo zypper install -y open-iscsi nfs-client cryptsetup go
 sudo systemctl -q enable iscsid
 sudo systemctl start iscsid
 
@@ -30,16 +30,49 @@ fi
 # TODO: It looks like "set -e" will break the intended functionality of the remaining code. Consider a refactor.
 set +e
 
-until (curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="agent --token ${k3s_cluster_secret}" K3S_URL="${k3s_server_url}" INSTALL_K3S_VERSION="${k3s_version}" sh -); do
+if [[ "${network_stack}" == "ipv6" ]]; then
+  tee /etc/sysctl.d/99-ipv6.conf > /dev/null <<EOF
+net.ipv6.conf.eth0.accept_ra = 2
+net.ipv6.conf.eth1.accept_ra = 2
+net.ipv6.conf.default.accept_ra = 2
+net.ipv6.conf.all.forwarding = 1
+net.ipv6.conf.eth1.forwarding = 1
+net.ipv6.conf.all.proxy_ndp = 1
+net.ipv6.conf.eth0.proxy_ndp = 1
+net.ipv6.conf.eth1.proxy_ndp = 1
+net.ipv6.conf.cni0.proxy_ndp = 1
+EOF
+  sysctl --system
+  cat <<EOF > /etc/resolv.conf
+nameserver 2606:4700:4700::1111
+nameserver 2001:4860:4860::8888
+nameserver 8.8.8.8
+nameserver 1.1.1.1
+EOF
+  chattr +i /etc/resolv.conf || true
+  IP=$(ip -6 addr show scope global | awk '/inet6/ && !/fe80/ {print $2}' | cut -d/ -f1 | head -n1)
+else
+  IP=$(hostname -I | awk '{print $1}')
+fi
+
+# TODO: It looks like "set -e" will break the intended functionality of the remaining code. Consider a refactor.
+set +e
+
+until (curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="agent --node-ip=$IP --token ${k3s_cluster_secret}" K3S_URL="${k3s_server_url}" INSTALL_K3S_VERSION="${k3s_version}" sh -); do
   echo 'k3s agent did not install correctly'
   sleep 2
 done
 
 if [[ "${thick_plugin}" == true ]]; then
-  ln -s /var/lib/rancher/k3s/agent/etc/cni/net.d /etc/cni
-  ln -s /var/lib/rancher/k3s/data/current/bin /opt/cni
+  mkdir -p /etc/cni
+  ln -s /var/lib/rancher/k3s/agent/etc/cni/net.d /etc/cni/net.d
+  mkdir -p /opt/cni
+  ln -s /var/lib/rancher/k3s/data/cni /opt/cni/bin
 fi
 
-curl -OL https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz
-tar -zxvf cni-plugins-linux-amd64-v1.3.0.tgz
-cp ipvlan /var/lib/rancher/k3s/data/current/bin/
+mkdir -p /tmp/gocache
+export GOCACHE=/tmp/gocache
+git clone https://github.com/c3y1huang/cni-plugins.git
+cd cni-plugins
+git checkout 6888978
+CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /var/lib/rancher/k3s/data/cni/ipvlan ./plugins/main/ipvlan