Added fixes to QE Jenkinsfiles and fixed flaky tests

jenkins/pipelines/QE/android/Jenkinsfile

@@ -28,10 +28,14 @@ pipeline {
             }
         }
         stage('Android') {
-            agent { label 'mac-mini-CBL-Android1' }
+            agent { label 'qe-mac-india' }
+            options {
+                lock('qe-mac-india,56141JEBF18456')
+            }
             environment {
-                KEYCHAIN_PASSWORD = credentials('mobile-qe')
-                ANDROID_HOME = "/Users/couchbase/Library/Android/sdk"
+                KEYCHAIN_PASSWORD = credentials('mobile-qe-india')
+                ANDROID_HOME = "${env.HOME}/Library/Android/sdk"
+                ANDROID_SERIAL = "56141JEBF18456"
                 JAVA_HOME = "/Library/Java/JavaVirtualMachines/temurin-17.jdk/Contents/Home"
                 PATH = "\$JAVA_HOME/bin:\$ANDROID_HOME/platform-tools:\$ANDROID_HOME/tools:\$ANDROID_HOME/tools/bin:/opt/homebrew/opt/python@3.10/bin:/opt/homebrew/bin:/usr/local/bin:${env.PATH}"
                 TS_ARTIFACTS_DIR = 'android'

jenkins/pipelines/QE/android/topology_single_device.json

@@ -4,7 +4,8 @@
         {
             "platform": "jak_android",
             "cbl_version": "{{version}}",
-            "location": "R5CT20FYA7R",
+            "location": "56141JEBF18456",
+            "ip_hint": "172.16.150.165",
             "download": true
         }
     ],

jenkins/pipelines/QE/c/Jenkinsfile

@@ -11,43 +11,160 @@ pipeline {
                 script {
                     if (params.CBL_VERSION == '') { error "CBL_VERSION is required" }
                     if (params.SGW_VERSION == '') { error "SGW_VERSION is required" }
-                    
+
                     currentBuild.displayName = "C Linux CBL:${params.CBL_VERSION} (#${currentBuild.number})"
                     currentBuild.description = "SGW: ${params.SGW_VERSION}"
                 }
             }
         }
         stage('Prebuild Servers') {
             steps {
-                build job: 'prebuild-test-server',
-                parameters: [
-                    string(name: 'TS_PLATFORM', value: 'c_linux_x86_64'),
-                    string(name: 'CBL_VERSION', value: params.CBL_VERSION),
-                ],
-                wait: true,
-                propagate: true
+                script {
+                    def platforms = [
+                        'c_windows', 'c_ios', 'c_android','c_linux_x86_64'
+                    ]
+                    def parallelBuilds = [:]
+                    for(p in platforms) {
+                        // Groovy stupidness.  Need to set a local variable here
+                        // to avoid late binding (all jobs use dotnet_macos).
+                        def platform = p
+                        parallelBuilds[platform] = {
+                            build job: 'prebuild-test-server',
+                            parameters: [
+                                string(name: 'TS_PLATFORM', value: platform),
+                                string(name: 'CBL_VERSION', value: params.CBL_VERSION),
+                                string(name: 'CBL_BUILD', value: params.CBL_BUILD)
+                            ],
+                            wait: true,
+                            propagate: true
+                        }
+                    }
+                    parallel parallelBuilds
+                }
             }
         }
         stage('Tests') {
-            agent { label 'linux-c' }
-            environment {
-                TS_ARTIFACTS_DIR = 'c'
-            }
-            steps {
-                echo "=== Run C Linux Tests"
-                timeout(time: 120, unit: 'MINUTES') {
-                    sh "jenkins/pipelines/QE/c/run_test.sh ${params.CBL_VERSION} linux_x86_64 ${params.SGW_VERSION} ${params.TS_DATASET_VERSION}"
+            parallel {
+                stage('Windows') {
+                    agent { label 'net-windows' }
+                    options {
+                        lock('net-windows')
+                    }
+                    environment {
+                        TS_DATASET_VERSION = "${params.TS_DATASET_VERSION}"
+                    }
+                    steps {
+                        echo "=== Run C Windows Tests"
+                        timeout(time: 120, unit: 'MINUTES') {
+                            bat '''
+                            "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe" ^
+                            -NoProfile ^
+                            -ExecutionPolicy Bypass ^
+                            -File jenkins\\pipelines\\QE\\c\\run_test.ps1 ^
+                            -Version %CBL_VERSION% ^
+                            -SgwVersion %SGW_VERSION% ^
+                            -DatasetVersion %TS_DATASET_VERSION%
+                            '''
+                        }
+                        echo "=== C Windows Tests Complete"
+                    }
+                    post {
+                        always {
+                            echo "=== Teardown C Windows Tests"
+                            timeout(time: 5, unit: 'MINUTES') {
+                                bat '''
+                                "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe" ^
+                                -NoProfile ^
+                                -ExecutionPolicy Bypass ^
+                                -File jenkins\\pipelines\\QE\\c\\teardown.ps1
+                                '''
+                            }
+                            archiveArtifacts artifacts: 'tests\\QE\\c_windows\\**\\*', fingerprint: true, allowEmptyArchive: true
+                            echo "=== C Windows Test Teardown Complete"
+                        }
+                    }
                 }
-                echo "=== C Linux Tests Complete"
-            }
-            post {
-                always {
-                    echo "=== Teardown C Linux Tests"
-                    timeout(time: 5, unit: 'MINUTES') {
-                        sh 'jenkins/pipelines/QE/c/teardown.sh'
-                    }
-                    archiveArtifacts artifacts: 'tests/QE/c_linux/**/*', fingerprint: true, allowEmptyArchive: true
-                    echo "=== C Linux Test Teardown Complete"
+                stage('Android') {
+                    agent { label 'qe-mac-india' }
+                    environment {
+                        KEYCHAIN_PASSWORD = credentials('mobile-qe-india')
+                        DOTNET_ROOT = "/opt/homebrew/opt/dotnet/libexec"
+                        PATH = "/opt/homebrew/opt/python@3.10/bin:/opt/homebrew/bin:/usr/local/bin:${env.PATH}:/Users/qe_mobile_india/Library/Android/sdk/cmdline-tools/latest/bin:/Users/qe_mobile_india/Library/Android/sdk/platform-tools:/Users/qe_mobile_india/Library/Android/sdk/emulator:${PATH}"
+                        AWS_PROFILE = "default"
+                        ANDROID_HOME = "/Users/qe_mobile_india/Library/Android/sdk"
+                    }
+                    steps {
+                        // Unlock keychain:
+                        withCredentials([string(credentialsId: 'mobile-qe', variable: 'KEYCHAIN_PASSWORD')]) {
+                            sh 'security unlock-keychain -p ${KEYCHAIN_PASSWORD} ~/Library/Keychains/login.keychain-db'
+                        }
+                        echo "=== Run C Android Tests"
+                        timeout(time: 120, unit: 'MINUTES') {
+                            sh "jenkins/pipelines/QE/c/run_test.sh ${params.CBL_VERSION} android ${params.SGW_VERSION} ${params.TS_DATASET_VERSION}"
+                        }
+                        echo "=== C Android Tests Complete"
+                    }
+                    post {
+                        always {
+                            echo "=== Teardown C Android Tests"
+                            timeout(time: 5, unit: 'MINUTES') {
+                                sh 'jenkins/pipelines/QE/c/teardown.sh'
+                            }
+                            archiveArtifacts artifacts: 'tests/QE/c_android/**/*', fingerprint: true, allowEmptyArchive: true
+                            echo "=== C Android Test Teardown Complete"
+                        }
+                    }
+                }
+                stage('iOS') {
+                    agent { label 'qe-mac-india' }
+                    environment {
+                        KEYCHAIN_PASSWORD = credentials('mobile-qe-india')
+                        PATH = "/opt/homebrew/opt/python@3.10/bin:/opt/homebrew/bin:/usr/local/bin:${env.PATH}"
+                        TS_ARTIFACTS_DIR = 'ios'
+                        TS_DATASET_VERSION = "${params.TS_DATASET_VERSION}"
+                    }
+                    steps {
+                        // Unlock keychain:
+                        sh 'security unlock-keychain -p ${KEYCHAIN_PASSWORD} ~/Library/Keychains/login.keychain-db'
+                        echo "=== Run C iOS Tests"
+                        timeout(time: 120, unit: 'MINUTES') {
+                            sh "jenkins/pipelines/QE/c/run_test.sh ${params.CBL_VERSION} ios ${params.SGW_VERSION} ${params.TS_DATASET_VERSION}"
+                        }
+                        echo "=== C iOS Tests Complete"
+                    }
+                    post {
+                        always {
+                            echo "=== Teardown C iOS Tests"
+                            timeout(time: 5, unit: 'MINUTES') {
+                                sh 'jenkins/pipelines/QE/c/teardown.sh'
+                            }
+                            archiveArtifacts artifacts: 'tests/QE/c_ios/**/*', fingerprint: true, allowEmptyArchive: true
+                            echo "=== C iOS Test Teardown Complete"
+                        }
+                    }
+                }
+                stage('linux_x86_64') {
+                    agent { label 'linux-c' }
+                    environment {
+                        TS_ARTIFACTS_DIR = 'c'
+                    }
+                    steps {
+                        echo "=== Run C Linux Tests"
+                        timeout(time: 120, unit: 'MINUTES') {
+                            sh "jenkins/pipelines/QE/c/run_test.sh ${params.CBL_VERSION} linux_x86_64 ${params.SGW_VERSION} ${params.TS_DATASET_VERSION}"
+                        }
+                        echo "=== C Linux Tests Complete"
+                    }
+                    post {
+                        always {
+                            echo "=== Teardown C Linux Tests"
+                            timeout(time: 5, unit: 'MINUTES') {
+                                sh 'jenkins/pipelines/QE/c/teardown.sh'
+                            }
+                            archiveArtifacts artifacts: 'tests/QE/c_linux/**/*', fingerprint: true, allowEmptyArchive: true
+                            echo "=== C Linux Test Teardown Complete"
+                        }
+                    }
                 }
             }
         }

jenkins/pipelines/QE/c/run_test.ps1

@@ -1,7 +1,7 @@
 param (
     [Parameter(Mandatory=$true)][string]$Version,
     [Parameter(Mandatory=$true)][string]$SgwVersion,
-    [Parameter][string]$DatasetVersion = "4.0"
+    [Parameter()][string]$DatasetVersion = "4.0"
 )
 
 $ErrorActionPreference = "Stop"
@@ -22,4 +22,4 @@ try {
 
 if($saved_exit -ne 0) {
     throw "Testing failed!"
-}
+}

jenkins/pipelines/QE/c/topologies/topology_single_android.json

@@ -4,7 +4,9 @@
         {
             "platform": "c_android",
             "cbl_version": "{{version}}",
-            "location": "1BCACS00001MX3"
+            "location": "56141JEBF18456",
+            "ip_hint": "172.16.150.165",
+            "download": true
         }
     ],
     "include": "../../../../../environment/aws/topology_setup/default_topology.json"

jenkins/pipelines/QE/c/topologies/topology_single_ios.json

@@ -4,7 +4,9 @@
         {
             "platform": "c_ios",
             "cbl_version": "{{version}}",
-            "location": "00008110-000E544201D1401E"
+            "location": "00008110-000E544201D1401E",
+            "ip_hint": "172.16.150.155",
+            "download": true
         }
     ],
     "include": "../../../../../environment/aws/topology_setup/default_topology.json"

jenkins/pipelines/QE/c/topologies/topology_single_windows.json

@@ -4,7 +4,8 @@
         {
             "platform": "c_windows",
             "cbl_version": "{{version}}",
-            "location": "localhost"
+            "location": "localhost",
+            "download": true
         }
     ],
     "include": "../../../../../environment/aws/topology_setup/default_topology.json"