Slack vitess v11.0.4.pre.v2

.github/CODEOWNERS

@@ -1,33 +1,44 @@
-* @sougou
-
+bootstrap.sh @deepthi
 /config/mycnf/ @askdba @shlomi-noach
 /docker/ @derekperkins @dkhenry
 /examples/compose @shlomi-noach
+/examples/demo @sougou
+/examples/legacy_local @deepthi
 /examples/local @rohit-nayak-ps
+/examples/operator @askdba
+/examples/region_sharding @deepthi
+/java/ @harshit-gangal
+/go/cache @vmg
 /go/cmd/vtadmin @ajm188 @doeg
 /go/cmd/vtctldclient @ajm188 @doeg
 /go/mysql @harshit-gangal @systay
+/go/protoutil @ajm188
 /go/test/endtoend/onlineddl @shlomi-noach
 /go/test/endtoend/orchestrator @deepthi @shlomi-noach
 /go/test/endtoend/vtgate @harshit-gangal @systay
 /go/vt/discovery @deepthi
 /go/vt/mysqlctl @deepthi
-/go/vt/proto/vtadmin @ajm188 @doeg
 /go/vt/orchestrator @deepthi @shlomi-noach
+/go/vt/proto/vtadmin @ajm188 @doeg
 /go/vt/schema @shlomi-noach
-/go/vt/sqlparser @harshit-gangal @systay
-/go/vt/vtadmin @ajm188 @doeg
+/go/vt/servenv @deepthi
+/go/vt/sqlparser @harshit-gangal @systay @GuptaManan100
+/go/vt/srvtopo @rafael
+/go/vt/topo @deepthi @rafael
+/go/vt/vtadmin @ajm188 @doeg @rohit-nayak-ps
 /go/vt/vtctl @deepthi
 /go/vt/vtctl/vtctl.go @ajm188 @doeg
 /go/vt/vtctl/grpcvtctldclient @ajm188 @doeg
 /go/vt/vtctl/grpcvtctldserver @ajm188 @doeg
 /go/vt/vtctl/vtctldclient @ajm188 @doeg
+/go/vt/vtctld @ajm188 @doeg @rohit-nayak-ps @deepthi
 /go/vt/vtgate @harshit-gangal @systay
 /go/vt/vttablet/tabletmanager @deepthi @shlomi-noach
 /go/vt/vttablet/tabletmanager/vreplication @rohit-nayak-ps
 /go/vt/vttablet/tabletmanager/vstreamer @rohit-nayak-ps
 /go/vt/vttablet/tabletserver @harshit-gangal @systay @shlomi-noach
 /go/vt/wrangler @deepthi @rohit-nayak-ps
+/go/vt/workflow @rohit-nayak-ps
 /helm/ @derekperkins @dkhenry
 /proto/vtadmin.proto @ajm188 @doeg
 /proto/vtctldata.proto @ajm188 @doeg

.github/pull_request_template.md

@@ -1,25 +1,23 @@
+<!--
+  Thank you for your contribution to the Vitess project.
+  How to contribute: https://vitess.io/docs/contributing/
+  Please first make sure there is an open Issue to discuss the feature/fix suggested in this PR.
+  If this is a new feature, please mark the Issue as "RFC".
+ -->
+
 <!-- if this PR is Work in Progress please create it as a Draft Pull Request -->
 
 ## Description
 <!-- A few sentences describing the overall goals of the pull request's commits. -->
+<!-- If this is a bug fix and you think the fix should be backported, please write so. -->
 
 ## Related Issue(s)
-<!-- List related issues and pull requests: -->
+<!-- List related issues and pull requests. If this PR fixes an issue, please add it using Fixes #????  -->
 
-- 
 
 ## Checklist
-- [ ] Should this PR be backported?
 - [ ] Tests were added or are not required
 - [ ] Documentation was added or is not required
 
 ## Deployment Notes
-<!-- Notes regarding deployment of the contained body of work. These should note any db migrations, etc. -->
-
-## Impacted Areas in Vitess
-Components that this PR will affect:
-
-- [ ]  Query Serving
-- [ ]  VReplication
-- [ ]  Cluster Management
-- [ ]  Build 
+<!-- Notes regarding deployment of the contained body of work. These should note any db migrations, etc. -->

.github/workflows/check_formatting.yml

@@ -4,13 +4,23 @@ jobs:
 
   build:
     name: Check Formatting
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
 
     - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: 1.15
+        go-version: 1.16
+
+    - name: Tune the OS
+      run: |
+        echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
+
+    # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185
+    - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file
+      run: |
+        echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+    # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED!
 
     - name: Check out code
       uses: actions/checkout@v2

.github/workflows/check_make_parser.yml

@@ -4,25 +4,26 @@ jobs:
 
   build:
     name: Check Make Parser
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
 
     - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: 1.15
+        go-version: 1.16
+
+    - name: Tune the OS
+      run: |
+        echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
 
     - name: Check out code
       uses: actions/checkout@v2
 
     - name: Get dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
-        sudo service mysql stop
+        sudo apt-get install -y make unzip g++ etcd curl git wget
         sudo service etcd stop
-        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
-        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
         go mod download
 
     - name: Run make minimaltools

.github/workflows/check_make_sizegen.yml

@@ -0,0 +1,36 @@
+name: check_make_sizegen
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Check Make Sizegen
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.16
+
+    - name: Tune the OS
+      run: |
+        echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y make unzip g++ etcd curl git wget
+        sudo service etcd stop
+        go mod download
+
+    - name: Run make minimaltools
+      run: |
+        make minimaltools
+
+    - name: check_make_sizegen
+      run: |
+        tools/check_make_sizegen.sh
+

.github/workflows/check_make_visitor.yml

@@ -4,25 +4,26 @@ jobs:
 
   build:
     name: Check Make Visitor
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
 
     - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: 1.15
+        go-version: 1.16
+
+    - name: Tune the OS
+      run: |
+        echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
 
     - name: Check out code
       uses: actions/checkout@v2
 
     - name: Get dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget
-        sudo service mysql stop
+        sudo apt-get install -y make unzip g++ etcd curl git wget
         sudo service etcd stop
-        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
-        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
         go mod download
 
     - name: Run make minimaltools
@@ -31,5 +32,5 @@ jobs:
 
     - name: check_make_visitor
       run: |
-        misc/git/hooks/visitorgen
+        misc/git/hooks/asthelpers
 

.github/workflows/check_runs_analysis.yaml

@@ -0,0 +1,59 @@
+on:
+  schedule:
+    - cron:  '*/10 * * * *'
+
+jobs:
+  analyze:
+    if: github.repository == 'vitessio/vitess'
+    name: analyze_check_runs
+    runs-on: ubuntu-latest
+    steps:
+
+    - uses: actions/checkout@v2
+      with:
+        ref: metrics
+    - name: analyze check runs
+      run: |
+        FAILED_RUNS_JSON="/tmp/failed_runs.json"
+        FAILED_RUNS_CSV="/tmp/failed_runs.csv"
+        WORKFLOW_FAILURES_FILE=".metrics/workflow-failures"
+
+        # Get latest failed check runs
+        curl \
+          -H 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
+          -H "Accept: application/vnd.github.v3+json" \
+          -H "Content-type: application/json" \
+          "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs?per_page=100&status=failure" \
+          > "$FAILED_RUNS_JSON"
+
+        # extract .created_at, .name, .html_url for each failed check run
+        cat "$FAILED_RUNS_JSON" |
+          jq -r '.workflow_runs[] | (.created_at + "," + .name + "," + .html_url)' > "$FAILED_RUNS_CSV"
+
+        # now combine back into $WORKFLOW_FAILURES_FILE,
+        # - skipping duplicates
+        # - limit file size to X last lines
+        # - encode with bas64
+        tmpfile="$(mktemp)"
+        cat $WORKFLOW_FAILURES_FILE $FAILED_RUNS_CSV | sort | uniq | tail -10000 | base64 -w 0 > $tmpfile
+        # sha of the file we're replacing:
+        sha="$(git hash-object ${WORKFLOW_FAILURES_FILE})"
+
+        json="
+          {
+            \"message\": \"automated merge of failed check runs\",
+            \"branch\": \"metrics\",
+            \"sha\": \"$sha\",
+            \"content\": \"$(cat $tmpfile)\"
+          }
+        "
+        json_tmpfile="$(mktemp)"
+        echo "$json" > $json_tmpfile
+
+        curl \
+          -X PUT \
+          -H 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
+          -H "Accept: application/vnd.github.v3+json" \
+          -H "Content-type: application/json" \
+          "https://api.github.com/repos/${GITHUB_REPOSITORY}/contents/${WORKFLOW_FAILURES_FILE}" \
+          -d "@${json_tmpfile}"

.github/workflows/cluster_endtoend_11.yml

@@ -0,0 +1,56 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows"
+
+name: Cluster (11)
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Run endtoend tests on Cluster (11)
+    runs-on: ubuntu-18.04
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.16
+
+    - name: Tune the OS
+      run: |
+        echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
+        # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio
+        echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf
+        sudo sysctl -p /etc/sysctl.conf
+
+    # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185
+    - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file
+      run: |
+        echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+    # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED!
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        # Get key to latest MySQL repo
+        sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29
+
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard 11