add the datastream syncer

Author: upodroid

infra/aws/terraform/registry.k8s.io/main.tf

@@ -18,35 +18,35 @@ locals {
   // aws ec2 describe-regions --all-regions --query "Regions[].RegionName" --output json | jq .[] | awk '{print $0","}' | sort --version-sort
   regions = [
     "af-south-1",
-    # "ap-east-1",
-    # "ap-east-2",
+    "ap-east-1",
+    "ap-east-2",
     "ap-northeast-1",
-    # "ap-northeast-2",
-    # "ap-northeast-3",
+    "ap-northeast-2",
+    "ap-northeast-3",
     "ap-southeast-1",
     "ap-southeast-2",
-    # "ap-southeast-3",
-    # "ap-southeast-4",
-    # "ap-southeast-5",
-    # "ap-southeast-6",
-    # "ap-southeast-7",
+    "ap-southeast-3",
+    "ap-southeast-4",
+    "ap-southeast-5",
+    "ap-southeast-6",
+    "ap-southeast-7",
     "ap-south-1",
-    # "ap-south-2",
-    # "ca-central-1",
-    # "ca-west-1",
+    "ap-south-2",
+    "ca-central-1",
+    "ca-west-1",
     "eu-central-1",
-    # "eu-central-2",
-    # "eu-north-1",
+    "eu-central-2",
+    "eu-north-1",
     "eu-south-1",
-    # "eu-south-2",
+    "eu-south-2",
     "eu-west-1",
-    # "eu-west-2",
+    "eu-west-2",
     "eu-west-3",
-    # "il-central-1",
-    # "me-central-1",
-    # "me-south-1",
-    # "mx-central-1",
-    # "sa-east-1",
+    "il-central-1",
+    "me-central-1",
+    "me-south-1",
+    "mx-central-1",
+    "sa-east-1",
     "us-east-1",
     "us-east-2",
     "us-west-1",
@@ -78,6 +78,9 @@ module "s3_buckets" {
   for_each = setsubtract(toset(local.regions), ["us-east-2"])
   source   = "./s3"
   region   = each.key
+  alternative_bucket_names = {
+    eu-west-2 = "767373bbdcb8270361b96548387bf2a9ad0d48758c35-eu-west-2"
+  }
 
   prefix = var.prefix
 }

infra/aws/terraform/registry.k8s.io/s3/bucket.tf

@@ -15,7 +15,7 @@ limitations under the License.
 */
 
 resource "aws_s3_bucket" "registry-k8s-io" {
-  bucket = "${var.prefix}registry-k8s-io-${var.region}"
+  bucket = lookup(var.alternative_bucket_names, var.region, "${var.prefix}registry-k8s-io-${var.region}")
   region = var.region
 }
 

infra/aws/terraform/registry.k8s.io/s3/variables.tf

@@ -36,3 +36,8 @@ variable "region" {
   type        = string
 }
 
+variable "alternative_bucket_names" {
+  description = "Map of region to alternative bucket name if the default naming convention is not used"
+  type        = map(string)
+  default     = {}
+}

infra/aws/terraform/registry.k8s.io/sync.tf

@@ -0,0 +1,71 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+locals {
+  // When we complete the initial datasync, remove the region from this list
+  // aws ec2 describe-regions --all-regions --query "Regions[].RegionName" --output json | jq .[] | awk '{print $0","}' | sort --version-sort
+  new_regions = [
+    # "foobar"
+  ]
+}
+
+module "s3_locations" {
+  source = "github.com/upodroid/terraform-aws-datasync//modules/datasync-locations?ref=main"
+  s3_locations = concat(
+    [
+      {
+        name                             = "source-bucket"
+        s3_bucket_arn                    = module.us-east-2.bucket_arn
+        subdirectory                     = "/"
+        create_role                      = false
+        s3_config_bucket_access_role_arn = "arn:aws:iam::513428760722:role/AWSDataSyncS3BucketAccess"
+        region                           = "us-east-2"
+      },
+    ],
+    [
+      for idx, region in local.new_regions :
+      {
+        name                             = "${region}"
+        s3_bucket_arn                    = module.s3_buckets[region].bucket_arn
+        subdirectory                     = "/"
+        create_role                      = false
+        s3_config_bucket_access_role_arn = "arn:aws:iam::513428760722:role/AWSDataSyncS3BucketAccess"
+        region                           = "${region}"
+      }
+    ]
+  )
+}
+
+module "s3_to_s3_tasks" {
+  source = "github.com/upodroid/terraform-aws-datasync//modules/datasync-task?ref=main"
+  datasync_tasks = [
+    for idx, region in local.new_regions :
+    {
+      name                     = "${region}"
+      source_location_arn      = "arn:aws:datasync:us-east-2:513428760722:location/loc-050a6ce9904237d71"
+      destination_location_arn = module.s3_locations.s3_locations[region].arn
+      task_mode                = "ENHANCED"
+      options = {
+        verify_mode       = "ONLY_FILES_TRANSFERRED" # Enhanced mode supports ONLY_FILES_TRANSFERRED or NONE
+        gid               = "NONE"
+        posix_permissions = "NONE"
+        uid               = "NONE"
+      }
+      schedule_expression = "cron(0 0 * * ? *)" # Run at 00:00 every day, we only need the first run.
+      region              = "us-east-2"
+    }
+  ]
+}