ccaf-tableflow-aws_glue-snowflake-kickstarter/setup-snowflake-objects.tf at main · j3-signalroom/ccaf-tableflow-aws_glue-snowflake-kickstarter · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
resource "snowflake_warehouse" "tableflow_kickstarter" {
  name           = local.warehouse_name
  warehouse_size = "xsmall"
  auto_suspend   = 60
  provider       = snowflake
}

resource "snowflake_database" "tableflow_kickstarter" {
  name     = local.database_name
  provider = snowflake
  comment  = "Database for Tableflow Kafka Topic data"

  depends_on = [
    snowflake_warehouse.tableflow_kickstarter
  ]
}

resource "snowflake_schema" "tableflow_kickstarter" {
  name       = local.schema_name
  database   = local.database_name
  provider   = snowflake
  comment    = "Schema for Tableflow Kafka Topic data"

  depends_on = [
    snowflake_database.tableflow_kickstarter
  ]
}

resource "snowflake_external_volume" "tableflow_kickstarter_volume" {
  provider     = snowflake.account_admin
  name         = local.volume_name
  allow_writes = false
  storage_location {
    storage_location_name = local.location_name
    storage_base_url      = local.tableflow_topics_s3_base_path
    storage_provider      = "S3"
    storage_aws_role_arn  = local.snowflake_aws_s3_glue_role_arn
  }

  depends_on = [
    confluent_tableflow_topic.stock_trades,
    confluent_tableflow_topic.stock_trades_with_totals
  ]
}

# Snowflake Terraform Provider 2.9.0 does not support the creation of a catalog integration
resource "snowflake_execute" "catalog_integration" {
  provider = snowflake.account_admin
  depends_on = [
    confluent_kafka_cluster.kafka_cluster,
    snowflake_external_volume.tableflow_kickstarter_volume
  ]

  execute = <<EOT
    CREATE OR REPLACE CATALOG INTEGRATION ${local.catalog_integration_name}
      CATALOG_SOURCE = GLUE
      TABLE_FORMAT = ICEBERG
      GLUE_AWS_ROLE_ARN = '${local.snowflake_aws_s3_glue_role_arn}'
      GLUE_CATALOG_ID = '${data.aws_caller_identity.current.account_id}'
      GLUE_REGION = '${var.aws_region}'
      CATALOG_NAMESPACE = '${confluent_kafka_cluster.kafka_cluster.id}'
      ENABLED = TRUE;
  EOT

  revert = <<EOT
    DROP CATALOG INTEGRATION ${local.catalog_integration_name};
  EOT

  query = <<EOT
    DESCRIBE CATALOG INTEGRATION ${local.catalog_integration_name};
  EOT
}

# Snowflake Terraform Provider 2.9.0 does not support the creation of an iceberg table
resource "snowflake_execute" "snowflake_stock_trades_iceberg_table" {
  provider = snowflake
  depends_on = [
    confluent_tableflow_topic.stock_trades,
    snowflake_external_volume.tableflow_kickstarter_volume,
    snowflake_execute.catalog_integration,
    aws_iam_role_policy_attachment.snowflake_s3_glue_policy_attachment,
    aws_iam_role.update_snowflake_s3_glue_role
  ]

  execute = <<EOT
    CREATE ICEBERG TABLE ${local.database_name}.${local.schema_name}.${confluent_tableflow_topic.stock_trades.display_name}
      EXTERNAL_VOLUME = '${local.volume_name}'
      CATALOG = '${local.catalog_integration_name}'
      CATALOG_TABLE_NAME = '${confluent_tableflow_topic.stock_trades.display_name}';
    EOT

  revert = <<EOT
    DROP ICEBERG TABLE ${local.database_name}.${local.schema_name}.${confluent_tableflow_topic.stock_trades.display_name};
  EOT

  query = <<EOT
    DESCRIBE ICEBERG TABLE ${local.database_name}.${local.schema_name}.${confluent_tableflow_topic.stock_trades.display_name};
  EOT
}

# Snowflake Terraform Provider 2.9.0 does not support the creation of an iceberg table
resource "snowflake_execute" "snowflake_stock_trades_with_totals_iceberg_table" {
  provider = snowflake
  depends_on = [
    module.create_set_1,
    confluent_tableflow_topic.stock_trades_with_totals,
    snowflake_external_volume.tableflow_kickstarter_volume,
    snowflake_execute.catalog_integration,
    aws_iam_role_policy_attachment.snowflake_s3_glue_policy_attachment,
    aws_iam_role.update_snowflake_s3_glue_role
  ]

  execute = <<EOT
    CREATE ICEBERG TABLE ${local.database_name}.${local.schema_name}.${confluent_tableflow_topic.stock_trades_with_totals.display_name}
      EXTERNAL_VOLUME = '${local.volume_name}'
      CATALOG = '${local.catalog_integration_name}'
      CATALOG_TABLE_NAME = '${confluent_tableflow_topic.stock_trades_with_totals.display_name}';
    EOT

  revert = <<EOT
    DROP ICEBERG TABLE ${local.database_name}.${local.schema_name}.${confluent_tableflow_topic.stock_trades_with_totals.display_name};
  EOT

  query = <<EOT
    DESCRIBE ICEBERG TABLE ${local.database_name}.${local.schema_name}.${confluent_tableflow_topic.stock_trades_with_totals.display_name};
  EOT
}