cms_mssp_pipeline/.env.example at main · tuva-health/cms_mssp_pipeline · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
# Copy this file to .env and fill in your values.
# .env is gitignored — never commit credentials.

# Shared
MSSP_ACO_ID=
MSSP_FILE_STORE=
MSSP_START_YEAR=2025
MSSP_DOWNLOAD_MODE=incremental

# Processing output
# Options: PARQUET, DUCKDB, MOTHERDUCK, SNOWFLAKE, DATABRICKS, BIGQUERY, REDSHIFT, FABRIC
MSSP_OUTPUT_TYPE=SNOWFLAKE
MSSP_OUTPUT_LOCATION=~/.data/mssp.duckdb
MSSP_FULL_REFRESH=false
MSSP_TEMP_LOCATION=./STAGED

# MotherDuck (required when MSSP_OUTPUT_TYPE=MOTHERDUCK)
MOTHERDUCK_DATABASE=
MOTHERDUCK_TOKEN=

# Snowflake (required when MSSP_OUTPUT_TYPE=SNOWFLAKE)
SNOWFLAKE_USERNAME=
SNOWFLAKE_DATABASE=
SNOWFLAKE_ACCOUNT=
SNOWFLAKE_SCHEMA=RAW_DATA
SNOWFLAKE_COMPUTE_WAREHOUSE=COMPUTE_WH
SNOWFLAKE_ACCOUNT_ROLE=ACCOUNTADMIN
SNOWFLAKE_RSA_KEY_PATH=~/.ssh/snowflake_rsa_key.p8
SNOWFLAKE_RSA_KEY_PASSPHRASE=

# Databricks (required when MSSP_OUTPUT_TYPE=DATABRICKS)
DATABRICKS_SERVER_HOSTNAME=
DATABRICKS_HTTP_PATH=
DATABRICKS_ACCESS_TOKEN=
DATABRICKS_SCHEMA=raw_data
DATABRICKS_CATALOG=
DATABRICKS_STAGING_PATH=

# BigQuery (required when MSSP_OUTPUT_TYPE=BIGQUERY)
BIGQUERY_PROJECT_ID=
BIGQUERY_DATASET_ID=raw_data
BIGQUERY_STAGING_BUCKET=
BIGQUERY_CREDENTIALS_PATH=
BIGQUERY_LOCATION=US

# Redshift (required when MSSP_OUTPUT_TYPE=REDSHIFT)
REDSHIFT_HOST=
REDSHIFT_DATABASE=
REDSHIFT_SCHEMA=raw_data
REDSHIFT_USER=
REDSHIFT_PASSWORD=
REDSHIFT_IAM_ROLE=
REDSHIFT_STAGING_BUCKET=
REDSHIFT_PORT=5439

# Fabric (required when MSSP_OUTPUT_TYPE=FABRIC)
FABRIC_ONELAKE_PATH=
FABRIC_TENANT_ID=
FABRIC_CLIENT_ID=
FABRIC_CLIENT_SECRET=

# AWS (required when MSSP_FILE_STORE is s3://)
# Use AWS_PROFILE (recommended) or AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY
AWS_PROFILE=
AWS_REGION=us-east-1
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=

# Azure (required when MSSP_FILE_STORE is az://, azure://, or abfss://)
AZURE_STORAGE_CONNECTION_STRING=
AZURE_STORAGE_ACCOUNT=

# GCS (required when MSSP_FILE_STORE is gs://)
GCS_PROJECT_ID=
MSSP_GCS_CREDENTIALS_PATH=
GCS_KEY_ID=
GCS_SECRET=

# Optional: S3 bucket for download state and file storage.
# Defaults to the bucket inferred from MSSP_FILE_STORE when that is an s3:// URL.
# Set only if you need to use a different bucket for downloads vs. the file store.
MSSP_S3_BUCKET=