Upgrading the repo to python 3.12 (#132)

Author: devinmatte

.github/workflows/deploy.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.11"]
+        python-version: ["3.12"]
     env:
       AWS_PROFILE: transitmatters
       AWS_DEFAULT_REGION: us-east-1

.github/workflows/lint.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.11"]
+        python-version: ["3.12"]
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4

.github/workflows/test.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.11"]
+        python-version: ["3.12"]
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 TransitMatters
+Copyright (c) 2025 TransitMatters
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,4 +1,5 @@
 # data-ingestion
+
 ![lint](https://github.com/transitmatters/data-ingestion/workflows/lint/badge.svg?branch=main)
 ![test](https://github.com/transitmatters/data-ingestion/workflows/test/badge.svg?branch=main)
 ![deploy](https://github.com/transitmatters/data-ingestion/workflows/deploy/badge.svg?branch=main)

devops/deploy.sh

@@ -25,19 +25,28 @@ pushd ingestor/
 poetry run chalice package --stage prod --merge-template .chalice/resources.json cfn/
 
 # Shrink the deployment package for the lambda layer https://stackoverflow.com/a/69355796
-echo "Shrinking the deployment package for the lambda layer"
+actualsize=$(wc -c <"cfn/layer-deployment.zip")
+echo "Shrinking the deployment package for the lambda layer. It's currently $actualsize bytes."
 
 source ../devops/helpers.sh
-shrink
+shrink > /dev/null 2>&1
+
+# Check the size of the new package
+newsize=$(wc -c <"cfn/layer-deployment.zip")
+echo "The deployment package is now $newsize bytes"
+diffsize=$(($actualsize - $newsize))
+echo "Difference: $diffsize bytes"
 
 # Check package size before deploying
 maximumsize=79100000
 actualsize=$(wc -c <"cfn/layer-deployment.zip")
+difference=$(expr $actualsize - $maximumsize)
 if [ $actualsize -ge $maximumsize ]; then
     echo ""
-    echo "layer-deployment.zip is over $maximumsize bytes. Shrink the package further to be able to deploy"
+    echo "layer-deployment.zip is over $maximumsize bytes. Shrink the package by $difference bytes to be able to deploy"
     exit 1
 fi
+echo "layer-deployment.zip is under the maximum size of $maximumsize bytes, by $difference bytes"
 
 aws cloudformation package --template-file cfn/sam.json --s3-bucket $BUCKET --output-template-file cfn/packaged.yaml
 aws cloudformation deploy --template-file cfn/packaged.yaml --stack-name $STACK_NAME \

devops/helpers.sh

@@ -7,9 +7,23 @@ function shrink {
     zip -d -qq cfn/layer-deployment.zip '**/LICENSE*'
     zip -d -qq cfn/layer-deployment.zip '**/AUTHOR*'
     zip -d -qq cfn/layer-deployment.zip '**/NOTICE*'
+    zip -d -qq cfn/layer-deployment.zip '**/*.md'
+    zip -d -qq cfn/layer-deployment.zip '**/*.c'
+    zip -d -qq cfn/layer-deployment.zip '**/*.cpp'
+    zip -d -qq cfn/layer-deployment.zip '**/*.h'
+    zip -d -qq cfn/layer-deployment.zip '**/*.pyx'
+    zip -d -qq cfn/layer-deployment.zip '**/*.pxd'
     zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/numpy*/tests/**/*
     zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/pandas*/tests/**/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/pandas/tests/**/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/pandas/_libs/tslibs/src/datetime/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/pandas/io/formats/templates/*
     zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/examples/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/sqlalchemy*/testing/**/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/sqlalchemy/dialects/oracle/**/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/sqlalchemy/dialects/mssql/**/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/sqlalchemy/dialects/mysql/**/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/sqlalchemy/dialects/postgresql/**/*
     zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/*/zoneinfo/Africa/*
     zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/*/zoneinfo/Asia/*
     zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/*/zoneinfo/Atlantic/*

ingestor/.chalice/config.json

@@ -3,7 +3,7 @@
   "app_name": "ingestor",
   "automatic_layer": true,
   "layers": [
-    "arn:aws:lambda:us-east-1:464622532012:layer:Datadog-Extension:76"
+    "arn:aws:lambda:us-east-1:464622532012:layer:Datadog-Extension:78"
   ],
   "stages": {
     "prod": {
@@ -124,4 +124,4 @@
       }
     }
   }
-}
+}

ingestor/chalicelib/parallel.py

@@ -31,5 +31,5 @@ def month_range(start, end):
     # So we generate a daily date_range and then resample it down (summing 0s as a no-op in the process) so it aligns.
     dates = pd.date_range(start, end, freq="1D", inclusive="both")
     series = pd.Series(0, index=dates)
-    months = series.resample("1M").sum().index
+    months = series.resample("ME").sum().index
     return months

ingestor/chalicelib/ridership/process.py

@@ -51,7 +51,7 @@ def format_ridership_csv(
     holidays = cal.holidays(start=df[date_key].min(), end=df[date_key].max())
 
     # mark as holiday and weekday
-    df["holiday"] = df[date_key].dt.date.astype("datetime64").isin(holidays.date)
+    df["holiday"] = df[date_key].dt.date.astype("datetime64[ns]").isin(holidays.date)
     df["weekday"] = df[date_key].dt.dayofweek
 
     # define peak, mark weekdays, convert service date back
@@ -99,7 +99,7 @@ def format_subway_data(path_to_csv_file: str):
     holidays = cal.holidays(start=df["servicedate"].min(), end=df["servicedate"].max())
 
     # mark as holiday and weekday
-    df["holiday"] = df["servicedate"].dt.date.astype("datetime64").isin(holidays.date)
+    df["holiday"] = df["servicedate"].dt.date.astype("datetime64[ns]").isin(holidays.date)
     df["weekday"] = df["servicedate"].dt.dayofweek
 
     # define peak, mark weekdays, convert service date back