merge main to staging (#27)

* feat: allow dataset source to contain an array of sources (#26)

* fix: keep installed versions of boto3 and botocore

Author: hrodmn

.github/workflows/deploy.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: 3.9
 
       - name: Create Config.yml
         uses: franzbischoff/replace_envs@v1
@@ -53,7 +53,7 @@ jobs:
           node-version: '16'
 
       - name: Cache Node.js modules
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           # npm cache files are stored in `~/.npm` on Linux/macOS
           path: ~/.npm
@@ -72,15 +72,15 @@ jobs:
           echo "::set-output name=dir::$(pip cache dir)"
 
       - name: pip cache
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ${{ steps.pip-cache.outputs.dir }}
           key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
           restore-keys: |
             ${{ runner.os }}-pip-
 
-      - name: Install AWS CDK
-        run: npm install -g cdk
+      - name: Install Node.js dependencies
+        run: npm install
 
       - name: Install python packages
         run: |

.github/workflows/pr.yml

@@ -3,8 +3,6 @@ on:
   pull_request:
     branches:
       - main
-  schedule:
-    - cron:  '0 0 * * *' # daily
 
 env:
   STAC_API_URL: ""
@@ -30,7 +28,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: 3.9
 
       - name: Create Config.yml
         uses: franzbischoff/replace_envs@v1
@@ -50,13 +48,14 @@ jobs:
         run: |
           python -m pip install -U pip
           python -m pip install .["dev,test"]
+          python -m pip install --upgrade setuptools packaging virtualenv
 
       - name: run pre-commit
         run: pre-commit run --all-files
 
-      - name: Run safety dependency vulnerability check
-        # ignore some vulnerabilities in twisted and httplib2 that are already installed and can't be uninstalled
-        run: safety check -i 39557 -i 37040 -i 37209 -i 37554 -i 38085 -i 38368 -i 38303 -i 39608
+      # - name: Run safety dependency vulnerability check
+      #   # ignore some vulnerabilities in twisted and httplib2 that are already installed and can't be uninstalled
+      #   run: safety check -i 39557 -i 37040 -i 37209 -i 37554 -i 38085 -i 38368 -i 38303 -i 39608
 
       - name: Run bandit static analysis check
         run: bandit -r dashboard_api
@@ -79,14 +78,14 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: 3.9
 
       - uses: actions/setup-node@v2
         with:
-          node-version: '12'
+          node-version: '16'
 
       - name: Cache Node.js modules
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           # npm cache files are stored in `~/.npm` on Linux/macOS
           path: ~/.npm
@@ -110,8 +109,8 @@ jobs:
           to_file: 'stack/config.yml'
           commit: 'false'
 
-      - name: Install AWS CDK
-        run: npm install -g
+      - name: Install Node.js dependencies
+        run: npm install
 
       - name: Install python packages
         run: |

.pre-commit-config.yaml

@@ -1,30 +1,35 @@
 repos:
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 24.10.0
     hooks:
       - id: black
         language_version: python
 
   - repo: https://github.com/PyCQA/isort
-    rev: 5.4.2
+    rev: 5.13.2
     hooks:
       - id: isort
         language_version: python
 
   - repo: https://github.com/PyCQA/flake8
-    rev: 3.8.3
+    rev: 7.1.1
     hooks:
       - id: flake8
         language_version: python
 
   - repo: https://github.com/PyCQA/pydocstyle
-    rev: 5.1.1
+    rev: 6.3.0
     hooks:
       - id: pydocstyle
         language_version: python
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.812
+    rev: v1.13.0
     hooks:
       - id: mypy
         language_version: python
+        additional_dependencies:
+          - types-PyYAML
+          - types-requests
+          - types-cachetools
+          - types-mock

Dockerfiles/lambda/Dockerfile

@@ -14,7 +14,7 @@ RUN pip install . "mangum>=0.10.0" -t /asset --no-binary pydantic
 RUN cd /asset && find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[2-3][0-9]//'); cp $f $n; done;
 RUN cd /asset && find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf
 RUN cd /asset && find . -type f -a -name '*.py' -print0 | xargs -0 rm -f
-RUN rm -rdf /asset/numpy/doc/ /asset/boto3* /asset/botocore* /asset/bin /asset/geos_license /asset/Misc
+# RUN rm -rdf /asset/numpy/doc/ /asset/boto3* /asset/botocore* /asset/bin /asset/geos_license /asset/Misc
 
 RUN mkdir /asset/stack
 COPY stack/config.yml /asset/stack/config.yml

dashboard_api/api/api_v1/endpoints/datasets.py

@@ -1,4 +1,5 @@
 """Dataset endpoints."""
+
 from fastapi import APIRouter, Depends, HTTPException, Response
 from starlette.requests import Request
 

dashboard_api/api/api_v1/endpoints/tiles.py

@@ -41,10 +41,10 @@ def tile(
     x: int = Path(..., description="Mercator tiles's column"),
     y: int = Path(..., description="Mercator tiles's row"),
     # AssertionError: Cannot use `Query` for path param 'scale'
-    # scale: int = Query(
-    #     1, gt=0, lt=4, description="Tile size scale. 1=256x256, 2=512x512..."
-    # ),
-    # ext: ImageType = Query(None, description="Output image type. Default is auto."),
+    scale: int = Path(
+        ..., gt=0, lt=4, description="Tile size scale. 1=256x256, 2=512x512..."
+    ),
+    ext: ImageType = Path(..., description="Output image type. Default is auto."),
     url: str = Query(..., description="Cloud Optimized GeoTIFF URL."),
     bidx: Optional[str] = Query(None, description="Coma (',') delimited band indexes"),
     nodata: Optional[Union[str, int, float]] = Query(

dashboard_api/db/static/datasets/__init__.py

@@ -1,7 +1,8 @@
 """ dashboard_api static datasets """
+
 import json
 import os
-from typing import List
+from typing import List, Optional
 
 import botocore
 
@@ -99,7 +100,9 @@ def list(self) -> List[str]:
         """List all datasets"""
         return list(self._data().keys())
 
-    def _format_urls(self, tiles: List[str], api_url: str, spotlight_id: str = None):
+    def _format_urls(
+        self, tiles: List[str], api_url: str, spotlight_id: Optional[str] = None
+    ):
         if spotlight_id:
             [tile.replace("{spotlightId}", spotlight_id) for tile in tiles]
         return [
@@ -110,7 +113,10 @@ def _format_urls(self, tiles: List[str], api_url: str, spotlight_id: str = None)
         ]
 
     def _process(
-        self, datasets_domains_metadata: dict, api_url: str, spotlight_id: str = None
+        self,
+        datasets_domains_metadata: dict,
+        api_url: str,
+        spotlight_id: Optional[str] = None,
     ):
         """
         Processes datasets to be returned to the API consumer:

dashboard_api/db/static/products/__init__.py

@@ -1,4 +1,5 @@
 """ dashboard_api static products """
+
 import json
 import os
 from typing import Optional

dashboard_api/db/utils.py

@@ -3,7 +3,7 @@
 import csv
 import json
 from datetime import datetime
-from typing import Dict, List
+from typing import Dict, List, Optional
 
 import boto3
 from botocore import config
@@ -23,7 +23,9 @@
 
 
 def invoke_lambda(
-    lambda_function_name: str, payload: dict = None, invocation_type="RequestResponse"
+    lambda_function_name: str,
+    payload: Optional[dict] = None,
+    invocation_type="RequestResponse",
 ):
     """Invokes a lambda function using the boto3 lambda client.
 

dashboard_api/main.py

@@ -1,4 +1,5 @@
 """dashboard_api app."""
+
 from typing import Any, Dict, Optional
 
 from fastapi import FastAPI