Feat add s3 metric spark

.github/workflows/tag_and_publish.yml

@@ -15,4 +15,3 @@ jobs:
       docker-tag: ${{ needs.tag.outputs.new_version }}
     secrets:
       repo-token: ${{ secrets.SERVICE_TOKEN }}
-

.github/workflows/test_and_lint.yml

@@ -8,15 +8,18 @@ on:
 jobs:
   ci:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [ '3.11', '3.12' ]
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+    - name: Checkout Code
+      uses: actions/checkout@v6
+    - name: Set up JDK 21
+      uses: actions/setup-java@v5
       with:
-        python-version: ${{ matrix.python-version }}
+        java-version: '21'
+        distribution: 'temurin'
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v6
+      with:
+        python-version: '3.11'
     - name: Install dependencies
       run: | 
         python -m pip install -e . --group dev --no-cache-dir

Dockerfile

@@ -1,9 +1,18 @@
-FROM python:3.12-slim
+FROM python:3.11-slim
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends openjdk-21-jre-headless && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk-amd64
+ENV PATH=$JAVA_HOME/bin:$PATH
+
 WORKDIR /app
 ADD src ./src
 ADD pyproject.toml .
 ADD setup.py .
 
-# Add git in case we need to install from branches
-RUN apt-get update && apt-get install -y git
+EXPOSE 4040
+
 RUN pip install . --no-cache-dir

pyproject.toml

@@ -17,18 +17,21 @@ readme = "README.md"
 dynamic = ["version"]
 
 dependencies = [
-    'aind-settings-utils>=0.1.0',
-    "pydantic>=2.0",
-    "vastpy",
-    "pandas",
-    "awswrangler",
-    "jinja2",
-    "requests",
-    "tenacity"
+    'aind-settings-utils>=0.1.0,<1.0.0',
+    "pydantic>=2.0,<3.0",
+    "vastpy>=0.3.0,<1.0.0",
+    "pandas>=2.3.0,<3.0",
+    "awswrangler>=3.14.0,<4.0",
+    "jinja2>=3.1.0,<4.0",
+    "requests>=2.32.0,<3.0",
+    "tenacity>=9.1.0,<10.0",
+    "pyspark>=3.5.0,<4.0",
+    "aind-data-access-api[docdb]>=1.10.0,<2.0"
 ]
 
 [dependency-groups]
 dev = [
+    'numpy>=1.26,<2.0.0',
     'black',
     'coverage',
     'flake8',