Deploy v2.8.0 to production

.github/PULL_REQUEST_TEMPLATE/production.md

@@ -0,0 +1,23 @@
+# :open_book: Changelog
+
+- list changes
+- list changes
+
+# Version bump required by the PR
+
+See [Semantic Versioning 2.0.0](https://semver.org/) for help discerning which is required.
+
+- [ ] Patch
+- [ ] Minor
+- [ ] Major
+
+# :rocket: Deployment Notes
+
+- Backward compatible API changes
+  - [ ] REST API
+  - [ ] Chat Websocket API
+- Backwards-incompatible API changes
+  - [ ] REST API
+  - [ ] Chat Websocket API
+- [ ] Specific deployment synchronization instructions with other apps/API's
+- [ ] Other specific instructions/tasks

.github/scripts/honeybadger_deploy_notification.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+curl \
+  --data "deploy[environment]=${DEPLOY_ENV}&deploy[local_username]=Github+Actions&deploy[revision]=${HONEYBADGER_REVISION}&api_key=${HONEYBADGER_API_KEY}" \
+  https://api.honeybadger.io/v1/deploys

.github/workflows/deploy.yml

@@ -9,15 +9,20 @@ on:
       - "node/**"
       - "chat/**"
       - "template.yaml"
+      - "**/template/yaml"
   workflow_dispatch:
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
+permissions:
+  contents: write
+  actions: write
 jobs:
   build-deploy:
     runs-on: ubuntu-latest
     permissions:
       id-token: write
-      contents: read
+      contents: write
+      actions: write
     environment: ${{ github.ref == 'refs/heads/main' && 'production' || 'staging' }}
     steps:
       - name: Set DEPLOY_ENV from Branch Name
@@ -44,6 +49,8 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: '3.12'
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
       - uses: aws-actions/setup-sam@v1
       - uses: aws-actions/configure-aws-credentials@master
         with:
@@ -65,3 +72,35 @@ jobs:
           HONEYBADGER_REVISION: ${{ github.sha }}
           SAM_CLI_BETA_BUILD_PERFORMANCE: 1
           SAM_CLI_BETA_PACKAGE_PERFORMANCE: 1
+      - name: Notify Honeybadger
+        run: .github/scripts/honeybadger_deploy_notification.sh
+        env:
+          DEPLOY_ENV: ${{ env.DEPLOY_ENV }}
+          HONEYBADGER_API_KEY: ${{ secrets.HONEYBADGER_API_KEY }}
+          HONEYBADGER_REVISION: ${{ github.sha }}
+      - name: Get current API version
+        if: ${{ github.ref == 'refs/heads/main' }}
+        run: echo "API_VERSION=$(make version)" >> $GITHUB_ENV
+      - name: Tag Release
+        if: ${{ github.ref == 'refs/heads/main' }}
+        run: |
+          git config --global user.email "$(git log --pretty=format:"%ae" | head -1)"
+          git config --global user.name "$(git log --pretty=format:"%an" | head -1)"
+          git tag -a v${API_VERSION} -m "Release ${API_VERSION}"
+      - name: Push changes
+        if: ${{ github.ref == 'refs/heads/main' }}
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          tags: true
+      - name: Dispatch New Production PR
+        if: ${{ github.ref == 'refs/heads/main' }}
+        uses: actions/github-script@v6
+        with:
+          script: |
+            github.rest.actions.createWorkflowDispatch({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: 'next_version.yml',
+              ref: 'deploy/staging',
+            })

.github/workflows/docs.yml

@@ -27,11 +27,13 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: '3.12'
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
       - name: Install dependencies
-        run: pip install -r requirements.txt
+        run: uv sync
         working-directory: ./docs
       - name: Build docs
-        run: mkdocs build --clean
+        run: uv run mkdocs build --clean
         working-directory: ./docs
       - name: Determine correct deploy domain for environment
         run: sed -i s/API_HOST/${HOSTNAME}/g docs/site/spec/openapi.*

.github/workflows/next_version.yml

@@ -0,0 +1,47 @@
+---
+name: Increment Version & Create Draft PR
+on:
+  workflow_dispatch
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+permissions:
+  contents: write
+  pull-requests: write
+jobs:
+  increment:
+    runs-on: ubuntu-latest
+    permissions: 
+      contents: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: deploy/staging
+      - name: Bump Version
+        id: increment
+        run: |
+          NEXT_VERSION=$(make version BUMP=patch)
+          git config --global user.name 'github-actions[bot]'
+          git config --global user.email '41898282+github-actions[bot]@users.noreply.github.com'
+          git commit -am "Bump version to ${NEXT_VERSION}"
+      - name: Push changes
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          branch: deploy/staging
+      - name: Read PR Template
+        id: template
+        uses: jaywcjlove/github-action-read-file@main
+        with:
+          path: .github/PULL_REQUEST_TEMPLATE/production.md
+      - name: Create New Production PR
+        uses: repo-sync/pull-request@v2
+        with:
+          source_branch: deploy/staging
+          destination_branch: main
+          pr_label: "release"
+          pr_title: Deploy vX.X.X to production
+          pr_body: |
+            ${{ steps.template.outputs.content }}
+          pr_draft: true

.github/workflows/test-python.yml

@@ -16,17 +16,17 @@ jobs:
       AWS_SECRET_ACCESS_KEY: ci
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
         with:
-          python-version: '3.12'
-          cache-dependency-path: chat/src/requirements.txt
-      - run: pip install -r requirements.txt && pip install -r requirements-dev.txt
-        working-directory: ./chat/src
+          enable-cache: true
+          cache-dependency-glob: "**/pyproject.toml"
+      - run: uv sync --group dev
       - name: Check code style
-        run: ruff check .
+        run: uv run ruff check .
       - name: Run tests
         run: |
-          coverage run --include='src/**/*' -m pytest -m ""
-          coverage report
+          uv run coverage run --include='src/**/*' -m pytest -m ""
+          uv run coverage report
         env:
           AWS_REGION: us-east-1

.github/workflows/validate-template.yml

@@ -16,11 +16,11 @@ jobs:
         - uses: actions/setup-python@v4
           with:
             python-version: '3.12' 
+        - name: Install uv
+          uses: astral-sh/setup-uv@v7
         - name: Install cfn-lint
           run: pip install cfn-lint
         - uses: aws-actions/setup-sam@v1
-        # - name: sam fix https://github.com/aws/aws-sam-cli/issues/4527
-        #   run: $(dirname $(readlink $(which sam)))/pip install --force-reinstall "cryptography==38.0.4"
         - uses: aws-actions/configure-aws-credentials@master
           with:
             role-to-assume: arn:aws:iam::${{ secrets.AwsAccount }}:role/github-actions-role

.husky/pre-commit

@@ -1,4 +1,4 @@
 #!/usr/bin/env sh
 . "$(dirname -- "$0")/_/husky.sh"
 cd api && npm run lint && npm run prettier && cd -
-cd chat/src && ruff check . && cd -
+cd chat/src && uv run ruff check . && cd -

.tool-versions

@@ -2,3 +2,4 @@ nodejs 20.15.0
 java corretto-19.0.1.10.1
 aws-sam-cli 1.135.0
 python 3.12.2
+uv 0.9.5

Makefile

@@ -33,6 +33,8 @@ help:
 	echo "make cover-node            | run node tests with coverage"
 	echo "make cover-python          | run python tests with coverage"
 
+./chat/dependencies/requirements.txt: ./chat/pyproject.toml
+	cd chat && uv export --format requirements-txt --no-hashes > dependencies/requirements.txt
 api: ./api/template.yaml ./api/src/package-lock.json $(wildcard ./api/src/**/*.js)
 chat: ./chat/template.yaml ./chat/dependencies/requirements.txt $(wildcard ./chat/src/**/*.py)
 av-download: ./av-download/template.yaml ./av-download/lambdas/package-lock.json $(wildcard ./av-download/lambdas/**/*.js)
@@ -74,20 +76,20 @@ style-node: deps-node
 test-node: deps-node
 	cd api && npm run test
 deps-python:
-	cd chat/src && pip install -r requirements.txt && pip install -r requirements-dev.txt
+	cd chat && uv sync --group dev
 cover-python: deps-python
-	cd chat && coverage run --source=src -m pytest -v && coverage report --skip-empty
+	cd chat && uv run coverage run --source=src -m pytest -v && uv run coverage report --skip-empty
 cover-html-python: deps-python
-	cd chat && coverage run --source=src -m pytest -v && coverage html --skip-empty
+	cd chat && uv run coverage run --source=src -m pytest -v && uv run coverage html --skip-empty
 style-python: deps-python
-	cd chat && ruff check . 
+	cd chat && uv run ruff check . 
 style-python-fix: deps-python
-	cd chat && ruff check --fix . 
+	cd chat && uv run ruff check --fix . 
 test-python: deps-python
-	cd chat && pytest
+	cd chat && uv run pytest
 python-version:
-	cd chat && python --version
-build: av-download/layers/ffmpeg/bin/ffmpeg .aws-sam/build.toml
+	cd chat && uv run python --version
+build: av-download/layers/ffmpeg/bin/ffmpeg api chat .aws-sam/build.toml
 validate:
 	cfn-lint template.yaml **/template.yaml --ignore-checks E3510 W1028 W8001
 serve-http: deps-node
@@ -140,6 +142,20 @@ sync-code: sync
 secrets:
 	ln -s ../tfvars/dc-api/*.yaml .
 clean:
-	rm -rf .aws-sam api/.aws-sam chat/.aws-sam av-download/.aws-sam api/node_modules api/src/node_modules chat/**/__pycache__ chat/.coverage chat/.ruff_cache
+	rm -rf .aws-sam api/.aws-sam chat/.aws-sam av-download/.aws-sam api/node_modules api/src/node_modules av-download/lambdas/node_modules chat/**/__pycache__ chat/.coverage chat/.ruff_cache
 reset:
 	for f in $$(find . -maxdepth 2 -name '*.orig'); do mv $$f $${f%%.orig}; done
+serve-docs:
+	cd docs && uv sync && uv run mkdocs serve -a 0.0.0.0:8000
+
+BUMP ?= ""
+version:
+	@if [[ -n "$(BUMP)" ]]; then \
+		for pkg in api api/dependencies api/src av-download/lambdas; do \
+			(cd $$pkg && npm version $(BUMP)) >/dev/null 2>&1; \
+		done; \
+		for pkg in chat docs; do \
+			(cd $$pkg && uv version --bump $(BUMP)) >/dev/null 2>&1; \
+		done; \
+	fi; \
+	node -e 'console.log(require("./api/package.json").version)'

README.md

@@ -142,11 +142,8 @@ In a nutshell:
 2. Edit the Markdown files in the `docs/docs` directory.
 3. To run `mkdocs` locally and preview your work:
    ```shell
-   cd docs
-   python -m venv ./.venv
-   pip install -r requirements.txt
    sg open all 8000
-   mkdocs serve -a 0.0.0.0:8000
+   make serve-docs
    ```
    Docs will be accessible at http://USER_PREFIX.dev.rdc.library.northwestern.edu:8000/
 
@@ -174,3 +171,11 @@ Typescript types for the schemas (Works, Collections, FileSets) are automaticall
 
 - If a deploy to the `deploy/staging` branch contains changes to the `docs/docs/spec/data-types.yaml` file, new types are generated and a commit is made to the `staging` branch of `nulib/dcapi`. This is intended to be for local testing by NUL devs against the private staging API.
 - If a deploy to production (`main` branch) contains changes to the `docs/docs/spec/data-types.yaml` file, new types are generated and a PR is opened into the `main` branch of `nulib/dcapi-types`. Also, an issue is created in `nulib/repodev_planning_and_docs` to review the PR and publish the types package (manually).
+
+## Versioning
+
+The current API version is maintained in several different project files. To increment the version, use
+```
+make version BUMP=<major|minor|patch>
+```
+If you don't specify a `BUMP` value, the command will simply print the current version.

api/src/handlers/oai.js

@@ -13,11 +13,20 @@ const { wrap } = require("./middleware");
 function invalidDateParameters(verb, dates) {
   if (!["ListRecords", "ListIdentifiers"].includes(verb)) return [];
 
-  const regex = new RegExp(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/);
+  // OAI-PMH spec allows three date formats:
+  // 1. YYYY-MM-DD (date only)
+  // 2. YYYY-MM-DDThh:mm:ssZ (no fractional seconds)
+  // 3. YYYY-MM-DDThh:mm:ss.fZ to YYYY-MM-DDThh:mm:ss.ffffffZ (1-6 fractional seconds)
+  const dateOnlyRegex = /^\d{4}-\d{2}-\d{2}$/;
+  const dateTimeRegex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,6})?Z$/;
   let invalidDates = [];
 
   for (const [dateParameter, dateValue] of Object.entries(dates)) {
-    if (dateValue && !regex.test(dateValue)) {
+    if (
+      dateValue &&
+      !dateOnlyRegex.test(dateValue) &&
+      !dateTimeRegex.test(dateValue)
+    ) {
       invalidDates.push(dateParameter);
     } else {
       continue;
@@ -56,7 +65,7 @@ exports.handler = wrap(async (event) => {
   if (invalidDateParameters(verb, dates).length > 0)
     return invalidOaiRequest(
       "badArgument",
-      "Invalid date -- make sure that 'from' or 'until' parameters are formatted as: 'YYYY-MM-DDThh:mm:ss.ffffffZ'"
+      "Invalid date -- make sure that 'from' or 'until' parameters are formatted as: 'YYYY-MM-DD' or 'YYYY-MM-DDThh:mm:ssZ' (with optional fractional seconds)"
     );
   if (!verb) return invalidOaiRequest("badArgument", "Missing required verb");
 

api/test/integration/oai.test.js

@@ -117,7 +117,7 @@ describe("Oai routes", () => {
         "badArgument"
       );
       expect(resultBody["OAI-PMH"].error["_text"]).to.eq(
-        "Invalid date -- make sure that 'from' or 'until' parameters are formatted as: 'YYYY-MM-DDThh:mm:ss.ffffffZ'"
+        "Invalid date -- make sure that 'from' or 'until' parameters are formatted as: 'YYYY-MM-DD' or 'YYYY-MM-DDThh:mm:ssZ' (with optional fractional seconds)"
       );
     });
 
@@ -137,6 +137,54 @@ describe("Oai routes", () => {
         .and.to.have.lengthOf(12);
     });
 
+    it("accepts OAI-PMH standard date formats without fractional seconds (Primo compatibility)", async () => {
+      const body =
+        "verb=ListRecords&metadataPrefix=oai_dc&from=1970-01-02T00:00:00Z";
+      mock
+        .post("/dc-v2-work/_search?scroll=2m")
+        .reply(200, helpers.testFixture("mocks/scroll.json"));
+      const event = helpers.mockEvent("POST", "/oai").body(body).render();
+      const result = await handler(event);
+      expect(result.statusCode).to.eq(200);
+      expect(result).to.have.header("content-type", /application\/xml/);
+      const resultBody = convert.xml2js(result.body, xmlOpts);
+      expect(resultBody["OAI-PMH"].ListRecords.record)
+        .to.be.an("array")
+        .and.to.have.lengthOf(12);
+    });
+
+    it("accepts OAI-PMH date-only format (YYYY-MM-DD)", async () => {
+      const body =
+        "verb=ListRecords&metadataPrefix=oai_dc&from=2022-01-01&until=2022-12-31";
+      mock
+        .post("/dc-v2-work/_search?scroll=2m")
+        .reply(200, helpers.testFixture("mocks/scroll.json"));
+      const event = helpers.mockEvent("POST", "/oai").body(body).render();
+      const result = await handler(event);
+      expect(result.statusCode).to.eq(200);
+      expect(result).to.have.header("content-type", /application\/xml/);
+      const resultBody = convert.xml2js(result.body, xmlOpts);
+      expect(resultBody["OAI-PMH"].ListRecords.record)
+        .to.be.an("array")
+        .and.to.have.lengthOf(12);
+    });
+
+    it("accepts OAI-PMH dates with varying fractional seconds (1-6 digits)", async () => {
+      const body =
+        "verb=ListRecords&metadataPrefix=oai_dc&from=2022-11-22T06:16:13.7Z&until=2022-11-22T06:16:13.79157Z";
+      mock
+        .post("/dc-v2-work/_search?scroll=2m")
+        .reply(200, helpers.testFixture("mocks/scroll.json"));
+      const event = helpers.mockEvent("POST", "/oai").body(body).render();
+      const result = await handler(event);
+      expect(result.statusCode).to.eq(200);
+      expect(result).to.have.header("content-type", /application\/xml/);
+      const resultBody = convert.xml2js(result.body, xmlOpts);
+      expect(resultBody["OAI-PMH"].ListRecords.record)
+        .to.be.an("array")
+        .and.to.have.lengthOf(12);
+    });
+
     it("uses an empty resumptionToken to tell harvesters that list requests are complete", async () => {
       mock
         .post(