Updated repository from deploy pipeline

Author: Automated GitHub deployer

.gitignore

@@ -1,5 +1,6 @@
-venv/
+*venv*
 .idea
+.pdm-python
 
 *.pyc
 __pycache__/
@@ -19,3 +20,6 @@ tests/flask_session/*
 
 scratch
 scratch/*
+
+*.pot
+data

.pre-commit-config.yaml

@@ -0,0 +1,26 @@
+repos:
+-   repo: local
+    hooks:
+    -   id: ruff
+        name: ruff
+        entry: ruff check
+        language: system
+        pass_filenames: false
+        always_run: true
+        stages: [pre-push]
+    -   id: ruff-format
+        name: ruff-format
+        entry: ruff format --check
+        language: system
+        pass_filenames: false
+        always_run: true
+        stages: [pre-push]
+    -   id: pdm-lock-check
+        name: pdm-lock-check
+        description: run pdm lock --check to validate config
+        entry: pdm lock --check
+        language: system
+        pass_filenames: false
+        always_run: true
+        files: ^pyproject.toml$
+        stages: [pre-push]

README.md

@@ -12,34 +12,39 @@ The intended use looks as follows:
 4. The application **shows the search results** in descending order of similarity (according to the AI model)
 5. The user compares the snippets and the shown wrappers to **establish a match**
 
-The application is developed and maintained by the Netherlands Forensic Institute (NFI).
-It is deployed on www.vuurwerkverkenner.nl.
+The application is developed and maintained by the Netherlands Forensic Institute (NFI) and is hosted at the following 
+domains: www.vuurwerkverkenner.nl, www.vuurwerkverkenner.com, www.vuurwerkverkenner.ai, 
+www.fireworksexplorer.eu, www.fireworksexplorer.com, www.fireworksexplorer.ai.
 
 ## Model and database
 The application relies on a **trained AI model** and a **background database** of fireworks, both maintained by the NFI.
 They are both stored on the HuggingFace organization page of the NFI (see [here](https://huggingface.co/NetherlandsForensicInstitute)).
 * The model, as well as a description of how it was trained, can be found [here](https://huggingface.co/NetherlandsForensicInstitute/vuurwerkverkenner).
 * The background database can be found [here](https://huggingface.co/datasets/NetherlandsForensicInstitute/vuurwerkverkenner-data).
 
-If you want to run the application locally, download a copy of the model and database to your machine.
-
-## Getting started
+## Running the app locally
 The following describes how to run the application on your own machine. 
-It can be run with a **dummy** model and database (which are contained within this repository), or with a copy of the **real** model and database which are stored on HuggingFace.
+It can be run with the live model and database, which are stored on HuggingFace.
 
 ### Prerequisites
-* Python 3.8
+* Python 3.12
 * Any web browser
-* Optionally, a local copy of the model and data (see above)
+* An internet connection (required for downloading the data when running the application for the first time)
+
+#### Recommended:  
+* A Python virtual environment
+* A HuggingFace account + access token  
+  Set the token as an environment variable named `HF_TOKEN`  
+  NOTE: It is possible to download the data without this token, but you will likely have to restart the process
+  a few times for it to fully complete te download.
 
 ### Requirements
-All package requirements for running the application are specified in `requirements.txt`.
-Packages can be installed from PyPi, e.g. through `pip install -r requirements.txt`.
+All package requirements for running the application are specified in `pyproject.toml`. 
 
-### Configuration
-Make sure parameters in `app/setup.cfg` (in particular `MODEL_CONFIG_FILE` and `META_DATA_DIR`) match the desired configuration.
-* For running the application with a **dummy** model and database, uncomment the indicated lines.
-* For running the application with the **real** model and database, make sure that the paths point to the correct locations on your machine. Additionally, make sure that the filepath for the weights file (in the model configuration file) matches the filepath on your machine.
+### Model and data
+The model and data are downloaded from HuggingFace when running the app for the first time. These data are saved 
+to the paths `MODEL_DIR` and `META_DATA_DIR`,
+which are listed in `setup.cfg`.
 
 ### Running the application
 Simply run `flask run` inside a terminal.

app/app.py

@@ -2,37 +2,62 @@
 import json
 import logging
 import os
+import re
+from collections.abc import Mapping
+from datetime import datetime
 from pathlib import Path
 from types import MappingProxyType
-from typing import Any, Iterable, Mapping, Optional, Union
+from typing import Any
 
-import confidence
 import numpy as np
-from flask import Flask, Response
+from flask import Flask, Response, request
+from flask_babel import Babel
+from huggingface_hub import snapshot_download
 from lru import LRU
+from speaklater import make_lazy_string
 from werkzeug.exceptions import HTTPException
 
 from app.blueprints.help import help
 from app.blueprints.index import index
-from app.blueprints.login import login
 from app.blueprints.results import results
-from app.blueprints.utils import redirect_to
-from app.calculations.models import ClassificationModel, load_model
+from app.calculations.models import EmbeddingClassifier, ViTEmbeddingModel, ViTModelConfig
 from app.requests.validate import clean_text
+from app.utils import get_locale, redirect_to
+from config.render.meta_data_mapping import (
+    META_DATA_KEY_MAPPING,
+    META_DATA_VALUE_MAPPING_STRING,
+    META_DATA_VALUE_MAPPING_WORDS,
+)
 
 APP_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 
 
-def get_model(app_config: Mapping[str, Any]) -> ClassificationModel:
-    config = confidence.loadf(app_config['MODEL_CONFIG_FILE'])
-    model = load_model(config.model)
-    if not isinstance(model, ClassificationModel):
-        raise ValueError('Classification model needed for prediction')
+def get_model(model_name: str, checkpoint_dir: str, meta_data: Mapping[str, Any]) -> EmbeddingClassifier:
+    snapshot_download(repo_id=model_name, local_dir=checkpoint_dir)
+
+    with open(os.path.join(checkpoint_dir, 'settings.json')) as f:
+        model_config = json.load(f)
+
+    if backbone := model_config.get("backbone"):
+        if backbone == "vit":
+            model_config["params"]["device"] = "cpu"
+            vit_config = ViTModelConfig(**model_config["params"])
+            embedding_model = ViTEmbeddingModel(config=vit_config)
+            embedding_model.load(os.path.join(checkpoint_dir, "best_weights.pt"), device="cpu")
+        else:
+            raise ValueError(f"backbone '{backbone}' not supported")
+    else:
+        raise ValueError("'backbone' field missing in `settings.json`")
+
+    model = EmbeddingClassifier(model=embedding_model)
+    model.update_embeddings(meta_data)
+
     return model
 
 
-def build_image_url(label: str, wrapper_index: int, filename: str) -> str:
-    return f'fireworks_{label}/wrappers/{wrapper_index}/{filename}'
+def build_image_url(category: str, label: str, filename: str) -> str:
+    """Build the url to the image."""
+    return f'{category}/{label}/{filename}'
 
 
 def parse_json(filename: str) -> Mapping[str, Any]:
@@ -44,49 +69,46 @@ def parse_json(filename: str) -> Mapping[str, Any]:
         return json.loads(f.read())
 
 
-def parse_meta_data(meta_data_dir: str, key_order: Optional[Iterable] = None) -> Mapping[str, Any]:
+def get_meta_data(app_config: Mapping[str, Any]) -> Mapping[str, Any]:
     """
-    Parse and return the fireworks meta-data from the meta-data directory.
-
-    :param meta_data_dir: the directory to parse the meta-data from.
-        It is assumed that the gzip-compressed JSON containing all the meta-data is
-        located in the root of this directory and has the filename "meta.json.gz".
-        It is also assumed that the JSON file follows the same structure as in the
-        example JSON that can be found in "data/demo_data/meta.json.gz"
-    :param key_order: optional parameter for defining the insertion order of the
-        meta-data keys. This defines the order in which the meta-data fields will be
-        displayed on the page
+    Download, parse and return the fireworks meta-data from the meta-data directory. The meta-data should consist of
+    a `meta.json.gz` file and nested folders each category/label containing the images.
 
+    :param app_config: Global constants used in the application
     :returns: an immutable mapping containing all the parsed meta-data
     """
-    filename = os.path.join(meta_data_dir, 'meta.json.gz')
-    meta_data = parse_json(filename=filename)
-    for label, group in meta_data.items():
-        for i, item in group["wrappers"].items():
-            # insert the keys in order of `key_order` if defined
-            if key_order:
-                item_sorted = {key: item[key] for key in key_order if key in item}
-                item_sorted.update({key: item[key] for key in item if key not in item_sorted})
-                item = item_sorted
+    if app_config.get('META_DATA_HF'):
+        snapshot_download(
+            repo_id=app_config['META_DATA_HF'], repo_type="dataset", local_dir=app_config['META_DATA_DIR']
+        )
+    meta_data = parse_json(filename=os.path.join(app_config['REFERENCE_DATA_DIR'], 'meta.json.gz'))
+    for category, articles in meta_data.items():
+        for label, article in articles.items():
+            article_meta_data = article.get('wrappers', {})
+            # insert the keys in order of `META_DATA_KEY_MAPPING.keys()`
+            article_meta_data_sorted = {
+                key: article_meta_data[key] for key in META_DATA_KEY_MAPPING.keys() if key in article_meta_data
+            }
+            article_meta_data_sorted.update(
+                {key: article_meta_data[key] for key in article_meta_data if key not in article_meta_data_sorted}
+            )
+            article_meta_data = article_meta_data_sorted  # noqa
             # clean the wrapper text
-            item["text"] = clean_text(item["text"])
+            article_meta_data["text"] = clean_text(article_meta_data.get("text", ""))
             # retrieve the image location for the wrapper
-            item["image"] = build_image_url(label, i, 'wrapper.jpg')
-            images = os.listdir(os.path.join(meta_data_dir, f'fireworks_{label}', 'wrappers', str(i)))
-            item["meta_images"] = tuple(build_image_url(label, i, img) for img in images if img != 'wrapper.jpg')
+            article_meta_data["image"] = build_image_url(category, label, app_config['WRAPPER_FILENAME'])
+            images = os.listdir(os.path.join(app_config['REFERENCE_DATA_DIR'], category, label))
+            article_meta_data["meta_images"] = tuple(
+                build_image_url(category, label, img) for img in images if img != app_config['WRAPPER_FILENAME']
+            )
             # overwrite entry
-            group["wrappers"][i] = item
-        # convert the embeddings to numpy arrays
-        group["embeddings"] = np.asarray(group["embeddings"])
+            article["wrappers"] = article_meta_data
+            # convert the embeddings to numpy arrays
+            article["embeddings"] = np.asarray(article["embeddings"])
     return MappingProxyType(meta_data)
 
 
-def parse_mapping(filename: str) -> Mapping[str, str]:
-    filename = os.path.join(APP_DIR, 'config', 'render', filename)
-    return MappingProxyType(parse_json(filename=filename))
-
-
-def create_app(config: Union[str, Mapping[str, Any]] = 'setup.cfg'):
+def create_app(config: str | Mapping[str, Any] = 'setup.cfg'):
     """
     Create the app. This method is automatically detected and triggered by Flask.
 
@@ -100,46 +122,65 @@ def create_app(config: Union[str, Mapping[str, Any]] = 'setup.cfg'):
     else:
         app.config.update(config)
 
+    Babel(app, locale_selector=get_locale)
+
     app.register_blueprint(index.index_page)
     app.register_blueprint(help.help_page)
     app.register_blueprint(results.results_page)
 
-    app.meta_data_key_mapping = parse_mapping('meta_data_key_mapping.json')
-    app.endangerment_mapping = parse_mapping('endangerment_mapping.json')
+    if not os.path.isabs(app.config['META_DATA_DIR']):
+        app.config['META_DATA_DIR'] = str(Path().absolute() / app.config['META_DATA_DIR'])
+    app.config['REFERENCE_DATA_DIR'] = os.path.join(app.config['META_DATA_DIR'], 'reference_data')
+    app.meta_data = get_meta_data(app_config=app.config)
 
-    if meta_data_dir := app.config.get("META_DATA_DIR", None):
-        # change relative path to absolute path if necessary
-        if not os.path.isabs(meta_data_dir):
-            app.config['META_DATA_DIR'] = str(Path().absolute() / Path(meta_data_dir))
-        app.meta_data = parse_meta_data(meta_data_dir=app.config['META_DATA_DIR'],
-                                        key_order=app.meta_data_key_mapping.keys())
-    else:
-        app.meta_data = {}
-
-    app.model = get_model(app.config)
+    if not os.path.isabs(app.config['MODEL_DIR']):
+        app.config['MODEL_DIR'] = str(Path().absolute() / app.config['MODEL_DIR'])
+    app.config["MODEL_DIR"] = str(Path().absolute() / Path(app.config['MODEL_DIR']))
+    app.model = get_model(app.config.get("MODEL_HF"), app.config["MODEL_DIR"], app.meta_data)
     app.cache = LRU(app.config.get('CACHE_SIZE'))
     app.jinja_env.filters['zip'] = zip
+    app.jinja_env.filters['translate_meta_data_values'] = translate_meta_data_values
 
     register_error_handlers(app)
 
     app.after_request(after_request)
 
-    if app.config.get("LOGIN_REQUIRED", None):
-        app.register_blueprint(login.login_page)
-        from flask_session import Session
-        Session(app)
-
     return app
 
 
+def translate_meta_data_values(input: str):
+    """
+    Translate the parts of the string defined in `META_DATA_VALUE_MAPPING_STRING` and then split the string in words and
+    translate individually from `META_DATA_VALUE_MAPPING_WORDS`. Return the translated string as a LazyString.
+    """
+    # Make the mapping keys lowercase
+    smap = {k.lower(): v for k, v in META_DATA_VALUE_MAPPING_STRING.items()}
+    wmap = {k.lower(): v for k, v in META_DATA_VALUE_MAPPING_WORDS.items()}
+
+    # Compile the patterns to replace
+    sub_pat = re.compile("|".join(map(re.escape, sorted(smap, key=len, reverse=True))), re.IGNORECASE)
+    word_pat = re.compile(rf"(?<![A-Za-z0-9])({'|'.join(map(re.escape, wmap))})(?![A-Za-z0-9])", re.IGNORECASE)
+
+    def apply_subs(string: str) -> str:
+        string = sub_pat.sub(lambda m: str(smap[m.group(0).lower()]), string)
+        string = word_pat.sub(lambda m: str(wmap[m.group(0).lower()]), string)
+        return string
+
+    # Apply the substitutions and return as a lazy string
+    return make_lazy_string(lambda t: apply_subs(t).capitalize(), input)
+
+
 def register_error_handlers(app):
     app.register_error_handler(404, redirect_bad_request)
     app.register_error_handler(405, redirect_bad_request)
     app.register_error_handler(Exception, handle_exception)
 
 
 def redirect_bad_request(ex: Exception):
-    logging.error(f"Bad request received (will be ignored with a redirect to the index page): {ex}")
+    logging.error(
+        f"{datetime.now()} Bad request received: {request.path} "
+        f"(will be ignored with a redirect to the index page): {ex}"
+    )
     return redirect_to("index")
 
 
@@ -155,13 +196,15 @@ def handle_exception(ex: Exception):
 
 def after_request(response: Response):
     response.headers['Strict-Transport-Security'] = 'max-age=31536000; includeSubDomains'
-    response.headers['Content-Security-Policy'] = "default-src 'none';" \
-                                                  "img-src 'self' data:;" \
-                                                  "font-src 'self';" \
-                                                  "connect-src 'self';" \
-                                                  "script-src 'self' 'unsafe-inline';" \
-                                                  "style-src 'self' 'unsafe-inline';" \
-                                                  "form-action 'self'; "
+    response.headers['Content-Security-Policy'] = (
+        "default-src 'none';"
+        "img-src 'self' data:;"
+        "font-src 'self';"
+        "connect-src 'self';"
+        "script-src 'self' 'unsafe-inline';"
+        "style-src 'self' 'unsafe-inline';"
+        "form-action 'self'; "
+    )
     response.headers['X-Content-Type-Options'] = 'nosniff'
     response.headers['X-Frame-Options'] = 'SAMEORIGIN'
     return response

app/blueprints/help/help.py

@@ -1,27 +1,17 @@
-import json
 import os
-from functools import lru_cache
 
-from flask import Blueprint, render_template
+from flask import Blueprint, current_app, render_template
 
-from app.blueprints.login import verify_authorization
+from app.blueprints.help.resources.help import get_faq
 
 RESOURCES_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'resources')
 help_page = Blueprint('help', __name__, template_folder='templates')
 
 
-def _parse_json_data(filename: str):
-    with open(filename, 'r') as f:
-        return json.loads(f.read())
-
-
-@lru_cache(maxsize=None)
 def _render_help_page():
-    json_data = _parse_json_data(filename=os.path.join(RESOURCES_DIR, 'help.json'))
-    return render_template('help.html', json_data=json_data)
+    return render_template('help.html', json_data=get_faq(current_app))
 
 
 @help_page.route('/help')
-@verify_authorization
 def show():
     return _render_help_page()