fixed upload bug and progress on multimedia

Author: kasperwelbers

backend/amcat4/config.py

@@ -113,6 +113,7 @@ class Settings(BaseSettings):
     s3_host: Annotated[str | None, Field(description="S3-compatible object storage host")] = None
     s3_access_key: Annotated[str | None, Field(description="S3 access key")] = None
     s3_secret_key: Annotated[str | None, Field(description="S3 secret key")] = None
+    s3_use_proxy: Annotated[str | None, Field(description="If true, use a proxy for s3_host at /s3. (proxy needs to be created by Caddy)")] = None
 
     oidc_url: Annotated[
         str | None,

backend/amcat4/connections.py

@@ -21,16 +21,18 @@ def __init__(
         self,
         elastic: AsyncElasticsearch | None = None,
         s3_client: S3Client | None = None,
+        s3_proxy_client: S3Client | None = None,
         s3_context_stack: AsyncExitStack | None = None,
         http_client: httpx.AsyncClient | None = None,
     ):
         self.elastic = elastic
         self.s3_client = s3_client
+        self.s3_proxy_client = s3_client
         self.s3_context_stack = s3_context_stack
         self.http_client = http_client
 
 
-CONNECTIONS = AmcatConnections(s3_client=None, elastic=None, http_client=None)  # type: ignore
+CONNECTIONS = AmcatConnections(s3_client=None, s3_proxy_client=None, elastic=None, http_client=None)  # type: ignore
 
 
 @asynccontextmanager
@@ -55,7 +57,7 @@ async def amcat_connections() -> AsyncGenerator[None, None]:
 
 def es() -> AsyncElasticsearch:
     """
-    Use this function to access the elasticsearch connection.
+    Access the elasticsearch connection.
     """
     if CONNECTIONS.elastic is None:
         raise ConnectionError("Elasticsearch connection not initialized")
@@ -64,22 +66,38 @@ def es() -> AsyncElasticsearch:
 
 def s3() -> S3Client:
     """
-    Use this function to access the s3 client.
+    Access the s3 client.
     """
     if CONNECTIONS.s3_client is None:
         raise ConnectionError("S3 client not started")
     return CONNECTIONS.s3_client
 
 
+def s3_public() -> S3Client:
+    """
+    Only use this for creating presigned requests for the public
+    s3 server. If the s3 server is not publicly accessible, set s3_use_proxy
+    to use the s3_proxy_client, which signs urls
+    """
+    settings = get_settings()
+    use_proxy = settings.s3_use_proxy and not settings.test_mode
+    s3 = CONNECTIONS.s3_proxy_client if use_proxy else CONNECTIONS.s3_client
+    if s3 is None:
+        raise ConnectionError("S3 client not started")
+    return s3
+
+
+def s3_enabled() -> bool:
+    settings = get_settings()
+    return all([settings.s3_host, settings.s3_access_key, settings.s3_secret_key])
+
+
 def http() -> httpx.AsyncClient:
     if CONNECTIONS.http_client is None:
         raise ConnectionError("HTTP client not started")
     return CONNECTIONS.http_client
 
 
-def s3_enabled() -> bool:
-    settings = get_settings()
-    return all([settings.s3_host, settings.s3_access_key, settings.s3_secret_key])
 
 
 async def _start_elastic():
@@ -127,8 +145,6 @@ async def _start_s3() -> None:
     if settings.s3_access_key is None or settings.s3_secret_key is None:
         raise ValueError("s3_access_key or s3_secret_key not specified")
 
-    ## It seems aioboto3 uses some of the sync boto3 types, so we need to hack around typing here.
-
     session = get_session()
     client = session.create_client(
         service_name="s3",
@@ -138,14 +154,28 @@ async def _start_s3() -> None:
         config=AioConfig(signature_version="s3v4"),
     )
 
+    # If the s3 server is hosted directly with docker compose, fastapi needs to
+    # use a client with the internal s3_host, but presigned requests need to be created
+    # for the /s3 proxy.
+    proxy_url = settings.host + '/s3' if settings.s3_use_proxy else settings.s3_host
+    proxy_client = session.create_client(
+        service_name="s3",
+        endpoint_url=proxy_url,
+        aws_access_key_id=settings.s3_access_key,
+        aws_secret_access_key=settings.s3_secret_key,
+        config=AioConfig(signature_version="s3v4"),
+    )
+
     CONNECTIONS.s3_context_stack = AsyncExitStack()
     CONNECTIONS.s3_client = await CONNECTIONS.s3_context_stack.enter_async_context(client)
+    CONNECTIONS.s3_proxy_client = await CONNECTIONS.s3_context_stack.enter_async_context(proxy_client)
 
 
 async def _close_s3():
     if CONNECTIONS.s3_context_stack is not None:
         await CONNECTIONS.s3_context_stack.aclose()
         CONNECTIONS.s3_client = None
+        CONNECTIONS.s3_proxy_client = None
         CONNECTIONS.s3_context_stack = None
 
 

backend/amcat4/objectstorage/s3bucket.py

@@ -16,7 +16,7 @@
 from typing_extensions import TypedDict
 
 from amcat4.config import get_settings
-from amcat4.connections import s3
+from amcat4.connections import s3, s3_public
 
 PRESIGNED_POST_HOURS_VALID = 6
 
@@ -188,7 +188,7 @@ async def presigned_post(
         conditions.append({"success_action_redirect": redirect})
         fields["success_action_redirect"] = redirect
 
-    pp = await s3().generate_presigned_post(
+    pp = await s3_public().generate_presigned_post(
         Bucket=bucket,
         Key=key,
         Fields=fields,
@@ -202,9 +202,9 @@ async def presigned_get(bucket: str, key: str, hours_valid=24, **kwargs) -> str:
     params = {"Bucket": bucket, "Key": key, **kwargs}
     params = {k: v for k, v in params.items() if v is not None}
 
-    return await s3().generate_presigned_url("get_object", Params=params, ExpiresIn=hours_valid * 3600)
+    return await s3_public().generate_presigned_url("get_object", Params=params, ExpiresIn=hours_valid * 3600)
 
 
 async def get_object_head(bucket: str, key: str) -> HeadObjectOutputTypeDef:
-    res = await s3().head_object(Bucket=bucket, Key=key)
+    res = await s3_public().head_object(Bucket=bucket, Key=key)
     return res

backend/uv.lock

@@ -45,8 +45,8 @@ services:
     volumes:
       - ${ELASTIC_SNAPSHOT_PATH:-snapshot_data}:/snapshots
 
-  elasticsearch:
-    profiles: [deploy, dev]
+  elasticsearch: &elastic_base
+    profiles: [deploy]
     image: docker.elastic.co/elasticsearch/elasticsearch:8.18.4
     depends_on:
       elasticsearch-init:
@@ -83,8 +83,14 @@ services:
       - ${ELASTIC_DATA_STORAGE:-elasticsearch_data}:/usr/share/elasticsearch/data:rw
       - ${ELASTIC_SNAPSHOT_PATH:-snapshot_data}:/usr/share/elasticsearch/data/snapshots:rw
 
-  # SeaweedFS S3-compatible storage — only used in local dev (run with --profile dev)
-  seaweedfs:
+  elasticsearch-dev:
+    <<: *elastic_base
+    profiles: [dev]
+    ports:
+      - "127.0.0.1:9200:9200"
+
+  # S3-compatible storage — only used in local dev (run with --profile dev)
+  s3:
     profiles: [dev]
     image: chrislusf/seaweedfs:latest
     container_name: seaweedfs01
@@ -99,8 +105,8 @@ services:
       - seaweedfs_data:/data
       - ./deploy/s3config.json:/etc/seaweedfs/s3config.json:ro
     ports:
-      - "8333:8333" # S3 Gateway
-      - "8889:8888" # Web UI
+      - "127.0.0.1:8333:8333" # S3 Gateway
+      # - "127.0.0.1:8889:8888" # Web UI
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8888/"]
       interval: 10s

deploy/docker-compose.yml

@@ -1,13 +1,19 @@
 {$AMCAT4_URL:localhost} {
     # API - Proxy to the FastAPI container
-
     handle /api {
         reverse_proxy api:5000
     }
     handle /api/* {
         reverse_proxy api:5000
     }
 
+    # S3 - Proxy to the S3 container
+    handle /s3/* {
+        reverse_proxy {$AMCAT4_S3_HOST} {
+            header_up Host {host}
+        }
+    }
+
     # UI - Static SPA logic
     handle {
         root * /srv

frontend/Caddyfile

@@ -105,7 +105,10 @@
     "vite-tsconfig-paths": "^6.1.1"
   },
   "pnpm": {
-    "onlyBuiltDependencies": ["esbuild", "sharp"],
+    "onlyBuiltDependencies": [
+      "esbuild",
+      "sharp"
+    ],
     "overrides": {
       "minimatch": "^10.0.0",
       "flatted": ">=3.4.0"

frontend/package.json

@@ -38,11 +38,11 @@ export function useFields(user?: AmcatSessionUser, projectId?: AmcatProjectId |
   });
 }
 
-async function getFields(user?: AmcatSessionUser, projectId?: AmcatProjectId) {
+async function getFields(user?: AmcatSessionUser, projectId?: AmcatProjectId): Promise<undefined | AmcatField[]> {
   if (!user || !projectId) return undefined;
   const res = await user.api.get(`/index/${projectId}/fields`);
   const fieldsArray = Object.keys(res.data).map((name) => ({ name, ...res.data[name] }));
-  const fields = z.array(amcatFieldSchema).parse(fieldsArray);
+  const fields: AmcatField[] = z.array(amcatFieldSchema).parse(fieldsArray);
   return fields.map((f) => {
     // set default values
     const default_settings = DEFAULT_CLIENT_SETTINGS[f.name] || DEFAULT_CLIENT_SETTINGS["__DEFAULT__"];

frontend/src/api/fields.tsx

@@ -19,19 +19,21 @@ export interface AmcatSession {
 }
 
 async function createUser(signOut: () => void): Promise<AmcatSessionUser> {
-  const sessionCookie = parseClientSessionCookie();
-  console.log(sessionCookie);
-
-  // if sessionCookie is invalid, break the session.
-  // This is strict, because the sessioncookie is the only way for the client to know
-  // if an amcat session cookie (httponly) is present
-  if (sessionCookie === "broken") signOut();
+  let sessionCookie = parseClientSessionCookie();
 
   const api = axios.create({
     baseURL: "/api/",
     withCredentials: true,
   });
 
+  // if sessionCookie is invalid, break the session.
+  // This is strict, because the sessioncookie is the only way for the client to know
+  // if an amcat session cookie (httponly) is present
+  if (sessionCookie === "broken") {
+    signOut();
+    sessionCookie = null;
+  }
+
   // We use a promise to avoid multiple requests performing the refresh flow
   let refreshQueue: Promise<null> | undefined;