feat: high level wrapper around Docker API

README.md

@@ -346,6 +346,39 @@ docker run \
   -d nextcloud-appapi-harp:local
 ```
 
+#### Debugging HaRP
+
+##### One time initializing steps:
+
+1. Create virtual environment
+2. Install `pydantic` (you can look at exact version in the **Dockerfile*) and `git+https://github.com/cloud-py-api/haproxy-python-spoa.git`
+3. Set next environment variables for running `haproxy_agent.py` script:
+    ```
+    HP_LOG_LEVEL=info;NC_INSTANCE_URL=http://nextcloud.local;HP_SHARED_KEY=some_very_secure_password;HP_FRP_DISABLE_TLS=true
+    ```
+4. Create folder `dev` at the root of repository, extract there content of the desired archive with the [FRP](https://github.com/fatedier/frp/releases/latest) archive which is located at `exapps_dev` folder of this repo.
+5. Edit the `data/nginx/vhost.d/nextcloud.local_location` file from the `nextcloud-docker-dev` to point `/exapps/` web route to the host:
+    ```
+    proxy_pass http://172.17.0.1:8780;
+    ```
+
+    > **Note:** my original content from my dev machine of file `nextcloud.local_location`:
+    > ```nginx
+    > location /exapps/ {
+    >   proxy_pass http://172.17.0.1:8780;
+    > }
+    > ```
+6. Use `docker compose up -d --force-recreate proxy` command from Julius `nextcloud-docker-dev` to recreate the proxy container.
+7. Register `HaRP` from the **Host** template. Replace `localhost` with `host.docker.internal` in `HaRP Host` field.
+
+##### Steps to run all parts of HaRP after initializing:
+
+1. Run FRP Server with `./dev/frps -c ./development/debugging/frps.toml` command.
+2. Run the FRP Client to connect Docker Engine to the FRP Server with `./dev/frpc -c ./development/debugging/frpc.toml` command.
+3. Run `./development/debugging/redeploy_haproxy_host.sh` command to redeploy `appapi-harp` container **with HaProxy only**.
+
+> **Note:** Existing `appapi-harp` container will be removed.
+
 ## Contributing
 
 Contributions to HaRP are welcome. Feel free to open issues, discussions or submit pull requests with improvements, bug fixes, or new features.

development/debugging/Dockerfile

@@ -0,0 +1,49 @@
+# SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+FROM haproxy:3.1.2-alpine3.21
+
+USER root
+
+# Bind addresses for 2 frontends (HTTP + HTTPS for exapps) and FRP Server.
+# If /certs/cert.pem does not exist, EXAPPS HTTPS frontend are disabled automatically.
+ENV HP_EXAPPS_ADDRESS="0.0.0.0:8780" \
+    HP_EXAPPS_HTTPS_ADDRESS="0.0.0.0:8781" \
+    HP_FRP_ADDRESS="0.0.0.0:8782" \
+    HP_FRP_DISABLE_TLS="false" \
+    HP_TIMEOUT_CONNECT="30s" \
+    HP_TIMEOUT_CLIENT="30s" \
+    HP_TIMEOUT_SERVER="1800s" \
+    NC_INSTANCE_URL="" \
+    HP_LOG_LEVEL="warning"
+
+RUN set -ex; \
+    apk add --no-cache \
+        git \
+        ca-certificates \
+        tzdata \
+        bash \
+        curl \
+        openssl \
+        bind-tools \
+        nano \
+        vim \
+        envsubst \
+        frp \
+        python3 \
+        py3-pip \
+        py3-aiohttp \
+        wget \
+        tar \
+        netcat-openbsd; \
+    chmod -R 777 /tmp;
+
+# Main haproxy config template
+COPY --chmod=664 ./development/debugging/haproxy.cfg /haproxy.cfg
+
+# SPOE config
+COPY --chmod=664 spoe-agent.conf /etc/haproxy/spoe-agent.conf
+
+ENTRYPOINT ["haproxy", "-f", "/haproxy.cfg", "-W", "-db"]
+
+LABEL com.centurylinklabs.watchtower.enable="false"

development/debugging/frpc.toml

@@ -0,0 +1,18 @@
+# SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+serverAddr = "127.0.0.1"                         # Replace with your HP_FRP_ADDRESS host
+serverPort = 8782                                # Default port for FRP or the port your reverse proxy listens on
+loginFailExit = false                            # If the FRP (HaRP) server is unavailable, continue trying to log in.
+
+metadatas.token = "some_very_secure_password"
+
+log.level = "info"
+
+[[proxies]]
+remotePort = 24000                               # we set it to 24000 as it is the basic Docker Engine
+name = "bundled-deploy-daemon"                   # Unique name for each Docker Engine
+type = "tcp"
+[proxies.plugin]
+type = "unix_domain_socket"
+unixPath = "/var/run/docker.sock"

development/debugging/frps.toml

@@ -0,0 +1,20 @@
+# SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+bindAddr = "127.0.0.1"
+bindPort = 8782
+
+transport.tls.force = false
+
+log.level = "info"
+
+maxPortsPerClient = 1
+allowPorts = [
+  { start = 23000, end = 23999 },
+  { start = 24000, end = 24099 }
+]
+
+[[httpPlugins]]
+addr = "127.0.0.1:8200"
+path = "/frp_handler"
+ops = ["Login"]

development/debugging/haproxy.cfg

@@ -0,0 +1,94 @@
+# SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+global
+    log stdout local0 info
+    maxconn 8192
+    ca-base /etc/ssl/certs
+
+defaults
+    log global
+    option httplog
+    option dontlognull
+    timeout connect 30s
+    timeout client 30s
+    timeout server 1800s
+
+
+###############################################################################
+# FRONTEND: ex_apps (HTTP)
+###############################################################################
+frontend ex_apps
+    mode http
+    bind 0.0.0.0:8780
+
+    filter spoe engine exapps-spoe config /etc/haproxy/spoe-agent.conf
+    http-request silent-drop if { var(txn.exapps.bad_request) -m int eq 1 }
+    http-request return status 401 content-type text/plain string "401 Unauthorized" if { var(txn.exapps.unauthorized) -m int eq 1 }
+    http-request return status 403 content-type text/plain string "403 Forbidden" if { var(txn.exapps.forbidden) -m int eq 1 }
+    http-request return status 404 content-type text/plain string "404 Not Found" if { var(txn.exapps.not_found) -m int eq 1 }
+    use_backend %[var(txn.exapps.backend)]
+
+###############################################################################
+# BACKENDS: ex_apps & ex_apps_backend_w_bruteforce
+###############################################################################
+backend ex_apps_backend
+    mode http
+    server frp_server 0.0.0.0
+    http-request set-path %[var(txn.exapps.target_path)]
+    http-request set-dst var(txn.exapps.target_ip)
+    http-request set-dst-port var(txn.exapps.target_port)
+    http-request set-header EX-APP-ID %[var(txn.exapps.exapp_id)]
+    http-request set-header EX-APP-VERSION %[var(txn.exapps.exapp_version)]
+    http-request set-header AUTHORIZATION-APP-API %[var(txn.exapps.exapp_token)]
+    http-request set-header AA-VERSION "32"  # TO-DO: temporary, remove it after we update all ExApps.
+
+backend ex_apps_backend_w_bruteforce
+    mode http
+    server frp_server 0.0.0.0
+    http-request set-path %[var(txn.exapps.target_path)]
+    http-request set-dst var(txn.exapps.target_ip)
+    http-request set-dst-port var(txn.exapps.target_port)
+    http-request set-header EX-APP-ID %[var(txn.exapps.exapp_id)]
+    http-request set-header EX-APP-VERSION %[var(txn.exapps.exapp_version)]
+    http-request set-header AUTHORIZATION-APP-API %[var(txn.exapps.exapp_token)]
+    http-request set-header AA-VERSION "32"  # TO-DO: temporary, remove it after we update all ExApps.
+    filter spoe engine exapps-bruteforce-protection-spoe config /etc/haproxy/spoe-agent.conf
+
+###############################################################################
+# BACKEND: nextcloud_control (HTTP)
+###############################################################################
+backend nextcloud_control_backend
+    mode http
+    server nextcloud_control 127.0.0.1:8200
+    http-request set-path %[var(txn.exapps.target_path)]
+
+###############################################################################
+# BACKEND: docker_engine (HTTP)
+###############################################################################
+backend docker_engine_backend
+    mode http
+    server frp_server 127.0.0.1
+    http-request set-dst-port var(txn.exapps.target_port)
+    http-request set-path %[var(txn.exapps.target_path)]
+
+    # docker system _ping
+    http-request allow if { path,url_dec -m reg -i ^(/v[\d\.]+)?/_ping$ } METH_GET
+    # docker inspect image
+    http-request allow if { path,url_dec -m reg -i ^(/v[\d\.]+)?/images/.*/json } METH_GET
+    # container inspect: GET containers/%s/json
+    http-request allow if { path,url_dec -m reg -i ^(/v[\d\.]+)?/containers/nc_app_[a-zA-Z0-9_.-]+/json } METH_GET
+    # container inspect: GET containers/%s/logs
+    http-request allow if { path,url_dec -m reg -i ^(/v[\d\.]+)?/containers/nc_app_[a-zA-Z0-9_.-]+/logs } METH_GET
+
+    # image pull: POST images/create?fromImage=%s
+    http-request allow if { path,url_dec -m reg -i ^(/v[\d\.]+)?/images/create } METH_POST
+    http-request deny
+
+
+backend agents
+    mode tcp
+    timeout connect 5s
+    timeout server  3m
+    option spop-check
+    server agent1 127.0.0.1:9600 check

development/debugging/redeploy_haproxy_host.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+# SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+# This file can be used for development for the "manual install" deployment type when FRP is disabled.
+# For Julius Docker-Dev, you need to additionally edit the `data/nginx/vhost.d/nextcloud.local_location` file,
+# changing `appapi-harp` to `172.17.0.1` and restart the "proxy" container.
+
+docker container remove --force appapi-harp
+
+docker build -f development/debugging/Dockerfile -t nextcloud-appapi-harp:debug .
+
+docker run \
+  --name appapi-harp -h appapi-harp \
+  --restart unless-stopped \
+  --network=host \
+  -d nextcloud-appapi-harp:debug

haproxy.cfg.template

@@ -108,44 +108,7 @@ backend docker_engine_backend
     http-request allow if { path,url_dec -m reg -i ^(/v[\d\.]+)?/containers/nc_app_[a-zA-Z0-9_.-]+/json } METH_GET
     # container inspect: GET containers/%s/logs
     http-request allow if { path,url_dec -m reg -i ^(/v[\d\.]+)?/containers/nc_app_[a-zA-Z0-9_.-]+/logs } METH_GET
-    # container start/stop: POST containers/%s/start containers/%s/stop
-    http-request allow if { path,url_dec -m reg -i ^(/v[\d\.]+)?/containers/nc_app_[a-zA-Z0-9_.-]+/((start)|(stop)) } METH_POST
-    # container rm: DELETE containers/%s
-    http-request allow if { path,url_dec -m reg -i ^(/v[\d\.]+)?/containers/nc_app_[a-zA-Z0-9_.-]+ } METH_DELETE
-    # container update/exec: POST containers/%s/update containers/%s/exec
-    http-request allow if { path,url_dec -m reg -i ^(/v[\d\.]+)?/containers/nc_app_[a-zA-Z0-9_.-]+/((update)|(exec)) } METH_POST
-    # container put: PUT containers/%s/archive
-    http-request allow if { path,url_dec -m reg -i ^(/v[\d\.]+)?/containers/nc_app_[a-zA-Z0-9_.-]+/archive } METH_PUT
-    # run exec instance: POST exec/%s
-    http-request allow if { path,url_dec -m reg -i ^(/v[\d\.]+)?/exec/[a-zA-Z0-9_.-]+/start } METH_POST
-
-    # container create: POST containers/create?name=%s
-    # ACL to restrict container name to nc_app_[a-zA-Z0-9_.-]+
-    acl nc_app_container_name url_param(name) -m reg -i "^nc_app_[a-zA-Z0-9_.-]+"
-
-    # ACL to restrict the number of Mounts to 1
-    acl one_mount_volume req.body -m reg -i "\"Mounts\"\s*:\s*\[\s*(?:(?!\"Mounts\"\s*:\s*\[)[^}]*)}[^}]*\]"
-    # ACL to deny if there are any binds
-    acl binds_present req.body -m reg -i "\"HostConfig\"\s*:.*\"Binds\"\s*:"
-    # ACL to restrict the type of Mounts to volume
-    acl type_not_volume req.body -m reg -i "\"Mounts\":\s*\[[^\]]*(\"Type\":\s*\"(?!volume\b)\w+\"[^\]]*)+\]"
-    http-request deny if { path,url_dec -m reg -i ^(/v[\d\.]+)?/containers/create } nc_app_container_name !one_mount_volume binds_present type_not_volume METH_POST
-
-    # ACL to restrict container creation, that it has HostConfig.Privileged(by searching for "Privileged" word in all payload) not set
-    acl no_privileged_flag req.body -m reg -i "\"Privileged\""
-    # ACL to allow mount volume with strict pattern for name: nc_app_[a-zA-Z0-9_.-]+_data
-    acl nc_app_volume_data_only req.body -m reg -i "\"Mounts\":\s?\[\s?{[^}]*\"Source\":\s?\"nc_app_[a-zA-Z0-9_.-]+_data\""
-    http-request allow if { path,url_dec -m reg -i ^(/v[\d\.]+)?/containers/create } nc_app_container_name !no_privileged_flag nc_app_volume_data_only METH_POST
-    # end of container create
-
-    # volume create: POST volumes/create
-    # restrict name
-    acl nc_app_volume_data req.body -m reg -i "\"Name\":\s?\"nc_app_[a-zA-Z0-9_.-]+_data\""
-    # do not allow to use "device" word e.g., "--opt device=:/path/to/dir"
-    acl volume_no_device req.body -m reg -i "\"device\""
-    http-request allow if { path,url_dec -m reg -i ^(/v[\d\.]+)?/volumes/create } nc_app_volume_data !volume_no_device METH_POST
-    # volume rm: DELETE volumes/%s
-    http-request allow if { path,url_dec -m reg -i ^(/v[\d\.]+)?/volumes/nc_app_[a-zA-Z0-9_.-]+_data } METH_DELETE
+
     # image pull: POST images/create?fromImage=%s
     http-request allow if { path,url_dec -m reg -i ^(/v[\d\.]+)?/images/create } METH_POST
     http-request deny