Skip to content

Commit 76bbc6f

Browse files
NotYuShengNotYuSheng
authored
Merge pull request #129 from NotYuSheng/dev
Merge
2 parents a421562 + a859a38 commit 76bbc6f

56 files changed

Lines changed: 1485 additions & 500 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/helm.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: Helm Lint
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- 'helm/**'
7+
push:
8+
paths:
9+
- 'helm/**'
10+
11+
jobs:
12+
helm-lint:
13+
name: Lint Helm Charts
14+
runs-on: ubuntu-latest
15+
16+
steps:
17+
- name: Checkout repo
18+
uses: actions/checkout@v4
19+
20+
- name: Set up Helm
21+
uses: azure/setup-helm@v4
22+
with:
23+
version: v3.18.3
24+
25+
- name: Lint all Helm charts in ./helm/
26+
run: |
27+
for chart in helm/*/; do
28+
echo "Linting chart: $chart"
29+
helm lint "$chart" || exit 1
30+
done

‎.github/workflows/integration-test.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,10 @@ jobs:
5454

5555
for NAME in $SERVICE_NAMES; do
5656
PREFIX=${NAME%_service}
57-
echo "Waiting for $NAME via http://localhost/$PREFIX/health..."
57+
echo "Waiting for $NAME via http://localhost:8080/$PREFIX/health..."
5858

5959
for i in {1..30}; do
60-
STATUS=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost/$PREFIX/health")
60+
STATUS=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:8080/$PREFIX/health")
6161
if [ "$STATUS" == "200" ]; then
6262
echo "$NAME is healthy"
6363
break

‎.github/workflows/publish-ghcr.yml‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
name: Publish to GHCR
2+
3+
on:
4+
# push:
5+
# branches: [dev]
6+
# release:
7+
# types: [created]
8+
workflow_dispatch:
9+
# pull_request:
10+
# branches:
11+
# - dev
12+
13+
jobs:
14+
build-and-push:
15+
runs-on: ubuntu-latest
16+
permissions:
17+
packages: write
18+
19+
steps:
20+
- name: Maximize build space
21+
uses: AdityaGarg8/remove-unwanted-software@v4.1
22+
with:
23+
remove-android: true
24+
remove-haskell: true
25+
remove-codeql: true
26+
27+
- name: Checkout repository
28+
uses: actions/checkout@v3
29+
30+
- name: Set up Docker Buildx
31+
uses: docker/setup-buildx-action@v3
32+
33+
- name: Set up QEMU
34+
uses: docker/setup-qemu-action@v3
35+
36+
- name: Extract metadata
37+
id: meta
38+
run: |
39+
SHORT_SHA=$(git rev-parse --short HEAD)
40+
echo "SHORT_SHA=$SHORT_SHA" >> $GITHUB_ENV
41+
42+
if [[ "${GITHUB_EVENT_NAME}" == "release" ]]; then
43+
TAG_NAME="${{ github.event.release.tag_name }}"
44+
echo "GIT_TAG=$TAG_NAME" >> $GITHUB_ENV
45+
echo "IS_RELEASE=true" >> $GITHUB_ENV
46+
else
47+
git fetch --tags
48+
LATEST_TAG=$(git tag --list 'v*.*.*' | sort -V | tail -n1)
49+
FALLBACK_TAG="${LATEST_TAG:-v0.0.0}"
50+
51+
PREFIX="dev"
52+
echo "GIT_TAG=${PREFIX}-${FALLBACK_TAG}-${SHORT_SHA}" >> $GITHUB_ENV
53+
echo "IS_RELEASE=false" >> $GITHUB_ENV
54+
fi
55+
56+
echo "REPO_OWNER_LC=${GITHUB_REPOSITORY_OWNER,,}" >> $GITHUB_ENV
57+
58+
- name: Log in to GitHub Container Registry
59+
uses: docker/login-action@v2
60+
with:
61+
registry: ghcr.io
62+
username: ${{ github.actor }}
63+
password: ${{ secrets.GITHUB_TOKEN }}
64+
65+
- name: Copy example.env to .env
66+
run: |
67+
echo "Copying example.env to .env..."
68+
find . -name "example.env" | while read f; do
69+
cp "$f" "$(dirname "$f")/.env"
70+
echo "Created $(dirname "$f")/.env"
71+
done
72+
73+
- name: Build, tag, and push all *_service Docker Compose services
74+
run: |
75+
set -e
76+
77+
SERVICES=$(docker compose config --services | grep '_service$')
78+
echo "Filtered services: $SERVICES"
79+
80+
for SERVICE in $SERVICES; do
81+
IMAGE_NAME="ghcr.io/${{ env.REPO_OWNER_LC }}/$SERVICE:${{ env.GIT_TAG }}"
82+
echo "Building $SERVICE"
83+
docker compose build $SERVICE
84+
85+
COMPOSE_IMAGE="omnipdf-${SERVICE}:latest"
86+
IMAGE_ID=$(docker image ls --filter=reference="${COMPOSE_IMAGE}" --format "{{.ID}}" | head -n 1)
87+
88+
if [[ -z "$IMAGE_ID" ]]; then
89+
echo "Build failed: No image found for $SERVICE (expected $COMPOSE_IMAGE)"
90+
docker image ls
91+
exit 1
92+
fi
93+
94+
echo "Tagging $IMAGE_ID as $IMAGE_NAME"
95+
docker tag "$IMAGE_ID" "$IMAGE_NAME"
96+
97+
echo "Pushing $IMAGE_NAME"
98+
docker push "$IMAGE_NAME"
99+
100+
if [[ "${{ env.IS_RELEASE }}" == "true" ]]; then
101+
LATEST_TAG="ghcr.io/${{ env.REPO_OWNER_LC }}/$SERVICE:latest"
102+
echo "Tagging and pushing as $LATEST_TAG"
103+
docker tag "$IMAGE_ID" "$LATEST_TAG"
104+
docker push "$LATEST_TAG"
105+
fi
106+
done

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,6 @@ build/
3636
# Uploaded files (if saved locally during dev)
3737
uploads/
3838
data/
39+
40+
llm_models/
41+
.llm_models

‎LICENSE‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2025 WebWork
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

‎chat_service/routers/chat.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,19 @@
11
from fastapi import APIRouter, HTTPException, Depends
22
from fastapi.concurrency import run_in_threadpool
33
from openai import OpenAI, APIError
4-
from shared_utils.client import get_openai_client
4+
from shared_utils.openai_client import get_openai_client
55
import logging
66
import os
77
from models.chat import ChatRequest
88

9-
router = APIRouter(prefix="/chat")
9+
router = APIRouter()
1010
logger = logging.getLogger(__name__)
1111

1212
_OPENAI_MODEL_DEFAULT = "qwen2.5-0.5b-instruct"
1313
OPENAI_MODEL_NAME = os.getenv("OPENAI_MODEL", _OPENAI_MODEL_DEFAULT)
1414

1515

16-
@router.post("/", status_code=201)
16+
@router.post("/chat", status_code=201)
1717
async def handle_chat(
1818
chat_request: ChatRequest,
1919
client: OpenAI = Depends(get_openai_client),

‎cleaner/Dockerfile‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
FROM python:3.13-slim
2+
3+
WORKDIR /app
4+
5+
ENV PYTHONPATH=/app/cleaner
6+
7+
RUN apt-get update && apt-get install -y build-essential && rm -rf /var/lib/apt/lists/*
8+
9+
# Copy requirements.txt from cleaner folder in root context
10+
COPY cleaner/requirements.txt .
11+
12+
RUN pip install --no-cache-dir -r requirements.txt
13+
14+
# Copy utils folder from root context into /app/utils
15+
COPY shared_utils ./cleaner/shared_utils
16+
17+
# Copy cleaner folder from root context into /app/cleaner
18+
COPY cleaner ./cleaner
19+
20+
CMD ["python3", "cleaner/main.py"]

‎cleaner/example.env‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# MinIO / S3-compatible storage
2+
MINIO_ENDPOINT=http://minio:9000
3+
MINIO_BUCKET=omnifiles
4+
MINIO_ACCESS_KEY=minioadmin
5+
MINIO_SECRET_KEY=minioadmin
6+
7+
# Redis storage
8+
REDIS_URL="redis://redis:6379/0?decode_responses=True&protocol=3"

‎cleaner/main.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import logging
2+
import signal
3+
4+
from utils.cleaner import setup_redis_watcher_thread
5+
6+
# Set up logger
7+
logging.basicConfig(
8+
level=logging.INFO,
9+
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s"
10+
)
11+
logger = logging.getLogger(__name__)
12+
13+
def main():
14+
# load the redis watcher
15+
watcher_thread = setup_redis_watcher_thread()
16+
logger.info(f"{watcher_thread}")
17+
18+
# Setup Graceful shutdown
19+
def exit_gracefully(signum, frame):
20+
watcher_thread.stop()
21+
signal.signal(signal.SIGINT, exit_gracefully)
22+
signal.signal(signal.SIGTERM, exit_gracefully)
23+
24+
# Wait till the thread is no longer running
25+
watcher_thread.join()
26+
logger.info("Cleaner stopped gracefully")
27+
28+
29+
if __name__ == "__main__":
30+
main()

‎cleaner/requirements.txt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
annotated-types==0.7.0
2+
async-timeout==5.0.1
3+
boto3==1.38.34
4+
botocore==1.38.36
5+
jmespath==1.0.1
6+
pydantic==2.11.7
7+
pydantic_core==2.33.2
8+
python-dateutil==2.9.0.post0
9+
redis==6.2.0
10+
s3transfer==0.13.0
11+
six==1.17.0
12+
typing_extensions==4.14.0
13+
urllib3==2.4.0

0 commit comments

Comments
 (0)