Start setup for search index in CI

Author: Hoff97

.github/workflows/develop.yml

@@ -6,43 +6,52 @@ on:
       - "develop"
 
 jobs:
-  clippy:
+  build-search-index:
+    name: "Build search index"
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+      - uses: prefix-dev/setup-pixi@v0.9.2
+        with:
+          run-install: false
       - name: Install toolchain
         run: rustup toolchain install stable
-      - name: Install clippy
-        run: rustup component add clippy
-      - name: Run clippy
+      - name: Configure SSH
         run: |
-          cd backend-rust
-          cargo clippy --all-targets --all-features -- -D warnings
+          mkdir -p ~/.ssh/
+          echo "$SSH_KEY" > ~/.ssh/prod.key
+          chmod 600 ~/.ssh/prod.key
+          cat >>~/.ssh/config <<END
+          Host prod
+            HostName $SSH_HOST
+            User $SSH_USER
+            IdentityFile ~/.ssh/prod.key
+            StrictHostKeyChecking no
+          END
+        env:
+          SSH_USER: ${{ secrets.SSH_USERNAME }}
+          SSH_KEY: ${{ secrets.SSHKEY }}
+          SSH_HOST: ${{ secrets.HOST }}
 
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Run tests
+      - name: Download index data
+        run: |
+          mkdir -p search_preprocess/data
+          cd search_preprocess/data
+          ../download_data.sh
+          ls -lh
+      - name: Build search index
         run: |
-          cd backend-rust
-          ./download_test_data.sh
-          cargo test --all-targets
+          cd search_preprocess
+          pixi install
+          pixi run python src/preprocess_search_data.py
+          pixi run python src/filter_search_data.py
+          pixi run python src/preprocess_flying_site_data.py
+          mkdir -p ../backend-rust/data
+          cp ./data/search_data_*.jsonl ../backend-rust/data/
+          cd ../backend-rust
+          cargo run --bin preprocess_search_index --release
 
-  docker:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build docker image
-        uses: docker/build-push-action@v4
-        with:
-          push: true
-          tags: hoff97/hikeandfly:develop
-          cache-from: type=registry,ref=hoff97/hikeandfly:develop
-          cache-to: type=inline
+          ls -lh data/
+      - name: Upload search index
+        run: |
+          scp -o StrictHostKeyChecking=no data/search_data_flying_sites.jsonl hikeandfly:~/data

search_preprocess/download_data.sh

@@ -1,10 +1,6 @@
 wget https://download1.graphhopper.com/public/europe/austria/photon-dump-austria-0.7-latest.jsonl.zst
-wget https://download1.graphhopper.com/public/europe/italy/photon-dump-italy-0.7-latest.jsonl.zst
-wget https://download1.graphhopper.com/public/europe/germany/photon-dump-germany-0.7-latest.jsonl.zst
-wget https://download1.graphhopper.com/public/europe/france-monacco/photon-dump-france-monacco-0.7-latest.jsonl.zst
-wget https://download1.graphhopper.com/public/europe/slovenia/photon-dump-slovenia-0.7-latest.jsonl.zst
 
 zstd -d photon-dump-*.jsonl.zst
 
-wget https://service.dhv.de/dbfiles/managed/gelaendedaten/dhvxml/dhvgelaende_dhvxml_alle.zip?format=dhvxml&land=alle&nutzung=on
-unzip unzip dhvgelaende_dhvxml_alle.zip\?format=dhvxml\&land=alle\&nutzung=on
+wget "https://service.dhv.de/dbfiles/managed/gelaendedaten/dhvxml/dhvgelaende_dhvxml_alle.zip?format=dhvxml&land=alle&nutzung=on"
+unzip "dhvgelaende_dhvxml_alle.zip?format=dhvxml&land=alle&nutzung=on"

search_preprocess/pixi.toml

@@ -2,7 +2,7 @@
 authors = ["Frithjof Winkelmann <fwinkelmann97@gmail.com>"]
 channels = ["conda-forge"]
 name = "search_preprocess"
-platforms = ["osx-arm64"]
+platforms = ["osx-arm64", "osx-64", "linux-64", "linux-aarch64", "win-64"]
 version = "0.1.0"
 
 [tasks]

search_preprocess/src/preprocess_search_data.py

@@ -210,26 +210,7 @@ def preprocess_data():
             else:
                 raise ValueError(f"Unsupported object type: {object_data['type']}")
 
-        c = list(categories.items())
-        c.sort(key=lambda x: x[1], reverse=True)
-        print(f"Found categories: {c}")
-        categories_contained_once = [k for k, v in categories.items() if v == 1]
-
-        second_level = defaultdict(int)
-        for name, count in categories.items():
-            splitted = name.split(".")
-            second_level[".".join(splitted[:2])] += count
-        sl = list(second_level.items())
-        sl.sort(key=lambda x: x[1], reverse=True)
-        print(f"Found categories: {sl}")
-
         df = pl.DataFrame(result)
-        df = df.filter(
-            pl.col("categories")
-            .list.set_intersection(categories_contained_once)
-            .list.len()
-            == 0
-        )
         print(df)
         df.write_ndjson(f"data/augmented_search_data_{region}.jsonl")
         df = df.drop("categories")