large test

Author: selsta

.dockerignore

@@ -1 +1,3 @@
 *
+!.github/
+!.github/qt_helper.py

.github/qt_helper.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+import argparse
+import fnmatch
 import hashlib
 import mmap
 import pathlib
@@ -10,7 +12,7 @@
 
 MAX_TRIES = 32
 
-def fetch_links_to_archives(os, target, major, minor, patch, toolchain):
+def fetch_links_to_archives(os, target, major, minor, patch, toolchain, packages):
   MAX_XML_SIZE = 1024 * 1024 * 1024
   MIRROR = 'download.qt.io'
   qt_dir = f'qt{major}_{major}{minor}{patch}'
@@ -30,12 +32,20 @@ def fetch_links_to_archives(os, target, major, minor, patch, toolchain):
   else:
     return
 
+  package_prefix = f'qt.qt{major}.{major}{minor}{patch}'
+  package_names = {
+    f'{package_prefix}.{package}.{toolchain}' if package else f'{package_prefix}.{toolchain}'
+    for package in packages
+  }
+
+  found_packages = set()
   for pkg in update_xml.findall('./PackageUpdate'):
     name = pkg.find('.//Name')
     if name == None:
       continue
-    if name.text != f'qt.qt{major}.{major}{minor}{patch}.{toolchain}':
+    if name.text not in package_names:
       continue
+    found_packages.add(name.text)
     version = pkg.find('.//Version')
     if version == None:
       continue
@@ -48,7 +58,11 @@ def fetch_links_to_archives(os, target, major, minor, patch, toolchain):
         continue
       url = f'{base_url}/{name.text}/{version.text}{archive}'
       file_name = pathlib.Path(urllib.parse.urlparse(url).path).name
-      yield {'name': file_name, 'url': url}
+      yield {'name': file_name, 'url': url, 'archive': archive}
+
+  missing_packages = package_names - found_packages
+  if missing_packages:
+    raise RuntimeError(f'Qt packages not found: {", ".join(sorted(missing_packages))}')
 
 def download(links):
   metalink = ET.Element('metalink', xmlns = "urn:ietf:params:xml:ns:metalink")
@@ -94,24 +108,47 @@ def calc_hash_sum(files):
 
 def extract_archives(files, out='.', targets=[]):
   for path in files:
-    if subprocess.Popen(['7z', 'x', '-bd', '-y', '-aoa', f'-o{out}', path] + targets,
+    print('extracting', path, flush=True)
+    if subprocess.Popen(['bsdtar', '-xf', path, '-C', out] + targets,
       stdout=subprocess.DEVNULL,
     ).wait() != 0:
       return False
   return True
 
 def main():
-  os, target, version, toolchain, expect = sys.argv[1:]
+  parser = argparse.ArgumentParser()
+  parser.add_argument('os')
+  parser.add_argument('target')
+  parser.add_argument('version')
+  parser.add_argument('toolchain')
+  parser.add_argument('expect')
+  parser.add_argument('--add-package', action='append', default=[],
+                      help='additional package below qt.qt<major>.<version>, such as addons.qtshadertools')
+  parser.add_argument('--archive', action='append', default=[],
+                      help='fnmatch pattern selecting archives from the requested packages')
+  args = parser.parse_args()
+
+  os, target, version, toolchain, expect = (
+    args.os, args.target, args.version, args.toolchain, args.expect
+  )
   major, minor, patch = version.split('.')
 
-  links = [*fetch_links_to_archives(os, target, major, minor, patch, toolchain)]
+  packages = [''] + args.add_package
+  links = [*fetch_links_to_archives(os, target, major, minor, patch, toolchain, packages)]
+  if args.archive:
+    links = [
+      link for link in links
+      if any(fnmatch.fnmatch(link['archive'], pattern) for pattern in args.archive)
+    ]
+  if not links:
+    raise RuntimeError('No Qt archives matched')
   print(*[l['url'].encode() for l in links], sep='\n', flush=True)
 
   assert download(links)
 
   result = calc_hash_sum([l['name'] for l in links])
   print('result', result, 'expect', expect, flush=True)
-  assert result == expect
+  assert expect == '-' or result == expect
 
   assert extract_archives([l['name'] for l in links], '.')
 

.github/workflows/build.yml

@@ -8,21 +8,21 @@ env:
 
 jobs:
   build-macos:
+    if: false
     runs-on: macos-latest
     steps:
     - uses: actions/checkout@v7
       with:
         submodules: recursive
     - name: install dependencies
       run: HOMEBREW_NO_AUTO_UPDATE=1 brew install boost ccache hidapi openssl zmq libsodium miniupnpc unbound libunwind-headers protobuf qt pkg-config
-    - name: configure
-      run: cmake -S . -B build/ci -DDEV_MODE=ON -DWITH_SCANNER=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$(brew --prefix qt)" -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
     - name: build
-      run: cmake --build build/ci --target monero-wallet-gui -j8
+      run: DEV_MODE=ON make release -j3
     - name: test qml
-      run: build/ci/bin/monero-wallet-gui.app/Contents/MacOS/monero-wallet-gui --test-qml
+      run: build/release/bin/monero-wallet-gui.app/Contents/MacOS/monero-wallet-gui --test-qml
 
   build-ubuntu:
+    if: false
     runs-on: ubuntu-26.04
     steps:
     - uses: actions/checkout@v7
@@ -38,15 +38,17 @@ jobs:
     - name: update apt
       run: sudo apt update
     - name: install monero dependencies
-      run: sudo apt -y install build-essential ccache cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler libgcrypt20-dev qt6-base-dev qt6-declarative-dev qt6-svg-dev qt6-tools-dev qt6-tools-dev-tools qml6-module-qtcore qml6-module-qtqml qml6-module-qtqml-models qml6-module-qtquick qml6-module-qtquick-controls qml6-module-qtquick-dialogs qml6-module-qtquick-effects qml6-module-qtquick-layouts qml6-module-qtquick-shapes qml6-module-qtquick-window qml6-module-qt-labs-folderlistmodel qml6-module-qt-labs-platform qml6-module-qtmultimedia xvfb libgl1 libegl1 libxkbcommon-x11-0 libxcb-cursor0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-render-util0
-    - name: configure
-      run: cmake -S . -B build/ci -DDEV_MODE=ON -DWITH_SCANNER=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+      run: sudo apt -y install build-essential ccache cmake libboost-all-dev miniupnpc libunbound-dev libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler libgcrypt20-dev qt6-base-dev qt6-declarative-dev qt6-svg-dev qt6-tools-dev qt6-tools-dev-tools qml6-module-qtcore qml6-module-qtqml qml6-module-qtqml-models qml6-module-qtquick qml6-module-qtquick-controls qml6-module-qtquick-dialogs qml6-module-qtquick-effects qml6-module-qtquick-layouts qml6-module-qtquick-shapes qml6-module-qtquick-window qml6-module-qt-labs-folderlistmodel qml6-module-qt-labs-platform
     - name: build
-      run: cmake --build build/ci --target monero-wallet-gui -j8
+      run: DEV_MODE=ON make release -j4
     - name: test qml
-      run: xvfb-run -a build/ci/bin/monero-wallet-gui --test-qml
+      env:
+        QT_QPA_PLATFORM: offscreen
+        QT_QUICK_BACKEND: software
+      run: build/release/bin/monero-wallet-gui --test-qml
 
   build-windows:
+    if: false
     runs-on: windows-latest
     defaults:
       run:
@@ -60,19 +62,21 @@ jobs:
         msystem: ucrt64
         update: true
         install: make
-        pacboy: toolchain:p ccache:p pcre:p cmake:p boost:p openssl:p zeromq:p libsodium:p hidapi:p protobuf:p libusb:p unbound:p git qt6-base:p qt6-declarative:p qt6-svg:p qt6-tools:p qt6-imageformats:p libgcrypt:p
+        pacboy: toolchain:p ccache:p pcre:p cmake:p boost:p openssl:p zeromq:p libsodium:p hidapi:p protobuf:p libusb:p unbound:p git qt6-base:p qt6-declarative:p qt6-svg:p qt6-tools:p libgcrypt:p
     - name: add qmlimportscanner.exe
       run: cp -f "$MSYSTEM_PREFIX/share/qt6/bin/qmlimportscanner.exe" "$MSYSTEM_PREFIX/bin/qmlimportscanner.exe"
     - name: configure
       run: cmake -S . -B build/ci -G "MSYS Makefiles" -DDEV_MODE=ON -DWITH_SCANNER=OFF -DARCH=x86-64 -DBUILD_64=ON -DBUILD_TAG=win-x64 -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
     - name: build
-      run: cmake --build build/ci --target monero-wallet-gui -j8
+      run: DEV_MODE=ON make release-win64 -j4
     - name: deploy
-      run: cmake --build build/ci --target deploy -j8
+      run: make deploy
+      working-directory: build/release
     - name: test qml
-      run: build/ci/bin/monero-wallet-gui --test-qml
+      run: build/release/bin/monero-wallet-gui --test-qml
 
   macos-bundle:
+    if: false
     name: macOS ${{ matrix.arch }} bundle
     runs-on: ${{ matrix.runner }}
     strategy:
@@ -87,8 +91,6 @@ jobs:
             artifact_name: macos-x64-bundle
             archive_name: monero-wallet-gui-macos-x64.tar.gz
             arch: Intel
-    env:
-      MACOSX_DEPLOYMENT_TARGET: "12.0"
     steps:
       - uses: actions/checkout@v7
         with:
@@ -100,54 +102,51 @@ jobs:
         working-directory: ../
       - name: configure
         run: >
-          cmake -S . -B build/bundle -G Ninja
-          -DDEV_MODE=ON
-          -DWITH_SCANNER=OFF
-          -DCMAKE_BUILD_TYPE=Release
-          -DCMAKE_PREFIX_PATH="/Users/runner/work/monero-gui/qt6.8.3/"
-          -DCMAKE_OSX_DEPLOYMENT_TARGET="${MACOSX_DEPLOYMENT_TARGET}"
+          cmake -S . -B build -G Ninja
+          -D CMAKE_BUILD_TYPE=Release
+          -D CMAKE_PREFIX_PATH="/Users/runner/work/monero-gui/qt6.8.3/"
+          -D CMAKE_OSX_DEPLOYMENT_TARGET="12.0"
+          -D ARCH=default
       - name: build
-        run: cmake --build build/bundle --target monero-wallet-gui --parallel 8
+        run: cmake --build build
       - name: deploy
-        run: cmake --build build/bundle --target deploy
+        run: cmake --build build --target deploy
       - name: test qml
-        run: build/bundle/bin/monero-wallet-gui.app/Contents/MacOS/monero-wallet-gui --test-qml
+        run: build/bin/monero-wallet-gui.app/Contents/MacOS/monero-wallet-gui --test-qml
       - name: create archive
-        run: tar -C build/bundle/bin -czf "${{ matrix.archive_name }}" monero-wallet-gui.app
+        run: tar -C build/bin -czf "${{ matrix.archive_name }}" monero-wallet-gui.app
       - uses: actions/upload-artifact@v7.0.1
         with:
           name: ${{ matrix.artifact_name }}
           path: ${{ matrix.archive_name }}
           archive: false
 
   docker-linux-static:
+    if: false
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v1
       with:
         submodules: recursive
-    - name: install dependencies
-      run: sudo apt -y install libopengl0 libglx0 libgl1
     - name: free up diskspace
       run: ${{env.FREE_DISKSPACE}}
     - name: prepare build environment
-      run: docker build --tag monero:build-env-linux --build-arg THREADS=3 --file Dockerfile.linux .
+      run: docker build --tag monero:build-env-linux --build-arg THREADS=4 --file Dockerfile.linux .
     - name: build
-      run: docker run --rm -v /home/runner/work/monero-gui/monero-gui:/monero-gui -w /monero-gui monero:build-env-linux sh -c 'make release-static -j3'
+      run: docker run --rm -v /home/runner/work/monero-gui/monero-gui:/monero-gui -w /monero-gui monero:build-env-linux sh -c 'make depends root=/depends target=x86_64-linux-gnu tag=linux-x64 -j4'
     - name: sha256sum
-      run: shasum -a256 /home/runner/work/monero-gui/monero-gui/build/release/bin/monero-wallet-gui
+      run: shasum -a256 /home/runner/work/monero-gui/monero-gui/build/x86_64-linux-gnu/release/bin/monero-wallet-gui
     - name: test qml
       env:
         QT_QPA_PLATFORM: offscreen
-        QT_QUICK_BACKEND: opengl
-        LIBGL_ALWAYS_SOFTWARE: "1"
-      run: /home/runner/work/monero-gui/monero-gui/build/release/bin/monero-wallet-gui --test-qml
+        QT_QUICK_BACKEND: software
+      run: /home/runner/work/monero-gui/monero-gui/build/x86_64-linux-gnu/release/bin/monero-wallet-gui --test-qml
     - uses: actions/upload-artifact@v4
       with:
         name: ${{ github.job }}
         path: |
-          /home/runner/work/monero-gui/monero-gui/build/release/bin/monero-wallet-gui
-          /home/runner/work/monero-gui/monero-gui/build/release/bin/monerod
+          /home/runner/work/monero-gui/monero-gui/build/x86_64-linux-gnu/release/bin/monero-wallet-gui
+          /home/runner/work/monero-gui/monero-gui/build/x86_64-linux-gnu/release/bin/monerod
 
   docker-windows-static:
     runs-on: ubuntu-22.04
@@ -158,9 +157,9 @@ jobs:
     - name: free up disk space
       run: ${{ env.FREE_DISKSPACE }}
     - name: prepare build environment
-      run: docker build --tag monero:build-env-windows --build-arg THREADS=3 --file Dockerfile.windows .
+      run: docker build --tag monero:build-env-windows --build-arg THREADS=4 --file Dockerfile.windows .
     - name: build
-      run: docker run --rm -v /home/runner/work/monero-gui/monero-gui:/monero-gui -w /monero-gui monero:build-env-windows sh -c 'make depends root=/depends target=x86_64-w64-mingw32 tag=win-x64 -j3'
+      run: docker run --rm -v /home/runner/work/monero-gui/monero-gui:/monero-gui -w /monero-gui monero:build-env-windows sh -c 'make depends root=/depends target=x86_64-w64-mingw32 tag=win-x64 -j4'
     - name: sha256sum
       run: shasum -a256 /home/runner/work/monero-gui/monero-gui/build/x86_64-w64-mingw32/release/bin/monero-wallet-gui.exe
     - uses: actions/upload-artifact@v4
@@ -179,9 +178,9 @@ jobs:
     - name: free up diskspace
       run: ${{env.FREE_DISKSPACE}}
     - name: prepare build environment
-      run: docker build --tag monero:build-env-android --build-arg THREADS=3 --file Dockerfile.android .
+      run: docker build --tag monero:build-env-android --build-arg THREADS=4 --file Dockerfile.android .
     - name: build
-      run: docker run --rm -v /home/runner/work/monero-gui/monero-gui:/monero-gui -e THREADS=3 monero:build-env-android
+      run: docker run --rm -v /home/runner/work/monero-gui/monero-gui:/monero-gui -e THREADS=4 monero:build-env-android
     - uses: actions/upload-artifact@v4
       with:
         name: ${{ github.job }}

Dockerfile.android

@@ -7,6 +7,8 @@ ARG ANDROID_SDK_REVISION=11076708_latest
 ARG ANDROID_SDK_HASH=2d2d50857e4eb553af5a6dc3ad507a17adf43d115264b1afc116f95c92e5e258
 ARG QT_VERSION=v6.8.3
 ARG QT_COMMIT=bab1fecd556ea561c4a89686293116741acfa1b4
+ARG QT_HOST_VERSION=6.8.3
+ARG QT_HOST_SHA256=8bbea76f5298021048f54c3e2934d7a73e2e6498ebc5a0cb30de16e24a66d847
 
 WORKDIR /opt/android
 ENV WORKDIR=/opt/android
@@ -18,15 +20,17 @@ ENV ANDROID_CLANGPP=aarch64-linux-android${ANDROID_NATIVE_API_LEVEL}-clang++
 ENV ANDROID_NDK_ROOT=${WORKDIR}/android-ndk-r${ANDROID_NDK_REVISION}
 ENV ANDROID_SDK_ROOT=${WORKDIR}/cmdline-tools
 ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64
-ENV PATH=${JAVA_HOME}/bin:${PATH}
+ENV PATH=${WORKDIR}/qt-host/bin:${WORKDIR}/qt-host/libexec:${JAVA_HOME}/bin:${PATH}
 ENV PREFIX=${WORKDIR}/prefix
 ENV QT_HOST_PREFIX=${WORKDIR}/qt-host
+ENV LD_LIBRARY_PATH=${QT_HOST_PREFIX}/lib
 ENV TOOLCHAIN_DIR=${ANDROID_NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update \
-    && apt-get install -y ant automake build-essential ca-certificates-java file gettext git libc6 libncurses5 \
-    libssl-dev libstdc++6 libtinfo5 libtool libz1 ninja-build openjdk-17-jdk-headless \
+    && apt-get install -y ant aria2 automake build-essential ca-certificates-java file gettext git libc6 libncurses5 \
+    libdbus-1-3 libegl1 libfontconfig1 libfreetype6 libgl1 libglib2.0-0 libgssapi-krb5-2 libssl-dev \
+    libstdc++6 libtinfo5 libtool libx11-6 libxkbcommon0 libz1 libzstd1 libarchive-tools ninja-build openjdk-17-jdk-headless \
     openjdk-17-jre-headless pkg-config python3 unzip wget \
     && rm -rf /var/lib/apt/lists/*
 
@@ -59,6 +63,25 @@ RUN git clone -b v4.3.3 --depth 1 https://github.com/Kitware/CMake \
 ENV HOST_PATH=${PATH}
 ENV PATH=${TOOLCHAIN_DIR}/aarch64-linux-android/bin:${TOOLCHAIN_DIR}/bin:${PATH}
 
+COPY .github/qt_helper.py /tmp/qt_helper.py
+
+RUN mkdir -p "${QT_HOST_PREFIX}" \
+    && cd "${QT_HOST_PREFIX}" \
+    && python3 /tmp/qt_helper.py linux_x64 desktop "${QT_HOST_VERSION}" linux_gcc_64 "${QT_HOST_SHA256}" \
+        --add-package addons.qtshadertools \
+        --archive 'qtbase-*.7z' \
+        --archive 'qtdeclarative-*.7z' \
+        --archive 'qttools-*.7z' \
+        --archive 'icu-*.7z' \
+        --archive 'qtshadertools-*.7z' \
+    && find "${QT_HOST_PREFIX}" -name 'libicu*.so*' -print \ # TODEL
+    && rm /tmp/qt_helper.py \
+    && echo "${QT_HOST_PREFIX}/lib" > /etc/ld.so.conf.d/qt-host.conf \
+    && ldconfig \
+    && ldd "${QT_HOST_PREFIX}/libexec/qvkgen" | tee /tmp/qvkgen.ldd \
+    && ! grep -q 'not found' /tmp/qvkgen.ldd \
+    && rm /tmp/qvkgen.ldd
+
 ARG ZLIB_VERSION=1.3.1
 ARG ZLIB_HASH=9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23
 RUN wget -q https://github.com/madler/zlib/releases/download/v${ZLIB_VERSION}/zlib-${ZLIB_VERSION}.tar.gz \
@@ -73,25 +96,10 @@ RUN wget -q https://github.com/madler/zlib/releases/download/v${ZLIB_VERSION}/zl
 
 RUN git clone https://code.qt.io/qt/qt5.git -b ${QT_VERSION} --depth 1 qt-sources \
     && git -C qt-sources reset --hard ${QT_COMMIT} \
-    && mkdir qt-build-host \
-    && cd qt-build-host \
-    && ../qt-sources/configure -init-submodules \
-        -submodules qtbase,qtdeclarative,qtimageformats,qtmultimedia,qtshadertools,qtsvg,qttools \
-        -prefix ${QT_HOST_PREFIX} \
-        -opensource -confirm-license -release \
-        -no-opengl -no-dbus -no-sql-sqlite \
-        -no-feature-assistant -no-feature-designer \
-        -no-feature-distancefieldgenerator -no-feature-pixeltool \
-        -no-feature-qtattributionsscanner -no-feature-qtdiag \
-        -no-feature-qtplugininfo \
-        -nomake examples -nomake tests \
-    && cmake --build . --parallel ${THREADS} \
-    && cmake --install . \
-    && cd .. \
     && mkdir qt-build-android \
     && cd qt-build-android \
-    && PATH=${HOST_PATH} ../qt-sources/configure \
-        -submodules qtbase,qtdeclarative,qtimageformats,qtmultimedia,qtshadertools,qtsvg,qttools \
+    && PATH=${HOST_PATH} ../qt-sources/configure -init-submodules \
+        -submodules qtbase,qtdeclarative,qtshadertools,qtsvg,qttools \
         -prefix ${PREFIX} \
         -qt-host-path ${QT_HOST_PREFIX} \
         -android-ndk ${ANDROID_NDK_ROOT} \
@@ -109,7 +117,7 @@ RUN git clone https://code.qt.io/qt/qt5.git -b ${QT_VERSION} --depth 1 qt-source
     && PATH=${HOST_PATH} cmake --build . --parallel ${THREADS} \
     && PATH=${HOST_PATH} cmake --install . \
     && cd .. \
-    && rm -rf qt-build-host qt-build-android qt-sources
+    && rm -rf qt-build-android qt-sources
 
 ARG ICONV_VERSION=1.18
 ARG ICONV_HASH=3b08f5f4f9b4eb82f151a7040bfd6fe6c6fb922efe4b1659c66ea933276965e8