Add docker custum

.github/workflows/cicd_docker.yml

@@ -30,8 +30,13 @@ jobs:
       id: build
       uses: docker/build-push-action@v5
       with:
+        file: ./Dockerfile.pdal
         load: true
         tags: ${{ env.TEST_TAG }}
+        build-args: |
+          GITHUB_REPOSITORY=alavenant/PDAL
+          GITHUB_SHA=master_28_05_25
+
 
     # run the test on the docker build
     - name: Run tests in docker image
@@ -63,7 +68,12 @@ jobs:
       if: ${{ github.event_name != 'pull_request' }}
       uses: docker/build-push-action@v5
       with:
+        file: ./Dockerfile.pdal
         context: .
         push: true
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
+        build-args: |
+          GITHUB_REPOSITORY=alavenant/PDAL
+          GITHUB_SHA=master_28_05_25
+

.github/workflows/cicd_test.yml

@@ -30,6 +30,7 @@ jobs:
         cache-environment: false
         cache-downloads: true
         generate-run-shell: true
+        shell: bash
 
     - name: compile_plugins
       shell: micromamba-shell {0}
@@ -45,4 +46,4 @@ jobs:
       run: |
         export PDAL_DRIVER_PATH=./install/lib
         echo $PDAL_DRIVER_PATH
-        python -m pytest -s
+        python -m pytest -s -m "not pdal_custom"

Dockerfile.pdal

@@ -0,0 +1,110 @@
+FROM condaforge/mambaforge:latest AS mamba_pdal
+
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+
+RUN conda create -n pdal_ign_plugin -y
+ARG GITHUB_SHA
+ARG GITHUB_REPOSITORY="PDAL/PDAL"
+ARG GITHUB_SERVER_URL="https://github.com"
+
+SHELL ["conda", "run", "-n", "pdal_ign_plugin", "/bin/bash", "-c"]
+
+RUN mamba install -c conda-forge git compilers conda-pack cmake make ninja sysroot_linux-64=2.17 && \
+    mamba install --yes -c conda-forge pdal --only-deps
+
+RUN rm -rf /opt/conda/envs/pdaltools/lib/libpdal*
+RUN rm -rf /opt/conda/envs/pdaltools/lib/libpdal_plugin*    
+
+RUN git clone "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" pdal && \
+    cd pdal ; \
+    git checkout ${GITHUB_SHA}
+
+RUN mkdir -p pdal/build && \
+    cd pdal/build  && \
+    CXXFLAGS="-Werror=strict-aliasing" LDFLAGS="-Wl,-rpath-link,$CONDA_PREFIX/lib" cmake -G Ninja  \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_LIBRARY_PATH:FILEPATH="$CONDA_PREFIX/lib" \
+        -DCMAKE_INCLUDE_PATH:FILEPATH="$CONDA_PREFIX/include" \
+        -DCMAKE_INSTALL_PREFIX="$CONDA_PREFIX" \
+        -DBUILD_PLUGIN_CPD=OFF \
+        -DBUILD_PLUGIN_PGPOINTCLOUD=ON \
+        -DBUILD_PLUGIN_NITF=ON \
+        -DBUILD_PLUGIN_ICEBRIDGE=ON \
+        -DBUILD_PLUGIN_HDF=ON \
+        -DBUILD_PLUGIN_TILEDB=ON \
+        -DBUILD_PLUGIN_E57=ON \
+        -DBUILD_PGPOINTCLOUD_TESTS=OFF \
+        -DWITH_ZSTD=ON \
+        ..
+
+RUN cd pdal/build  && \
+    ninja
+
+RUN cd pdal/build  && \
+    ctest -V
+
+RUN cd pdal/build  && \
+    ninja install
+
+RUN git clone https://github.com/PDAL/python.git
+
+RUN mamba install --yes -c conda-forge pybind11
+
+RUN mkdir -p python/build && \
+    cd python/build  && \
+    CXXFLAGS="-Werror=strict-aliasing" LDFLAGS="-Wl,-rpath-link,$CONDA_PREFIX/lib" cmake -G Ninja  \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_LIBRARY_PATH:FILEPATH="$CONDA_PREFIX/lib" \
+        -DCMAKE_INCLUDE_PATH:FILEPATH="$CONDA_PREFIX/include" \
+        -DCMAKE_INSTALL_PREFIX="$CONDA_PREFIX/lib/python3.13/site-packages/" \
+        ..
+
+RUN cd python/build  && ninja
+RUN cd python/build  && ctest -V
+RUN cd python/build  && ninja install
+
+RUN git clone https://github.com/PDAL/python-plugins.git pdal-python && \
+    cd pdal-python && git checkout 1.6.5  && \
+    pip install -vv . --no-deps
+
+COPY src src
+COPY CMakeLists.txt CMakeLists.txt
+RUN mkdir -p build && \
+    cd build  && \
+    CXXFLAGS="-Werror=strict-aliasing" LDFLAGS="-Wl,-rpath-link,$CONDA_PREFIX/lib" cmake -G Ninja  \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_LIBRARY_PATH:FILEPATH="$CONDA_PREFIX/lib" \
+        -DCMAKE_INCLUDE_PATH:FILEPATH="$CONDA_PREFIX/include" \
+        ..
+
+RUN cd build  && ninja
+RUN cd python/build  && ctest -V
+RUN cd build  && ninja install
+
+# Add our environment
+RUN mamba install numpy requests gdal geopandas pytest pip
+RUN pip install laspy[lazrs]
+RUN pip install ign-pdal-tools==1.11.1
+
+FROM debian:bullseye-slim
+
+# install PDAL
+COPY --from=mamba_pdal /opt/conda/envs/pdal_ign_plugin/bin/pdal /opt/conda/envs/pdal_ign_plugin/bin/pdal
+COPY --from=mamba_pdal /opt/conda/envs/pdal_ign_plugin/bin/python /opt/conda/envs/pdal_ign_plugin/bin/python
+COPY --from=mamba_pdal /opt/conda/envs/pdal_ign_plugin/lib/ /opt/conda/envs/pdal_ign_plugin/lib/
+COPY --from=mamba_pdal /opt/conda/envs/pdal_ign_plugin/ssl /opt/conda/envs/pdal_ign_plugin/ssl
+COPY --from=mamba_pdal /opt/conda/envs/pdal_ign_plugin/share/proj/proj.db /opt/conda/envs/pdal_ign_plugin/share/proj/proj.db
+COPY --from=mamba_pdal /install/lib /plugins/lib
+
+ENV PATH=$PATH:/opt/conda/envs/pdal_ign_plugin/bin/
+ENV PROJ_LIB=/opt/conda/envs/pdal_ign_plugin/share/proj/
+ENV GDAL_DATA=/opt/conda/envs/pdal_ign_plugin/share/gdal/
+ENV PDAL_DRIVER_PATH=/plugins/lib
+
+# Install python macro module
+COPY pdal_ign_macro /pdal_ign_plugin/pdal_ign_macro
+COPY pyproject.toml /pdal_ign_plugin/pyproject.toml
+WORKDIR /pdal_ign_plugin
+
+# Add example scripts + test data (to be able to test inside the docker image)
+COPY test /pdal_ign_plugin/test

Makefile

@@ -0,0 +1,66 @@
+# Makefile to manage main tasks
+# cf. https://blog.ianpreston.ca/conda/python/bash/2020/05/13/conda_envs.html#makefile
+
+# Oneshell means I can run multiple lines in a recipe in the same shell, so I don't have to
+# chain commands together with semicolon
+.ONESHELL:
+SHELL = /bin/bash
+
+
+##############################
+# Install
+##############################
+install:
+	mamba env update -n pdal_ign_plugin -f environment.yml
+
+
+##############################
+# Dev/Contrib tools
+##############################
+
+testing:
+	python -m pytest ./test -s --log-cli-level DEBUG
+
+install-precommit:
+	pre-commit install
+
+
+##############################
+# Build/deploy pip lib
+##############################
+
+deploy: check
+	twine upload dist/*
+
+check: dist/pdal_ign_plugin*.tar.gz
+	twine check dist/*
+
+dist/pdal_ign_plugin*.tar.gz:
+	python -m build
+
+build: clean
+	python -m build
+
+clean:
+	rm -rf tmp
+	rm -rf pdal_ign_plugin.egg-info
+	rm -rf dist
+
+##############################
+# Build/deploy Docker image
+##############################
+
+IMAGE_NAME=pdal_ign_plugin
+VERSION=`python -m pdal_ign_macro.version`
+
+docker-build: clean
+	docker build --no-cache -t ${IMAGE_NAME}:${VERSION} -f Dockerfile .
+
+docker-build-pdal: clean
+	docker build --build-arg GITHUB_REPOSITORY=alavenant/PDAL --build-arg GITHUB_SHA=master_28_05_25 -t ${IMAGE_NAME}:${VERSION} -f Dockerfile.pdal .
+
+docker-test-pdal: clean
+	docker run --rm  -v `pwd`/tmp_data:/output -t ${IMAGE_NAME}:${VERSION} python -m pytest --log-cli-level=debug 
+
+docker-test:
+	docker run --rm -it ${IMAGE_NAME}:${VERSION} python -m pytest -s

README.md

@@ -117,6 +117,13 @@ marco.my_macro(...)
 
 See the `scripts` folder for example usages of this module.
 
+## Docker
+
+There are two docker files. 
+
+    - The main Dockerfile use the official pdal version. 
+    - The Dockerfile.pdal permit to use non official version of pdal (the master or a fork for exemple). See the "docker-build-pdal" command in  Makefile 
+
 ### Usage from outside the docker image:
 
 If you have a python script on your computer, you can mount its containing folder as a volume in order to

RELEASE_NOTES.md

@@ -1,5 +1,7 @@
 
-- Update to ign-pdal-tools 1.10.0 for fix in add_points_in_pointcloud
+- Update to ign-pdal-tools 1.11.1 for fix in add_points_in_pointcloud
+- Add Dockerfile.pdal for use non official version of pdal
+- fix tests to next version of pdal 
 
 ### 0.4.1
 

environment.yml

@@ -22,5 +22,5 @@ dependencies:
   - pip
   - pip:
     - build # Installed via pip after issues when creating the environment ("build does not exist")
-    - ign-pdal-tools==1.10.0
+    - ign-pdal-tools==1.11.1
     - laspy[lazrs]

test/pdal_ign_macro/test_mark_points_to_use_for_digital_models_with_new_dimension.py

@@ -20,16 +20,16 @@ def test_mark_points_to_use_for_digital_models_with_new_dimension():
         mark_points_to_use_for_digital_models_with_new_dimension(
             ini_las, las_output.name, dsm_dimension, dtm_dimension, "", ""
         )
-        pipeline = pdal.Pipeline()
-        pipeline |= pdal.Reader.las(ini_las)
-        input_dimensions = set(pipeline.quickinfo["readers.las"]["dimensions"].split(", "))
-        pipeline = pdal.Pipeline()
-        pipeline |= pdal.Reader.las(las_output.name)
-        output_dimensions = set(pipeline.quickinfo["readers.las"]["dimensions"].split(", "))
+        pipeline_input = pdal.Reader.las(ini_las).pipeline()
+        input_dimensions = set(pipeline_input.quickinfo["readers.las"]["dimensions"].split(", "))
+        pipeline_output = pdal.Reader.las(las_output.name).pipeline()
+        output_dimensions = set(pipeline_output.quickinfo["readers.las"]["dimensions"].split(", "))
         assert output_dimensions == input_dimensions.union([dsm_dimension, dtm_dimension])
 
-        pipeline.execute()
-        arr = pipeline.arrays[0]
+        # pipeline.quickinfo done before need that we re create the pipeline
+        pipeline_output = pdal.Reader.las(las_output.name).pipeline()
+        pipeline_output.execute()
+        arr = pipeline_output.arrays[0]
         assert np.any(arr[dsm_dimension] == 1)
         assert np.any(arr[dtm_dimension] == 1)
 
@@ -48,12 +48,10 @@ def test_mark_points_to_use_for_digital_models_with_new_dimension_keep_dimension
             "",
             keep_temporary_dimensions=True,
         )
-        pipeline = pdal.Pipeline()
-        pipeline |= pdal.Reader.las(las_output.name)
-        output_dimensions = set(pipeline.quickinfo["readers.las"]["dimensions"].split(", "))
+        pipeline_mtd = pdal.Reader.las(las_output.name).pipeline()
+        output_dimensions = set(pipeline_mtd.quickinfo["readers.las"]["dimensions"].split(", "))
         assert dsm_dimension in output_dimensions
         assert dtm_dimension in output_dimensions
-
         assert all(
             [
                 dim in output_dimensions
@@ -65,7 +63,8 @@ def test_mark_points_to_use_for_digital_models_with_new_dimension_keep_dimension
                 ]
             ]
         )
-
+        # pipeline.quickinfo done before need that we re create the pipeline
+        pipeline = pdal.Reader.las(las_output.name).pipeline()
         pipeline.execute()
         arr = pipeline.arrays[0]
         assert np.any(arr[dsm_dimension] == 1)
@@ -87,12 +86,13 @@ def test_main_no_buffer():
             keep_temporary_dims=False,
             skip_buffer=True,
         )
-        pipeline = pdal.Pipeline()
-        pipeline |= pdal.Reader.las(las_output.name)
-        output_dimensions = pipeline.quickinfo["readers.las"]["dimensions"].split(", ")
-        assert dsm_dimension in output_dimensions
-        assert dtm_dimension in output_dimensions
 
+        pipeline_mtd = pdal.Reader.las(filename=las_output.name).pipeline()
+        metadata = pipeline_mtd.quickinfo["readers.las"]["dimensions"]
+        assert dsm_dimension in metadata
+        assert dtm_dimension in metadata
+
+        pipeline = pdal.Reader.las(filename=las_output.name).pipeline()
         pipeline.execute()
         arr = pipeline.arrays[0]
         assert np.any(arr[dsm_dimension] == 1)
@@ -117,14 +117,15 @@ def test_main_with_buffer():
             tile_width=50,
             tile_coord_scale=10,
         )
-        pipeline = pdal.Pipeline()
-        pipeline |= pdal.Reader.las(las_output.name)
-        output_dimensions = pipeline.quickinfo["readers.las"]["dimensions"].split(", ")
+
+        pipeline_mtd = pdal.Reader.las(las_output.name).pipeline()
+        output_dimensions = pipeline_mtd.quickinfo["readers.las"]["dimensions"]
         assert dsm_dimension in output_dimensions
         assert dtm_dimension in output_dimensions
 
-        pipeline.execute()
-        arr = pipeline.arrays[0]
+        pipeline_out = pdal.Reader.las(las_output.name).pipeline()
+        pipeline_out.execute()
+        arr = pipeline_out.arrays[0]
         assert np.any(arr[dsm_dimension] == 1)
         assert np.any(arr[dtm_dimension] == 1)
 

test/pdal_ign_macro/test_preprocessing_mnx.py

@@ -72,6 +72,7 @@ def test_preprocess_mnx(
     dtm_dimension = "dtm_marker"
 
     with tempfile.NamedTemporaryFile(suffix="_preprocessed_output.laz") as las_output:
+
         preprocess_mnx(
             input_las=ini_las,
             input_geometry=ini_geojson,
@@ -91,8 +92,13 @@ def test_preprocess_mnx(
             tile_coord_scale=10,
         )
 
-        pipeline_output = pdal.Pipeline()
-        pipeline_output |= pdal.Reader.las(las_output.name)
+        pipeline_mtd = pdal.Reader.las(las_output.name).pipeline()
+        metadata = pipeline_mtd.quickinfo["readers.las"]["dimensions"]
+        assert dsm_dimension in metadata, "DSM marker dimension not found"
+        assert dtm_dimension in metadata, "DTM marker dimension not found"
+
+        # pipeline.quickinfo done before need that we re create the pipeline
+        pipeline_output = pdal.Reader.las(las_output.name).pipeline()
         pipeline_output.execute()
         arr = pipeline_output.arrays[0]
 
@@ -102,9 +108,6 @@ def test_preprocess_mnx(
             count_66 == nb_points_added
         ), f"Expected {nb_points_added} points added with class 66, got {count_66}"
 
-        output_dimensions = set(pipeline_output.quickinfo["readers.las"]["dimensions"].split(", "))
-        assert dsm_dimension in output_dimensions, "DSM marker dimension not found"
-        assert dtm_dimension in output_dimensions, "DTM marker dimension not found"
         assert np.any(arr[dsm_dimension] == 1), "No points marked for DSM"
         assert np.any(arr[dtm_dimension] == 1), "No points marked for DTM"
         assert not np.all(arr["Intensity"] == 0), "Lost Intensity value"