[CICD] Use system nlohmann-json (#530)

Author: shiptux

Makefile

@@ -226,6 +226,8 @@ BUILD_INCDIR := $(BUILDDIR)/include
 PREFIX ?= /usr/local
 DESTDIR  ?= $(PREFIX)/lib
 INC_DESTDIR ?= $(PREFIX)/include
+# Source builds use the submodule; distro builds override this with /usr/include.
+JSON_INCLUDE_DIR ?= $(abspath third-party/json/single_include)
 
 # Public headers exported alongside libflagcx.so
 PUBLIC_HEADERS := \
@@ -242,7 +244,7 @@ INCLUDEDIR := \
 	$(abspath flagcx/runner/include) \
 	$(abspath flagcx/core/include) \
 	$(abspath flagcx/service/include) \
-	$(abspath third-party/json/single_include)
+	$(JSON_INCLUDE_DIR)
 
 # Append NVSHMEM include path (must come after INCLUDEDIR := assignment)
 ifeq ($(USE_SHMEM), 1)

packaging/debian/control

@@ -6,6 +6,7 @@ Build-Depends: debhelper-compat (= 13),
                g++ (>= 9),
                make,
                cmake,
+               nlohmann-json3-dev,
                patchelf
 Standards-Version: 4.6.0
 Homepage: https://github.com/flagos-ai/FlagCX
@@ -21,8 +22,8 @@ X-Build-Environment: This package is designed to be built in backend-specific
   - MetaX backend: Requires maca_sdk (from repos.metax-tech.com)
     Built in Ubuntu 22.04 with MetaX APT repository configured
  .
- The Build-Depends listed above are for the Debian packaging tools only.
- Backend-specific dependencies are provided by the container environment.
+ Common build dependencies are declared above. Backend-specific dependencies
+ are provided by the container environment.
 
 Package: libflagcx-metax
 Build-Profiles: <!pkg.flagcx.nvidia-only>

packaging/debian/rules

@@ -6,6 +6,9 @@ export DH_VERBOSE = 1
 # Usage: FLAGCX_BUILD_BACKEND=metax dpkg-buildpackage ...
 # Valid values: metax, nvidia, all (default)
 FLAGCX_BUILD_BACKEND ?= all
+# Distro packages build against nlohmann-json3-dev, not the git submodule.
+JSON_INCLUDE_DIR ?= /usr/include
+export JSON_INCLUDE_DIR
 
 # Build directories for different backends
 BUILD_DIR_METAX = $(CURDIR)/build-metax

packaging/rpm/specs/flagcx.spec

@@ -35,20 +35,12 @@ BuildRequires:  make
 BuildRequires:  cmake
 BuildRequires:  patchelf
 # nlohmann-json package name varies by distro:
-#   - RHEL/Rocky 8 (via EPEL): json-devel
-#   - RHEL/Rocky 9 (via EPEL): nlohmann-json-devel
-#   - OpenEuler / others: nlohmann-json-devel (fallback)
-# TODO: verify Rocky 9 / RHEL 9 build path end-to-end; the EPEL 9 package
-# name is nlohmann-json-devel, but this has only been smoke-tested.
-%if 0%{?rhel} == 8
+#   - Fedora and RHEL/Rocky 8/9 (via EPEL): json-devel
+#   - openEuler: nlohmann-json-devel
+%if 0%{?fedora} || 0%{?rhel}
 BuildRequires:  json-devel
 %else
-%if 0%{?rhel} >= 9
 BuildRequires:  nlohmann-json-devel
-%else
-# Non-RHEL (OpenEuler, etc.) – assume upstream nlohmann-json-devel package name.
-BuildRequires:  nlohmann-json-devel
-%endif
 %endif
 
 %description
@@ -80,7 +72,7 @@ Development files (headers and libraries) for libflagcx-%{backend}.
 %setup -q
 
 %build
-make USE_%{backend_upper}=1 PREFIX=%{_prefix}
+make USE_%{backend_upper}=1 PREFIX=%{_prefix} JSON_INCLUDE_DIR=%{_includedir}
 
 %install
 rm -rf %{buildroot}

setup.py

@@ -48,10 +48,15 @@
     os.path.join("plugin", "torch", "flagcx", "src", "utils_flagcx.cpp"),
 ]
 
+VENDORED_JSON_INCLUDE_DIR = os.path.join(
+    ROOT_DIR, "third-party", "json", "single_include"
+)
+JSON_INCLUDE_DIR = os.environ.get("JSON_INCLUDE_DIR") or VENDORED_JSON_INCLUDE_DIR
+
 include_dirs = [
     os.path.join(PLUGIN_DIR, "flagcx", "include"),
     os.path.join(ROOT_DIR, "flagcx", "include"),
-    os.path.join(ROOT_DIR, "third-party", "json", "single_include"),
+    JSON_INCLUDE_DIR,
 ]
 
 # Will be updated in build_ext to point at the built libflagcx.so
@@ -77,22 +82,31 @@
 if BuildExtension is not None:
     class BuildExtWithMake(BuildExtension):
         def build_extensions(self):
-            # -- Step 0: Ensure git submodules are initialized --
-            submodule_marker = os.path.join(
-                ROOT_DIR, "third-party", "json", "single_include"
-            )
-            if not os.path.isdir(submodule_marker):
+            # -- Step 0: Resolve nlohmann-json headers --
+            json_header = os.path.join(JSON_INCLUDE_DIR, "nlohmann", "json.hpp")
+            if (
+                JSON_INCLUDE_DIR == VENDORED_JSON_INCLUDE_DIR
+                and not os.path.isfile(json_header)
+            ):
                 print("[flagcx] Initializing git submodules ...")
                 subprocess.check_call(
                     ["git", "submodule", "update", "--init", "--recursive"],
                     cwd=ROOT_DIR,
                 )
+            if not os.path.isfile(json_header):
+                raise RuntimeError(
+                    f"nlohmann/json.hpp not found under JSON_INCLUDE_DIR={JSON_INCLUDE_DIR}"
+                )
 
             # -- Step 1: Build libflagcx.so via make --
             build_dir = os.path.join(ROOT_DIR, "build")
             lib_dir = os.path.join(build_dir, "lib")
 
-            make_args = [f"BUILDDIR={build_dir}", f"{adaptor_make_flag}=1"]
+            make_args = [
+                f"BUILDDIR={build_dir}",
+                f"{adaptor_make_flag}=1",
+                f"JSON_INCLUDE_DIR={JSON_INCLUDE_DIR}",
+            ]
 
             # Forward additional env vars to make
             env_to_make = [
@@ -185,4 +199,4 @@ def build_extensions(self):
     ext_modules=ext_modules,
     cmdclass=cmdclass,
     entry_points={"torch.backends": ["flagcx = flagcx:init"]},
-)
+)