[Merge] Merge latest main into libcudf-reuse-prebuilt

Sync the branch with `origin/main` so PR #4885 merges cleanly without a force-push on the open PR.

Resolve the `build/buildcpp.sh` conflict: keep the `cudf.reuse.prebuilt` skip-guard around the `libcudfjni` cmake and add the `CMAKE_NETRC` and `CMAKE_NETRC_FILE` flags from main. The merge also adopts the `thirdparty/cudf` pin from main.

---------

Signed-off-by: Nghia Truong <nghiat@nvidia.com>

Author: ttnghia

build/buildcpp.sh

@@ -73,6 +73,9 @@ fi
 
 source "$PROJECT_BUILD_DIR/buildcpp-env.sh"
 
+CMAKE_NETRC=${CMAKE_NETRC:-IGNORED}
+CMAKE_NETRC_FILE=${CMAKE_NETRC_FILE:-$HOME/.netrc}
+
 if [[ "$GPU_ARCHS" != "DEPRECATED" ]]; then
     CMAKE_CUDA_ARCHITECTURES="$GPU_ARCHS"    
     echo "==========================================================================================
@@ -288,6 +291,8 @@ else
     -DCUDF_JNI_LIBCUDF_STATIC=ON \
     -DCUDF_USE_PER_THREAD_DEFAULT_STREAM="$CUDF_USE_PER_THREAD_DEFAULT_STREAM" \
     -DCMAKE_CUDA_ARCHITECTURES="$CMAKE_CUDA_ARCHITECTURES" \
+    -DCMAKE_NETRC="$CMAKE_NETRC" \
+    -DCMAKE_NETRC_FILE="$CMAKE_NETRC_FILE" \
     -DRMM_LOGGING_LEVEL="$RMM_LOGGING_LEVEL" \
     -DUSE_GDS="$USE_GDS" \
     -C="$CUDF_PIN_PATH/setup.cmake"

ci/Jenkinsfile.premerge

@@ -228,7 +228,7 @@ git --no-pager diff --name-only HEAD \$BASE -- ${PREMERGE_DOCKERFILE} || true"""
                                     def jniVer = readMavenPom(file: 'pom.xml').version
                                     common.restoreJniCcache(this, jniVer, 'cuda12', 'ART_CREDS')
                                     try {
-                                        sh 'source build/env.sh && ${sclCMD} "ci/premerge-build.sh"'
+                                        runPremergeBuild()
                                         sh 'bash ci/fuzz-test.sh'
                                     } catch (e) {
                                         common.printJVMCoreDumps(this)
@@ -266,7 +266,7 @@ git --no-pager diff --name-only HEAD \$BASE -- ${PREMERGE_DOCKERFILE} || true"""
                                     def jniVer = readMavenPom(file: 'pom.xml').version
                                     common.restoreJniCcache(this, jniVer, 'cuda13', 'ART_CREDS')
                                     try {
-                                        sh 'source build/env.sh && ${sclCMD} "ci/premerge-build.sh"'
+                                        runPremergeBuild()
                                         sh 'bash ci/fuzz-test.sh'
                                     } catch (e) {
                                         common.printJVMCoreDumps(this)
@@ -312,6 +312,16 @@ git --no-pager diff --name-only HEAD \$BASE -- ${PREMERGE_DOCKERFILE} || true"""
     }
 } // end of pipeline
 
+void runPremergeBuild() {
+    common.prepareArtNetrc(this)
+    withEnv([
+        "ARROW_THRIFT_MIRROR_URL=https://${env.ARTIFACTORY_NAME}/artifactory/sw-spark-maven/apache-remote",
+        "CMAKE_NETRC=OPTIONAL",
+    ]) {
+        sh 'source build/env.sh && ${sclCMD} "ci/premerge-build.sh"'
+    }
+}
+
 void uploadDocker(String IMAGE_NAME) {
     def DOCKER_CMD = "docker --config $WORKSPACE/.docker"
     retry(3) {

patches/0001-use-static-nvcomp-from-libcudf.patch

@@ -0,0 +1,14 @@
+diff --git a/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java b/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java
+index 0ab9ec79d1..3d42449128 100755
+--- a/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java
++++ b/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java
+@@ -69,9 +69,6 @@ public class NativeDepsLoader {
+    * subsequent stages are loaded.
+    */
+   private static final String[][] loadOrder = new String[][]{
+-      new String[]{
+-          "nvcomp"
+-      },
+       new String[]{
+           "cudf"
+       },

patches/noop.patch

@@ -578,14 +578,6 @@
                     <include>libprofilerjni.so</include>
                   </includes>
                 </resource>
-                <resource>
-                  <directory>${libcudfjni.build.path}</directory>
-                  <includes>
-                    <include>libnvcomp.so</include>
-                    <include>libnvcomp_gdeflate.so</include>
-                    <include>libnvcomp_bitcomp.so</include>
-                  </includes>
-                </resource>
               </resources>
             </configuration>
           </execution>

pom.xml

@@ -62,6 +62,7 @@ option(BUILD_TESTS "Configure CMake to build tests" OFF)
 option(BUILD_BENCHMARKS "Configure CMake to build (google) benchmarks" OFF)
 option(BUILD_FAULTINJ "Configure CMake to build fault injection" ON)
 option(BUILD_PROFILER "Configure CMake to build profiler" ON)
+option(HIDE_PRIVATE_CUDA_SYMBOLS "Hide private symbols from static CUDA compiler libraries" ON)
 
 message(
   VERBOSE "SPARK_RAPIDS_JNI: Build with per-thread default stream:
@@ -70,6 +71,9 @@ message(
 message(VERBOSE "SPARK_RAPIDS_JNI: Configure CMake to build tests: ${BUILD_TESTS}")
 message(VERBOSE "SPARK_RAPIDS_JNI: Configure CMake to build (nvbench) benchmarks: ${BUILD_BENCHMARKS}")
 message(VERBOSE "SPARK_RAPIDS_JNI: Configure CMake to build fault injection: ${BUILD_FAULTINJ}")
+message(VERBOSE
+        "SPARK_RAPIDS_JNI: Hide private static CUDA compiler symbols: ${HIDE_PRIVATE_CUDA_SYMBOLS}"
+)
 
 set(SPARK_RAPIDS_JNI_BUILD_TESTS ${BUILD_TESTS})
 set(SPARK_RAPIDS_JNI_BUILD_BENCHMARKS ${BUILD_BENCHMARKS})
@@ -349,6 +353,18 @@ target_link_libraries(
     ${PARQUET_LIB}
     ${THRIFT_LIB}
 )
+if(HIDE_PRIVATE_CUDA_SYMBOLS)
+  # These toolkit implementation symbols are not part of the cuDF or JNI ABI. Localizing them keeps
+  # the regular ELF symbol table for diagnostics while removing them from the dynamic export table.
+  target_link_options(
+    spark_rapids_jni
+    PRIVATE "LINKER:--exclude-libs,libnvrtc_static.a"
+            "LINKER:--exclude-libs,libnvrtc-builtins_static.a"
+            "LINKER:--exclude-libs,libnvptxcompiler_static.a"
+            "LINKER:--exclude-libs,libnvJitLink_static.a"
+            "LINKER:--exclude-libs,libnvcomp_static.a"
+  )
+endif()
 rapids_cuda_set_runtime(spark_rapids_jni USE_STATIC ON)
 set_target_properties(spark_rapids_jni PROPERTIES LINK_LANGUAGE "CXX")
 # For backwards-compatibility with the cudf Java bindings and RAPIDS accelerated UDFs,

src/main/cpp/CMakeLists.txt

@@ -686,8 +686,12 @@ void convert_to_nvtxt(std::ifstream& in, std::ostream& out, program_options cons
               color    = it->second->color();
               category = it->second->category();
             }
-            marker_start ms{
-              m->timestamp(), process_id, thread_id, color, category, m->name()->str()};
+            marker_start ms{m->timestamp(),
+                            process_id,
+                            thread_id,
+                            color,
+                            category,
+                            m->name() ? m->name()->str() : ""};
             auto [ignored, inserted] = marker_start_map.insert({m->id(), ms});
             if (not inserted) {
               std::ostringstream oss;
@@ -970,8 +974,8 @@ int convert_to_nvtxw(std::ifstream& in,
                             thread_id,
                             color,
                             category,
-                            m->name()->str(),
-                            m->domain()->str()};
+                            m->name() ? m->name()->str() : "",
+                            m->domain() ? m->domain()->str() : ""};
             auto [ignored, inserted] = marker_start_map.insert({m->id(), ms});
             if (not inserted) {
               std::ostringstream oss;

src/main/cpp/profiler/spark_rapids_profile_converter.cpp

@@ -140,7 +140,8 @@ JNIEXPORT jlong JNICALL Java_com_nvidia_spark_rapids_jni_GpuTimeZoneDB_convertOr
   jlong reader_tz_info_table,
   jint reader_tz_initial_offset,
   jint reader_tz_raw_offset,
-  jintArray reader_dst_rule)
+  jintArray reader_dst_rule,
+  jboolean writer_reader_rules_differ)
 {
   JNI_NULL_CHECK(env, input_handle, "input column is null", 0);
 
@@ -160,7 +161,38 @@ JNIEXPORT jlong JNICALL Java_com_nvidia_spark_rapids_jni_GpuTimeZoneDB_convertOr
     auto const reader = spark_rapids_jni::orc_tz_side{
       reader_tz_info_tab, reader_tz_initial_offset, reader_tz_raw_offset, reader_dst};
     return cudf::jni::release_as_jlong(spark_rapids_jni::convert_orc_writer_reader_timezones(
-      *input, static_cast<int64_t>(writer_tz_offset_at_orc_2015_base_us), writer, reader));
+      *input,
+      static_cast<int64_t>(writer_tz_offset_at_orc_2015_base_us),
+      writer,
+      reader,
+      cudf::get_default_stream(),
+      cudf::get_current_device_resource_ref(),
+      writer_reader_rules_differ));
+  }
+  JNI_CATCH(env, 0);
+}
+
+JNIEXPORT jlong JNICALL Java_com_nvidia_spark_rapids_jni_GpuTimeZoneDB_convertOrcFromUtcWithRules(
+  JNIEnv* env,
+  jclass,
+  jlong input_handle,
+  jlong reader_tz_info_table,
+  jint reader_tz_initial_offset,
+  jint reader_tz_raw_offset,
+  jintArray reader_dst_rule)
+{
+  JNI_NULL_CHECK(env, input_handle, "input column is null", 0);
+
+  JNI_TRY
+  {
+    cudf::jni::auto_set_device(env);
+    auto const input              = reinterpret_cast<cudf::column_view const*>(input_handle);
+    auto const reader_tz_info_tab = reinterpret_cast<cudf::table_view const*>(reader_tz_info_table);
+    auto const reader_dst         = parse_dst_rule(env, reader_dst_rule);
+    cudf::jni::check_java_exception(env);
+    auto const reader = spark_rapids_jni::orc_tz_side{
+      reader_tz_info_tab, reader_tz_initial_offset, reader_tz_raw_offset, reader_dst};
+    return cudf::jni::release_as_jlong(spark_rapids_jni::convert_orc_from_utc(*input, reader));
   }
   JNI_CATCH(env, 0);
 }