Switch to memory64 for the Wasm build to enable greater memory usage.

.github/workflows/publish-npm.yaml

@@ -7,10 +7,25 @@ on:
 name: Build and publish to NPM 
 
 jobs:
+  prepare-version:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repo
+      uses: actions/checkout@v4
+
+    - id: find-version
+      run: |
+        version=$(cat .image-version)
+        echo "version=${version}" >> $GITHUB_OUTPUT
+
+    outputs:
+      version: ${{ steps.find-version.outputs.version }}
+
   build:
     runs-on: ubuntu-latest
+    needs: [prepare-version]
     if: ${{ github.event.workflow_run.conclusion == 'success' }}
-    container: ghcr.io/kanaverse/emcmake-docker/builder:master
+    container: ghcr.io/kanaverse/emcmake-docker/builder:${{ needs.prepare-version.outputs.version }}
 
     steps:
     - name: Checkout repo
@@ -19,7 +34,7 @@ jobs:
     - name: Set up Node
       uses: actions/setup-node@v4
       with:
-        node-version: 18
+        node-version: 24
         registry-url: 'https://registry.npmjs.org'
 
     # We only run the remaining (remote-touching) steps if the version has actually changed.
@@ -38,7 +53,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: extern/installed
-        key: deps-${{ hashFiles('extern/**/build.sh') }}-${{ env.IMAGE_VERSION }}
+        key: deps-${{ hashFiles('extern/**/build.sh') }}-${{ needs.prepare-version.outputs.version }}
 
     - name: Cache Modules
       if: env.UPDATE_SCRANJS == 1
@@ -64,7 +79,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: tests/rds/*.rds
-        key: rds-${{ hashFiles('tests/rds/generate.R') }}
+        key: rds-${{ hashFiles('tests/rds/generate.R') }}-${{ needs.prepare-version.outputs.version }}
 
     - name: Double-checking tests
       if: env.UPDATE_SCRANJS == 1

.github/workflows/run-tests.yaml

@@ -6,16 +6,20 @@ on:
 
 name: Test JS bindings
 
-env:
-  IMAGE_VERSION: master
-
 jobs:
   prepare-version:
     runs-on: ubuntu-latest
     steps:
-    - run: echo "null"
+    - name: Checkout repo
+      uses: actions/checkout@v4
+
+    - id: find-version
+      run: |
+        version=$(cat .image-version)
+        echo "version=${version}" >> $GITHUB_OUTPUT
+
     outputs:
-      version: ${{ env.IMAGE_VERSION }}
+      version: ${{ steps.find-version.outputs.version }}
 
   # Building the RDS files for the various RDS-reading utilities.
   create_rds:
@@ -53,7 +57,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: extern/installed
-        key: deps-${{ hashFiles('extern/**/build.sh') }}-${{ env.IMAGE_VERSION }}
+        key: deps-${{ hashFiles('extern/**/build.sh') }}-${{ needs.prepare-version.outputs.version }}
 
     - name: Build HDF5
       if: steps.installed.outputs.cache-hit != 'true'
@@ -79,7 +83,7 @@ jobs:
     - name: Set up Node
       uses: actions/setup-node@v4
       with:
-        node-version: 18
+        node-version: 24
 
     - name: Cache Modules
       uses: actions/cache@v4
@@ -94,7 +98,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: extern/installed
-        key: deps-${{ hashFiles('extern/**/build.sh') }}-${{ env.IMAGE_VERSION }}
+        key: deps-${{ hashFiles('extern/**/build.sh') }}-${{ needs.prepare-version.outputs.version }}
 
     - name: Cache RDS files
       uses: actions/cache@v4
@@ -107,7 +111,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: js/wasm
-        key: wasm-${{ hashFiles('build.sh', 'extern/**/build.sh', 'CMakeLists.txt', 'extern/CMakeLists.txt', 'src/*') }}-${{ env.IMAGE_VERSION }}
+        key: wasm-${{ hashFiles('build.sh', 'extern/**/build.sh', 'CMakeLists.txt', 'extern/CMakeLists.txt', 'src/*') }}-${{ needs.prepare-version.outputs.version }}
 
     - name: Update node build 
       if: steps.wasm-build.outputs.cache-hit != 'true'

.image-version

@@ -0,0 +1 @@
+2025-11-25

.npmrc

@@ -0,0 +1 @@
+engine-strict=true

CMakeLists.txt

@@ -71,6 +71,7 @@ target_compile_options(
     scran_wasm PUBLIC 
     -O3
     -pthread
+    -sMEMORY64
 )
 
 target_link_libraries(
@@ -121,7 +122,8 @@ target_link_options(scran_wasm PRIVATE
     -O3
     --bind 
     -sALLOW_MEMORY_GROWTH=1 
-    -sMAXIMUM_MEMORY=4GB 
+    -sMEMORY64
+    -sMAXIMUM_MEMORY=16GB # current maximum, otherwise Emscripten complains.
     -sSTACK_SIZE=2MB
     -sUSE_ZLIB=1 
     -sMODULARIZE=1 
@@ -137,14 +139,15 @@ set_target_properties(scran_wasm PROPERTIES OUTPUT_NAME scran)
 
 set(COMPILE_NODE OFF CACHE BOOL "Compile for Node.js")
 if (COMPILE_NODE)
+    # Exporting HEAP8 for compatibility with old wasmarrays.js.
     target_link_options(scran_wasm PRIVATE
         -sENVIRONMENT=node 
         -sNODERAWFS=1
-        -sEXPORTED_RUNTIME_METHODS=wasmMemory,PThread
+        -sEXPORTED_RUNTIME_METHODS=wasmMemory,HEAP8,PThread
     )
 else ()
     target_link_options(scran_wasm PRIVATE 
         -sENVIRONMENT=web,worker 
-        -sEXPORTED_RUNTIME_METHODS=wasmMemory,PThread,FS
+        -sEXPORTED_RUNTIME_METHODS=wasmMemory,HEAP8,PThread,FS
     )
 endif()

build.sh

@@ -45,12 +45,16 @@ if [ ! -e $builddir ]
 then
     mkdir $builddir
     echo "{}" > $builddir/package.json # avoid assuming ES6 syntax for igraph config scripts.
+
+    # Setting the C flags below, even though we don't have any C;
+    # this ensures that emcmake knows to operate in 64-bit mode for find_package().
     emcmake cmake \
         -S . \
         -B $builddir \
         -DCOMPILE_NODE=${node_flag} \
         -DCMAKE_BUILD_TYPE=Release \
         -DCMAKE_PREFIX_PATH=extern/installed \
+        -DCMAKE_C_FLAGS="-sMEMORY64" \
         -DTATAMI_HDF5_FIND_HDF5=OFF
 fi
 

extern/CMakeLists.txt

@@ -95,7 +95,7 @@ FetchContent_Declare(
 FetchContent_Declare(
   scran_qc
   GIT_REPOSITORY https://github.com/libscran/scran_qc
-  GIT_TAG master
+  GIT_TAG v0.1.1
 )
 
 FetchContent_Declare(

extern/hdf5/build.sh

@@ -30,7 +30,7 @@ BUILD_DIR=build-${HDF5_VERSION}
 if [ ! -e ${BUILD_DIR} ]
 then
     mkdir -p ../installed
-    coreflags="-pthread" # propagating compile flags from the root scran.js CMakeLists.txt.
+    coreflags="-pthread -sMEMORY64" # propagating compile flags from the root scran.js CMakeLists.txt.
     echo "{}" > package.json # avoid assuming ES6 module syntax from the root scran.js package.json.
     emcmake cmake \
         -S ${SOURCE_DIR} \

extern/igraph/build.sh

@@ -23,7 +23,7 @@ BUILD_DIR=build-${IGRAPH_VERSION}
 if [ ! -e ${BUILD_DIR} ]
 then
     mkdir -p ../installed
-    coreflags="-pthread" # propagating compile flags from the root scran.js CMakeLists.txt.
+    coreflags="-pthread -sMEMORY64" # propagating compile flags from the root scran.js CMakeLists.txt.
     echo "{}" > package.json # avoid assuming ES6 module syntax from the root scran.js package.json.
     emcmake cmake \
         -S ${SOURCE_DIR} \

js/wasm.js

@@ -8,9 +8,7 @@ const cache = {};
  * @param {object} [options={}] - Optional parameters.
  * @param {number} [options.numberOfThreads=4] - Number of threads to use for calculations.
  * This will spin up the requested number of Web Workers during module initialization.
- * @param {boolean} [options.localFile=false] - Whether or not to look for the Wasm and worker scripts locally.
- * This should only be `true` when using old versions of Node.js where file URLs are not supported, 
- * and is ignored completely outside of Node.js contexts.
+ * @param {boolean} [options.localFile=false] - Deprecated and ignored.
  *
  * @return {boolean}
  * The Wasm bindings are initialized and `true` is returned.
@@ -25,10 +23,6 @@ export async function initialize({ numberOfThreads = 4, localFile = false } = {}
         scran_custom_nthreads: numberOfThreads
     };
 
-    if (localFile) {                                                                /** NODE ONLY **/  
-        options.locateFile = (x) => import.meta.url.substring(7) + "/../wasm/" + x; /** NODE ONLY **/
-    }                                                                               /** NODE ONLY **/
-
     cache.module = await loadScran(options);
     cache.space = register(cache.module);
 

package.json

@@ -43,5 +43,8 @@
     "homepage": "https://github.com/kanaverse/scran.js#readme",
     "dependencies": {
         "wasmarrays.js": "^0.1.3"
+    },
+    "engines" : { 
+        "node" : ">=24.0.0" 
     }
 }

src/NeighborIndex.h

@@ -10,12 +10,12 @@
 struct NeighborIndex {
     std::unique_ptr<knncolle::Prebuilt<int32_t, int32_t, double> > index;
 
-    size_t num_obs() const {
-        return index->num_observations();
+    double num_obs() const {
+        return static_cast<double>(index->num_observations());
     }
 
-    size_t num_dim() const {
-        return index->num_dimensions();
+    double num_dim() const {
+        return static_cast<double>(index->num_dimensions());
     }
 };
 
@@ -43,20 +43,20 @@ struct NeighborResults {
     }
 
 public:
-    size_t size(int32_t truncate) const {
+    double size(int32_t truncate) const {
         size_t out = 0;
         size_t long_truncate = truncate;
         for (const auto& current : neighbors) {
             out += std::min(long_truncate, current.size());
         }
-        return out;
+        return static_cast<double>(out);
     }
 
-    size_t num_obs() const {
-        return neighbors.size();
+    double num_obs() const {
+        return static_cast<double>(neighbors.size());
     }
 
-    int32_t num_neighbors() const {
+    double num_neighbors() const {
         return (neighbors.empty() ? 0 : neighbors.front().size());
     }
 

src/hdf5_utils.cpp

@@ -6,6 +6,7 @@
 #include <cstddef>
 #include <algorithm>
 #include <unordered_map>
+#include <iostream>
 
 emscripten::val extract_attribute_names(const H5::H5Object& handle) {
     auto output = emscripten::val::array();
@@ -17,6 +18,9 @@ emscripten::val extract_attribute_names(const H5::H5Object& handle) {
     return output;
 }
 
+// Don't return size_t's directly, instead convert them to doubles so that we get Numbers in javascript.
+// Otherwise we have to deal with BigInts and those are a  pain.
+
 template<typename Handle_>
 emscripten::val extract_shape(const Handle_& handle) {
     auto dspace = handle.getSpace();
@@ -79,9 +83,9 @@ emscripten::val format_string_type(const H5::StrType& stype) {
         output.set("encoding", emscripten::val("UTF-8"));
     }
     if (stype.isVariableStr()) {
-        output.set("length", emscripten::val(-1));
+        output.set("length", emscripten::val(static_cast<double>(-1)));
     } else {
-        output.set("length", emscripten::val(stype.getSize()));
+        output.set("length", emscripten::val(static_cast<double>(stype.getSize())));
     }
     return output;
 }
@@ -674,12 +678,12 @@ H5::PredType choose_numeric_type(const std::string& type) {
     }
 }
 
-H5::StrType choose_string_type(const std::string& encoding, std::int32_t strlen) {
+H5::StrType choose_string_type(const std::string& encoding, std::int64_t strlen_or_var) {
     H5::StrType stype;
-    if (strlen < 0) {
+    if (strlen_or_var < 0) {
         stype = H5::StrType(0, H5T_VARIABLE);
     } else {
-        stype = H5::StrType(0, std::max(1, strlen)); // Make sure that is at least of length 1.
+        stype = H5::StrType(0, std::max(static_cast<std::int64_t>(1), strlen_or_var)); // Make sure that is at least of length 1.
     }
     if (encoding == "ASCII") {
         stype.setCset(H5T_CSET_ASCII);
@@ -762,7 +766,7 @@ H5::CompType choose_compound_type(const emscripten::val& members) {
         auto mode = type["mode"].template as<std::string>();
         if (mode == "string") {
             auto encoding = type["encoding"].template as<std::string>(); 
-            auto str_len = type["length"].template as<double>(); 
+            auto str_len = type["length"].template as<std::size_t>(); 
             all_types.emplace_back(std::move(name), offset, choose_string_type(encoding, str_len));
         } else if (mode == "numeric") {
             auto type2 = type["type"].template as<std::string>(); 
@@ -817,9 +821,9 @@ void create_numeric_hdf5_dataset(std::string path, std::string name, emscripten:
     }
 }
 
-void create_string_hdf5_dataset(std::string path, std::string name, emscripten::val shape, int32_t deflate_level, emscripten::val chunks, std::string encoding, std::int32_t strlen) {
+void create_string_hdf5_dataset(std::string path, std::string name, emscripten::val shape, std::int32_t deflate_level, emscripten::val chunks, std::string encoding, std::int64_t strlen_or_var) {
     try {
-        create_hdf5_dataset(path, name, choose_string_type(encoding, strlen), shape, deflate_level, chunks);
+        create_hdf5_dataset(path, name, choose_string_type(encoding, strlen_or_var), shape, deflate_level, chunks);
     } catch (H5::Exception& e) {
         throw std::runtime_error(e.getCDetailMsg());
     }
@@ -875,9 +879,9 @@ void create_numeric_hdf5_attribute(std::string path, std::string name, std::stri
     }
 }
 
-void create_string_hdf5_attribute(std::string path, std::string name, std::string attr, emscripten::val shape, std::string encoding, std::int32_t strlen) {
+void create_string_hdf5_attribute(std::string path, std::string name, std::string attr, emscripten::val shape, std::string encoding, std::int64_t strlen_or_var) {
     try {
-        create_hdf5_attribute(path, name, attr, choose_string_type(encoding, strlen), shape);
+        create_hdf5_attribute(path, name, attr, choose_string_type(encoding, strlen_or_var), shape);
     } catch (H5::Exception& e) {
         throw std::runtime_error(e.getCDetailMsg());
     }
@@ -1200,7 +1204,7 @@ void write_compound_hdf5_attribute(std::string path, std::string name, std::stri
 
 /************* String length guessers **************/
 
-std::size_t get_max_str_len(emscripten::val x) {
+double get_max_str_len(emscripten::val x) {
     std::size_t strlen = 0;
     for (auto y : x) {
         if (y.isString()) {
@@ -1210,7 +1214,7 @@ std::size_t get_max_str_len(emscripten::val x) {
             }
         }
     }
-    return strlen;
+    return static_cast<double>(strlen); // hopefully it fits, who knows?
 }
 
 emscripten::val get_max_str_len_compound(emscripten::val x, emscripten::val fields) {
@@ -1219,7 +1223,6 @@ emscripten::val get_max_str_len_compound(emscripten::val x, emscripten::val fiel
         to_access.emplace_back(f.template as<std::string>(), 0);
     }
 
-    std::size_t strlen = 0;
     for (auto y : x) {
         for (auto& t : to_access) {
             auto current_raw = y[t.first];
@@ -1234,7 +1237,7 @@ emscripten::val get_max_str_len_compound(emscripten::val x, emscripten::val fiel
 
     auto output = emscripten::val::array();
     for (const auto& t : to_access) {
-        output.call<void>("push", emscripten::val(t.second));
+        output.call<void>("push", emscripten::val(static_cast<double>(t.second))); // hopefully it fits, who knows.
     }
     return output;
 }

src/run_singlepp.cpp

@@ -174,8 +174,8 @@ struct SingleppIntegratedReferences {
     SingleppIntegratedReferences(Store s) : store(std::move(s)) {};
 
 public:
-    size_t num_references() const {
-        return store.num_references();
+    double num_references() const {
+        return static_cast<double>(store.num_references());
     }
 };