Streamlined the handling of non-empty blocks of sparse LHS vectors.

Author: LTLA

include/tatami_mult/dense_matrix/sparse_column/column_to_column.hpp

@@ -131,15 +131,12 @@ void multiply_sparse_column_with_dense_column_matrix_to_column_output(
                     options.block_size,
                     /* zero = */ [&](const LeftIndex_) -> void {} // buffers should already be zeroed.
                 );
-                cd = left_block_info.position;
-
                 const auto cd_num = left_block_info.num_non_empty;
-                if (cd_num == 0) {
-                    break;
-                }
 
-                if (left_block_info.consecutive) {
-                    const LeftIndex_ cd_base = start + left_non_empty.front();
+                // If the LHS rows are all non-empty, we can speed up the loops by just using a simple counter to get the column indices.
+                // Otherwise, we'll have to access the 'left_non_empty' vector to figure out the indices of each non-empty column.
+                if (left_block_info.all_non_empty) {
+                    const LeftIndex_ cd_base = cd + start; 
                     for (RightIndex_ rc = 0; rc < right_NC; ++rc) {
                         const auto rightcol = right_ptrs[rc];
                         for (LeftIndex_ cd_counter = 0; cd_counter < cd_num; ++cd_counter) {
@@ -166,6 +163,8 @@ void multiply_sparse_column_with_dense_column_matrix_to_column_output(
                         }
                     }
                 }
+
+                cd = left_block_info.position;
             }
         }
 

include/tatami_mult/dense_matrix/sparse_row/column_to_row.hpp

@@ -123,18 +123,12 @@ void multiply_sparse_row_with_dense_column_matrix_to_row_output(
                     std::fill_n(output + sanisizer::product_unsafe<std::size_t>(start + lr_copy, right_NC), right_NC, 0);
                 }
             );
-            lr = left_block_info.position;
-
             const auto lr_num = left_block_info.num_non_empty;
-            if (lr_num == 0) {
-                break;
-            }
 
-            // If the LHS columns are consecutive, we can speed up the loops by just using a simple counter to get the column indices.
+            // If the LHS columns are all non-empty, we can speed up the loops by just using a simple counter to get the column indices.
             // Otherwise, we'll have to access the 'left_non_empty' vector to figure out the indices of each non-empty column.
-            if (left_block_info.consecutive) {
-                const auto lr_base = start + left_non_empty.front();
-
+            if (left_block_info.all_non_empty) {
+                const auto lr_base = lr + start; 
                 // Yes, we deliberately iterate over the RHS columns in the outer loop to keep the dense column in cache across multiple LHS vectors.
                 // If we did it the other way around, this would defeat the purpose of blocking.
                 for (RightIndex_ rc = 0; rc < right_NC; ++rc) {
@@ -169,6 +163,8 @@ void multiply_sparse_row_with_dense_column_matrix_to_row_output(
                     }
                 }
             }
+
+            lr = left_block_info.position;
         }
     }, left_NR, options.num_threads);
 }

include/tatami_mult/multiple_vectors/sparse_column.hpp

@@ -144,17 +144,12 @@ void multiply_sparse_column_with_multiple_vectors(
                     options.block_size,
                     /* zero = */ [](Index_) -> void {} // (No need to worry about zeroing as the buffers have already been zeroed.)
                 );
-                cd = left_block_info.position;
-
                 const auto cd_num = left_block_info.num_non_empty;
-                if (cd_num == 0) {
-                    break;
-                }
 
-                // If the LHS columns are consecutive, we can speed up the loops by just using a simple counter to get the column indices.
+                // If the LHS columns are all non-empty, we can speed up the loops by just using a simple counter to get the column indices.
                 // Otherwise, we'll have to access the 'left_non_empty' vector to figure out the indices of each non-empty column.
-                if (left_block_info.consecutive) {
-                    const Index_ cd_base = start + left_non_empty.front();
+                if (left_block_info.all_non_empty) {
+                    const Index_ cd_base = start + cd;
                     for (RightIndex rc = 0; rc < right_NC; ++rc) {
                         const auto outcol = outptrs[rc];
                         const auto rightcol = right[rc];
@@ -183,6 +178,8 @@ void multiply_sparse_column_with_multiple_vectors(
                         }
                     }
                 }
+
+                cd = left_block_info.position;
             }
         }
 

include/tatami_mult/utils.hpp

@@ -42,10 +42,10 @@ void populate_dense_buffers(
 
 template<typename Index_>
 struct FetchNonEmptySparseBlockInfo {
-    FetchNonEmptySparseBlockInfo(const Index_ position, const Index_ num_non_empty, const bool consecutive) : 
-        position(position), num_non_empty(num_non_empty), consecutive(consecutive) {}
+    FetchNonEmptySparseBlockInfo(const Index_ position, const Index_ num_non_empty, const bool all_non_empty) : 
+        position(position), num_non_empty(num_non_empty), all_non_empty(all_non_empty) {}
     Index_ position, num_non_empty;
-    bool consecutive;
+    bool all_non_empty;
 };
 
 template<typename Value_, typename Index_, class Zero_>
@@ -62,7 +62,7 @@ FetchNonEmptySparseBlockInfo<Index_> fetch_non_empty_sparse_block(
 ) { 
     non_empty.clear();
     Index_ num_non_empty = 0;
-    bool consecutive = true;
+    bool all_non_empty = true;
 
     // We assume that a check for remaining dimension elements was performed before continuing the blocked loop.
     assert(position < length);
@@ -71,7 +71,7 @@ FetchNonEmptySparseBlockInfo<Index_> fetch_non_empty_sparse_block(
         auto lrange = ext.fetch(vbuffers[num_non_empty].data(), ibuffers[num_non_empty].data());
         if (lrange.number == 0) {
             zero(position);
-            consecutive = false;
+            all_non_empty = false;
             ++position;
             continue;
         }
@@ -86,7 +86,7 @@ FetchNonEmptySparseBlockInfo<Index_> fetch_non_empty_sparse_block(
         }
     } while (position < length);
 
-    return FetchNonEmptySparseBlockInfo(position, num_non_empty, consecutive);
+    return FetchNonEmptySparseBlockInfo(position, num_non_empty, all_non_empty);
 }
 
 template<typename Output_, typename DenseValue_, typename Value_, typename Index_>