string: strlength() workaround for unicode2native() buffer-length bug in pre-10.2.0 Octave

In Octave 9.x and earlier, and 10.1.0, strlength() would return the wrong length for short strings, because Octave's unicode2native() will return an incorrect result with extra bytes, due to a buffer length calculation bug. This adds a disgusting hack to work around that particular bug and produce correct strlength results. Uses a conservative check to only activate that workaround when it detects that particular buggy strlength() behavior.

See #143.

Author: apjanke

CHANGES.txt

@@ -4,7 +4,9 @@ Tablicious Changelog
 0.4.6 (unreleased)
 ------------------
 
-(nothing here yet)
+* Work around `unicode2native` buffer length bug in `string.strlength`. ([#143](https://github.com/apjanke/octave-tablicious/issues/143))
+* Internal changes.
+  * Change some more code to GNU Octave style.
 
 0.4.5 (2025-07-03)
 ------------------

inst/@string/string.m

@@ -555,13 +555,52 @@ function disp (this)
     ##
     ## @end deftypefn
     function out = strlength (this)
+      persistent do_u2n_buflen_bug_workaround
+      if isempty (do_u2n_buflen_bug_workaround)
+        # Workaround for unicode2native() buffer-length bug in pre-10.2.0 Octave that
+        # results in extra bytes in its output for short strings.
+        # See:
+        #   * https://github.com/apjanke/octave-tablicious/issues/143
+        #   * https://octave.discourse.group/t/wrong-number-of-bytes-returned-from-unicode2native/6418
+        # There's a bug in Octave that causes unicode2native() conversions to UTF-16 or
+        # UTF-32 to add extra bogus bytes when converting short strings, which throws off
+        # the character count here. The bug doesn't seem to affect strings length 4 or
+        # longer. So as a hacky workaround, in affected Octave versions, we'll just add
+        # some padding to all the strings passed to unicode2native to get out of the danger
+        # zone for that bug, and then back them out of the resulting char counts.
+        # I'm not sure which versions of Octave were affected, or exactly what the scope of
+        # the bug is, so probe for the specific buggy behavior I'm familiar with, and bail
+        # if we get something that doesn't closely match that or the good-case behavior.
+        good_strlens = [0 2 4 6 8 10];
+        buflenbug_strlens = [0 5 6 7 8 10];
+        got_strlens = arrayfun(@(n) numel (unicode2native (repmat ('a', [1 n]), 'UTF-16LE')), 0:5);
+        if (isequal (got_strlens, good_strlens))
+          do_u2n_buflen_bug_workaround = false;
+        elseif (isequal (got_strlens, buflenbug_strlens))
+          do_u2n_buflen_bug_workaround = true;
+        else
+          error ('tablicious:BUG', ['Got an unexpected strlen probe value for unicode2native test ' ...
+            'test in strlength() initialization. This indicates a behavior of core Octave that ' ...
+            'Tablicious is unprepared for. This is a bug that Tablicious cannot handle. Please ' ...
+            'report this to the Tablicious maintainers if you are so inclined. Cannot produce ' ...
+            'reasonable strlength() results in these conditions. Aborting. Sorry.'])
+        endif
+      endif
       out = NaN (size (this));
       for i = 1:numel (out)
         if (this.tfMissing(i))
           continue
         endif
-        utf16 = unicode2native (this.strs{i}, 'UTF-16LE');
-        out(i) = numel (utf16) / 2;
+        if (do_u2n_buflen_bug_workaround)
+          # unicode2native bug workaround case for pre-10.2.0 Octave; see above.
+          # Yes, this is a terrible hack. Sorry about that.
+          utf16 = unicode2native (['aaaa' this.strs{i}], 'UTF-16LE');
+          out(i) = (numel (utf16) / 2) - 4;
+        else
+          # regular case
+          utf16 = unicode2native (this.strs{i}, 'UTF-16LE');
+          out(i) = numel (utf16) / 2;
+        endif
       endfor
     endfunction
 

inst/t/t_02_string.m

@@ -0,0 +1,62 @@
+# Covered by the 3-clause BSD License (see LICENSES/LICENSE-MATPOWER file for details).
+#
+# Copyright (c) 2025, Andrew Janke
+
+function obj = t_02_string (quiet)
+# Tests for string array.
+
+if nargin < 1 || isempty (quiet);   quiet = false;  endif
+
+verbose = ! quiet;
+
+# Test fixtures
+
+# strlength tests
+
+# scalars: { str, strlen; ...}
+fixture_1 = {
+  ''      0
+  'a'     1
+  'aa'    2
+  'aaa'   3
+  'aaaa'  4
+  'aaaaa'   5
+};
+
+# nonscalars: { strs, strlens; ...}
+fixture_2 = {
+  {''}      [0]
+  {'a'}     [1]
+  {'a', 'aa', 'aaa', ''}    [1, 2, 3, 0]
+};
+
+# Run strlength tests
+
+t_begin (size (fixture_1, 1) + size (fixture_2, 1), quiet);
+
+# scalar strlength tests
+
+max_prec = 16;
+for i_test = 1:size (fixture_1, 1)
+  [str_c, exp_len] = fixture_1{i_test,:};
+  str = string (str_c);
+  t_is (strlength (str), exp_len, max_prec, sprintf ('strlength ("%s")\n', str_c));
+endfor
+
+# nonscalar strlength tests
+
+for i_test = 1:size (fixture_2, 1)
+  [strs_c, exp_len] = fixture_1{i_test,:};
+  strs = string (strs_c);
+  t_is (strlength (strs), exp_len, max_prec, sprintf ('strlength (<item %d>)\n', i_test));
+endfor
+
+# Wrap up
+
+if nargout
+  obj = T;
+endif
+
+t_end;
+
+endfunction