fix: Parse character, octal, and hex string escapes

Support additional protobuf character escapes, including escaped quotes, plus bounded octal and hexadecimal escapes. Add tests for parsed defaults and numeric escape boundaries.

Unicode escapes and validation of malformed or unknown escapes remain unsupported.

Author: jiang1997

src/tokenize.js

@@ -9,13 +9,19 @@ var setCommentRe = /^ *[*/]+ */,
     setCommentAltRe = /^\s*\*?\/*/,
     setCommentSplitRe = /\n/g,
     whitespaceRe = /\s/,
-    unescapeRe = /\\(.?)/g;
+    unescapeRe = /\\([0-7]{1,3}|[xX][0-9a-fA-F]{1,2}|.?)/g;
 
 var unescapeMap = {
-    "0": "\0",
+    "a": "\x07",
+    "b": "\b",
+    "f": "\f",
     "r": "\r",
     "n": "\n",
-    "t": "\t"
+    "t": "\t",
+    "v": "\v",
+    "?": "?",
+    "'": "'",
+    "\"": "\""
 };
 
 /**
@@ -27,6 +33,11 @@ var unescapeMap = {
  */
 function unescape(str) {
     return str.replace(unescapeRe, function($0, $1) {
+        var first = $1.charAt(0);
+        if (first >= "0" && first <= "7")
+            return String.fromCharCode(parseInt($1, 8) & 255);
+        if (first === "x" || first === "X")
+            return String.fromCharCode(parseInt($1.substring(1), 16));
         switch ($1) {
             case "\\":
             case "":

tests/api_tokenize.js

@@ -7,16 +7,26 @@ var tokenize = protobuf.tokenize;
 tape.test("tokenize", function(test) {
 
     test.test(test.name + " - unescape", function(test) {
-        test.equal(tokenize.unescape("\\\\0 \\\0 \\0 \0"), "\\0  \0 \0", "should propery unescape zero-sequences");
-        test.equal(tokenize.unescape("\\\t\\t\\r\\n"), "\t\r\n", "should propery unescape tabs and line feeds");
+        test.equal(tokenize.unescape("\\\\0\\0"), "\\0\0", "should properly unescape zero-sequences");
+        test.equal(tokenize.unescape("\\t\\r\\n"), "\t\r\n", "should properly unescape tabs and line feeds");
+        test.equal(tokenize.unescape("\\a\\b\\f\\v\\?\\'\\\""), "\x07\b\f\v?'\"", "should unescape single-character sequences");
+        test.equal(tokenize.unescape("\\101\\102\\103"), "ABC", "should unescape octal sequences");
+        test.equal(tokenize.unescape("\\x41\\x42\\X43"), "ABC", "should unescape hexadecimal sequences");
+        test.equal(tokenize.unescape("\\1x\\12x\\123x\\x1x\\x12x"), "\x01x\nxSx\x01x\x12x", "should consume bounded numeric escapes");
         test.end();
     });
 
     test.ok(expect("", [null]), "should instantly finish for an empty source");
     test.ok(expect("'hello\\nworld'", ["'", "hello\nworld", "'", null]), "should parse single quoted strings");
     test.ok(expect("\"hello\\nworld\"", ["\"", "hello\nworld", "\"", null]), "should parse double quoted strings");
+    test.ok(expect("'a\\'b'", ["'", "a'b", "'", null]), "should parse escaped single quotes");
+    test.ok(expect("\"a\\\"b\"", ["\"", "a\"b", "\"", null]), "should parse escaped double quotes");
     test.ok(expectError("\"as\"d\""), "should throw for invalid strings");
 
+    var escapedDefault = protobuf.parse("syntax = \"proto2\"; message M { optional string value = 1 [default = \"default<>\\'\\\"abc\\101\\x42\"]; }")
+        .root.lookupType("M").fields.value.options.default;
+    test.equal(escapedDefault, "default<>'\"abcAB", "should preserve escaped string defaults");
+
     var tn = tokenize("message Test {}");
     test.throws(function() {
         tn.skip("somethingelse", false);

tests/data/test.js

@@ -4489,7 +4489,7 @@ $root.jspb = (function() {
              * @memberof jspb.test.DefaultValues
              * @instance
              */
-            DefaultValues.prototype.stringField = "default<>abc";
+            DefaultValues.prototype.stringField = "default<>'\"abc";
 
             /**
              * DefaultValues boolField.
@@ -4818,7 +4818,7 @@ $root.jspb = (function() {
                     throw $Error("max depth exceeded");
                 var object = {};
                 if (options.defaults) {
-                    object.stringField = "default<>abc";
+                    object.stringField = "default<>'\"abc";
                     object.boolField = true;
                     if ($util.Long) {
                         var long = new $util.Long(11, 0, false);