Allow pretokenise decoder to decode raw strings - fix espruino/EspruinoWebIDE#297

Author: gfwilliams

plugins/pretokenise.js

@@ -225,8 +225,21 @@
       var ch = code.charCodeAt(i);
       if (needSpaceBetween(lastCh, ch))
         resultCode += " ";
-      if (ch>=LEX_OPERATOR_START && ch<LEX_OPERATOR_START+TOKENS.length) {
-        resultCode += TOKENS[ch-LEX_OPERATOR_START];
+      if (ch>=LEX_OPERATOR_START) {
+        if (ch==LEX_RAW_STRING8) { // decode raw strings
+          var len = code.charCodeAt(i+1);
+          resultCode += JSON.stringify(code.substring(i+2, i+2+len));
+          i+=1+len;
+        } else if (ch==LEX_RAW_STRING16) {
+          var len = code.charCodeAt(i+1) | (code.charCodeAt(i+2)<<8);
+          resultCode += JSON.stringify(code.substring(i+3, i+3+len));
+          i+=2+len;
+        } else if (ch<LEX_OPERATOR_START+TOKENS.length) // decoded other tokens
+          resultCode += TOKENS[ch-LEX_OPERATOR_START];
+        else {
+          console.warn("Unexpected pretokenised string code:", ch);
+          resultCode += code[i];
+        }
       } else resultCode += code[i];
       lastCh = ch;
     }