Added E.asUTF8 and extra functionality to work with UTF8 strings in Espruino

Author: gfwilliams

libs/graphics/jswrap_graphics.c

@@ -2091,7 +2091,7 @@ void _jswrap_graphics_stringMetrics(JsGraphics *gfx, JsVar *var, int lineStartIn
   int height = fontHeight;
   int maxWidth = 0;
   while (jsvStringIteratorHasChar(&it)) {
-    char ch = jsvStringIteratorGetCharAndNext(&it);
+    int ch = jsvStringIteratorGetUTF8CharAndNext(&it);
     if (ch=='\n') {
       if (width>maxWidth) maxWidth=width;
       width = 0;
@@ -2211,7 +2211,7 @@ JsVar *jswrap_graphics_wrapString(JsVar *parent, JsVar *str, int maxWidth) {
   jsvStringIteratorNew(&it, str, 0);
 
   while (jsvStringIteratorHasChar(&it) || endOfText) {
-    char ch = jsvStringIteratorGetCharAndNext(&it);
+    int ch = jsvStringIteratorGetUTF8CharAndNext(&it);
     if (endOfText || ch=='\n' || ch==' ') { // newline or space
       int currentPos = jsvStringIteratorGetIndex(&it);
       if ((lineWidth + spaceWidth + wordWidth <= maxWidth) &&
@@ -2377,7 +2377,7 @@ JsVar *jswrap_graphics_drawString(JsVar *parent, JsVar *var, int x, int y, bool
   JsvStringIterator it;
   jsvStringIteratorNew(&it, str, 0);
   while (jsvStringIteratorHasChar(&it)) {
-    char ch = jsvStringIteratorGetCharAndNext(&it);
+    int ch = jsvStringIteratorGetUTF8CharAndNext(&it);
     if (ch=='\n') {
       x = startx;
 #ifndef SAVE_ON_FLASH

src/jsparse.c

@@ -19,6 +19,7 @@
 #include "jswrap_functions.h" // insane check for eval in jspeFunctionCall
 #include "jswrap_json.h" // for jsfPrintJSON
 #include "jswrap_espruino.h" // for jswrap_espruino_memoryArea
+#include "jswrap_string.h" // for jswrap_string_charAt
 #ifndef ESPR_NO_REGEX
 #include "jswrap_regexp.h" // for jswrap_regexp_constructor
 #endif
@@ -1061,8 +1062,7 @@ JsVar *jspGetVarNamedField(JsVar *object, JsVar *nameVar, bool returnName) {
     } else if (jsvIsString(object) && jsvIsInt(nameVar)) {
       JsVarInt idx = jsvGetInteger(nameVar);
       if (idx>=0 && idx<(JsVarInt)jsvGetStringLength(object)) {
-        char ch = jsvGetCharInString(object, (size_t)idx);
-        child = jsvNewStringOfLength(1, &ch);
+        return jswrap_string_charAt(object, idx);
       } else if (returnName)
         child = jsvCreateNewChild(object, nameVar, 0); // just return *something* to show this is handled
     } else {

src/jsutils.c

@@ -990,10 +990,38 @@ unsigned short int int_sqrt32(unsigned int x) {
 }
 
 /// Gets the length of a unicode char sequence by looking at the first char
-int jsUnicodeCharLength(char c) {
+int jsUTF8LengthFromChar(char c) {
   if ((c&0x80)==0) return 1; // ASCII - definitely just one byte
   if ((c&0xE0)==0xC0) return 2; // 2-byte code starts with 0b110xxxxx
   if ((c&0xF0)==0xE0) return 3; // 3-byte code starts with 0b1110xxxx
   if ((c&0xF8)==0xF0) return 4; // 4-byte code starts with 0b11110xxx
   return 1;
 }
+
+/// Given a codepoint, figure hot how many bytes it needs for UTF8 encoding
+int jsUTF8Bytes(int codepoint) {
+  if (codepoint <= 0x7F) return 1;
+  if (codepoint <= 0x7FF) return 2;
+  if (codepoint <= 0xFFFF) return 3;
+  if (codepoint <= 0x10FFFF) return 4;
+  return 0;
+}
+
+// encode a codepoint as a string, NOT null terminated (utf8 min size=4)
+int jsUTF8Encode(int codepoint, char* utf8) {
+  static const uint8_t masks[] = {
+    0x80, // 10000000
+    0xE0, // 11100000
+    0xF0, // 11110000
+    0xF8  // 11111000
+  };
+  int size = jsUTF8Bytes(codepoint);
+  if (!size) return 0;
+  for (int i = size - 1; i > 0; --i) {
+      utf8[i] = (char)((codepoint & ~0xC0) | 0x80);
+      codepoint >>= 6;
+  }
+  utf8[0] = (char)((codepoint & ~(masks[size - 1])) | (masks[size - 1] << 1));
+  return size;
+}
+

src/jsutils.h

@@ -625,6 +625,12 @@ typedef struct {
 } Vector3;
 
 /// Gets the length of a unicode char sequence by looking at the first char
-int jsUnicodeCharLength(char c);
+int jsUTF8LengthFromChar(char c);
+
+/// Given a codepoint, figure hot how many bytes it needs for UTF8 encoding
+int jsUTF8Bytes(int codepoint);
+
+// encode a codepoint as a string, NOT null terminated (utf8 min size=4)
+int jsUTF8Encode(int codepoint, char* utf8);
 
 #endif /* JSUTILS_H_ */

src/jsvar.c

@@ -99,7 +99,7 @@ unsigned char jsvGetLocks(JsVar *v) { return (unsigned char)((v->flags>>JSV_LOCK
 #define JSV_IS_INT(f) ((f)==JSV_INTEGER || JSV_IS_PIN(f) || (f)==JSV_NAME_INT || (f)==JSV_NAME_INT_INT || (f)==JSV_NAME_INT_BOOL)
 #define JSV_IS_NUMERIC(f) ((f)>=_JSV_NUMERIC_START && (f)<=_JSV_NUMERIC_END)
 #define JSV_IS_STRING(f) ((f)>=_JSV_STRING_START && (f)<=_JSV_STRING_END)
-#define JSV_IS_UNICODE_STRING(f)  (f)==JSV_UNICODE_STRING
+#define JSV_IS_UNICODE_STRING(f)  (f)==JSV_UTF8_STRING
 #define JSV_IS_STRING_EXT(f) ((f)>=JSV_STRING_EXT_0 && (f)<=JSV_STRING_EXT_MAX)
 #define JSV_IS_FLAT_STRING(f) (f)==JSV_FLAT_STRING
 #define JSV_IS_NATIVE_STRING(f) (f)==JSV_NATIVE_STRING
@@ -128,7 +128,7 @@ bool jsvIsInt(const JsVar *v) { if (!v) return false; char f = v->flags&JSV_VART
 bool jsvIsFloat(const JsVar *v) { return v && (v->flags&JSV_VARTYPEMASK)==JSV_FLOAT; }
 bool jsvIsBoolean(const JsVar *v) { if (!v) return false; char f = v->flags&JSV_VARTYPEMASK; return JSV_IS_BOOL(f); }
 bool jsvIsString(const JsVar *v) { if (!v) return false; char f = v->flags&JSV_VARTYPEMASK; return JSV_IS_STRING(f); } ///< String, or a NAME too
-bool jsvIsUnicodeString(const JsVar *v) { if (!v) return false; char f = v->flags&JSV_VARTYPEMASK; return JSV_IS_UNICODE_STRING(f); } ///< Just a unicode string (Unicode JsVar, pointing to a string)
+bool jsvIsUTF8String(const JsVar *v) { if (!v) return false; char f = v->flags&JSV_VARTYPEMASK; return JSV_IS_UNICODE_STRING(f); } ///< Just a unicode string (UTF8 JsVar, pointing to a string)
 bool jsvIsBasicString(const JsVar *v) { if (!v) return false; char f = v->flags&JSV_VARTYPEMASK; return f>=JSV_STRING_0 && f<=JSV_STRING_MAX; } ///< Just a string (NOT a name/flatstr/nativestr or flashstr)
 bool jsvIsStringExt(const JsVar *v) { if (!v) return false; char f = v->flags&JSV_VARTYPEMASK; return JSV_IS_STRING_EXT(f); } ///< The extra bits dumped onto the end of a string to store more data
 bool jsvIsFlatString(const JsVar *v) { if (!v) return false; char f = v->flags&JSV_VARTYPEMASK; return JSV_IS_FLAT_STRING(f); }
@@ -1096,9 +1096,9 @@ JsVar *jsvNewStringOfLength(unsigned int byteLength, const char *initialData) {
   return first;
 }
 
-JsVar *jsvNewUnicodeString(JsVar* dataString) {
+JsVar *jsvNewUTF8String(JsVar* dataString) {
   assert(jsvIsString(dataString));
-   JsVar *var = jsvNewWithFlags(JSV_UNICODE_STRING);
+   JsVar *var = jsvNewWithFlags(JSV_UTF8_STRING);
   if (!var) return 0; // no memory
   jsvSetFirstChild(var, jsvGetRef(jsvRef(dataString)));
   return var;
@@ -1643,6 +1643,18 @@ bool jsvIsEmptyString(JsVar *v) {
 
 size_t jsvGetStringLength(const JsVar *v) {
   size_t strLength = 0;
+  // For unicode, we just have to iterate to get a length
+  if (jsvIsUTF8String(v)) {
+    JsvStringIterator it;
+    jsvStringIteratorNew(&it, v, 0);
+    while (jsvStringIteratorHasChar(&it)) {
+      jsvStringIteratorNextUTF8(&it);
+      strLength++;
+    }
+    jsvStringIteratorFree(&it);
+    return strLength;
+  }
+
   const JsVar *var = v;
   JsVar *newVar = 0;
   if (!jsvHasCharacterData(v)) return 0;
@@ -1895,12 +1907,12 @@ void jsvAppendStringVarComplete(JsVar *var, const JsVar *str) {
   jsvAppendStringVar(var, str, 0, JSVAPPENDSTRINGVAR_MAXLENGTH);
 }
 
-char jsvGetCharInString(JsVar *v, size_t idx) {
+int jsvGetCharInString(JsVar *v, size_t idx) {
   if (!jsvIsString(v)) return 0;
 
   JsvStringIterator it;
   jsvStringIteratorNew(&it, v, idx);
-  char ch = jsvStringIteratorGetChar(&it);
+  int ch = jsvStringIteratorGetUTF8CharAndNext(&it);
   jsvStringIteratorFree(&it);
   return ch;
 }
@@ -2493,9 +2505,8 @@ int jsvCompareString(JsVar *va, JsVar *vb, size_t starta, size_t startb, bool eq
   jsvStringIteratorNew(&itb, vb, startb);
   // step to first positions
   while (true) {
-    int ca = jsvStringIteratorGetCharOrMinusOne(&ita);
-    int cb = jsvStringIteratorGetCharOrMinusOne(&itb);
-
+    int ca = jsvStringIteratorGetUTF8CharAndNext(&ita);
+    int cb = jsvStringIteratorGetUTF8CharAndNext(&itb);
     if (ca != cb) {
       jsvStringIteratorFree(&ita);
       jsvStringIteratorFree(&itb);
@@ -2507,8 +2518,6 @@ int jsvCompareString(JsVar *va, JsVar *vb, size_t starta, size_t startb, bool eq
       jsvStringIteratorFree(&itb);
       return 0;
     }
-    jsvStringIteratorNext(&ita);
-    jsvStringIteratorNext(&itb);
   }
   // never get here, but the compiler warns...
   return true;
@@ -2522,14 +2531,12 @@ JsVar *jsvGetCommonCharacters(JsVar *va, JsVar *vb) {
   JsvStringIterator ita, itb;
   jsvStringIteratorNew(&ita, va, 0);
   jsvStringIteratorNew(&itb, vb, 0);
-  int ca = jsvStringIteratorGetCharOrMinusOne(&ita);
-  int cb = jsvStringIteratorGetCharOrMinusOne(&itb);
+  int ca = jsvStringIteratorGetUTF8CharAndNext(&ita);
+  int cb = jsvStringIteratorGetUTF8CharAndNext(&itb);
   while (ca>0 && cb>0 && ca == cb) {
     jsvAppendCharacter(v, (char)ca);
-    jsvStringIteratorNext(&ita);
-    jsvStringIteratorNext(&itb);
-    ca = jsvStringIteratorGetCharOrMinusOne(&ita);
-    cb = jsvStringIteratorGetCharOrMinusOne(&itb);
+    ca = jsvStringIteratorGetUTF8CharAndNext(&ita);
+    cb = jsvStringIteratorGetUTF8CharAndNext(&itb);
   }
   jsvStringIteratorFree(&ita);
   jsvStringIteratorFree(&itb);

src/jsvar.h

@@ -72,9 +72,9 @@ typedef enum {
     JSV_STRING_MAX  = JSV_STRING_0+JSVAR_DATA_STRING_LEN,
     JSV_FLAT_STRING = JSV_STRING_MAX+1, ///< Flat strings store the length (in chars) as an int, and then the subsequent JsVars (in memory) store data
     JSV_NATIVE_STRING = JSV_FLAT_STRING+1, ///< Native strings store an address and length, and reference the underlying data directly
-    JSV_UNICODE_STRING = JSV_NATIVE_STRING+1, ///< Unicode just point to a normal string with firstChild, but just tag that the string is a unicode one
+    JSV_UTF8_STRING = JSV_NATIVE_STRING+1, ///< UTF8 just point to a normal string with firstChild, but just tag that the string is a unicode one
 #ifdef SPIFLASH_BASE
-    JSV_FLASH_STRING = JSV_UNICODE_STRING+1, ///< Like a native String, but not writable and uses jshFlashRead
+    JSV_FLASH_STRING = JSV_UTF8_STRING+1, ///< Like a native String, but not writable and uses jshFlashRead
     _JSV_STRING_END = JSV_FLASH_STRING,
 #else
     _JSV_STRING_END = JSV_NATIVE_STRING,
@@ -302,7 +302,7 @@ JsVar *jsvNewFromString(const char *str); ///< Create a new string
 JsVar *jsvNewNameFromString(const char *str); ///< Create a new name from a string
 JsVar *jsvNewStringOfLength(unsigned int byteLength, const char *initialData); ///< Create a new string of the given length - full of 0s (or initialData if specified)
 static ALWAYS_INLINE JsVar *jsvNewFromEmptyString() { return jsvNewWithFlags(JSV_STRING_0); } ;///< Create a new empty string
-JsVar *jsvNewUnicodeString(JsVar* dataString); ///< Create a new unicode string using the given data string for backing
+JsVar *jsvNewUTF8String(JsVar* dataString); ///< Create a new unicode string using the given data string for backing
 static ALWAYS_INLINE JsVar *jsvNewNull() { return jsvNewWithFlags(JSV_NULL); } ;///< Create a new null variable
 /** Create a new variable from a substring. argument must be a string. stridx = start char or str, maxLength = max number of characters (can be JSVAPPENDSTRINGVAR_MAXLENGTH)  */
 JsVar *jsvNewFromStringVar(const JsVar *str, size_t stridx, size_t maxLength);
@@ -386,7 +386,7 @@ bool jsvIsInt(const JsVar *v);
 bool jsvIsFloat(const JsVar *v);
 bool jsvIsBoolean(const JsVar *v);
 bool jsvIsString(const JsVar *v); ///< String, or a NAME too
-bool jsvIsUnicodeString(const JsVar *v); ///< Just a unicode string (Unicode JsVar, pointing to a string)
+bool jsvIsUTF8String(const JsVar *v); ///< Just a unicode string (UTF8 JsVar, pointing to a string)
 bool jsvIsBasicString(const JsVar *v); ///< Just a string (NOT a name)
 bool jsvIsStringExt(const JsVar *v); ///< The extra bits dumped onto the end of a string to store more data
 bool jsvIsFlatString(const JsVar *v);
@@ -513,7 +513,7 @@ static ALWAYS_INLINE void jsvAppendCharacter(JsVar *var, char ch) { jsvAppendStr
 #define JSVAPPENDSTRINGVAR_MAXLENGTH (0x7FFFFFFF)
 void jsvAppendStringVar(JsVar *var, const JsVar *str, size_t stridx, size_t maxLength); ///< Append str to var. Both must be strings. stridx = start char or str, maxLength = max number of characters (can be JSVAPPENDSTRINGVAR_MAXLENGTH)
 void jsvAppendStringVarComplete(JsVar *var, const JsVar *str); ///< Append all of str to var. Both must be strings.
-char jsvGetCharInString(JsVar *v, size_t idx); ///< Get a character at the given index in the String
+int jsvGetCharInString(JsVar *v, size_t idx); ///< Get a character at the given index in the String (handles unicode)
 void jsvSetCharInString(JsVar *v, size_t idx, char ch, bool bitwiseOR); ///< Set a character at the given index in the String. If bitwiseOR, ch will be ORed with the character already at that position.
 int jsvGetStringIndexOf(JsVar *str, char ch); ///< Get the index of a character in a string, or -1