Implement NativeCall wide string support

docs/ops.markdown

@@ -2817,6 +2817,14 @@ following structure (times are in microseconds, sizes are in bytes):
 ## nativecallrefresh
 Refresh the C-based data backing the Perl 6 object. This op should only be used if changes have been made to the C-data, and these changes are not being reflected in the Perl 6 object.
 
+## iswcharunsigned
+
+Returns 1 if wchar_t is unsigned, otherwise returns 0.
+
+## iswintunsigned
+
+Returns 1 if wint_t is unsigned, otherwise returns 0.
+
 # <a id="async"></a> Asynchronous Operations
 
 The various asynchronous operations, such as timers and asynchronous I/O, take

src/how/NQPAttribute.nqp

@@ -2,6 +2,7 @@ knowhow NQPAttribute {
     has $!name;
     has $!type;
     has $!has_type;
+    has $!char_type;
     has $!box_target;
     has $!default;
     has $!has_default;
@@ -16,12 +17,13 @@ knowhow NQPAttribute {
         $attr
     }
 
-    method BUILD(:$name, :$type, :$has_type, :$box_target, :$default, :$has_default) {
-        $!name := $name;
-        $!type := $type;
-        $!has_type := $has_type;
-        $!box_target := $box_target;
-        $!default := $default;
+    method BUILD(:$name, :$type, :$has_type, :$char_type, :$box_target, :$default, :$has_default) {
+        $!name        := $name;
+        $!type        := $type;
+        $!has_type    := $has_type;
+        $!char_type   := $char_type;
+        $!box_target  := $box_target;
+        $!default     := $default;
         $!has_default := $has_default;
     }
 
@@ -33,6 +35,14 @@ knowhow NQPAttribute {
         $!has_type ?? $!type !! nqp::null()
     }
 
+    method set_char_type($char_type) {
+        $!char_type := $char_type;
+    }
+
+    method char_type() {
+        $!char_type
+    }
+
     method has_accessor() {
         0
     }

src/vm/js/Operations.nqp

@@ -1753,24 +1753,21 @@ class QAST::OperationsJS {
     add_simple_op('callercode', :!inlinable, $T_OBJ, [], sub () {"caller_ctx.codeRef()"});
 
     # Native Call
+
     add_simple_op('buildnativecall',  $T_INT, [$T_OBJ, $T_STR, $T_STR, $T_STR, $T_OBJ, $T_OBJ], :side_effects, :ctx);
     add_simple_op('nativecall', $T_OBJ, [$T_OBJ, $T_OBJ, $T_OBJ], :side_effects);
-
     add_simple_op('nativecallsizeof', $T_INT, [$T_OBJ], :decont(0));
-
     add_simple_op('nativecallglobal', $T_OBJ, [$T_STR, $T_STR, $T_OBJ, $T_OBJ]);
-
     add_simple_op('nativecallcast', $T_OBJ, [$T_OBJ, $T_OBJ, $T_OBJ]);
-
     add_simple_op('nativecallrefresh', $T_OBJ, [$T_INT], :side_effects);
-
     add_simple_op('initnativecall', $T_INT, [], :side_effects);
+    add_simple_op('iswcharunsigned', $T_INT, []);
+    add_simple_op('iswintunsigned', $T_INT, []);
 
     # Continuations
 
     add_simple_op('continuationreset', $T_OBJ, [$T_OBJ, $T_OBJ], :side_effects, :ctx, :takes_hll, :await);
     add_simple_op('continuationinvoke', $T_OBJ, [$T_OBJ, $T_OBJ], :side_effects, :ctx, :takes_hll, :await);
-
     add_simple_op('continuationcontrol', $T_OBJ, [$T_INT, $T_OBJ, $T_OBJ], :side_effects, :ctx, :takes_hll, :await);
 
 

src/vm/js/const_map.nqp

@@ -13,15 +13,15 @@ my %const_map := nqp::hash(
     'CCLASS_ALPHANUMERIC',  2048,
     'CCLASS_NEWLINE',       4096,
     'CCLASS_WORD',          8192,
-    
+
     'HLL_ROLE_NONE',        0,
     'HLL_ROLE_INT',         1,
     'HLL_ROLE_NUM',         2,
     'HLL_ROLE_STR',         3,
     'HLL_ROLE_ARRAY',       4,
     'HLL_ROLE_HASH',        5,
     'HLL_ROLE_CODE',        6,
-    
+
     'CONTROL_ANY',          2,
     'CONTROL_NEXT',         4,
     'CONTROL_REDO',         8,
@@ -35,7 +35,7 @@ my %const_map := nqp::hash(
     'CONTROL_AWAIT',        8192,
     'CONTROL_EMIT',         16384,
     'CONTROL_DONE',         32768,
-    
+
     'STAT_EXISTS',             0,
     'STAT_FILESIZE',           1,
     'STAT_ISDIR',              2,
@@ -57,17 +57,27 @@ my %const_map := nqp::hash(
     'STAT_PLATFORM_BLOCKSIZE', -6,
     'STAT_PLATFORM_BLOCKS',    -7,
 
-    'C_TYPE_CHAR',              -1,
-    'C_TYPE_SHORT',             -2,
-    'C_TYPE_INT',               -3,
-    'C_TYPE_LONG',              -4,
-    'C_TYPE_LONGLONG',          -5,
-    'C_TYPE_SIZE_T',            -6,
-    'C_TYPE_BOOL',              -7,
-    'C_TYPE_ATOMIC_INT',        -8,
-    'C_TYPE_FLOAT',             -1,
-    'C_TYPE_DOUBLE',            -2,
-    'C_TYPE_LONGDOUBLE',        -3,
+    'P6INT_C_TYPE_CHAR',        -1,
+    'P6INT_C_TYPE_SHORT',       -2,
+    'P6INT_C_TYPE_INT',         -3,
+    'P6INT_C_TYPE_LONG',        -4,
+    'P6INT_C_TYPE_LONGLONG',    -5,
+    'P6INT_C_TYPE_BOOL',        -6,
+    'P6INT_C_TYPE_SIZE_T',      -7,
+    'P6INT_C_TYPE_ATOMIC_INT',  -8,
+    'P6INT_C_TYPE_WCHAR_T',     -9,
+    'P6INT_C_TYPE_WINT_T',      -10,
+    'P6INT_C_TYPE_CHAR16_T',    -11,
+    'P6INT_C_TYPE_CHAR32_T',    -12,
+
+    'P6NUM_C_TYPE_FLOAT',       -32,
+    'P6NUM_C_TYPE_DOUBLE',      -33,
+    'P6NUM_C_TYPE_LONGDOUBLE',  -34,
+
+    'P6STR_C_TYPE_CHAR',        -64,
+    'P6STR_C_TYPE_WCHAR_T',     -65,
+    'P6STR_C_TYPE_CHAR16_T',    -66,
+    'P6STR_C_TYPE_CHAR32_T',    -67,
 
     'TYPE_CHECK_CACHE_DEFINITIVE',  0,
     'TYPE_CHECK_CACHE_THEN_METHOD', 1,

src/vm/js/nqp-runtime/nativecall.js

@@ -112,3 +112,15 @@ op.nativecallsizeof = function(obj) {
 op.initnativecall = function() {
   return 1;
 };
+
+op.iswcharunsigned = function() {
+  /* XXX: we can't implement this properly until ref-napi supports wide
+   * strings. */
+  return process.platform === 'win32';
+};
+
+op.iswintunsigned = function() {
+  /* XXX: we can't implement this properly until ref-napi supports wide
+   * strings. */
+  return 0;
+};

src/vm/js/nqp-runtime/reprs.js

@@ -699,45 +699,62 @@ class Uninstantiable extends REPR {
 reprs.Uninstantiable = Uninstantiable;
 
 
-const C_TYPE_CHAR = -1;
-const C_TYPE_SHORT = -2;
-const C_TYPE_INT = -3;
-const C_TYPE_LONG = -4;
-const C_TYPE_LONGLONG = -5;
-const C_TYPE_SIZE_T = -6;
-const C_TYPE_BOOL = -7;
-const C_TYPE_ATOMIC_INT = -8;
+const P6INT_C_TYPE_CHAR       = -1;
+const P6INT_C_TYPE_SHORT      = -2;
+const P6INT_C_TYPE_INT        = -3;
+const P6INT_C_TYPE_LONG       = -4;
+const P6INT_C_TYPE_LONGLONG   = -5;
+const P6INT_C_TYPE_SIZE_T     = -6;
+const P6INT_C_TYPE_BOOL       = -7;
+const P6INT_C_TYPE_ATOMIC_INT = -8;
+const P6INT_C_TYPE_WCHAR_T    = -9;
+const P6INT_C_TYPE_WINT_T     = -10;
+const P6INT_C_TYPE_CHAR16_T   = -11;
+const P6INT_C_TYPE_CHAR32_T   = -12;
 
 function cType(ctype) {
   switch (ctype) {
-    case C_TYPE_CHAR:
+    case P6INT_C_TYPE_CHAR:
       return ref.types.char;
-    case C_TYPE_SHORT:
+    case P6INT_C_TYPE_SHORT:
       return ref.types.short;
-    case C_TYPE_ATOMIC_INT:
-    case C_TYPE_INT:
+    case P6INT_C_TYPE_ATOMIC_INT:
+    case P6INT_C_TYPE_INT:
       return ref.types.int;
-    case C_TYPE_LONG:
+    case P6INT_C_TYPE_LONG:
       return ref.types.long;
-    case C_TYPE_LONGLONG:
+    case P6INT_C_TYPE_LONGLONG:
       return ref.types.longlong;
-    case C_TYPE_SIZE_T:
+    case P6INT_C_TYPE_SIZE_T:
       return ref.types.size_t;
-    case C_TYPE_BOOL:
+    case P6INT_C_TYPE_BOOL:
       return ref.types.bool;
+    case P6INT_C_TYPE_WCHAR_T:
+      // XXX: we have no way of telling what type wchar_t actually is without
+      // making changes to ref-napi.
+      return ref.types.int;
+    case P6INT_C_TYPE_WINT_T:
+      // XXX: we have no way of telling what type wint_t actually is without
+      // making changes to ref-napi.
+      return ref.types.int;
+    case P6INT_C_TYPE_CHAR16_T:
+      return ref.types.uint16;
+    case P6INT_C_TYPE_CHAR32_T:
+      return ref.types.uint32;
   }
 }
 
 
 class P6int extends REPR {
   constructor() {
     super();
-    this.bits = 32;
+    this.type       = P6INT_C_TYPE_INT;
+    this.bits       = 32;
     this.isUnsigned = 0;
   }
 
   nativeCallSize() {
-    return this.bits/8;
+    return this.bits / 8;
   }
 
   asRefType() {
@@ -748,7 +765,7 @@ class P6int extends REPR {
     } else if (this.bits === 32) {
       return this.isUnsigned ? ref.types.uint32 : ref.types.int32;
     } else {
-      throw new NQPException(`Unsupported use in lowlevel contex, bits: ${this.bits}`);
+      throw new NQPException(`Unsupported use in lowlevel context, bits: ${this.bits}`);
     }
   }
 
@@ -771,24 +788,40 @@ class P6int extends REPR {
   compose(STable, reprInfoHash) {
     const integer = reprInfoHash.content.get('integer');
     if (integer) {
+      const type = integer.content.get('nativetype');
       const bits = integer.content.get('bits');
-      if (bits === undefined) {
-      } else if (bits instanceof NQPInt) {
-        this.bits = bits.value < 0 ? cType(bits.value).size * 8 : bits.value;
+      if (type === undefined) {
+        if (bits === undefined) {
+          // ...
+        } else if (bits instanceof NQPInt) {
+          this.bits = bits.value;
+          switch (this.bits) {
+            case 8:  this.type = P6INT_C_TYPE_CHAR;     break;
+            case 16: this.type = P6INT_C_TYPE_SHORT;    break;
+            case 32: this.type = P6INT_C_TYPE_INT;      break;
+            case 64: this.type = P6INT_C_TYPE_LONGLONG; break;
+          }
+        } else {
+          throw 'bits to P6int.compose must be a native int';
+        }
+      } else if (type instanceof NQPInt) {
+        this.type = type.value;
+        this.bits = cType(this.type).size * 8;
       } else {
-        throw 'bits to P6int.compose must be a native int';
+        throw 'nativetype to P6int.compose must be a native int';
       }
 
       const unsigned = integer.content.get('unsigned');
-      if (unsigned) {
-        if (unsigned instanceof NQPInt) {
-          this.isUnsigned = unsigned.value;
-        } else {
-          throw 'unsigned to P6int.compose must be a native int';
-        }
+      if (unsigned === undefined) {
+        // ...
+      } else if (unsigned instanceof NQPInt) {
+        this.isUnsigned = unsigned.value;
+      } else {
+        throw 'unsigned to P6int.compose must be a native int';
       }
     }
   }
+}
 
   deserializeFinish(obj, data) {
     // TODO integers bigger than 32bit
@@ -864,12 +897,14 @@ class P6int extends REPR {
   }
 
   serializeReprData(st, cursor) {
+    cursor.varint(this.type);
     cursor.varint(this.bits);
     cursor.varint(this.isUnsigned);
   }
 
   deserializeReprData(cursor, STable) {
-    this.bits = cursor.varint();
+    this.type       = cursor.varint();
+    this.bits       = cursor.varint();
     this.isUnsigned = cursor.varint();
   }
 

src/vm/jvm/QAST/Compiler.nqp

@@ -2093,17 +2093,27 @@ my %const_map := nqp::hash(
     'TYPE_CHECK_CACHE_THEN_METHOD', 1,
     'TYPE_CHECK_NEEDS_ACCEPTS',     2,
 
-    'C_TYPE_CHAR',              -1,
-    'C_TYPE_SHORT',             -2,
-    'C_TYPE_INT',               -3,
-    'C_TYPE_LONG',              -4,
-    'C_TYPE_LONGLONG',          -5,
-    'C_TYPE_SIZE_T',            -6,
-    'C_TYPE_BOOL',              -7,
-    'C_TYPE_ATOMIC_INT',        -8,
-    'C_TYPE_FLOAT',             -1,
-    'C_TYPE_DOUBLE',            -2,
-    'C_TYPE_LONGDOUBLE',        -3,
+    'P6INT_C_TYPE_CHAR',        -1,
+    'P6INT_C_TYPE_SHORT',       -2,
+    'P6INT_C_TYPE_INT',         -3,
+    'P6INT_C_TYPE_LONG',        -4,
+    'P6INT_C_TYPE_LONGLONG',    -5,
+    'P6INT_C_TYPE_BOOL',        -6,
+    'P6INT_C_TYPE_SIZE_T',      -7,
+    'P6INT_C_TYPE_ATOMIC_INT',  -8,
+    'P6INT_C_TYPE_WCHAR_T',     -9,
+    'P6INT_C_TYPE_WINT_T',      -10,
+    'P6INT_C_TYPE_CHAR16_T',    -11,
+    'P6INT_C_TYPE_CHAR32_T',    -12,
+
+    'P6NUM_C_TYPE_FLOAT',       -32,
+    'P6NUM_C_TYPE_DOUBLE',      -33,
+    'P6NUM_C_TYPE_LONGDOUBLE',  -34,
+
+    'P6STR_C_TYPE_CHAR',        -64,
+    'P6STR_C_TYPE_WCHAR_T',     -65,
+    'P6STR_C_TYPE_CHAR16_T',    -66,
+    'P6STR_C_TYPE_CHAR32_T',    -67,
 
     'NORMALIZE_NONE',            0,
     'NORMALIZE_NFC',             1,
@@ -2925,6 +2935,8 @@ QAST::OperationsJAST.map_classlib_core_op('nativecallrefresh', $TYPE_NATIVE_OPS,
 QAST::OperationsJAST.map_classlib_core_op('nativecallsizeof', $TYPE_NATIVE_OPS, 'nativecallsizeof', [$RT_OBJ], $RT_INT, :tc);
 QAST::OperationsJAST.map_classlib_core_op('nativecallcast', $TYPE_NATIVE_OPS, 'nativecallcast', [$RT_OBJ, $RT_OBJ, $RT_OBJ], $RT_OBJ, :tc);
 QAST::OperationsJAST.map_classlib_core_op('nativecallglobal', $TYPE_NATIVE_OPS, 'nativecallglobal', [$RT_STR, $RT_STR, $RT_OBJ, $RT_OBJ], $RT_OBJ, :tc);
+QAST::OperationsJAST.map_classlib_core_op('iswcharunsigned', $TYPE_NATIVE_OPS, 'iswcharunsigned', [], $RT_INT);
+QAST::OperationsJAST.map_classlib_core_op('iswintunsigned', $TYPE_NATIVE_OPS, 'iswintunsigned', [], $RT_INT);
 
 QAST::OperationsJAST.add_core_op('getcodelocation', -> $qastcomp, $op {
     $qastcomp.as_jast(QAST::Op.new(