From 015b852a3d7e77e00195e0fd1a4174f53904e915 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Fri, 7 Feb 2025 15:14:13 +0800 Subject: [PATCH] chore: cherry pick to 0.10 (#2048) ## What does this PR do? ## Related issues ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --------- Co-authored-by: weipeng Co-authored-by: orisgarno <42269241+orisgarno@users.noreply.github.com> Co-authored-by: hn <72974271+Hen1ng@users.noreply.github.com> Co-authored-by: hening --- docs/guide/java_serialization_guide.md | 187 ++-- java/benchmark/pom.xml | 10 - .../src/main/java/org/apache/fury/Fury.java | 5 + .../apache/fury/resolver/ClassResolver.java | 7 +- java/fury-extensions/pom.xml | 94 ++ .../apache/fury/meta/ZstdMetaCompressor.java | 69 ++ .../apache/fury/meta/ClassDefEncoderTest.java | 54 + java/pom.xml | 1 + javascript/.gitignore | 1 + javascript/benchmark/index.js | 2 +- javascript/jest.config.js | 7 +- .../packages/fury/lib/platformBuffer.ts | 86 +- javascript/packages/fury/lib/reader/index.ts | 21 +- javascript/packages/fury/lib/type.ts | 5 +- javascript/packages/fury/lib/util.ts | 5 +- javascript/packages/fury/lib/writer/index.ts | 18 +- javascript/packages/hps/binding.gyp | 6 +- javascript/packages/hps/index.ts | 27 +- javascript/packages/hps/package.json | 10 +- javascript/packages/hps/scripts/build.js | 41 + javascript/packages/hps/scripts/preinstall.js | 56 + javascript/packages/hps/src/fastcall.cc | 146 ++- .../packages/hps/src/v8-fast-api-calls.h | 957 ------------------ javascript/test/hps.test.ts | 33 +- javascript/test/platformBuffer.test.ts | 30 +- 25 files changed, 674 insertions(+), 1204 deletions(-) create mode 100644 java/fury-extensions/pom.xml create mode 100644 java/fury-extensions/src/main/java/org/apache/fury/meta/ZstdMetaCompressor.java create mode 100644 java/fury-extensions/src/test/java/org/apache/fury/meta/ClassDefEncoderTest.java create mode 100644 javascript/packages/hps/scripts/build.js create mode 100644 javascript/packages/hps/scripts/preinstall.js delete mode 100644 javascript/packages/hps/src/v8-fast-api-calls.h diff --git a/docs/guide/java_serialization_guide.md b/docs/guide/java_serialization_guide.md index 9a792d7b00..c4ee8f1d5a 100644 --- a/docs/guide/java_serialization_guide.md +++ b/docs/guide/java_serialization_guide.md @@ -102,7 +102,7 @@ public class Example { | `compressLong` | Enables or disables long compression for smaller size. | `true` | | `compressString` | Enables or disables string compression for smaller size. | `false` | | `classLoader` | The classloader should not be updated; Fury caches class metadata. Use `LoaderBinding` or `ThreadSafeFury` for classloader updates. | `Thread.currentThread().getContextClassLoader()` | -| `compatibleMode` | Type forward/backward compatibility config. Also Related to `checkClassVersion` config. `SCHEMA_CONSISTENT`: Class schema must be consistent between serialization peer and deserialization peer. `COMPATIBLE`: Class schema can be different between serialization peer and deserialization peer. They can add/delete fields independently. [See more](#class-inconsistency-and-class-version-check). | `CompatibleMode.SCHEMA_CONSISTENT` | +| `compatibleMode` | Type forward/backward compatibility config. Also Related to `checkClassVersion` config. `SCHEMA_CONSISTENT`: Class schema must be consistent between serialization peer and deserialization peer. `COMPATIBLE`: Class schema can be different between serialization peer and deserialization peer. They can add/delete fields independently. [See more](#class-inconsistency-and-class-version-check). | `CompatibleMode.SCHEMA_CONSISTENT` | | `checkClassVersion` | Determines whether to check the consistency of the class schema. If enabled, Fury checks, writes, and checks consistency using the `classVersionHash`. It will be automatically disabled when `CompatibleMode#COMPATIBLE` is enabled. Disabling is not recommended unless you can ensure the class won't evolve. | `false` | | `checkJdkClassSerializable` | Enables or disables checking of `Serializable` interface for classes under `java.*`. If a class under `java.*` is not `Serializable`, Fury will throw an `UnsupportedOperationException`. | `true` | | `registerGuavaTypes` | Whether to pre-register Guava types such as `RegularImmutableMap`/`RegularImmutableList`. These types are not public API, but seem pretty stable. | `true` | @@ -125,7 +125,7 @@ public class Example { Single thread fury: ```java -Fury fury=Fury.builder() +Fury fury = Fury.builder() .withLanguage(Language.JAVA) // enable reference tracking for shared/circular reference. // Disable it will have better performance if no duplicate reference. @@ -137,14 +137,14 @@ Fury fury=Fury.builder() // enable async multi-threaded compilation. .withAsyncCompilation(true) .build(); - byte[]bytes=fury.serialize(object); - System.out.println(fury.deserialize(bytes)); +byte[] bytes = fury.serialize(object); +System.out.println(fury.deserialize(bytes)); ``` Thread-safe fury: ```java -ThreadSafeFury fury=Fury.builder() +ThreadSafeFury fury = Fury.builder() .withLanguage(Language.JAVA) // enable reference tracking for shared/circular reference. // Disable it will have better performance if no duplicate reference. @@ -160,10 +160,45 @@ ThreadSafeFury fury=Fury.builder() // enable async multi-threaded compilation. .withAsyncCompilation(true) .buildThreadSafeFury(); - byte[]bytes=fury.serialize(object); - System.out.println(fury.deserialize(bytes)); +byte[] bytes = fury.serialize(object); +System.out.println(fury.deserialize(bytes)); ``` +### Handling Class Schema Evolution in Serialization + +In many systems, the schema of a class used for serialization may change over time. For instance, fields within a class +may be added or removed. When serialization and deserialization processes use different versions of jars, the schema of +the class being deserialized may differ from the one used during serialization. + +By default, Fury serializes objects using the `CompatibleMode.SCHEMA_CONSISTENT` mode. This mode assumes that the +deserialization process uses the same class schema as the serialization process, minimizing payload overhead. +However, if there is a schema inconsistency, deserialization will fail. + +If the schema is expected to change, to make deserialization succeed, i.e. schema forward/backward compatibility. +Users must configure Fury to use `CompatibleMode.COMPATIBLE`. This can be done using the +`FuryBuilder#withCompatibleMode(CompatibleMode.COMPATIBLE)` method. +In this compatible mode, deserialization can handle schema changes such as missing or extra fields, allowing it to +succeed even when the serialization and deserialization processes have different class schemas. + +Here is an example of creating Fury to support schema evolution: + +```java +Fury fury = Fury.builder() + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .build(); + +byte[] bytes = fury.serialize(object); +System.out.println(fury.deserialize(bytes)); +``` + +This compatible mode involves serializing class metadata into the serialized output. Despite Fury's use of +sophisticated compression techniques to minimize overhead, there is still some additional space cost associated with +class metadata. + +To further reduce metadata costs, Fury introduces a class metadata sharing mechanism, which allows the metadata to be +sent to the deserialization process only once. For more details, please refer to the [Meta Sharing](#MetaSharing) +section. + ### Smaller size `FuryBuilder#withIntCompressed`/`FuryBuilder#withLongCompressed` can be used to compress int/long for smaller size. @@ -184,9 +219,9 @@ For long compression, fury support two encoding: - Otherwise write as 9 bytes: `| 0b1 | little-endian 8bytes long |` - Fury PVL(Progressive Variable-length Long) Encoding: - First bit in every byte indicate whether has next byte. if first bit is set, then next byte will be read util - first bit of next byte is unset. + first bit of next byte is unset. - Negative number will be converted to positive number by `(v << 1) ^ (v >> 63)` to reduce cost of small negative - numbers. + numbers. If a number are `long` type, it can't be represented by smaller bytes mostly, the compression won't get good enough result, @@ -199,22 +234,18 @@ space savings. Deep copy example: ```java -Fury fury=Fury.builder() - ... - .withRefCopy(true).build(); - SomeClass a=xxx; - SomeClass copied=fury.copy(a) +Fury fury = Fury.builder().withRefCopy(true).build(); +SomeClass a = xxx; +SomeClass copied = fury.copy(a); ``` Make fury deep copy ignore circular and shared reference, this deep copy mode will ignore circular and shared reference. Same reference of an object graph will be copied into different objects in one `Fury#copy`. ```java -Fury fury=Fury.builder() - ... - .withRefCopy(false).build(); - SomeClass a=xxx; - SomeClass copied=fury.copy(a) +Fury fury = Fury.builder().withRefCopy(false).build(); +SomeClass a = xxx; +SomeClass copied = fury.copy(a); ``` ### Implement a customized serializer @@ -257,8 +288,8 @@ class FooSerializer extends Serializer { Register serializer: ```java -Fury fury=getFury(); - fury.registerSerializer(Foo.class,new FooSerializer(fury)); +Fury fury = getFury(); +fury.registerSerializer(Foo.class, new FooSerializer(fury)); ``` ### Security & Class Registration @@ -279,9 +310,9 @@ Note that class registration order is important, serialization and deserializati should have same registration order. ```java -Fury fury=xxx; - fury.register(SomeClass.class); - fury.register(SomeClass1.class,200); +Fury fury = xxx; +fury.register(SomeClass.class); +fury.register(SomeClass1.class,200); ``` If you invoke `FuryBuilder#requireClassRegistration(false)` to disable class registration check, @@ -290,19 +321,20 @@ allowed for serialization. For example, you can allow classes started with `org.example.*` by: ```java -Fury fury=xxx; - fury.getClassResolver().setClassChecker((classResolver,className)->className.startsWith("org.example.")); +Fury fury = xxx; +fury.getClassResolver().setClassChecker( + (classResolver, className) -> className.startsWith("org.example.")); ``` ```java -AllowListChecker checker=new AllowListChecker(AllowListChecker.CheckLevel.STRICT); - ThreadSafeFury fury=new ThreadLocalFury(classLoader->{ - Fury f=Fury.builder().requireClassRegistration(true).withClassLoader(classLoader).build(); +AllowListChecker checker = new AllowListChecker(AllowListChecker.CheckLevel.STRICT); +ThreadSafeFury fury = new ThreadLocalFury(classLoader -> { + Fury f = Fury.builder().requireClassRegistration(true).withClassLoader(classLoader).build(); f.getClassResolver().setClassChecker(checker); checker.addListener(f.getClassResolver()); return f; - }); - checker.allowClass("org.example.*"); +}); +checker.allowClass("org.example.*"); ``` Fury also provided a `org.apache.fury.resolver.AllowListChecker` which is allowed/disallowed list based checker to @@ -360,30 +392,30 @@ forward/backward compatibility automatically. // // share meta across serialization. // .withMetaContextShare(true) // Not thread-safe fury. -MetaContext context=xxx; - fury.getSerializationContext().setMetaContext(context); - byte[]bytes=fury.serialize(o); +MetaContext context = xxx; +fury.getSerializationContext().setMetaContext(context); +byte[] bytes = fury.serialize(o); // Not thread-safe fury. - MetaContext context=xxx; - fury.getSerializationContext().setMetaContext(context); - fury.deserialize(bytes) +MetaContext context = xxx; +fury.getSerializationContext().setMetaContext(context); +fury.deserialize(bytes); // Thread-safe fury - fury.setClassLoader(beanA.getClass().getClassLoader()); - byte[]serialized=fury.execute( - f->{ - f.getSerializationContext().setMetaContext(context); - return f.serialize(beanA); +fury.setClassLoader(beanA.getClass().getClassLoader()); +byte[] serialized = fury.execute( + f -> { + f.getSerializationContext().setMetaContext(context); + return f.serialize(beanA); } - ); +); // thread-safe fury - fury.setClassLoader(beanA.getClass().getClassLoader()); - Object newObj=fury.execute( - f->{ - f.getSerializationContext().setMetaContext(context); - return f.deserialize(serialized); +fury.setClassLoader(beanA.getClass().getClassLoader()); +Object newObj = fury.execute( + f -> { + f.getSerializationContext().setMetaContext(context); + return f.deserialize(serialized); } - ); +); ``` ### Deserialize non-existent classes @@ -404,10 +436,10 @@ Fury support mapping object from one type to another type. > Notes: > > 1. This mapping will execute a deep copy, all mapped fields are serialized into binary and -deserialized from that binary to map into another type. + deserialized from that binary to map into another type. > 2. All struct types must be registered with same ID, otherwise Fury can not mapping to correct struct type. -> Be careful when you use `Fury#register(Class)`, because fury will allocate an auto-grown ID which might be -> inconsistent if you register classes with different order between Fury instance. + > Be careful when you use `Fury#register(Class)`, because fury will allocate an auto-grown ID which might be + > inconsistent if you register classes with different order between Fury instance. ```java public class StructMappingExample { @@ -460,12 +492,12 @@ the binary are generated by jdk serialization, you use following pattern to make then upgrade serialization to fury in an async rolling-up way: ```java -if(JavaSerializer.serializedByJDK(bytes)){ +if (JavaSerializer.serializedByJDK(bytes)) { ObjectInputStream objectInputStream=xxx; return objectInputStream.readObject(); - }else{ +} else { return fury.deserialize(bytes); - } +} ``` ### Upgrade fury @@ -482,18 +514,18 @@ serialized data using code like following to keep binary compatibility: ```java -MemoryBuffer buffer=xxx; - buffer.writeVarInt32(2); - fury.serialize(buffer,obj); +MemoryBuffer buffer = xxx; +buffer.writeVarInt32(2); +fury.serialize(buffer, obj); ``` Then for deserialization, you need: ```java -MemoryBuffer buffer=xxx; - int furyVersion=buffer.readVarInt32() - Fury fury=getFury(furyVersion); - fury.deserialize(buffer); +MemoryBuffer buffer = xxx; +int furyVersion = buffer.readVarInt32(); +Fury fury = getFury(furyVersion); +fury.deserialize(buffer); ``` `getFury` is a method to load corresponding fury, you can shade and relocate different version of fury to different @@ -520,9 +552,38 @@ consistent between serialization and deserialization. ### Deserialize POJO into another type -Fury allows you to serialize one POJO and deserialize it into a different POJO. To achieve this, configure Fury with +Fury allows you to serialize one POJO and deserialize it into a different POJO. The different POJO means the schema inconsistency. Users must to configure Fury with `CompatibleMode` set to `org.apache.fury.config.CompatibleMode.COMPATIBLE`. +```java +public class DeserializeIntoType { + static class Struct1 { + int f1; + String f2; + + public Struct1(int f1, String f2) { + this.f1 = f1; + this.f2 = f2; + } + } + + static class Struct2 { + int f1; + String f2; + double f3; + } + + static ThreadSafeFury fury = Fury.builder() + .withCompatibleMode(CompatibleMode.COMPATIBLE).buildThreadSafeFury(); + + public static void main(String[] args) { + Struct1 struct1 = new Struct1(10, "abc"); + byte[] data = fury.serializeJavaObject(struct1); + Struct2 struct2 = (Struct2) fury.deserializeJavaObject(bytes, Struct2.class); + } +} +``` + ### Use wrong API for deserialization If you serialize an object by invoking `Fury#serialize`, you should invoke `Fury#deserialize` for deserialization diff --git a/java/benchmark/pom.xml b/java/benchmark/pom.xml index 8d4e1a80c3..6e9eb5bfd3 100644 --- a/java/benchmark/pom.xml +++ b/java/benchmark/pom.xml @@ -139,16 +139,6 @@ smoothie-map 2.0.2 - - com.google.protobuf - protobuf-java - ${protobuf.version} - - - com.google.flatbuffers - flatbuffers-java - 2.0.3 - org.slf4j slf4j-api diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index 939340fe90..cfc01656c8 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -208,6 +208,11 @@ public SerializerFactory getSerializerFactory() { return classResolver.getSerializerFactory(); } + public Serializer getSerializer(Class cls) { + Preconditions.checkNotNull(cls); + return classResolver.getSerializer(cls); + } + @Override public MemoryBuffer serialize(Object obj, long address, int size) { MemoryBuffer buffer = MemoryUtils.buffer(address, size); diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java index 840f32857f..b6cf515785 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java @@ -1410,11 +1410,10 @@ private ClassInfo readClassInfoWithMetaShare(MetaContext metaContext, int index) return classInfo; } - public ClassInfo readClassInfoWithMetaShare(MemoryBuffer buffer, Class targetClass) { assert metaContextShareEnabled; ClassInfo classInfo = - readClassInfoWithMetaShare(buffer, fury.getSerializationContext().getMetaContext()); + readClassInfoWithMetaShare(buffer, fury.getSerializationContext().getMetaContext()); Class readClass = classInfo.getCls(); // replace target class if needed if (targetClass != readClass) { @@ -1423,8 +1422,8 @@ public ClassInfo readClassInfoWithMetaShare(MemoryBuffer buffer, Class target if (newClassInfo == null) { // similar to create serializer for `NonexistentMetaShared` newClassInfo = - getMetaSharedClassInfo( - classInfo.classDef.replaceRootClassTo(this, targetClass), targetClass); + getMetaSharedClassInfo( + classInfo.classDef.replaceRootClassTo(this, targetClass), targetClass); extRegistry.transformedClassInfo.put(key, newClassInfo); } return newClassInfo; diff --git a/java/fury-extensions/pom.xml b/java/fury-extensions/pom.xml new file mode 100644 index 0000000000..6f6acdb72f --- /dev/null +++ b/java/fury-extensions/pom.xml @@ -0,0 +1,94 @@ + + + + + org.apache.fury + fury-parent + 0.9.0 + + 4.0.0 + + fury-extensions + + + Apache Fury™ is a blazingly fast multi-language serialization framework powered by jit and zero-copy. + + Apache Fury (incubating) is an effort undergoing incubation at the Apache + Software Foundation (ASF), sponsored by the Apache Incubator PMC. + + Incubation is required of all newly accepted projects until a further review + indicates that the infrastructure, communications, and decision making process + have stabilized in a manner consistent with other successful ASF projects. + + While incubation status is not necessarily a reflection of the completeness + or stability of the code, it does indicate that the project has yet to be + fully endorsed by the ASF. + + + + 1.5.6-9 + 8 + 8 + ${basedir}/.. + + + + + com.github.luben + zstd-jni + ${zstd.version} + provided + + + org.apache.fury + fury-core + ${project.version} + + + org.apache.fury + fury-test-core + ${project.version} + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + org.apache.fury.extension + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + diff --git a/java/fury-extensions/src/main/java/org/apache/fury/meta/ZstdMetaCompressor.java b/java/fury-extensions/src/main/java/org/apache/fury/meta/ZstdMetaCompressor.java new file mode 100644 index 0000000000..d61f6052dd --- /dev/null +++ b/java/fury-extensions/src/main/java/org/apache/fury/meta/ZstdMetaCompressor.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fury.meta; + +import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdException; +import java.util.Arrays; + +public class ZstdMetaCompressor implements MetaCompressor { + + @Override + public byte[] compress(byte[] data, int offset, int size) { + long maxCompressedSize = Zstd.compressBound(size); + if (maxCompressedSize > Integer.MAX_VALUE) { + throw new ZstdException(Zstd.errGeneric(), "Max output size is greater than MAX_INT"); + } + byte[] compressedData = new byte[(int) maxCompressedSize]; + long originalSize = + Zstd.compressByteArray( + compressedData, + 0, + (int) maxCompressedSize, + data, + offset, + size, + Zstd.defaultCompressionLevel()); + + return Arrays.copyOf(compressedData, (int) originalSize); + } + + @Override + public byte[] decompress(byte[] data, int offset, int size) { + int decompressedSize = (int) Zstd.getFrameContentSize(data, offset, size, false); + byte[] decompressedBytes = new byte[decompressedSize]; + long originalSize = + Zstd.decompressByteArray(decompressedBytes, 0, decompressedSize, data, offset, size); + return Arrays.copyOf(decompressedBytes, (int) originalSize); + } + + @Override + public int hashCode() { + return ZstdMetaCompressor.class.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + return o != null && getClass() == o.getClass(); + } +} diff --git a/java/fury-extensions/src/test/java/org/apache/fury/meta/ClassDefEncoderTest.java b/java/fury-extensions/src/test/java/org/apache/fury/meta/ClassDefEncoderTest.java new file mode 100644 index 0000000000..d581e94169 --- /dev/null +++ b/java/fury-extensions/src/test/java/org/apache/fury/meta/ClassDefEncoderTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fury.meta; + +import static org.apache.fury.meta.ClassDefEncoder.buildFieldsInfo; +import static org.apache.fury.meta.ClassDefEncoder.getClassFields; + +import java.util.List; +import org.apache.fury.Fury; +import org.apache.fury.memory.MemoryBuffer; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class ClassDefEncoderTest { + + static class TestFieldsOrderClass1 { + private int intField2; + private boolean booleanField; + private Object objField; + private long longField; + } + + @Test + public void testBasicClassDef_zstdMetaCompressor() throws Exception { + Fury fury = + Fury.builder().withMetaShare(true).withMetaCompressor(new ZstdMetaCompressor()).build(); + Class type = TestFieldsOrderClass1.class; + List fieldsInfo = buildFieldsInfo(fury.getClassResolver(), type); + MemoryBuffer buffer = + ClassDefEncoder.encodeClassDef( + fury.getClassResolver(), type, getClassFields(type, fieldsInfo), true); + ClassDef classDef = ClassDef.readClassDef(fury.getClassResolver(), buffer); + Assert.assertEquals(classDef.getClassName(), type.getName()); + Assert.assertEquals(classDef.getFieldsInfo().size(), type.getDeclaredFields().length); + Assert.assertEquals(classDef.getFieldsInfo(), fieldsInfo); + } +} diff --git a/java/pom.xml b/java/pom.xml index ad42a73482..0314fe88e9 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -72,6 +72,7 @@ fury-format fury-core + fury-extensions fury-test-core fury-testsuite diff --git a/javascript/.gitignore b/javascript/.gitignore index 8216c3805c..40b7ff8f52 100644 --- a/javascript/.gitignore +++ b/javascript/.gitignore @@ -1,2 +1,3 @@ package-lock.json coverage +packages/hps/includes \ No newline at end of file diff --git a/javascript/benchmark/index.js b/javascript/benchmark/index.js index 52d0b267be..9de0cb0ac3 100644 --- a/javascript/benchmark/index.js +++ b/javascript/benchmark/index.js @@ -19,7 +19,7 @@ const Fury = require("@furyjs/fury"); const utils = require("@furyjs/fury/dist/lib/util"); -const hps = require('@furyjs/hps'); +const hps = require('@furyjs/hps').default; const fury = new Fury.default({ hps, refTracking: false, useSliceString: true }); const Benchmark = require("benchmark"); const protobuf = require("protobufjs"); diff --git a/javascript/jest.config.js b/javascript/jest.config.js index ce81426919..363bdd2bfc 100644 --- a/javascript/jest.config.js +++ b/javascript/jest.config.js @@ -18,11 +18,9 @@ */ /** @type {import('ts-jest').JestConfigWithTsJest} */ -const semver = require("semver"); -const hpsEnable = semver.gt(process.versions.node, '20.0.0') && process.platform !== 'win32'; module.exports = { - collectCoverage: hpsEnable, + collectCoverage: true, preset: 'ts-jest', testEnvironment: 'node', collectCoverageFrom: [ @@ -31,9 +29,6 @@ module.exports = { "!**/build/**", "!packages/fury/lib/murmurHash3.ts" ], - "testPathIgnorePatterns" : [ - hpsEnable ? null : "(.*)/hps.test.ts$", - ].filter(Boolean), transform: { '\\.ts$': ['ts-jest', { tsconfig: { diff --git a/javascript/packages/fury/lib/platformBuffer.ts b/javascript/packages/fury/lib/platformBuffer.ts index e3a0978d62..2cecadacd6 100644 --- a/javascript/packages/fury/lib/platformBuffer.ts +++ b/javascript/packages/fury/lib/platformBuffer.ts @@ -19,10 +19,11 @@ import { hasBuffer } from "./util"; -let utf8Encoder: TextEncoder | null; -let textDecoder: TextDecoder | null; +const utf8Encoder = new TextEncoder(); +const utf8Decoder = new TextDecoder("utf-8"); +const utf16LEDecoder = new TextDecoder("utf-16le"); -export type SupportedEncodings = "latin1" | "utf8"; +export type SupportedEncodings = "latin1" | "utf8" | "utf16le"; export interface PlatformBuffer extends Uint8Array { toString(encoding?: SupportedEncodings, start?: number, end?: number): string; @@ -32,59 +33,55 @@ export interface PlatformBuffer extends Uint8Array { export class BrowserBuffer extends Uint8Array implements PlatformBuffer { write(string: string, offset: number, encoding: SupportedEncodings = "utf8"): void { - if (encoding === "latin1") { - return this.latin1Write(string, offset); + switch (encoding) { + case "utf8": + return this.utf8Write(string, offset); + case "utf16le": + return this.ucs2Write(string, offset); + case "latin1": + return this.latin1Write(string, offset); + default: + break; + } + } + + private ucs2Write(string: string, offset: number): void { + for (let i = 0; i < string.length; i++) { + const codePoint = string.charCodeAt(i); + this[offset++] = codePoint & 0xFF; + this[offset++] = (codePoint >> 8) & 0xFF; } - return this.utf8Write(string, offset); } toString(encoding: SupportedEncodings = "utf8", start = 0, end = this.length): string { - if (encoding === "latin1") { - return this.latin1Slice(start, end); + switch (encoding) { + case "utf8": + return this.utf8Slice(start, end); + case "utf16le": + return this.utf16LESlice(start, end); + case "latin1": + return this.latin1Slice(start, end); + default: + return ""; } - return this.utf8Slice(start, end); } static alloc(size: number) { return new BrowserBuffer(new Uint8Array(size)); } - latin1Write(string: string, offset: number) { + private latin1Write(string: string, offset: number) { let index = 0; for (; index < string.length; index++) { this[offset++] = string.charCodeAt(index); } } - utf8Write(string: string, offset: number) { - let c1: number; - let c2: number; - for (let i = 0; i < string.length; ++i) { - c1 = string.charCodeAt(i); - if (c1 < 128) { - this[offset++] = c1; - } else if (c1 < 2048) { - this[offset++] = (c1 >> 6) | 192; - this[offset++] = (c1 & 63) | 128; - } else if ( - (c1 & 0xfc00) === 0xd800 - && ((c2 = string.charCodeAt(i + 1)) & 0xfc00) === 0xdc00 - ) { - c1 = 0x10000 + ((c1 & 0x03ff) << 10) + (c2 & 0x03ff); - ++i; - this[offset++] = (c1 >> 18) | 240; - this[offset++] = ((c1 >> 12) & 63) | 128; - this[offset++] = ((c1 >> 6) & 63) | 128; - this[offset++] = (c1 & 63) | 128; - } else { - this[offset++] = (c1 >> 12) | 224; - this[offset++] = ((c1 >> 6) & 63) | 128; - this[offset++] = (c1 & 63) | 128; - } - } + private utf8Write(string: string, offset: number) { + utf8Encoder.encodeInto(string, this.subarray(offset)); } - latin1Slice(start: number, end: number) { + private latin1Slice(start: number, end: number) { if (end - start < 1) { return ""; } @@ -95,16 +92,18 @@ export class BrowserBuffer extends Uint8Array implements PlatformBuffer { return str; } - utf8Slice(start: number, end: number) { + private utf16LESlice(start: number, end: number) { if (end - start < 1) { return ""; } + return utf16LEDecoder.decode(this.subarray(start, end)); + } - if (!textDecoder) { - textDecoder = new TextDecoder("utf-8"); + private utf8Slice(start: number, end: number) { + if (end - start < 1) { + return ""; } - - return textDecoder.decode(this.subarray(start, end)); + return utf8Decoder.decode(this.subarray(start, end)); } copy(target: Uint8Array, targetStart?: number, sourceStart?: number, sourceEnd?: number) { @@ -150,8 +149,5 @@ export const fromString = hasBuffer ? (str: string) => Buffer.from(str) as unknown as PlatformBuffer : (str: string) => { - if (!utf8Encoder) { - utf8Encoder = new TextEncoder(); - } return new BrowserBuffer(utf8Encoder.encode(str)); }; diff --git a/javascript/packages/fury/lib/reader/index.ts b/javascript/packages/fury/lib/reader/index.ts index 5731127a4a..46fe1aa71f 100644 --- a/javascript/packages/fury/lib/reader/index.ts +++ b/javascript/packages/fury/lib/reader/index.ts @@ -17,7 +17,7 @@ * under the License. */ -import { Config, LATIN1 } from "../type"; +import { Config, LATIN1, UTF16, UTF8 } from "../type"; import { isNodeEnv } from "../util"; import { PlatformBuffer, alloc, fromUint8Array } from "../platformBuffer"; import { readLatin1String } from "./string"; @@ -124,10 +124,23 @@ export class BinaryReader { return result; } + stringUtf16LE(len: number) { + const result = this.platformBuffer.toString("utf16le", this.cursor, this.cursor + len); + this.cursor += len; + return result; + } + stringOfVarUInt32() { - const isLatin1 = this.uint8() === LATIN1; - const len = this.varUInt32(); - return isLatin1 ? this.stringLatin1(len) : this.stringUtf8(len); + switch (this.uint8()) { + case LATIN1: + return this.stringLatin1(this.varUInt32()); + case UTF8: + return this.stringUtf8(this.varUInt32()); + case UTF16: + return this.stringUtf16LE(this.varUInt32()); + default: + break; + } } stringLatin1(len: number) { diff --git a/javascript/packages/fury/lib/type.ts b/javascript/packages/fury/lib/type.ts index 23956da896..6c76e9c0e2 100644 --- a/javascript/packages/fury/lib/type.ts +++ b/javascript/packages/fury/lib/type.ts @@ -94,10 +94,9 @@ export const HalfMinInt32 = MinInt32 / 2; export const LATIN1 = 0; export const UTF8 = 1; - +export const UTF16 = 2; export interface Hps { - isLatin1: (str: string) => boolean; - stringCopy: (str: string, dist: Uint8Array, offset: number) => void; + serializeString: (str: string, dist: Uint8Array, offset: number) => number; } export enum Mode { diff --git a/javascript/packages/fury/lib/util.ts b/javascript/packages/fury/lib/util.ts index 390427048b..cba7392741 100644 --- a/javascript/packages/fury/lib/util.ts +++ b/javascript/packages/fury/lib/util.ts @@ -17,8 +17,7 @@ * under the License. */ -import { ObjectTypeDescription, Type, TypeDescription } from "./description"; -import { InternalSerializerType } from "./type"; +import { Type, TypeDescription } from "./description"; export const isNodeEnv: boolean = typeof process !== "undefined" @@ -107,7 +106,7 @@ export const data2Description = ( .map(([key, value]) => { return [key, data2Description(value, `${tag}.${key}`)]; }) - .filter(([_, v]) => Boolean(v)), + .filter(([, v]) => Boolean(v)), ), ); } diff --git a/javascript/packages/fury/lib/writer/index.ts b/javascript/packages/fury/lib/writer/index.ts index 69ef309026..d8984ac3cd 100644 --- a/javascript/packages/fury/lib/writer/index.ts +++ b/javascript/packages/fury/lib/writer/index.ts @@ -188,22 +188,8 @@ export class BinaryWriter { } stringOfVarUInt32Fast(v: string) { - const { isLatin1: detectIsLatin1, stringCopy } = this.config.hps!; - const isLatin1 = detectIsLatin1(v); - const len = isLatin1 ? v.length : strByteLength(v); - this.dataView.setUint8(this.cursor++, isLatin1 ? LATIN1 : UTF8); - this.varUInt32(len); - this.reserve(len); - if (isLatin1) { - stringCopy(v, this.platformBuffer, this.cursor); - } else { - if (len < 40) { - this.fastWriteStringUtf8(v, this.platformBuffer, this.cursor); - } else { - this.platformBuffer.write(v, this.cursor, "utf8"); - } - } - this.cursor += len; + const { serializeString } = this.config.hps!; + this.cursor = serializeString(v, this.platformBuffer, this.cursor); } stringOfVarUInt32Slow(v: string) { diff --git a/javascript/packages/hps/binding.gyp b/javascript/packages/hps/binding.gyp index 73b11b41e7..a9ba8c717a 100644 --- a/javascript/packages/hps/binding.gyp +++ b/javascript/packages/hps/binding.gyp @@ -20,11 +20,11 @@ { "target_name": "hps", "sources": [ - "src/fastcall.cc", - "src/v8-fast-api-calls.h" + "src/fastcall.cc" ], "include_dirs" : [ - " boolean; - stringCopy: (str: string, dist: Uint8Array, offset: number) => void; + serializeString: (dist: Uint8Array, str: string, offset: number, maxLength: number) => number; } -const { isLatin1, stringCopy } = hps; +const build = () => { + try { + const hps: Hps = require("bindings")("hps.node"); + const { serializeString: _serializeString } = hps; -export { - isLatin1, - stringCopy, + return { + serializeString: (v: string, dist: Uint8Array, offset: number) => { + if (typeof v !== "string") { + throw new Error(`isLatin1 requires string but got ${typeof v}`); + } + // todo boundary check + return _serializeString(dist, v, offset, 0); + }, + }; + } catch (error) { + return null; + } }; + +export default build(); diff --git a/javascript/packages/hps/package.json b/javascript/packages/hps/package.json index b235a3180f..0dca75709c 100644 --- a/javascript/packages/hps/package.json +++ b/javascript/packages/hps/package.json @@ -6,19 +6,21 @@ "files": [ "dist", "src", + "scripts/preinstall.js", "binding.gyp" ], "gypfile": false, "scripts": { - "postinstall": "node -e \"if (process.version.match(/v(\\d+)/)[1] >= 20 && process.platform !== 'win32') { require('child_process').execSync('npx node-gyp rebuild') } \"", - "build": "node -e \"if (process.version.match(/v(\\d+)/)[1] >= 20 && process.platform !== 'win32') { require('child_process').execSync('npx node-gyp rebuild && tsc') } \"", + "preinstall": "node ./scripts/preinstall.js", + "build": "node ./scripts/build.js", "prepublishOnly": "npm run build" }, "license": "Apache-2.0", "dependencies": { "bindings": "~1.2.1", - "nan": "^2.17.0", - "node-gyp": "^9.4.0" + "fs-extra": "^11.3.0", + "nan": "^2.22.0", + "node-gyp": "^11.0.0" }, "engines": { "node": "^20.0.0" diff --git a/javascript/packages/hps/scripts/build.js b/javascript/packages/hps/scripts/build.js new file mode 100644 index 0000000000..0967363622 --- /dev/null +++ b/javascript/packages/hps/scripts/build.js @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const { spawn } = require("node:child_process"); +const semver = require("semver"); +const { engines } = require("../package.json"); +const versionValid = semver.satisfies(process.version, engines.node); + +function watchError(child) { + child.on("error", (error) => { + console.error(error); + process.exit(1); + }); + child.on("exit", (code, signal) => { + if (code !== 0) { + process.exit(code); + } + }); +} + +if (versionValid) { + const gyp = spawn("npx", ["node-gyp", "rebuild"], { stdio: 'inherit', shell: true }); + watchError(gyp); +} +watchError(spawn("npx", ["tsc"], { stdio: 'inherit', shell: true })); diff --git a/javascript/packages/hps/scripts/preinstall.js b/javascript/packages/hps/scripts/preinstall.js new file mode 100644 index 0000000000..9a1ad291b1 --- /dev/null +++ b/javascript/packages/hps/scripts/preinstall.js @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const { spawn } = require("node:child_process"); +const path = require("node:path"); +const version = process.version; +const fs = require('fs-extra'); +const semver = require("semver"); +const { engines } = require("../package.json"); +const versionValid = semver.satisfies(process.version, engines.node); + +async function downloadDeps(urls) { + await Promise.all(urls.map(({url, dist}) => { + new Promise((resolve, reject) => { + fs.ensureDirSync(path.dirname(dist)); + const process = spawn('curl', [url, '-o', dist]); + process.on("close", () => { + console.log("finish") + resolve(); + }); + process.on("error", () => { + console.error(`download from ${url} failed`); + reject(); + }); + }) + })) +} + +async function main() { + await downloadDeps([ + { + url: `https://raw.githubusercontent.com/nodejs/node/refs/tags/${version}/deps/v8/include/v8-fast-api-calls.h`, + dist: path.join(__dirname, "..", "includes/v8-fast-api-calls.h") + } + ]); +} + +if (versionValid) { + main(); +} diff --git a/javascript/packages/hps/src/fastcall.cc b/javascript/packages/hps/src/fastcall.cc index 38f1cda6f7..2edd27ad3b 100644 --- a/javascript/packages/hps/src/fastcall.cc +++ b/javascript/packages/hps/src/fastcall.cc @@ -19,46 +19,124 @@ #include "v8-fast-api-calls.h" #include +using v8::Local; +using v8::String; +using v8::Value; -void IsLatin1Slow(const v8::FunctionCallbackInfo &info) { - v8::Local input = info[0].As(); - info.GetReturnValue().Set(Nan::New(input->IsOneByte())); +template constexpr T RoundUp(T a, T b) { + return a % b != 0 ? a + b - (a % b) : a; } -bool IsLatin1Fast(v8::Local receiver, - const v8::FastOneByteString &source) { - return true; +template constexpr T *AlignUp(T *ptr, U alignment) { + return reinterpret_cast( + RoundUp(reinterpret_cast(ptr), alignment)); } -static v8::CFunction fast_is_latin1(v8::CFunction::Make(IsLatin1Fast)); +uint32_t writeVarUint32(uint8_t *dst, uint32_t offset, int32_t value) { + if (value >> 7 == 0) { + dst[offset] = (uint8_t)value; + return 1; + } + if (value >> 14 == 0) { + dst[offset++] = (uint8_t)((value & 0x7F) | 0x80); + dst[offset++] = (uint8_t)(value >> 7); + return 2; + } + if (value >> 21 == 0) { + dst[offset++] = (uint8_t)((value & 0x7F) | 0x80); + dst[offset++] = (uint8_t)(value >> 7 | 0x80); + dst[offset++] = (uint8_t)(value >> 14); + return 3; + } + if (value >> 28 == 0) { + dst[offset++] = (uint8_t)((value & 0x7F) | 0x80); + dst[offset++] = (uint8_t)(value >> 7 | 0x80); + dst[offset++] = (uint8_t)(value >> 14 | 0x80); + dst[offset++] = (uint8_t)(value >> 21); + return 4; + } + dst[offset++] = (uint8_t)((value & 0x7F) | 0x80); + dst[offset++] = (uint8_t)(value >> 7 | 0x80); + dst[offset++] = (uint8_t)(value >> 14 | 0x80); + dst[offset++] = (uint8_t)(value >> 21 | 0x80); + dst[offset++] = (uint8_t)(value >> 28); + return 5; +} -void StringCopySlow(const v8::FunctionCallbackInfo &info) { - v8::Isolate *isolate = info.GetIsolate(); - v8::Local source = info[0].As(); - char *type_array = - (char *)info[1].As()->Buffer()->GetBackingStore()->Data(); - int32_t offset = - info[2]->Int32Value(info.GetIsolate()->GetCurrentContext()).FromJust(); +enum Encoding { LATIN1, UTF8, UTF16 }; - v8::String::Utf8Value source_value(isolate, source); +uint32_t writeUCS2(v8::Isolate *isolate, uint8_t *buf, Local str, + int flags) { + uint16_t *const dst = reinterpret_cast(buf); - for (size_t i = 0; i < source_value.length(); i++) { - *(type_array + (offset++)) = *((*source_value) + i); + uint16_t *const aligned_dst = AlignUp(dst, sizeof(*dst)); + size_t nchars; + if (aligned_dst == dst) { + nchars = str->Write(isolate, dst, 0, -1, flags); + return nchars * sizeof(*dst); } - info.GetReturnValue().Set(Nan::New(100)); + + nchars = str->Write(isolate, aligned_dst, 0, str->Length() - 1, flags); + + memmove(dst, aligned_dst, nchars * sizeof(*dst)); + + uint16_t last; + str->Write(isolate, &last, nchars, 1, flags); + memcpy(buf + nchars * sizeof(*dst), &last, sizeof(last)); + nchars++; + + return nchars * sizeof(*dst); +} + +static void serializeString(const v8::FunctionCallbackInfo &args) { + auto isolate = args.GetIsolate(); + auto context = isolate->GetCurrentContext(); + v8::Local dst = args[0].As(); + v8::Local str = args[1].As(); + uint32_t offset = args[2].As()->Uint32Value(context).ToChecked(); + + bool is_one_byte = str->IsOneByte(); + uint8_t *dst_data = + reinterpret_cast(dst->Buffer()->GetBackingStore()->Data()); + + if (is_one_byte && str->IsExternalOneByte()) { + dst_data[offset++] = Encoding::LATIN1; // encoding + offset += writeVarUint32(dst_data, offset, str->Length()); // length + const auto src = str->GetExternalOneByteStringResource()->data(); + memcpy(dst_data + offset, src, str->Length()); + offset += str->Length(); + } else { + v8::HandleScope scope(isolate); + int flags = String::HINT_MANY_WRITES_EXPECTED | + String::NO_NULL_TERMINATION | String::REPLACE_INVALID_UTF8; + if (is_one_byte) { + dst_data[offset++] = Encoding::LATIN1; // encoding + offset += writeVarUint32(dst_data, offset, str->Length()); // length + offset += str->WriteOneByte(isolate, dst_data + offset, 0, str->Length(), + flags); + } else { + dst_data[offset++] = Encoding::UTF16; // encoding + offset += writeVarUint32(dst_data, offset, str->Length() * 2); // length + offset += writeUCS2(isolate, dst_data + offset, str, flags); + } + } + + args.GetReturnValue().Set(offset); } -void StringCopyFast(v8::Local receiver, - const v8::FastOneByteString &source, - const v8::FastApiTypedArray &ab, - u_int32_t offset) { - uint8_t *ptr; - ab.getStorageIfAligned(&ptr); - int32_t i = 0; - memcpy(ptr + offset, source.data, source.length); +static uint32_t serializeStringFast(Local receiver, + const v8::FastApiTypedArray &dst, + const v8::FastOneByteString &src, + uint32_t offset, uint32_t max_length) { + uint8_t *dst_data; + dst.getStorageIfAligned(&dst_data); + dst_data[offset++] = Encoding::LATIN1; // encoding + offset += writeVarUint32(dst_data, src.length, offset); // length + memcpy(dst_data + offset, src.data, src.length); + return offset + src.length; } -static v8::CFunction string_copy_fast(v8::CFunction::Make(StringCopyFast)); +v8::CFunction fast_serialize_string(v8::CFunction::Make(serializeStringFast)); v8::Local FastFunction(v8::Isolate *isolate, v8::FunctionCallback callback, @@ -72,14 +150,12 @@ v8::Local FastFunction(v8::Isolate *isolate, void Init(v8::Local exports) { v8::Local context = exports->GetCreationContextChecked(); v8::Isolate *isolate = context->GetIsolate(); - exports->Set(context, Nan::New("isLatin1").ToLocalChecked(), - FastFunction(isolate, IsLatin1Slow, &fast_is_latin1) - ->GetFunction(context) - .ToLocalChecked()); - exports->Set(context, Nan::New("stringCopy").ToLocalChecked(), - FastFunction(isolate, StringCopySlow, &string_copy_fast) - ->GetFunction(context) - .ToLocalChecked()); + exports + ->Set(context, Nan::New("serializeString").ToLocalChecked(), + FastFunction(isolate, serializeString, &fast_serialize_string) + ->GetFunction(context) + .ToLocalChecked()) + .Check(); } NODE_MODULE(fury_util, Init) \ No newline at end of file diff --git a/javascript/packages/hps/src/v8-fast-api-calls.h b/javascript/packages/hps/src/v8-fast-api-calls.h deleted file mode 100644 index 0fe7cd2489..0000000000 --- a/javascript/packages/hps/src/v8-fast-api-calls.h +++ /dev/null @@ -1,957 +0,0 @@ -// Copyright 2020 the V8 project authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -/** - * This file provides additional API on top of the default one for making - * API calls, which come from embedder C++ functions. The functions are being - * called directly from optimized code, doing all the necessary typechecks - * in the compiler itself, instead of on the embedder side. Hence the "fast" - * in the name. Example usage might look like: - * - * \code - * void FastMethod(int param, bool another_param); - * - * v8::FunctionTemplate::New(isolate, SlowCallback, data, - * signature, length, constructor_behavior - * side_effect_type, - * &v8::CFunction::Make(FastMethod)); - * \endcode - * - * By design, fast calls are limited by the following requirements, which - * the embedder should enforce themselves: - * - they should not allocate on the JS heap; - * - they should not trigger JS execution. - * To enforce them, the embedder could use the existing - * v8::Isolate::DisallowJavascriptExecutionScope and a utility similar to - * Blink's NoAllocationScope: - * https://source.chromium.org/chromium/chromium/src/+/master:third_party/blink/renderer/platform/heap/thread_state_scopes.h;l=16 - * - * Due to these limitations, it's not directly possible to report errors by - * throwing a JS exception or to otherwise do an allocation. There is an - * alternative way of creating fast calls that supports falling back to the - * slow call and then performing the necessary allocation. When one creates - * the fast method by using CFunction::MakeWithFallbackSupport instead of - * CFunction::Make, the fast callback gets as last parameter an output variable, - * through which it can request falling back to the slow call. So one might - * declare their method like: - * - * \code - * void FastMethodWithFallback(int param, FastApiCallbackOptions& options); - * \endcode - * - * If the callback wants to signal an error condition or to perform an - * allocation, it must set options.fallback to true and do an early return from - * the fast method. Then V8 checks the value of options.fallback and if it's - * true, falls back to executing the SlowCallback, which is capable of reporting - * the error (either by throwing a JS exception or logging to the console) or - * doing the allocation. It's the embedder's responsibility to ensure that the - * fast callback is idempotent up to the point where error and fallback - * conditions are checked, because otherwise executing the slow callback might - * produce visible side-effects twice. - * - * An example for custom embedder type support might employ a way to wrap/ - * unwrap various C++ types in JSObject instances, e.g: - * - * \code - * - * // Helper method with a check for field count. - * template - * inline T* GetInternalField(v8::Local wrapper) { - * assert(offset < wrapper->InternalFieldCount()); - * return reinterpret_cast( - * wrapper->GetAlignedPointerFromInternalField(offset)); - * } - * - * class CustomEmbedderType { - * public: - * // Returns the raw C object from a wrapper JS object. - * static CustomEmbedderType* Unwrap(v8::Local wrapper) { - * return GetInternalField(wrapper); - * } - * static void FastMethod(v8::Local receiver_obj, int param) { - * CustomEmbedderType* receiver = static_cast( - * receiver_obj->GetAlignedPointerFromInternalField( - * kV8EmbedderWrapperObjectIndex)); - * - * // Type checks are already done by the optimized code. - * // Then call some performance-critical method like: - * // receiver->Method(param); - * } - * - * static void SlowMethod( - * const v8::FunctionCallbackInfo& info) { - * v8::Local instance = - * v8::Local::Cast(info.Holder()); - * CustomEmbedderType* receiver = Unwrap(instance); - * // TODO: Do type checks and extract {param}. - * receiver->Method(param); - * } - * }; - * - * // TODO(mslekova): Clean-up these constants - * // The constants kV8EmbedderWrapperTypeIndex and - * // kV8EmbedderWrapperObjectIndex describe the offsets for the type info - * // struct and the native object, when expressed as internal field indices - * // within a JSObject. The existance of this helper function assumes that - * // all embedder objects have their JSObject-side type info at the same - * // offset, but this is not a limitation of the API itself. For a detailed - * // use case, see the third example. - * static constexpr int kV8EmbedderWrapperTypeIndex = 0; - * static constexpr int kV8EmbedderWrapperObjectIndex = 1; - * - * // The following setup function can be templatized based on - * // the {embedder_object} argument. - * void SetupCustomEmbedderObject(v8::Isolate* isolate, - * v8::Local context, - * CustomEmbedderType* embedder_object) { - * isolate->set_embedder_wrapper_type_index( - * kV8EmbedderWrapperTypeIndex); - * isolate->set_embedder_wrapper_object_index( - * kV8EmbedderWrapperObjectIndex); - * - * v8::CFunction c_func = - * MakeV8CFunction(CustomEmbedderType::FastMethod); - * - * Local method_template = - * v8::FunctionTemplate::New( - * isolate, CustomEmbedderType::SlowMethod, v8::Local(), - * v8::Local(), 1, v8::ConstructorBehavior::kAllow, - * v8::SideEffectType::kHasSideEffect, &c_func); - * - * v8::Local object_template = - * v8::ObjectTemplate::New(isolate); - * object_template->SetInternalFieldCount( - * kV8EmbedderWrapperObjectIndex + 1); - * object_template->Set(isolate, "method", method_template); - * - * // Instantiate the wrapper JS object. - * v8::Local object = - * object_template->NewInstance(context).ToLocalChecked(); - * object->SetAlignedPointerInInternalField( - * kV8EmbedderWrapperObjectIndex, - * reinterpret_cast(embedder_object)); - * - * // TODO: Expose {object} where it's necessary. - * } - * \endcode - * - * For instance if {object} is exposed via a global "obj" variable, - * one could write in JS: - * function hot_func() { - * obj.method(42); - * } - * and once {hot_func} gets optimized, CustomEmbedderType::FastMethod - * will be called instead of the slow version, with the following arguments: - * receiver := the {embedder_object} from above - * param := 42 - * - * Currently supported return types: - * - void - * - bool - * - int32_t - * - uint32_t - * - float32_t - * - float64_t - * Currently supported argument types: - * - pointer to an embedder type - * - JavaScript array of primitive types - * - bool - * - int32_t - * - uint32_t - * - int64_t - * - uint64_t - * - float32_t - * - float64_t - * - * The 64-bit integer types currently have the IDL (unsigned) long long - * semantics: https://heycam.github.io/webidl/#abstract-opdef-converttoint - * In the future we'll extend the API to also provide conversions from/to - * BigInt to preserve full precision. - * The floating point types currently have the IDL (unrestricted) semantics, - * which is the only one used by WebGL. We plan to add support also for - * restricted floats/doubles, similarly to the BigInt conversion policies. - * We also differ from the specific NaN bit pattern that WebIDL prescribes - * (https://heycam.github.io/webidl/#es-unrestricted-float) in that Blink - * passes NaN values as-is, i.e. doesn't normalize them. - * - * To be supported types: - * - TypedArrays and ArrayBuffers - * - arrays of embedder types - * - * - * The API offers a limited support for function overloads: - * - * \code - * void FastMethod_2Args(int param, bool another_param); - * void FastMethod_3Args(int param, bool another_param, int third_param); - * - * v8::CFunction fast_method_2args_c_func = - * MakeV8CFunction(FastMethod_2Args); - * v8::CFunction fast_method_3args_c_func = - * MakeV8CFunction(FastMethod_3Args); - * const v8::CFunction fast_method_overloads[] = {fast_method_2args_c_func, - * fast_method_3args_c_func}; - * Local method_template = - * v8::FunctionTemplate::NewWithCFunctionOverloads( - * isolate, SlowCallback, data, signature, length, - * constructor_behavior, side_effect_type, - * {fast_method_overloads, 2}); - * \endcode - * - * In this example a single FunctionTemplate is associated to multiple C++ - * functions. The overload resolution is currently only based on the number of - * arguments passed in a call. For example, if this method_template is - * registered with a wrapper JS object as described above, a call with two - * arguments: - * obj.method(42, true); - * will result in a fast call to FastMethod_2Args, while a call with three or - * more arguments: - * obj.method(42, true, 11); - * will result in a fast call to FastMethod_3Args. Instead a call with less than - * two arguments, like: - * obj.method(42); - * would not result in a fast call but would fall back to executing the - * associated SlowCallback. - */ - -#ifndef INCLUDE_V8_FAST_API_CALLS_H_ -#define INCLUDE_V8_FAST_API_CALLS_H_ - -#include -#include - -#include -#include - -#include "v8-internal.h" // NOLINT(build/include_directory) -#include "v8-local-handle.h" // NOLINT(build/include_directory) -#include "v8-typed-array.h" // NOLINT(build/include_directory) -#include "v8-value.h" // NOLINT(build/include_directory) -#include "v8config.h" // NOLINT(build/include_directory) - -namespace v8 { - -class Isolate; - -class CTypeInfo { - public: - enum class Type : uint8_t { - kVoid, - kBool, - kUint8, - kInt32, - kUint32, - kInt64, - kUint64, - kFloat32, - kFloat64, - kPointer, - kV8Value, - kSeqOneByteString, - kApiObject, // This will be deprecated once all users have - // migrated from v8::ApiObject to v8::Local. - kAny, // This is added to enable untyped representation of fast - // call arguments for test purposes. It can represent any of - // the other types stored in the same memory as a union (see - // the AnyCType struct declared below). This allows for - // uniform passing of arguments w.r.t. their location - // (in a register or on the stack), independent of their - // actual type. It's currently used by the arm64 simulator - // and can be added to the other simulators as well when fast - // calls having both GP and FP params need to be supported. - }; - - // kCallbackOptionsType is not part of the Type enum - // because it is only used internally. Use value 255 that is larger - // than any valid Type enum. - static constexpr Type kCallbackOptionsType = Type(255); - - enum class SequenceType : uint8_t { - kScalar, - kIsSequence, // sequence - kIsTypedArray, // TypedArray of T or any ArrayBufferView if T - // is void - kIsArrayBuffer // ArrayBuffer - }; - - enum class Flags : uint8_t { - kNone = 0, - kAllowSharedBit = 1 << 0, // Must be an ArrayBuffer or TypedArray - kEnforceRangeBit = 1 << 1, // T must be integral - kClampBit = 1 << 2, // T must be integral - kIsRestrictedBit = 1 << 3, // T must be float or double - }; - - explicit constexpr CTypeInfo( - Type type, SequenceType sequence_type = SequenceType::kScalar, - Flags flags = Flags::kNone) - : type_(type), sequence_type_(sequence_type), flags_(flags) {} - - typedef uint32_t Identifier; - explicit constexpr CTypeInfo(Identifier identifier) - : CTypeInfo(static_cast(identifier >> 16), - static_cast((identifier >> 8) & 255), - static_cast(identifier & 255)) {} - constexpr Identifier GetId() const { - return static_cast(type_) << 16 | - static_cast(sequence_type_) << 8 | - static_cast(flags_); - } - - constexpr Type GetType() const { return type_; } - constexpr SequenceType GetSequenceType() const { return sequence_type_; } - constexpr Flags GetFlags() const { return flags_; } - - static constexpr bool IsIntegralType(Type type) { - return type == Type::kUint8 || type == Type::kInt32 || - type == Type::kUint32 || type == Type::kInt64 || - type == Type::kUint64; - } - - static constexpr bool IsFloatingPointType(Type type) { - return type == Type::kFloat32 || type == Type::kFloat64; - } - - static constexpr bool IsPrimitive(Type type) { - return IsIntegralType(type) || IsFloatingPointType(type) || - type == Type::kBool; - } - - private: - Type type_; - SequenceType sequence_type_; - Flags flags_; -}; - -struct FastApiTypedArrayBase { - public: - // Returns the length in number of elements. - size_t V8_EXPORT length() const { return length_; } - // Checks whether the given index is within the bounds of the collection. - void V8_EXPORT ValidateIndex(size_t index) const; - - protected: - size_t length_ = 0; -}; - -template -struct FastApiTypedArray : public FastApiTypedArrayBase { - public: - V8_INLINE T get(size_t index) const { -#ifdef DEBUG - ValidateIndex(index); -#endif // DEBUG - T tmp; - memcpy(&tmp, reinterpret_cast(data_) + index, sizeof(T)); - return tmp; - } - - bool getStorageIfAligned(T** elements) const { - if (reinterpret_cast(data_) % alignof(T) != 0) { - return false; - } - *elements = reinterpret_cast(data_); - return true; - } - - private: - // This pointer should include the typed array offset applied. - // It's not guaranteed that it's aligned to sizeof(T), it's only - // guaranteed that it's 4-byte aligned, so for 8-byte types we need to - // provide a special implementation for reading from it, which hides - // the possibly unaligned read in the `get` method. - void* data_; -}; - -// Any TypedArray. It uses kTypedArrayBit with base type void -// Overloaded args of ArrayBufferView and TypedArray are not supported -// (for now) because the generic “any” ArrayBufferView doesn’t have its -// own instance type. It could be supported if we specify that -// TypedArray always has precedence over the generic ArrayBufferView, -// but this complicates overload resolution. -struct FastApiArrayBufferView { - void* data; - size_t byte_length; -}; - -struct FastApiArrayBuffer { - void* data; - size_t byte_length; -}; - -struct FastOneByteString { - const char* data; - uint32_t length; -}; - -class V8_EXPORT CFunctionInfo { - public: - // Construct a struct to hold a CFunction's type information. - // |return_info| describes the function's return type. - // |arg_info| is an array of |arg_count| CTypeInfos describing the - // arguments. Only the last argument may be of the special type - // CTypeInfo::kCallbackOptionsType. - CFunctionInfo(const CTypeInfo& return_info, unsigned int arg_count, - const CTypeInfo* arg_info); - - const CTypeInfo& ReturnInfo() const { return return_info_; } - - // The argument count, not including the v8::FastApiCallbackOptions - // if present. - unsigned int ArgumentCount() const { - return HasOptions() ? arg_count_ - 1 : arg_count_; - } - - // |index| must be less than ArgumentCount(). - // Note: if the last argument passed on construction of CFunctionInfo - // has type CTypeInfo::kCallbackOptionsType, it is not included in - // ArgumentCount(). - const CTypeInfo& ArgumentInfo(unsigned int index) const; - - bool HasOptions() const { - // The options arg is always the last one. - return arg_count_ > 0 && arg_info_[arg_count_ - 1].GetType() == - CTypeInfo::kCallbackOptionsType; - } - - private: - const CTypeInfo return_info_; - const unsigned int arg_count_; - const CTypeInfo* arg_info_; -}; - -struct FastApiCallbackOptions; - -// Provided for testing. -struct AnyCType { - AnyCType() : int64_value(0) {} - - union { - bool bool_value; - int32_t int32_value; - uint32_t uint32_value; - int64_t int64_value; - uint64_t uint64_value; - float float_value; - double double_value; - void* pointer_value; - Local object_value; - Local sequence_value; - const FastApiTypedArray* uint8_ta_value; - const FastApiTypedArray* int32_ta_value; - const FastApiTypedArray* uint32_ta_value; - const FastApiTypedArray* int64_ta_value; - const FastApiTypedArray* uint64_ta_value; - const FastApiTypedArray* float_ta_value; - const FastApiTypedArray* double_ta_value; - const FastOneByteString* string_value; - FastApiCallbackOptions* options_value; - }; -}; - -static_assert( - sizeof(AnyCType) == 8, - "The AnyCType struct should have size == 64 bits, as this is assumed " - "by EffectControlLinearizer."); - -class V8_EXPORT CFunction { - public: - constexpr CFunction() : address_(nullptr), type_info_(nullptr) {} - - const CTypeInfo& ReturnInfo() const { return type_info_->ReturnInfo(); } - - const CTypeInfo& ArgumentInfo(unsigned int index) const { - return type_info_->ArgumentInfo(index); - } - - unsigned int ArgumentCount() const { return type_info_->ArgumentCount(); } - - const void* GetAddress() const { return address_; } - const CFunctionInfo* GetTypeInfo() const { return type_info_; } - - enum class OverloadResolution { kImpossible, kAtRuntime, kAtCompileTime }; - - // Returns whether an overload between this and the given CFunction can - // be resolved at runtime by the RTTI available for the arguments or at - // compile time for functions with different number of arguments. - OverloadResolution GetOverloadResolution(const CFunction* other) { - // Runtime overload resolution can only deal with functions with the - // same number of arguments. Functions with different arity are handled - // by compile time overload resolution though. - if (ArgumentCount() != other->ArgumentCount()) { - return OverloadResolution::kAtCompileTime; - } - - // The functions can only differ by a single argument position. - int diff_index = -1; - for (unsigned int i = 0; i < ArgumentCount(); ++i) { - if (ArgumentInfo(i).GetSequenceType() != - other->ArgumentInfo(i).GetSequenceType()) { - if (diff_index >= 0) { - return OverloadResolution::kImpossible; - } - diff_index = i; - - // We only support overload resolution between sequence types. - if (ArgumentInfo(i).GetSequenceType() == - CTypeInfo::SequenceType::kScalar || - other->ArgumentInfo(i).GetSequenceType() == - CTypeInfo::SequenceType::kScalar) { - return OverloadResolution::kImpossible; - } - } - } - - return OverloadResolution::kAtRuntime; - } - - template - static CFunction Make(F* func) { - return ArgUnwrap::Make(func); - } - - // Provided for testing purposes. - template - static CFunction Make(R (*func)(Args...), - R_Patch (*patching_func)(Args_Patch...)) { - CFunction c_func = ArgUnwrap::Make(func); - static_assert( - sizeof...(Args_Patch) == sizeof...(Args), - "The patching function must have the same number of arguments."); - c_func.address_ = reinterpret_cast(patching_func); - return c_func; - } - - CFunction(const void* address, const CFunctionInfo* type_info); - - private: - const void* address_; - const CFunctionInfo* type_info_; - - template - class ArgUnwrap { - static_assert(sizeof(F) != sizeof(F), - "CFunction must be created from a function pointer."); - }; - - template - class ArgUnwrap { - public: - static CFunction Make(R (*func)(Args...)); - }; -}; - -/** - * A struct which may be passed to a fast call callback, like so: - * \code - * void FastMethodWithOptions(int param, FastApiCallbackOptions& options); - * \endcode - */ -struct FastApiCallbackOptions { - /** - * Creates a new instance of FastApiCallbackOptions for testing purpose. The - * returned instance may be filled with mock data. - */ - static FastApiCallbackOptions CreateForTesting(Isolate* isolate) { - return {false, {0}, nullptr}; - } - - /** - * If the callback wants to signal an error condition or to perform an - * allocation, it must set options.fallback to true and do an early return - * from the fast method. Then V8 checks the value of options.fallback and if - * it's true, falls back to executing the SlowCallback, which is capable of - * reporting the error (either by throwing a JS exception or logging to the - * console) or doing the allocation. It's the embedder's responsibility to - * ensure that the fast callback is idempotent up to the point where error and - * fallback conditions are checked, because otherwise executing the slow - * callback might produce visible side-effects twice. - */ - bool fallback; - - /** - * The `data` passed to the FunctionTemplate constructor, or `undefined`. - * `data_ptr` allows for default constructing FastApiCallbackOptions. - */ - union { - uintptr_t data_ptr; - v8::Local data; - }; - - /** - * When called from WebAssembly, a view of the calling module's memory. - */ - FastApiTypedArray* const wasm_memory; -}; - -namespace internal { - -// Helper to count the number of occurances of `T` in `List` -template -struct count : std::integral_constant {}; -template -struct count - : std::integral_constant::value> {}; -template -struct count : count {}; - -template -class CFunctionInfoImpl : public CFunctionInfo { - static constexpr int kOptionsArgCount = - count(); - static constexpr int kReceiverCount = 1; - - static_assert(kOptionsArgCount == 0 || kOptionsArgCount == 1, - "Only one options parameter is supported."); - - static_assert(sizeof...(ArgBuilders) >= kOptionsArgCount + kReceiverCount, - "The receiver or the options argument is missing."); - - public: - constexpr CFunctionInfoImpl() - : CFunctionInfo(RetBuilder::Build(), sizeof...(ArgBuilders), - arg_info_storage_), - arg_info_storage_{ArgBuilders::Build()...} { - constexpr CTypeInfo::Type kReturnType = RetBuilder::Build().GetType(); - static_assert(kReturnType == CTypeInfo::Type::kVoid || - kReturnType == CTypeInfo::Type::kBool || - kReturnType == CTypeInfo::Type::kInt32 || - kReturnType == CTypeInfo::Type::kUint32 || - kReturnType == CTypeInfo::Type::kFloat32 || - kReturnType == CTypeInfo::Type::kFloat64 || - kReturnType == CTypeInfo::Type::kPointer || - kReturnType == CTypeInfo::Type::kAny, - "64-bit int, string and api object values are not currently " - "supported return types."); - } - - private: - const CTypeInfo arg_info_storage_[sizeof...(ArgBuilders)]; -}; - -template -struct TypeInfoHelper { - static_assert(sizeof(T) != sizeof(T), "This type is not supported"); -}; - -#define SPECIALIZE_GET_TYPE_INFO_HELPER_FOR(T, Enum) \ - template <> \ - struct TypeInfoHelper { \ - static constexpr CTypeInfo::Flags Flags() { \ - return CTypeInfo::Flags::kNone; \ - } \ - \ - static constexpr CTypeInfo::Type Type() { return CTypeInfo::Type::Enum; } \ - static constexpr CTypeInfo::SequenceType SequenceType() { \ - return CTypeInfo::SequenceType::kScalar; \ - } \ - }; - -template -struct CTypeInfoTraits {}; - -#define DEFINE_TYPE_INFO_TRAITS(CType, Enum) \ - template <> \ - struct CTypeInfoTraits { \ - using ctype = CType; \ - }; - -#define PRIMITIVE_C_TYPES(V) \ - V(bool, kBool) \ - V(uint8_t, kUint8) \ - V(int32_t, kInt32) \ - V(uint32_t, kUint32) \ - V(int64_t, kInt64) \ - V(uint64_t, kUint64) \ - V(float, kFloat32) \ - V(double, kFloat64) \ - V(void*, kPointer) - -// Same as above, but includes deprecated types for compatibility. -#define ALL_C_TYPES(V) \ - PRIMITIVE_C_TYPES(V) \ - V(void, kVoid) \ - V(v8::Local, kV8Value) \ - V(v8::Local, kV8Value) \ - V(AnyCType, kAny) - -// ApiObject was a temporary solution to wrap the pointer to the v8::Value. -// Please use v8::Local in new code for the arguments and -// v8::Local for the receiver, as ApiObject will be deprecated. - -ALL_C_TYPES(SPECIALIZE_GET_TYPE_INFO_HELPER_FOR) -PRIMITIVE_C_TYPES(DEFINE_TYPE_INFO_TRAITS) - -#undef PRIMITIVE_C_TYPES -#undef ALL_C_TYPES - -#define SPECIALIZE_GET_TYPE_INFO_HELPER_FOR_TA(T, Enum) \ - template <> \ - struct TypeInfoHelper&> { \ - static constexpr CTypeInfo::Flags Flags() { \ - return CTypeInfo::Flags::kNone; \ - } \ - \ - static constexpr CTypeInfo::Type Type() { return CTypeInfo::Type::Enum; } \ - static constexpr CTypeInfo::SequenceType SequenceType() { \ - return CTypeInfo::SequenceType::kIsTypedArray; \ - } \ - }; - -#define TYPED_ARRAY_C_TYPES(V) \ - V(uint8_t, kUint8) \ - V(int32_t, kInt32) \ - V(uint32_t, kUint32) \ - V(int64_t, kInt64) \ - V(uint64_t, kUint64) \ - V(float, kFloat32) \ - V(double, kFloat64) - -TYPED_ARRAY_C_TYPES(SPECIALIZE_GET_TYPE_INFO_HELPER_FOR_TA) - -#undef TYPED_ARRAY_C_TYPES - -template <> -struct TypeInfoHelper> { - static constexpr CTypeInfo::Flags Flags() { return CTypeInfo::Flags::kNone; } - - static constexpr CTypeInfo::Type Type() { return CTypeInfo::Type::kVoid; } - static constexpr CTypeInfo::SequenceType SequenceType() { - return CTypeInfo::SequenceType::kIsSequence; - } -}; - -template <> -struct TypeInfoHelper> { - static constexpr CTypeInfo::Flags Flags() { return CTypeInfo::Flags::kNone; } - - static constexpr CTypeInfo::Type Type() { return CTypeInfo::Type::kUint32; } - static constexpr CTypeInfo::SequenceType SequenceType() { - return CTypeInfo::SequenceType::kIsTypedArray; - } -}; - -template <> -struct TypeInfoHelper { - static constexpr CTypeInfo::Flags Flags() { return CTypeInfo::Flags::kNone; } - - static constexpr CTypeInfo::Type Type() { - return CTypeInfo::kCallbackOptionsType; - } - static constexpr CTypeInfo::SequenceType SequenceType() { - return CTypeInfo::SequenceType::kScalar; - } -}; - -template <> -struct TypeInfoHelper { - static constexpr CTypeInfo::Flags Flags() { return CTypeInfo::Flags::kNone; } - - static constexpr CTypeInfo::Type Type() { - return CTypeInfo::Type::kSeqOneByteString; - } - static constexpr CTypeInfo::SequenceType SequenceType() { - return CTypeInfo::SequenceType::kScalar; - } -}; - -#define STATIC_ASSERT_IMPLIES(COND, ASSERTION, MSG) \ - static_assert(((COND) == 0) || (ASSERTION), MSG) - -} // namespace internal - -template -class V8_EXPORT CTypeInfoBuilder { - public: - using BaseType = T; - - static constexpr CTypeInfo Build() { - constexpr CTypeInfo::Flags kFlags = - MergeFlags(internal::TypeInfoHelper::Flags(), Flags...); - constexpr CTypeInfo::Type kType = internal::TypeInfoHelper::Type(); - constexpr CTypeInfo::SequenceType kSequenceType = - internal::TypeInfoHelper::SequenceType(); - - STATIC_ASSERT_IMPLIES( - uint8_t(kFlags) & uint8_t(CTypeInfo::Flags::kAllowSharedBit), - (kSequenceType == CTypeInfo::SequenceType::kIsTypedArray || - kSequenceType == CTypeInfo::SequenceType::kIsArrayBuffer), - "kAllowSharedBit is only allowed for TypedArrays and ArrayBuffers."); - STATIC_ASSERT_IMPLIES( - uint8_t(kFlags) & uint8_t(CTypeInfo::Flags::kEnforceRangeBit), - CTypeInfo::IsIntegralType(kType), - "kEnforceRangeBit is only allowed for integral types."); - STATIC_ASSERT_IMPLIES( - uint8_t(kFlags) & uint8_t(CTypeInfo::Flags::kClampBit), - CTypeInfo::IsIntegralType(kType), - "kClampBit is only allowed for integral types."); - STATIC_ASSERT_IMPLIES( - uint8_t(kFlags) & uint8_t(CTypeInfo::Flags::kIsRestrictedBit), - CTypeInfo::IsFloatingPointType(kType), - "kIsRestrictedBit is only allowed for floating point types."); - STATIC_ASSERT_IMPLIES(kSequenceType == CTypeInfo::SequenceType::kIsSequence, - kType == CTypeInfo::Type::kVoid, - "Sequences are only supported from void type."); - STATIC_ASSERT_IMPLIES( - kSequenceType == CTypeInfo::SequenceType::kIsTypedArray, - CTypeInfo::IsPrimitive(kType) || kType == CTypeInfo::Type::kVoid, - "TypedArrays are only supported from primitive types or void."); - - // Return the same type with the merged flags. - return CTypeInfo(internal::TypeInfoHelper::Type(), - internal::TypeInfoHelper::SequenceType(), kFlags); - } - - private: - template - static constexpr CTypeInfo::Flags MergeFlags(CTypeInfo::Flags flags, - Rest... rest) { - return CTypeInfo::Flags(uint8_t(flags) | uint8_t(MergeFlags(rest...))); - } - static constexpr CTypeInfo::Flags MergeFlags() { return CTypeInfo::Flags(0); } -}; - -namespace internal { -template -class CFunctionBuilderWithFunction { - public: - explicit constexpr CFunctionBuilderWithFunction(const void* fn) : fn_(fn) {} - - template - constexpr auto Ret() { - return CFunctionBuilderWithFunction< - CTypeInfoBuilder, - ArgBuilders...>(fn_); - } - - template - constexpr auto Arg() { - // Return a copy of the builder with the Nth arg builder merged with - // template parameter pack Flags. - return ArgImpl( - std::make_index_sequence()); - } - - // Provided for testing purposes. - template - auto Patch(Ret (*patching_func)(Args...)) { - static_assert( - sizeof...(Args) == sizeof...(ArgBuilders), - "The patching function must have the same number of arguments."); - fn_ = reinterpret_cast(patching_func); - return *this; - } - - auto Build() { - static CFunctionInfoImpl instance; - return CFunction(fn_, &instance); - } - - private: - template - struct GetArgBuilder; - - // Returns the same ArgBuilder as the one at index N, including its flags. - // Flags in the template parameter pack are ignored. - template - struct GetArgBuilder { - using type = - typename std::tuple_element>::type; - }; - - // Returns an ArgBuilder with the same base type as the one at index N, - // but merges the flags with the flags in the template parameter pack. - template - struct GetArgBuilder { - using type = CTypeInfoBuilder< - typename std::tuple_element>::type::BaseType, - std::tuple_element>::type::Build() - .GetFlags(), - Flags...>; - }; - - // Return a copy of the CFunctionBuilder, but merges the Flags on - // ArgBuilder index N with the new Flags passed in the template parameter - // pack. - template - constexpr auto ArgImpl(std::index_sequence) { - return CFunctionBuilderWithFunction< - RetBuilder, typename GetArgBuilder::type...>(fn_); - } - - const void* fn_; -}; - -class CFunctionBuilder { - public: - constexpr CFunctionBuilder() {} - - template - constexpr auto Fn(R (*fn)(Args...)) { - return CFunctionBuilderWithFunction, - CTypeInfoBuilder...>( - reinterpret_cast(fn)); - } -}; - -} // namespace internal - -// static -template -CFunction CFunction::ArgUnwrap::Make(R (*func)(Args...)) { - return internal::CFunctionBuilder().Fn(func).Build(); -} - -using CFunctionBuilder = internal::CFunctionBuilder; - -static constexpr CTypeInfo kTypeInfoInt32 = CTypeInfo(CTypeInfo::Type::kInt32); -static constexpr CTypeInfo kTypeInfoFloat64 = - CTypeInfo(CTypeInfo::Type::kFloat64); - -/** - * Copies the contents of this JavaScript array to a C++ buffer with - * a given max_length. A CTypeInfo is passed as an argument, - * instructing different rules for conversion (e.g. restricted float/double). - * The element type T of the destination array must match the C type - * corresponding to the CTypeInfo (specified by CTypeInfoTraits). - * If the array length is larger than max_length or the array is of - * unsupported type, the operation will fail, returning false. Generally, an - * array which contains objects, undefined, null or anything not convertible - * to the requested destination type, is considered unsupported. The operation - * returns true on success. `type_info` will be used for conversions. - */ -template -bool V8_EXPORT V8_WARN_UNUSED_RESULT TryToCopyAndConvertArrayToCppBuffer( - Local src, T* dst, uint32_t max_length); - -template <> -bool V8_EXPORT V8_WARN_UNUSED_RESULT -TryToCopyAndConvertArrayToCppBuffer::Build().GetId(), - int32_t>(Local src, int32_t* dst, - uint32_t max_length); - -template <> -bool V8_EXPORT V8_WARN_UNUSED_RESULT -TryToCopyAndConvertArrayToCppBuffer::Build().GetId(), - uint32_t>(Local src, uint32_t* dst, - uint32_t max_length); - -template <> -bool V8_EXPORT V8_WARN_UNUSED_RESULT -TryToCopyAndConvertArrayToCppBuffer::Build().GetId(), - float>(Local src, float* dst, - uint32_t max_length); - -template <> -bool V8_EXPORT V8_WARN_UNUSED_RESULT -TryToCopyAndConvertArrayToCppBuffer::Build().GetId(), - double>(Local src, double* dst, - uint32_t max_length); - -} // namespace v8 - -#endif // INCLUDE_V8_FAST_API_CALLS_H_ diff --git a/javascript/test/hps.test.ts b/javascript/test/hps.test.ts index 8d8bc1bce6..40be8d4e1f 100644 --- a/javascript/test/hps.test.ts +++ b/javascript/test/hps.test.ts @@ -17,28 +17,27 @@ * under the License. */ -import { isLatin1, stringCopy } from '../packages/hps/index'; +import { BinaryReader } from '@furyjs/fury/dist/lib/reader'; +import hps from '../packages/hps/index'; import { describe, expect, test } from '@jest/globals'; -describe('hps', () => { - test('should isLatin1 work', () => { - for (let index = 0; index < 10000; index++) { - var is = isLatin1("hello"); - expect(is).toBe(true) - - var is = isLatin1("😁"); - expect(is).toBe(false) - } - }); +const skipableDescribe = (hps ? describe : describe.skip); - test('should stringCopy work', () => { +skipableDescribe('hps', () => { + test.only('should isLatin1 work', () => { + const { serializeString } = hps!; for (let index = 0; index < 10000; index++) { - const dist = new Uint8Array(5); - stringCopy("hello", dist, 0); - expect([...dist]).toEqual([104, 101, 108, 108, 111]) + const bf = Buffer.alloc(100); + serializeString("hello", bf, 0); + var reader = new BinaryReader({}); + reader.reset(bf); + expect(reader.stringOfVarUInt32()).toBe("hello") + + serializeString("😁", bf, 0); + var reader = new BinaryReader({}); + reader.reset(bf); + expect(reader.stringOfVarUInt32()).toBe("😁") } }); }); - - diff --git a/javascript/test/platformBuffer.test.ts b/javascript/test/platformBuffer.test.ts index cc633fc436..e0f6362ec5 100644 --- a/javascript/test/platformBuffer.test.ts +++ b/javascript/test/platformBuffer.test.ts @@ -40,37 +40,20 @@ describe('platformBuffer', () => { test('should latin1Write work', () => { const bb = BrowserBuffer.alloc(100); - bb.latin1Write("hello, world", 0); + bb.write("hello, world", 0, "latin1"); - const str = bb.latin1Slice(0, 12); + const str = bb.toString("latin1", 0, 12); expect(str).toBe("hello, world"); - - const str2 = bb.toString('latin1', 0, 12); - expect(str2).toBe("hello, world"); - }); - - test('should write latin1 work', () => { - const bb = BrowserBuffer.alloc(100); - bb.write("hello, world", 0, 'latin1'); - - const str = bb.latin1Slice(0, 12); - expect(str).toBe("hello, world"); - - const str2 = bb.toString('latin1', 0, 12); - expect(str2).toBe("hello, world"); }); test('should utf8Write work', () => { const rawStr = "我是Fury, 你好！😁א"; const bb = BrowserBuffer.alloc(100); - bb.utf8Write(rawStr, 0); + bb.write(rawStr, 0); - const str = bb.utf8Slice(0, 27); + const str = bb.toString("utf8", 0, 27); expect(str).toBe(rawStr); - - const str2 = bb.toString('utf8', 0, 27); - expect(str2).toBe(rawStr); }); test('should utf8 work', () => { @@ -78,9 +61,6 @@ describe('platformBuffer', () => { const bb = BrowserBuffer.alloc(100); bb.write(rawStr, 0, 'utf8'); - const str = bb.utf8Slice(0, 27); - expect(str).toBe(rawStr); - const str2 = bb.toString('utf8', 0, 27); expect(str2).toBe(rawStr); }); @@ -93,7 +73,7 @@ describe('platformBuffer', () => { test('should copy work', () => { const bb = BrowserBuffer.alloc(100); - bb.latin1Write("hello", 0); + bb.write("hello", 0, 'latin1'); const target = new Uint8Array(5); bb.copy(target, 0, 0, 5); expect([...target]).toEqual([ 104, 101, 108, 108, 111 ])