From d97aae71e33dbc14ee064fe940e3547998da74ad Mon Sep 17 00:00:00 2001 From: sean Date: Sat, 2 Mar 2024 18:51:13 +0100 Subject: [PATCH 1/9] Add: GltfFileStream and rewritten GltfDataBuffer --- include/fastgltf/core.hpp | 132 ++++++++++----------- src/fastgltf.cpp | 236 +++++++++++++++++++------------------- tests/accessor_tests.cpp | 14 ++- tests/base64_tests.cpp | 13 ++- tests/basic_test.cpp | 173 ++++++++++++++-------------- tests/benchmarks.cpp | 48 ++++---- tests/extension_tests.cpp | 108 ++++++++--------- tests/glb_tests.cpp | 17 ++- tests/uri_tests.cpp | 13 ++- tests/write_tests.cpp | 79 +++++++------ 10 files changed, 418 insertions(+), 415 deletions(-) diff --git a/include/fastgltf/core.hpp b/include/fastgltf/core.hpp index 76ec83d11..882420a50 100644 --- a/include/fastgltf/core.hpp +++ b/include/fastgltf/core.hpp @@ -26,6 +26,7 @@ #pragma once +#include #include #include @@ -67,7 +68,7 @@ namespace std { namespace fastgltf { struct BinaryGltfChunk; - class GltfDataBuffer; + class GltfDataGetter; enum class Error : std::uint64_t { None = 0, @@ -259,15 +260,8 @@ namespace fastgltf { */ DecomposeNodeMatrices = 1 << 5, - /** - * This option makes fastgltf minimise the JSON file before parsing. In most cases, - * minimising it beforehand actually reduces the time spent. However, there are plenty - * of cases where this option slows down parsing drastically, which from my testing seem - * to all be glTFs which contain embedded buffers and/or are already minimised. Note that - * fastgltf only minimises the string if the data was loaded using GltfDataBuffer::loadFromFile - * or GltfDataBuffer::copyBytes, and that the bytes will also be overwritten. - */ - MinimiseJsonBeforeParsing = 1 << 6, + // TODO: Remove or keep for any compatibility? + MinimiseJsonBeforeParsing [[deprecated]] = 1 << 6, /** * Loads all external images into CPU memory. It does not decode any texture data. Complementary @@ -606,72 +600,80 @@ namespace fastgltf { * @return The type of the glTF file, either glTF, GLB, or Invalid if it was not determinable. If this function * returns Invalid it is highly likely that the buffer does not actually represent a valid glTF file. */ - GltfType determineGltfFileType(GltfDataBuffer* buffer); + GltfType determineGltfFileType(GltfDataGetter& data); - /** - * Gets the amount of byte padding required on the GltfDataBuffer, as simdjson requires to be - * able to overflow as it uses SIMD to load N bytes at a time. - */ - std::size_t getGltfBufferPadding() noexcept; + /** + * This interface defines how the parser can read the bytes making up a glTF or GLB file. + */ + class GltfDataGetter { + public: + virtual void read(void* ptr, std::size_t count) = 0; + [[nodiscard]] virtual span read(std::size_t count, std::size_t padding) = 0; - /** - * This class holds a chunk of data that makes up a JSON string that the glTF parser will use - * and read from. - */ - class GltfDataBuffer { - friend class Parser; - friend GltfType determineGltfFileType(GltfDataBuffer* buffer); + /** + * Reset is used to put the offset index back to the start of the buffer/file. + * This is only used with determineGltfFileType, as it needs to peek into the beginning of the file. + */ + virtual void reset() = 0; - protected: - std::size_t allocatedSize = 0; - std::size_t dataSize = 0; - std::byte* bufferPointer = nullptr; + [[nodiscard]] virtual std::size_t bytesRead() = 0; + [[nodiscard]] virtual std::size_t totalSize() = 0; + }; - std::unique_ptr buffer; + class GltfDataBuffer : public GltfDataGetter { + std::unique_ptr buffer; - std::filesystem::path filePath = {}; + std::size_t allocatedSize = 0; + std::size_t dataSize = 0; - public: - explicit GltfDataBuffer() noexcept; + std::size_t idx = 0; - /** - * Constructs a new GltfDataBuffer from a span object, copying its data as there - * is no guarantee for the allocation size to have the adequate padding. - */ - explicit GltfDataBuffer(span data) noexcept; + void allocateAndCopy(const std::byte* bytes); - virtual ~GltfDataBuffer() noexcept; + public: + GltfDataBuffer(const std::filesystem::path& path); + GltfDataBuffer(const std::byte* bytes, std::size_t count); - /** - * Saves the given pointer including the given range. - * If the capacity of the allocation minus the used size is smaller than fastgltf::getGltfBufferPadding, - * this function will re-allocate and copy the bytes. - * Also, it will set the padding bytes all to 0, so be sure to not use that for any other data. - */ - bool fromByteView(std::uint8_t* bytes, std::size_t byteCount, std::size_t capacity) noexcept; +#if FASTGLTF_CPP_20 + GltfDataBuffer(std::span span); +#endif - /** - * This will create a copy of the passed bytes and allocate an adequately sized buffer. - */ - bool copyBytes(const std::uint8_t* bytes, std::size_t byteCount) noexcept; + void read(void* ptr, std::size_t count) override; - /** - * Loads the file with a optional byte offset into a memory buffer. - */ - bool loadFromFile(const std::filesystem::path& path, std::uint64_t byteOffset = 0) noexcept; + span read(std::size_t count, std::size_t padding) override; - /** - * Returns the size, in bytes, - * @return - */ - [[nodiscard]] std::size_t getBufferSize() const noexcept { - return dataSize; + void reset() override; + + std::size_t bytesRead() override; + + std::size_t totalSize() override; + + [[nodiscard]] explicit operator span() { + return span(buffer.get(), dataSize); } + }; - [[nodiscard]] explicit operator span() { - return span(bufferPointer, dataSize); - } - }; + class GltfFileStream : public GltfDataGetter { + std::ifstream fileStream; + std::vector buf; + + std::size_t fileSize; + + public: + GltfFileStream(const std::filesystem::path& path); + + bool isOpen() const; + + void read(void* ptr, std::size_t count) override; + + span read(std::size_t count, std::size_t padding) override; + + void reset() override; + + std::size_t bytesRead() override; + + std::size_t totalSize() override; + }; #if defined(__ANDROID__) void setAndroidAssetManager(AAssetManager* assetManager) noexcept; @@ -774,21 +776,21 @@ namespace fastgltf { * * @return An Asset wrapped in an Expected type, which may contain an error if one occurred. */ - [[nodiscard]] Expected loadGltf(GltfDataBuffer* buffer, std::filesystem::path directory, Options options = Options::None, Category categories = Category::All); + [[nodiscard]] Expected loadGltf(GltfDataGetter& buffer, std::filesystem::path directory, Options options = Options::None, Category categories = Category::All); /** * Loads a glTF file from pre-loaded bytes representing a JSON file. * * @return An Asset wrapped in an Expected type, which may contain an error if one occurred. */ - [[nodiscard]] Expected loadGltfJson(GltfDataBuffer* buffer, std::filesystem::path directory, Options options = Options::None, Category categories = Category::All); + [[nodiscard]] Expected loadGltfJson(GltfDataGetter& buffer, std::filesystem::path directory, Options options = Options::None, Category categories = Category::All); /** * Loads a glTF file embedded within a GLB container, which may contain the first buffer of the glTF asset. * * @return An Asset wrapped in an Expected type, which may contain an error if one occurred. */ - [[nodiscard]] Expected loadGltfBinary(GltfDataBuffer* buffer, std::filesystem::path directory, Options options = Options::None, Category categories = Category::All); + [[nodiscard]] Expected loadGltfBinary(GltfDataGetter& buffer, std::filesystem::path directory, Options options = Options::None, Category categories = Category::All); /** * This function can be used to set callbacks so that you can control memory allocation for diff --git a/src/fastgltf.cpp b/src/fastgltf.cpp index 105a5a61e..5e6a0a4f9 100644 --- a/src/fastgltf.cpp +++ b/src/fastgltf.cpp @@ -3732,86 +3732,106 @@ fg::Error fg::Parser::parseTextures(simdjson::dom::array& textures, Asset& asset #pragma endregion #pragma region GltfDataBuffer -std::size_t fg::getGltfBufferPadding() noexcept { - return simdjson::SIMDJSON_PADDING; +fg::GltfDataBuffer::GltfDataBuffer(const std::filesystem::path& path) { + std::error_code ec; + dataSize = static_cast(std::filesystem::file_size(path, ec)); + if (ec) { + // TODO: Fail? + } + + // Open the file and determine the size. + std::ifstream file(path, std::ios::binary); + if (!file.is_open() || file.bad()) + return; + + allocatedSize = dataSize + simdjson::SIMDJSON_PADDING; + buffer = decltype(buffer)(new std::byte[allocatedSize]); // To mimic std::make_unique_for_overwrite (C++20) + if (!buffer) + return; + + // Copy the data and fill the padding region with zeros. + file.read(reinterpret_cast(buffer.get()), static_cast(dataSize)); + std::memset(buffer.get() + dataSize, 0, allocatedSize - dataSize); } -fg::GltfDataBuffer::GltfDataBuffer() noexcept = default; -fg::GltfDataBuffer::~GltfDataBuffer() noexcept = default; +fg::GltfDataBuffer::GltfDataBuffer(const std::byte *bytes, std::size_t count) { + dataSize = count; + allocateAndCopy(bytes); +} -fg::GltfDataBuffer::GltfDataBuffer(span data) noexcept { - dataSize = data.size(); +#if FASTGLTF_CPP_20 +fg::GltfDataBuffer::GltfDataBuffer(std::span span) { + dataSize = span.size_bytes(); + allocateAndCopy(span.data()); +} +#endif - allocatedSize = data.size() + getGltfBufferPadding(); +void fg::GltfDataBuffer::allocateAndCopy(const std::byte *bytes) { + allocatedSize = dataSize + simdjson::SIMDJSON_PADDING; buffer = decltype(buffer)(new std::byte[allocatedSize]); - auto* ptr = buffer.get(); - std::memcpy(ptr, data.data(), dataSize); - std::memset(ptr + dataSize, 0, allocatedSize - dataSize); + std::memcpy(buffer.get(), bytes, dataSize); + std::memset(buffer.get() + dataSize, 0, allocatedSize - dataSize); +} - bufferPointer = ptr; +void fg::GltfDataBuffer::read(void *ptr, std::size_t count) { + std::memcpy(ptr, buffer.get() + idx, count); + idx += count; } -bool fg::GltfDataBuffer::fromByteView(std::uint8_t* bytes, std::size_t byteCount, std::size_t capacity) noexcept { - using namespace simdjson; - if (bytes == nullptr || byteCount == 0 || capacity == 0) - return false; +fg::span fg::GltfDataBuffer::read(std::size_t count, std::size_t padding) { + span sub(buffer.get() + idx, count); + idx += count; + return sub; +} - if (capacity - byteCount < getGltfBufferPadding()) - return copyBytes(bytes, byteCount); +void fg::GltfDataBuffer::reset() { + idx = 0; +} - dataSize = byteCount; - bufferPointer = reinterpret_cast(bytes); - allocatedSize = capacity; - std::memset(bufferPointer + dataSize, 0, getGltfBufferPadding()); - return true; +std::size_t fg::GltfDataBuffer::bytesRead() { + return idx; } -bool fg::GltfDataBuffer::copyBytes(const std::uint8_t* bytes, std::size_t byteCount) noexcept { - using namespace simdjson; - if (bytes == nullptr || byteCount == 0) - return false; +std::size_t fg::GltfDataBuffer::totalSize() { + return dataSize; +} - // Allocate a byte array with a bit of padding. - dataSize = byteCount; - allocatedSize = byteCount + getGltfBufferPadding(); - buffer = decltype(buffer)(new std::byte[allocatedSize]); // To mimic std::make_unique_for_overwrite (C++20) - bufferPointer = buffer.get(); +fg::GltfFileStream::GltfFileStream(const std::filesystem::path &path) : fileStream(path, std::ios::binary) { + fileSize = std::filesystem::file_size(path); +} - // Copy the data and fill the padding region with zeros. - std::memcpy(bufferPointer, bytes, dataSize); - std::memset(bufferPointer + dataSize, 0, allocatedSize - dataSize); - return true; +bool fg::GltfFileStream::isOpen() const { + return fileStream.is_open(); } -bool fg::GltfDataBuffer::loadFromFile(const fs::path& path, std::uint64_t byteOffset) noexcept { - using namespace simdjson; - std::error_code ec; - auto length = static_cast(fs::file_size(path, ec)); - if (ec) { - return false; - } +void fg::GltfFileStream::read(void *ptr, std::size_t count) { + fileStream.read( + reinterpret_cast(ptr), + static_cast(count)); +} - // Open the file and determine the size. - std::ifstream file(path, std::ios::binary); - if (!file.is_open() || file.bad()) - return false; +fg::span fg::GltfFileStream::read(std::size_t count, std::size_t padding) { + static_assert(sizeof(decltype(buf)::value_type) == sizeof(std::byte)); - filePath = path; + buf.resize(count + padding); + fileStream.read( + reinterpret_cast(buf.data()), + static_cast(count)); - file.seekg(static_cast(byteOffset), std::ifstream::beg); + return span(reinterpret_cast(buf.data()), buf.size()); +} - dataSize = static_cast(length) - byteOffset; - allocatedSize = dataSize + getGltfBufferPadding(); - buffer = decltype(buffer)(new std::byte[allocatedSize]); // To mimic std::make_unique_for_overwrite (C++20) - if (!buffer) - return false; - bufferPointer = buffer.get(); +void fg::GltfFileStream::reset() { + fileStream.seekg(0, std::ifstream::beg); +} - // Copy the data and fill the padding region with zeros. - file.read(reinterpret_cast(bufferPointer), static_cast(dataSize)); - std::memset(bufferPointer + dataSize, 0, allocatedSize - dataSize); - return true; +std::size_t fg::GltfFileStream::bytesRead() { + return fileStream.tellg(); +} + +std::size_t fg::GltfFileStream::totalSize() { + return fileSize; } #pragma endregion @@ -3868,27 +3888,24 @@ bool fg::AndroidGltfDataBuffer::loadFromAndroidAsset(const fs::path& path, std:: #pragma endregion #pragma region Parser -fastgltf::GltfType fg::determineGltfFileType(GltfDataBuffer* buffer) { - if (buffer->bufferPointer == nullptr) - return GltfType::Invalid; - - if (buffer->dataSize > sizeof(BinaryGltfHeader)) { - // We'll try and read a BinaryGltfHeader from the buffer to see if the magic is correct. - BinaryGltfHeader header = {}; - std::memcpy(&header, buffer->bufferPointer, sizeof header); - if (header.magic == binaryGltfHeaderMagic) { - return GltfType::GLB; - } +fastgltf::GltfType fg::determineGltfFileType(GltfDataGetter& data) { + // We'll try and read a BinaryGltfHeader from the buffer to see if the magic is correct. + BinaryGltfHeader header = {}; + data.read(&header, sizeof header); + data.reset(); + if (header.magic == binaryGltfHeaderMagic) { + return GltfType::GLB; } - if (buffer->dataSize > sizeof(std::uint8_t) * 4) { - // First, check if any of the first four characters is a '{'. - std::array begin = {}; - std::memcpy(begin.data(), buffer->bufferPointer, sizeof begin); - for (const auto& i : begin) { - if ((char)i == '{') - return GltfType::glTF; - } + // First, check if any of the first four characters is a '{'. + std::array begin = {}; + data.read(begin.data(), begin.size()); + data.reset(); + for (const auto& i : begin) { + if ((char)i == ' ') + continue; + if ((char)i == '{') + return GltfType::glTF; } return GltfType::Invalid; @@ -3910,21 +3927,21 @@ fg::Parser& fg::Parser::operator=(Parser&& other) noexcept { fg::Parser::~Parser() = default; -fg::Expected fg::Parser::loadGltf(GltfDataBuffer* buffer, fs::path _directory, Options _options, Category categories) { - auto type = fastgltf::determineGltfFileType(buffer); +fg::Expected fg::Parser::loadGltf(GltfDataGetter& data, fs::path _directory, Options _options, Category categories) { + auto type = fastgltf::determineGltfFileType(data); if (type == fastgltf::GltfType::glTF) { - return loadGltfJson(buffer, std::move(_directory), _options, categories); + return loadGltfJson(data, std::move(_directory), _options, categories); } if (type == fastgltf::GltfType::GLB) { - return loadGltfBinary(buffer, std::move(_directory), _options, categories); + return loadGltfBinary(data, std::move(_directory), _options, categories); } return Error::InvalidFileData; } -fg::Expected fg::Parser::loadGltfJson(GltfDataBuffer* buffer, fs::path _directory, Options _options, Category categories) { +fg::Expected fg::Parser::loadGltfJson(GltfDataGetter& data, fs::path _directory, Options _options, Category categories) { using namespace simdjson; options = _options; @@ -3937,20 +3954,10 @@ fg::Expected fg::Parser::loadGltfJson(GltfDataBuffer* buffer, fs::pat } #endif - // If we own the allocation of the JSON data, we'll try to minify the JSON, which, in most cases, - // will speed up the parsing by a small amount. - std::size_t jsonLength = buffer->getBufferSize(); - if (buffer->buffer != nullptr && hasBit(options, Options::MinimiseJsonBeforeParsing)) { - std::size_t newLength = 0; - auto result = simdjson::minify(reinterpret_cast(buffer->bufferPointer), buffer->getBufferSize(), - reinterpret_cast(buffer->bufferPointer), newLength); - if (result != SUCCESS || newLength == 0) { - return Error::InvalidJson; - } - buffer->dataSize = jsonLength = newLength; - } - - auto view = padded_string_view(reinterpret_cast(buffer->bufferPointer), jsonLength, buffer->allocatedSize); + auto jsonSpan = data.read(data.totalSize(), SIMDJSON_PADDING); + simdjson::padded_string_view view(reinterpret_cast(jsonSpan.data()), + data.totalSize(), + data.totalSize() + SIMDJSON_PADDING); simdjson::dom::object root; if (auto error = jsonParser->parse(view).get(root); error != SUCCESS) FASTGLTF_UNLIKELY { return Error::InvalidJson; @@ -3959,7 +3966,7 @@ fg::Expected fg::Parser::loadGltfJson(GltfDataBuffer* buffer, fs::pat return parse(root, categories); } -fg::Expected fg::Parser::loadGltfBinary(GltfDataBuffer* buffer, fs::path _directory, Options _options, Category categories) { +fg::Expected fg::Parser::loadGltfBinary(GltfDataGetter& data, fs::path _directory, Options _options, Category categories) { using namespace simdjson; options = _options; @@ -3970,21 +3977,15 @@ fg::Expected fg::Parser::loadGltfBinary(GltfDataBuffer* buffer, fs::p return Error::InvalidPath; } - std::size_t offset = 0UL; - auto read = [&buffer, &offset](void* dst, std::size_t size) mutable { - std::memcpy(dst, buffer->bufferPointer + offset, size); - offset += size; - }; - BinaryGltfHeader header = {}; - read(&header, sizeof header); + data.read(&header, sizeof header); if (header.magic != binaryGltfHeaderMagic) { return Error::InvalidGLB; } if (header.version != 2) { return Error::UnsupportedVersion; } - if (header.length >= buffer->allocatedSize) { + if (header.length > data.totalSize()) { return Error::InvalidGLB; } @@ -3992,17 +3993,17 @@ fg::Expected fg::Parser::loadGltfBinary(GltfDataBuffer* buffer, fs::p // 1. JSON chunk // 2. BIN chunk (optional) BinaryGltfChunk jsonChunk = {}; - read(&jsonChunk, sizeof jsonChunk); + data.read(&jsonChunk, sizeof jsonChunk); if (jsonChunk.chunkType != binaryGltfJsonChunkMagic) { return Error::InvalidGLB; } // Create a string view of the JSON chunk in the GLB data buffer. The documentation of parse() // says the padding can be initialised to anything, apparently. Therefore, this should work. - simdjson::padded_string_view jsonChunkView(reinterpret_cast(buffer->bufferPointer) + offset, + auto jsonSpan = data.read(jsonChunk.chunkLength, SIMDJSON_PADDING); + simdjson::padded_string_view jsonChunkView(reinterpret_cast(jsonSpan.data()), jsonChunk.chunkLength, jsonChunk.chunkLength + SIMDJSON_PADDING); - offset += jsonChunk.chunkLength; simdjson::dom::object root; if (jsonParser->parse(jsonChunkView).get(root) != SUCCESS) FASTGLTF_UNLIKELY { @@ -4010,31 +4011,28 @@ fg::Expected fg::Parser::loadGltfBinary(GltfDataBuffer* buffer, fs::p } // Is there enough room for another chunk header? - if (header.length > (offset + sizeof(BinaryGltfChunk))) { + if (header.length > (data.bytesRead() + sizeof(BinaryGltfChunk))) { BinaryGltfChunk binaryChunk = {}; - read(&binaryChunk, sizeof binaryChunk); + data.read(&binaryChunk, sizeof binaryChunk); if (binaryChunk.chunkType != binaryGltfDataChunkMagic) { return Error::InvalidGLB; } - // The binary chunk is allowed to be empty: - // When the binary buffer is empty or when it is stored by other means, - // this chunk SHOULD be omitted. if (binaryChunk.chunkLength != 0) { if (hasBit(options, Options::LoadGLBBuffers)) { if (config.mapCallback != nullptr) { auto info = config.mapCallback(binaryChunk.chunkLength, config.userPointer); if (info.mappedMemory != nullptr) { - read(info.mappedMemory, binaryChunk.chunkLength); + data.read(info.mappedMemory, binaryChunk.chunkLength); if (config.unmapCallback != nullptr) { config.unmapCallback(&info, config.userPointer); } - glbBuffer = sources::CustomBuffer{info.customId}; + glbBuffer = sources::CustomBuffer{info.customId, MimeType::None}; } } else { StaticVector binaryData(binaryChunk.chunkLength); - read(binaryData.data(), binaryChunk.chunkLength); + data.read(binaryData.data(), binaryChunk.chunkLength); sources::Array vectorData = { std::move(binaryData), @@ -4043,10 +4041,10 @@ fg::Expected fg::Parser::loadGltfBinary(GltfDataBuffer* buffer, fs::p glbBuffer = std::move(vectorData); } } else { - const span glbBytes(reinterpret_cast(buffer->bufferPointer + offset), - binaryChunk.chunkLength); + // TODO: What to do with this? + //const span glbBytes(reinterpret_cast(buffer->bufferPointer + offset), binaryChunk.chunkLength); sources::ByteView glbByteView = {}; - glbByteView.bytes = glbBytes; + //glbByteView.bytes = glbBytes; glbByteView.mimeType = MimeType::GltfBuffer; glbBuffer = glbByteView; } diff --git a/tests/accessor_tests.cpp b/tests/accessor_tests.cpp index b631c9112..49e5d5bb4 100644 --- a/tests/accessor_tests.cpp +++ b/tests/accessor_tests.cpp @@ -113,11 +113,12 @@ TEST_CASE("Test matrix data padding", "[gltf-tools]") { TEST_CASE("Test accessor", "[gltf-tools]") { auto lightsLamp = sampleModels / "2.0" / "LightsPunctualLamp" / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(lightsLamp / "LightsPunctualLamp.gltf")); + + fastgltf::GltfFileStream jsonData(lightsLamp / "LightsPunctualLamp.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser(fastgltf::Extensions::KHR_lights_punctual); - auto asset = parser.loadGltfJson(&jsonData, lightsLamp, fastgltf::Options::LoadExternalBuffers, + auto asset = parser.loadGltfJson(jsonData, lightsLamp, fastgltf::Options::LoadExternalBuffers, fastgltf::Category::Buffers | fastgltf::Category::BufferViews | fastgltf::Category::Accessors); REQUIRE(asset.error() == fastgltf::Error::None); @@ -189,11 +190,12 @@ TEST_CASE("Test accessor", "[gltf-tools]") { TEST_CASE("Test sparse accessor", "[gltf-tools]") { auto simpleSparseAccessor = sampleModels / "2.0" / "SimpleSparseAccessor" / "glTF"; - auto jsonData = std::make_unique(); - REQUIRE(jsonData->loadFromFile(simpleSparseAccessor / "SimpleSparseAccessor.gltf")); + + fastgltf::GltfFileStream jsonData(simpleSparseAccessor / "SimpleSparseAccessor.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser; - auto asset = parser.loadGltfJson(jsonData.get(), simpleSparseAccessor, fastgltf::Options::LoadExternalBuffers, + auto asset = parser.loadGltfJson(jsonData, simpleSparseAccessor, fastgltf::Options::LoadExternalBuffers, fastgltf::Category::Buffers | fastgltf::Category::BufferViews | fastgltf::Category::Accessors); REQUIRE(asset.error() == fastgltf::Error::None); diff --git a/tests/base64_tests.cpp b/tests/base64_tests.cpp index e6e4e1998..aa54ba5a3 100644 --- a/tests/base64_tests.cpp +++ b/tests/base64_tests.cpp @@ -72,13 +72,14 @@ TEST_CASE("Test base64 buffer decoding", "[base64]") { auto cylinderEngine = sampleModels / "2.0" / "2CylinderEngine" / "glTF-Embedded"; auto boxTextured = sampleModels / "2.0" / "BoxTextured" / "glTF-Embedded"; - auto tceJsonData = std::make_unique(); - REQUIRE(tceJsonData->loadFromFile(cylinderEngine / "2CylinderEngine.gltf")); - auto btJsonData = std::make_unique(); - REQUIRE(btJsonData->loadFromFile(boxTextured / "BoxTextured.gltf")); + + fastgltf::GltfFileStream tceJsonData(cylinderEngine / "2CylinderEngine.gltf"); + REQUIRE(tceJsonData.isOpen()); + fastgltf::GltfFileStream btJsonData(boxTextured / "BoxTextured.gltf"); + REQUIRE(btJsonData.isOpen()); SECTION("Validate large buffer load from glTF") { - auto asset = parser.loadGltfJson(tceJsonData.get(), cylinderEngine, fastgltf::Options::None, fastgltf::Category::Buffers); + auto asset = parser.loadGltfJson(tceJsonData, cylinderEngine, fastgltf::Options::None, fastgltf::Category::Buffers); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(asset->buffers.size() == 1); @@ -93,7 +94,7 @@ TEST_CASE("Test base64 buffer decoding", "[base64]") { } SECTION("Validate base64 buffer and image load from glTF") { - auto asset = parser.loadGltfJson(btJsonData.get(), boxTextured, fastgltf::Options::None, fastgltf::Category::Images | fastgltf::Category::Buffers); + auto asset = parser.loadGltfJson(btJsonData, boxTextured, fastgltf::Options::None, fastgltf::Category::Images | fastgltf::Category::Buffers); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(asset->buffers.size() == 1); diff --git a/tests/basic_test.cpp b/tests/basic_test.cpp index 4d05fbbda..38c1cebfd 100644 --- a/tests/basic_test.cpp +++ b/tests/basic_test.cpp @@ -85,11 +85,12 @@ TEST_CASE("Test if glTF type detection works", "[gltf-loader]") { SECTION("glTF") { auto gltfPath = sampleModels / "2.0" / "ABeautifulGame" / "glTF"; REQUIRE(std::filesystem::exists(gltfPath)); - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(gltfPath / "ABeautifulGame.gltf")); - REQUIRE(fastgltf::determineGltfFileType(&jsonData) == fastgltf::GltfType::glTF); + fastgltf::GltfFileStream jsonData(gltfPath / "ABeautifulGame.gltf"); + REQUIRE(jsonData.isOpen()); - auto model = parser.loadGltfJson(&jsonData, gltfPath); + REQUIRE(fastgltf::determineGltfFileType(jsonData) == fastgltf::GltfType::glTF); + + auto model = parser.loadGltfJson(jsonData, gltfPath); REQUIRE(model.error() == fastgltf::Error::None); REQUIRE(model.get_if() != nullptr); REQUIRE(fastgltf::validate(model.get()) == fastgltf::Error::None); @@ -98,48 +99,51 @@ TEST_CASE("Test if glTF type detection works", "[gltf-loader]") { SECTION("GLB") { auto glbPath = sampleModels / "2.0" / "BoomBox" / "glTF-Binary"; REQUIRE(std::filesystem::exists(glbPath)); - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(glbPath / "BoomBox.glb")); - REQUIRE(fastgltf::determineGltfFileType(&jsonData) == fastgltf::GltfType::GLB); + fastgltf::GltfFileStream jsonData(glbPath / "BoomBox.glb"); + REQUIRE(jsonData.isOpen()); + + REQUIRE(fastgltf::determineGltfFileType(jsonData) == fastgltf::GltfType::GLB); - auto model = parser.loadGltfBinary(&jsonData, glbPath); + auto model = parser.loadGltfBinary(jsonData, glbPath); REQUIRE(model.error() == fastgltf::Error::None); REQUIRE(model.get_if() != nullptr); } SECTION("Invalid") { - auto gltfPath = path / "base64.txt"; // Random file in the test directory that's not a glTF file. - REQUIRE(std::filesystem::exists(gltfPath)); - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(gltfPath)); - REQUIRE(fastgltf::determineGltfFileType(&jsonData) == fastgltf::GltfType::Invalid); + auto fakePath = path / "base64.txt"; // Random file in the test directory that's not a glTF file. + REQUIRE(std::filesystem::exists(fakePath)); + fastgltf::GltfFileStream jsonData(fakePath); + REQUIRE(jsonData.isOpen()); + + REQUIRE(fastgltf::determineGltfFileType(jsonData) == fastgltf::GltfType::Invalid); } } TEST_CASE("Loading some basic glTF", "[gltf-loader]") { fastgltf::Parser parser; SECTION("Loading basic invalid glTF files") { - auto jsonData = std::make_unique(); - REQUIRE(jsonData->loadFromFile(path / "empty_json.gltf")); - auto emptyGltf = parser.loadGltfJson(jsonData.get(), path); + fastgltf::GltfFileStream jsonData(path / "empty_json.gltf"); + REQUIRE(jsonData.isOpen()); + + auto emptyGltf = parser.loadGltfJson(jsonData, path); REQUIRE(emptyGltf.error() == fastgltf::Error::InvalidOrMissingAssetField); } SECTION("Load basic glTF file") { - auto basicJsonData = std::make_unique(); - REQUIRE(basicJsonData->loadFromFile(path / "basic_gltf.gltf")); + fastgltf::GltfFileStream jsonData(path / "basic_gltf.gltf"); + REQUIRE(jsonData.isOpen()); - auto basicGltf = parser.loadGltfJson(basicJsonData.get(), path); + auto basicGltf = parser.loadGltfJson(jsonData, path); REQUIRE(basicGltf.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(basicGltf.get()) == fastgltf::Error::None); } SECTION("Loading basic Cube.gltf") { auto cubePath = sampleModels / "2.0" / "Cube" / "glTF"; - auto cubeJsonData = std::make_unique(); - REQUIRE(cubeJsonData->loadFromFile(cubePath / "Cube.gltf")); + fastgltf::GltfFileStream jsonData(cubePath / "Cube.gltf"); + REQUIRE(jsonData.isOpen()); - auto cube = parser.loadGltfJson(cubeJsonData.get(), cubePath, noOptions, fastgltf::Category::OnlyRenderable); + auto cube = parser.loadGltfJson(jsonData, cubePath, noOptions, fastgltf::Category::OnlyRenderable); REQUIRE(cube.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(cube.get()) == fastgltf::Error::None); @@ -174,10 +178,10 @@ TEST_CASE("Loading some basic glTF", "[gltf-loader]") { SECTION("Loading basic Box.gltf") { auto boxPath = sampleModels / "2.0" / "Box" / "glTF"; - auto boxJsonData = std::make_unique(); - REQUIRE(boxJsonData->loadFromFile(boxPath / "Box.gltf")); + fastgltf::GltfFileStream jsonData(boxPath / "Box.gltf"); + REQUIRE(jsonData.isOpen()); - auto box = parser.loadGltfJson(boxJsonData.get(), boxPath, noOptions, fastgltf::Category::OnlyRenderable); + auto box = parser.loadGltfJson(jsonData, boxPath, noOptions, fastgltf::Category::OnlyRenderable); REQUIRE(box.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(box.get()) == fastgltf::Error::None); @@ -201,12 +205,11 @@ TEST_CASE("Loading some basic glTF", "[gltf-loader]") { TEST_CASE("Loading glTF animation", "[gltf-loader]") { auto animatedCube = sampleModels / "2.0" / "AnimatedCube" / "glTF"; - - auto jsonData = std::make_unique(); - REQUIRE(jsonData->loadFromFile(animatedCube / "AnimatedCube.gltf")); + fastgltf::GltfFileStream jsonData(animatedCube / "AnimatedCube.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser; - auto asset = parser.loadGltfJson(jsonData.get(), animatedCube, noOptions, fastgltf::Category::OnlyAnimations); + auto asset = parser.loadGltfJson(jsonData, animatedCube, noOptions, fastgltf::Category::OnlyAnimations); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -228,12 +231,11 @@ TEST_CASE("Loading glTF animation", "[gltf-loader]") { TEST_CASE("Loading glTF skins", "[gltf-loader]") { auto simpleSkin = sampleModels / "2.0" / "SimpleSkin" / "glTF"; - - auto jsonData = std::make_unique(); - REQUIRE(jsonData->loadFromFile(simpleSkin / "SimpleSkin.gltf")); + fastgltf::GltfFileStream jsonData(simpleSkin / "SimpleSkin.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser; - auto asset = parser.loadGltfJson(jsonData.get(), simpleSkin, noOptions, fastgltf::Category::Skins | fastgltf::Category::Nodes); + auto asset = parser.loadGltfJson(jsonData, simpleSkin, noOptions, fastgltf::Category::Skins | fastgltf::Category::Nodes); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -255,11 +257,11 @@ TEST_CASE("Loading glTF skins", "[gltf-loader]") { TEST_CASE("Loading glTF cameras", "[gltf-loader]") { auto cameras = sampleModels / "2.0" / "Cameras" / "glTF"; - auto jsonData = std::make_unique(); - REQUIRE(jsonData->loadFromFile(cameras / "Cameras.gltf")); + fastgltf::GltfFileStream jsonData(cameras / "Cameras.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser; - auto asset = parser.loadGltfJson(jsonData.get(), cameras, noOptions, fastgltf::Category::Cameras); + auto asset = parser.loadGltfJson(jsonData, cameras, noOptions, fastgltf::Category::Cameras); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -283,29 +285,34 @@ TEST_CASE("Loading glTF cameras", "[gltf-loader]") { REQUIRE(pOrthographic->znear == 0.01f); } -TEST_CASE("Validate whole glTF", "[gltf-loader]") { - auto sponza = sampleModels / "2.0" / "Sponza" / "glTF"; - auto jsonData = std::make_unique(); - REQUIRE(jsonData->loadFromFile(sponza / "Sponza.gltf")); +TEST_CASE("Validate models with re-used parser", "[gltf-loader]") { + auto sponza = sampleModels / "2.0" / "Sponza" / "glTF"; + fastgltf::Parser parser; - fastgltf::Parser parser; - auto model = parser.loadGltfJson(jsonData.get(), sponza); - REQUIRE(model.error() == fastgltf::Error::None); - REQUIRE(fastgltf::validate(model.get()) == fastgltf::Error::None); + SECTION("Validate Sponza.gltf") { + fastgltf::GltfFileStream jsonData(sponza / "Sponza.gltf"); + REQUIRE(jsonData.isOpen()); + + auto model = parser.loadGltfJson(jsonData, sponza); + REQUIRE(model.error() == fastgltf::Error::None); + REQUIRE(fastgltf::validate(model.get()) == fastgltf::Error::None); + } - auto brainStem = sampleModels / "2.0" / "BrainStem" / "glTF"; - jsonData = std::make_unique(); - REQUIRE(jsonData->loadFromFile(brainStem / "BrainStem.gltf")); + SECTION("Validate BrainStem.gltf") { + auto brainStem = sampleModels / "2.0" / "BrainStem" / "glTF"; + fastgltf::GltfFileStream jsonData(brainStem / "BrainStem.gltf"); + REQUIRE(jsonData.isOpen()); - auto model2 = parser.loadGltfJson(jsonData.get(), brainStem); - REQUIRE(model2.error() == fastgltf::Error::None); - REQUIRE(fastgltf::validate(model2.get()) == fastgltf::Error::None); + auto model = parser.loadGltfJson(jsonData, brainStem); + REQUIRE(model.error() == fastgltf::Error::None); + REQUIRE(fastgltf::validate(model.get()) == fastgltf::Error::None); + } } TEST_CASE("Test allocation callbacks for embedded buffers", "[gltf-loader]") { auto boxPath = sampleModels / "2.0" / "Box" / "glTF-Embedded"; - auto jsonData = std::make_unique(); - REQUIRE(jsonData->loadFromFile(boxPath / "Box.gltf")); + fastgltf::GltfFileStream jsonData(boxPath / "Box.gltf"); + REQUIRE(jsonData.isOpen()); std::vector allocations; @@ -322,7 +329,7 @@ TEST_CASE("Test allocation callbacks for embedded buffers", "[gltf-loader]") { fastgltf::Parser parser; parser.setUserPointer(&allocations); parser.setBufferAllocationCallback(mapCallback, nullptr); - auto asset = parser.loadGltfJson(jsonData.get(), boxPath, noOptions, fastgltf::Category::Buffers); + auto asset = parser.loadGltfJson(jsonData, boxPath, noOptions, fastgltf::Category::Buffers); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -342,8 +349,8 @@ TEST_CASE("Test allocation callbacks for embedded buffers", "[gltf-loader]") { TEST_CASE("Test base64 decoding callbacks", "[gltf-loader]") { auto boxPath = sampleModels / "2.0" / "Box" / "glTF-Embedded"; - auto jsonData = std::make_unique(); - REQUIRE(jsonData->loadFromFile(boxPath / "Box.gltf")); + fastgltf::GltfFileStream jsonData(boxPath / "Box.gltf"); + REQUIRE(jsonData.isOpen()); size_t decodeCounter = 0; auto decodeCallback = [](std::string_view encodedData, uint8_t* outputData, size_t padding, size_t outputSize, void* userPointer) { @@ -354,7 +361,7 @@ TEST_CASE("Test base64 decoding callbacks", "[gltf-loader]") { fastgltf::Parser parser; parser.setUserPointer(&decodeCounter); parser.setBase64DecodeCallback(decodeCallback); - auto model = parser.loadGltfJson(jsonData.get(), boxPath, noOptions, fastgltf::Category::Buffers); + auto model = parser.loadGltfJson(jsonData, boxPath, noOptions, fastgltf::Category::Buffers); REQUIRE(model.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(model.get()) == fastgltf::Error::None); REQUIRE(decodeCounter != 0); @@ -362,16 +369,16 @@ TEST_CASE("Test base64 decoding callbacks", "[gltf-loader]") { TEST_CASE("Test TRS parsing and optional decomposition", "[gltf-loader]") { SECTION("Test decomposition on glTF asset") { - auto jsonData = std::make_unique(); - REQUIRE(jsonData->loadFromFile(path / "transform_matrices.gltf")); + fastgltf::GltfFileStream jsonData(path / "transform_matrices.gltf"); + REQUIRE(jsonData.isOpen()); // Parse once without decomposing, once with decomposing the matrix. fastgltf::Parser parser; - auto assetWithMatrix = parser.loadGltfJson(jsonData.get(), path, noOptions, fastgltf::Category::Nodes | fastgltf::Category::Cameras); + auto assetWithMatrix = parser.loadGltfJson(jsonData, path, noOptions, fastgltf::Category::Nodes | fastgltf::Category::Cameras); REQUIRE(assetWithMatrix.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(assetWithMatrix.get()) == fastgltf::Error::None); - auto assetDecomposed = parser.loadGltfJson(jsonData.get(), path, fastgltf::Options::DecomposeNodeMatrices, fastgltf::Category::Nodes | fastgltf::Category::Cameras); + auto assetDecomposed = parser.loadGltfJson(jsonData, path, fastgltf::Options::DecomposeNodeMatrices, fastgltf::Category::Nodes | fastgltf::Category::Cameras); REQUIRE(assetDecomposed.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(assetDecomposed.get()) == fastgltf::Error::None); @@ -446,11 +453,11 @@ TEST_CASE("Test TRS parsing and optional decomposition", "[gltf-loader]") { TEST_CASE("Validate sparse accessor parsing", "[gltf-loader]") { auto simpleSparseAccessor = sampleModels / "2.0" / "SimpleSparseAccessor" / "glTF"; - auto jsonData = std::make_unique(); - REQUIRE(jsonData->loadFromFile(simpleSparseAccessor / "SimpleSparseAccessor.gltf")); + fastgltf::GltfFileStream jsonData(simpleSparseAccessor / "SimpleSparseAccessor.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser; - auto asset = parser.loadGltfJson(jsonData.get(), simpleSparseAccessor, noOptions, fastgltf::Category::Accessors); + auto asset = parser.loadGltfJson(jsonData, simpleSparseAccessor, noOptions, fastgltf::Category::Accessors); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -468,11 +475,11 @@ TEST_CASE("Validate sparse accessor parsing", "[gltf-loader]") { TEST_CASE("Validate morph target parsing", "[gltf-loader]") { auto simpleMorph = sampleModels / "2.0" / "SimpleMorph" / "glTF"; - auto jsonData = std::make_unique(); - REQUIRE(jsonData->loadFromFile(simpleMorph / "SimpleMorph.gltf")); + fastgltf::GltfFileStream jsonData(simpleMorph / "SimpleMorph.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser; - auto asset = parser.loadGltfJson(jsonData.get(), simpleMorph, noOptions, fastgltf::Category::Meshes); + auto asset = parser.loadGltfJson(jsonData, simpleMorph, noOptions, fastgltf::Category::Meshes); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -499,11 +506,11 @@ TEST_CASE("Validate morph target parsing", "[gltf-loader]") { TEST_CASE("Test accessors min/max", "[gltf-loader]") { auto lightsLamp = sampleModels / "2.0" / "LightsPunctualLamp" / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(lightsLamp / "LightsPunctualLamp.gltf")); + fastgltf::GltfFileStream jsonData(lightsLamp / "LightsPunctualLamp.gltf"); + REQUIRE(jsonData.isOpen()); - fastgltf::Parser parser(fastgltf::Extensions::KHR_lights_punctual); - auto asset = parser.loadGltfJson(&jsonData, lightsLamp, noOptions, fastgltf::Category::Accessors); + fastgltf::Parser parser(fastgltf::Extensions::KHR_lights_punctual); + auto asset = parser.loadGltfJson(jsonData, lightsLamp, noOptions, fastgltf::Category::Accessors); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -563,16 +570,16 @@ TEST_CASE("Test accessors min/max", "[gltf-loader]") { TEST_CASE("Test unicode characters", "[gltf-loader]") { #if FASTGLTF_CPP_20 auto unicodePath = sampleModels / "2.0" / std::filesystem::path(u8"Unicode❤♻Test") / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(unicodePath / std::filesystem::path(u8"Unicode❤♻Test.gltf"))); + fastgltf::GltfFileStream jsonData(unicodePath / std::filesystem::path(u8"Unicode❤♻Test.gltf")); + REQUIRE(jsonData.isOpen()); #else auto unicodePath = sampleModels / "2.0" / std::filesystem::u8path(u8"Unicode❤♻Test") / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(unicodePath / std::filesystem::u8path(u8"Unicode❤♻Test.gltf"))); + fastgltf::GltfFileStream jsonData(unicodePath / std::filesystem::u8path(u8"Unicode❤♻Test.gltf")); + REQUIRE(jsonData.isOpen()); #endif fastgltf::Parser parser; - auto asset = parser.loadGltfJson(&jsonData, unicodePath); + auto asset = parser.loadGltfJson(jsonData, unicodePath); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -586,13 +593,10 @@ TEST_CASE("Test unicode characters", "[gltf-loader]") { TEST_CASE("Test extras callback", "[gltf-loader]") { auto materialVariants = sampleModels / "2.0" / "MaterialsVariantsShoe" / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(materialVariants / "MaterialsVariantsShoe.gltf")); - std::vector nodeNames; // lambda callback to parse the glTF JSON from the jsonData buffer - auto parseJson = [&]() { + auto parseJson = [&](fastgltf::GltfDataGetter& data) { auto extrasCallback = [](simdjson::dom::object *extras, std::size_t objectIndex, fastgltf::Category category, void *userPointer) { if (category != fastgltf::Category::Nodes) @@ -609,11 +613,13 @@ TEST_CASE("Test extras callback", "[gltf-loader]") { fastgltf::Parser parser; parser.setExtrasParseCallback(extrasCallback); parser.setUserPointer(&nodeNames); - return parser.loadGltfJson(&jsonData, materialVariants); + return parser.loadGltfJson(data, materialVariants); }; // The asset has to be reused for exporting in the next section. - auto asset = parseJson(); + fastgltf::GltfFileStream jsonData(materialVariants / "MaterialsVariantsShoe.gltf"); + REQUIRE(jsonData.isOpen()); + auto asset = parseJson(jsonData); SECTION("Validate node names from extras") { REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -645,10 +651,11 @@ TEST_CASE("Test extras callback", "[gltf-loader]") { // Update the data buffer auto& string = json.get().output; - jsonData.fromByteView(reinterpret_cast(string.data()), string.size(), string.capacity()); + fastgltf::GltfDataBuffer reexportedJson(reinterpret_cast(string.data()), + string.size()); nodeNames.clear(); - auto reparsed = parseJson(); + auto reparsed = parseJson(reexportedJson); REQUIRE(reparsed.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(reparsed.get()) == fastgltf::Error::None); diff --git a/tests/benchmarks.cpp b/tests/benchmarks.cpp index 0145e41f8..16d95afbe 100644 --- a/tests/benchmarks.cpp +++ b/tests/benchmarks.cpp @@ -82,7 +82,7 @@ fastgltf::StaticVector readFileAsBytes(const std::filesystem::path throw std::runtime_error(std::string { "Failed to open file: " } + filePath.string()); auto fileSize = file.tellg(); - fastgltf::StaticVector bytes(static_cast(fileSize) + fastgltf::getGltfBufferPadding()); + fastgltf::StaticVector bytes(static_cast(fileSize)); file.seekg(0, std::ifstream::beg); file.read(reinterpret_cast(bytes.data()), fileSize); file.close(); @@ -103,11 +103,10 @@ TEST_CASE("Benchmark loading of NewSponza", "[gltf-benchmark]") { #endif auto bytes = readFileAsBytes(intelSponza / "NewSponza_Main_glTF_002.gltf"); - auto jsonData = std::make_unique(); - REQUIRE(jsonData->fromByteView(bytes.data(), bytes.size() - fastgltf::getGltfBufferPadding(), bytes.size())); + fastgltf::GltfDataBuffer jsonData(reinterpret_cast(bytes.data()), bytes.size()); BENCHMARK("Parse NewSponza") { - return parser.loadGltfJson(jsonData.get(), intelSponza, benchmarkOptions); + return parser.loadGltfJson(jsonData, intelSponza, benchmarkOptions); }; #ifdef HAS_TINYGLTF @@ -131,14 +130,14 @@ TEST_CASE("Benchmark loading of NewSponza", "[gltf-benchmark]") { #ifdef HAS_GLTFRS auto padding = fastgltf::getGltfBufferPadding(); BENCHMARK("Parse NewSponza with gltf-rs") { - auto slice = rust::Slice(reinterpret_cast(bytes.data()), bytes.size() - padding); + auto slice = rust::Slice(reinterpret_cast(bytes.data()), bytes.size()); return rust::gltf::run(slice); }; #endif #ifdef HAS_ASSIMP BENCHMARK("Parse NewSponza with assimp") { - return aiImportFileFromMemory(reinterpret_cast(bytes.data()), jsonData->getBufferSize(), 0, nullptr); + return aiImportFileFromMemory(reinterpret_cast(bytes.data()), bytes.size(), 0, nullptr); }; #endif } @@ -153,11 +152,10 @@ TEST_CASE("Benchmark base64 decoding from glTF file", "[gltf-benchmark]") { auto cylinderEngine = sampleModels / "2.0" / "2CylinderEngine" / "glTF-Embedded"; auto bytes = readFileAsBytes(cylinderEngine / "2CylinderEngine.gltf"); - auto jsonData = std::make_unique(); - REQUIRE(jsonData->fromByteView(bytes.data(), bytes.size() - fastgltf::getGltfBufferPadding(), bytes.size())); + fastgltf::GltfDataBuffer jsonData(reinterpret_cast(bytes.data()), bytes.size()); BENCHMARK("Parse 2CylinderEngine and decode base64") { - return parser.loadGltfJson(jsonData.get(), cylinderEngine, benchmarkOptions); + return parser.loadGltfJson(jsonData, cylinderEngine, benchmarkOptions); }; #ifdef HAS_TINYGLTF @@ -183,14 +181,14 @@ TEST_CASE("Benchmark base64 decoding from glTF file", "[gltf-benchmark]") { #ifdef HAS_GLTFRS auto padding = fastgltf::getGltfBufferPadding(); BENCHMARK("2CylinderEngine with gltf-rs") { - auto slice = rust::Slice(reinterpret_cast(bytes.data()), bytes.size() - padding); + auto slice = rust::Slice(reinterpret_cast(bytes.data()), bytes.size()); return rust::gltf::run(slice); }; #endif #ifdef HAS_ASSIMP BENCHMARK("2CylinderEngine with assimp") { - const auto* scene = aiImportFileFromMemory(reinterpret_cast(bytes.data()), jsonData->getBufferSize(), 0, nullptr); + const auto* scene = aiImportFileFromMemory(reinterpret_cast(bytes.data()), bytes.size(), 0, nullptr); REQUIRE(scene != nullptr); return scene; }; @@ -207,11 +205,10 @@ TEST_CASE("Benchmark raw JSON parsing", "[gltf-benchmark]") { auto buggyPath = sampleModels / "2.0" / "Buggy" / "glTF"; auto bytes = readFileAsBytes(buggyPath / "Buggy.gltf"); - auto jsonData = std::make_unique(); - REQUIRE(jsonData->fromByteView(bytes.data(), bytes.size() - fastgltf::getGltfBufferPadding(), bytes.size())); + fastgltf::GltfDataBuffer jsonData(reinterpret_cast(bytes.data()), bytes.size()); BENCHMARK("Parse Buggy.gltf") { - return parser.loadGltfJson(jsonData.get(), buggyPath, benchmarkOptions); + return parser.loadGltfJson(jsonData, buggyPath, benchmarkOptions); }; #ifdef HAS_TINYGLTF @@ -236,14 +233,14 @@ TEST_CASE("Benchmark raw JSON parsing", "[gltf-benchmark]") { #ifdef HAS_GLTFRS auto padding = fastgltf::getGltfBufferPadding(); BENCHMARK("Parse Buggy.gltf with gltf-rs") { - auto slice = rust::Slice(reinterpret_cast(bytes.data()), bytes.size() - padding); + auto slice = rust::Slice(reinterpret_cast(bytes.data()), bytes.size()); return rust::gltf::run(slice); }; #endif #ifdef HAS_ASSIMP BENCHMARK("Parse Buggy.gltf with assimp") { - return aiImportFileFromMemory(reinterpret_cast(bytes.data()), jsonData->getBufferSize(), 0, nullptr); + return aiImportFileFromMemory(reinterpret_cast(bytes.data()), bytes.size(), 0, nullptr); }; #endif } @@ -262,11 +259,10 @@ TEST_CASE("Benchmark massive gltf file", "[gltf-benchmark]") { #endif auto bytes = readFileAsBytes(bistroPath / "bistro.gltf"); - auto jsonData = std::make_unique(); - REQUIRE(jsonData->fromByteView(bytes.data(), bytes.size() - fastgltf::getGltfBufferPadding(), bytes.size())); + fastgltf::GltfDataBuffer jsonData(reinterpret_cast(bytes.data()), bytes.size()); BENCHMARK("Parse Bistro") { - return parser.loadGltfJson(jsonData.get(), bistroPath, benchmarkOptions); + return parser.loadGltfJson(jsonData, bistroPath, benchmarkOptions); }; #ifdef HAS_TINYGLTF @@ -291,14 +287,14 @@ TEST_CASE("Benchmark massive gltf file", "[gltf-benchmark]") { #ifdef HAS_GLTFRS auto padding = fastgltf::getGltfBufferPadding(); BENCHMARK("Parse Bistro with gltf-rs") { - auto slice = rust::Slice(reinterpret_cast(bytes.data()), bytes.size() - padding); + auto slice = rust::Slice(reinterpret_cast(bytes.data()), bytes.size()); return rust::gltf::run(slice); }; #endif #ifdef HAS_ASSIMP BENCHMARK("Parse Bistro with assimp") { - return aiImportFileFromMemory(reinterpret_cast(bytes.data()), jsonData->getBufferSize(), 0, nullptr); + return aiImportFileFromMemory(reinterpret_cast(bytes.data()), bytes.size(), 0, nullptr); }; #endif } @@ -306,8 +302,7 @@ TEST_CASE("Benchmark massive gltf file", "[gltf-benchmark]") { TEST_CASE("Compare parsing performance with minified documents", "[gltf-benchmark]") { auto buggyPath = sampleModels / "2.0" / "Buggy" / "glTF"; auto bytes = readFileAsBytes(buggyPath / "Buggy.gltf"); - auto jsonData = std::make_unique(); - REQUIRE(jsonData->fromByteView(bytes.data(), bytes.size() - fastgltf::getGltfBufferPadding(), bytes.size())); + fastgltf::GltfDataBuffer jsonData(reinterpret_cast(bytes.data()), bytes.size()); // Create a minified JSON string std::vector minified(bytes.size()); @@ -325,16 +320,15 @@ TEST_CASE("Compare parsing performance with minified documents", "[gltf-benchmar return result; }; - auto minifiedJsonData = std::make_unique(); - REQUIRE(minifiedJsonData->fromByteView(minified.data(), minified.size() - fastgltf::getGltfBufferPadding(), minified.size())); + fastgltf::GltfDataBuffer minifiedJsonData(reinterpret_cast(minified.data()), minified.size()); fastgltf::Parser parser; BENCHMARK("Parse Buggy.gltf with normal JSON") { - return parser.loadGltfJson(jsonData.get(), buggyPath, benchmarkOptions); + return parser.loadGltfJson(jsonData, buggyPath, benchmarkOptions); }; BENCHMARK("Parse Buggy.gltf with minified JSON") { - return parser.loadGltfJson(minifiedJsonData.get(), buggyPath, benchmarkOptions); + return parser.loadGltfJson(minifiedJsonData, buggyPath, benchmarkOptions); }; } diff --git a/tests/extension_tests.cpp b/tests/extension_tests.cpp index b8fd622ac..e417c7c9b 100644 --- a/tests/extension_tests.cpp +++ b/tests/extension_tests.cpp @@ -13,12 +13,11 @@ TEST_CASE("Extension KHR_texture_transform", "[gltf-loader]") { auto transformTest = sampleModels / "2.0" / "TextureTransformMultiTest" / "glTF"; - - auto jsonData = std::make_unique(); - REQUIRE(jsonData->loadFromFile(transformTest / "TextureTransformMultiTest.gltf")); + fastgltf::GltfFileStream jsonData(transformTest / "TextureTransformMultiTest.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser(fastgltf::Extensions::KHR_texture_transform); - auto asset = parser.loadGltfJson(jsonData.get(), transformTest, fastgltf::Options::DontRequireValidAssetMember, fastgltf::Category::Materials); + auto asset = parser.loadGltfJson(jsonData, transformTest, fastgltf::Options::DontRequireValidAssetMember, fastgltf::Category::Materials); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -33,13 +32,12 @@ TEST_CASE("Extension KHR_texture_transform", "[gltf-loader]") { TEST_CASE("Extension KHR_texture_basisu", "[gltf-loader]") { auto stainedLamp = sampleModels / "2.0" / "StainedGlassLamp" / "glTF-KTX-BasisU"; - - auto jsonData = std::make_unique(); - REQUIRE(jsonData->loadFromFile(stainedLamp / "StainedGlassLamp.gltf")); + fastgltf::GltfFileStream jsonData(stainedLamp / "StainedGlassLamp.gltf"); + REQUIRE(jsonData.isOpen()); SECTION("Loading KHR_texture_basisu") { fastgltf::Parser parser(fastgltf::Extensions::KHR_texture_basisu); - auto asset = parser.loadGltfJson(jsonData.get(), path, fastgltf::Options::DontRequireValidAssetMember, + auto asset = parser.loadGltfJson(jsonData, path, fastgltf::Options::DontRequireValidAssetMember, fastgltf::Category::Textures | fastgltf::Category::Images); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -64,7 +62,7 @@ TEST_CASE("Extension KHR_texture_basisu", "[gltf-loader]") { SECTION("Testing requiredExtensions") { // We specify no extensions, yet the StainedGlassLamp requires KHR_texture_basisu. fastgltf::Parser parser(fastgltf::Extensions::None); - auto stainedGlassLamp = parser.loadGltfJson(jsonData.get(), path, fastgltf::Options::DontRequireValidAssetMember); + auto stainedGlassLamp = parser.loadGltfJson(jsonData, path, fastgltf::Options::DontRequireValidAssetMember); REQUIRE(stainedGlassLamp.error() == fastgltf::Error::MissingExtensions); } } @@ -73,11 +71,11 @@ TEST_CASE("Extension KHR_texture_basisu", "[gltf-loader]") { TEST_CASE("Extension EXT_meshopt_compression", "[gltf-loader]") { auto brainStem = sampleModels / "2.0" / "BrainStem" / "glTF-Meshopt"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(brainStem / "BrainStem.gltf")); + fastgltf::GltfFileStream jsonData(brainStem / "BrainStem.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser(fastgltf::Extensions::EXT_meshopt_compression | fastgltf::Extensions::KHR_mesh_quantization); - auto asset = parser.loadGltfJson(&jsonData, brainStem, fastgltf::Options::None); + auto asset = parser.loadGltfJson(jsonData, brainStem, fastgltf::Options::None); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -109,11 +107,11 @@ TEST_CASE("Extension EXT_meshopt_compression", "[gltf-loader]") { TEST_CASE("Extension KHR_lights_punctual", "[gltf-loader]") { SECTION("Point light") { auto lightsLamp = sampleModels / "2.0" / "LightsPunctualLamp" / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(lightsLamp / "LightsPunctualLamp.gltf")); + fastgltf::GltfFileStream jsonData(lightsLamp / "LightsPunctualLamp.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser(fastgltf::Extensions::KHR_lights_punctual); - auto asset = parser.loadGltfJson(&jsonData, lightsLamp, fastgltf::Options::None, fastgltf::Category::Nodes); + auto asset = parser.loadGltfJson(jsonData, lightsLamp, fastgltf::Options::None, fastgltf::Category::Nodes); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -135,11 +133,11 @@ TEST_CASE("Extension KHR_lights_punctual", "[gltf-loader]") { SECTION("Directional light") { auto directionalLight = sampleModels / "2.0" / "DirectionalLight" / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(directionalLight / "DirectionalLight.gltf")); + fastgltf::GltfFileStream jsonData(directionalLight / "DirectionalLight.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser(fastgltf::Extensions::KHR_lights_punctual); - auto asset = parser.loadGltfJson(&jsonData, directionalLight, fastgltf::Options::None, fastgltf::Category::Nodes); + auto asset = parser.loadGltfJson(jsonData, directionalLight, fastgltf::Options::None, fastgltf::Category::Nodes); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -157,11 +155,11 @@ TEST_CASE("Extension KHR_lights_punctual", "[gltf-loader]") { TEST_CASE("Extension KHR_materials_specular", "[gltf-loader]") { auto specularTest = sampleModels / "2.0" / "SpecularTest" / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(specularTest / "SpecularTest.gltf")); + fastgltf::GltfFileStream jsonData(specularTest / "SpecularTest.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser(fastgltf::Extensions::KHR_materials_specular); - auto asset = parser.loadGltfJson(&jsonData, specularTest, fastgltf::Options::None, fastgltf::Category::Materials); + auto asset = parser.loadGltfJson(jsonData, specularTest, fastgltf::Options::None, fastgltf::Category::Materials); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -186,11 +184,11 @@ TEST_CASE("Extension KHR_materials_specular", "[gltf-loader]") { TEST_CASE("Extension KHR_materials_ior and KHR_materials_iridescence", "[gltf-loader]") { auto specularTest = sampleModels / "2.0" / "IridescenceDielectricSpheres" / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(specularTest / "IridescenceDielectricSpheres.gltf")); + fastgltf::GltfFileStream jsonData(specularTest / "IridescenceDielectricSpheres.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser(fastgltf::Extensions::KHR_materials_iridescence | fastgltf::Extensions::KHR_materials_ior); - auto asset = parser.loadGltfJson(&jsonData, specularTest, fastgltf::Options::None, fastgltf::Category::Materials); + auto asset = parser.loadGltfJson(jsonData, specularTest, fastgltf::Options::None, fastgltf::Category::Materials); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -214,11 +212,12 @@ TEST_CASE("Extension KHR_materials_ior and KHR_materials_iridescence", "[gltf-lo TEST_CASE("Extension KHR_materials_volume and KHR_materials_transmission", "[gltf-loader]") { auto beautifulGame = sampleModels / "2.0" / "ABeautifulGame" / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(beautifulGame / "ABeautifulGame.gltf")); + + fastgltf::GltfFileStream jsonData(beautifulGame / "ABeautifulGame.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser(fastgltf::Extensions::KHR_materials_volume | fastgltf::Extensions::KHR_materials_transmission); - auto asset = parser.loadGltfJson(&jsonData, beautifulGame, fastgltf::Options::None, fastgltf::Category::Materials); + auto asset = parser.loadGltfJson(jsonData, beautifulGame, fastgltf::Options::None, fastgltf::Category::Materials); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -237,11 +236,11 @@ TEST_CASE("Extension KHR_materials_volume and KHR_materials_transmission", "[glt TEST_CASE("Extension KHR_materials_clearcoat", "[gltf-loader]") { auto clearcoatTest = sampleModels / "2.0" / "ClearCoatTest" / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(clearcoatTest / "ClearCoatTest.gltf")); + fastgltf::GltfFileStream jsonData(clearcoatTest / "ClearCoatTest.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser(fastgltf::Extensions::KHR_materials_clearcoat); - auto asset = parser.loadGltfJson(&jsonData, clearcoatTest, fastgltf::Options::None, fastgltf::Category::Materials); + auto asset = parser.loadGltfJson(jsonData, clearcoatTest, fastgltf::Options::None, fastgltf::Category::Materials); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -262,11 +261,11 @@ TEST_CASE("Extension KHR_materials_clearcoat", "[gltf-loader]") { TEST_CASE("Extension KHR_materials_emissive_strength", "[gltf-loader]") { auto emissiveStrengthTest = sampleModels / "2.0" / "EmissiveStrengthTest" / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(emissiveStrengthTest / "EmissiveStrengthTest.gltf")); + fastgltf::GltfFileStream jsonData(emissiveStrengthTest / "EmissiveStrengthTest.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser(fastgltf::Extensions::KHR_materials_emissive_strength); - auto asset = parser.loadGltfJson(&jsonData, emissiveStrengthTest); + auto asset = parser.loadGltfJson(jsonData, emissiveStrengthTest); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -278,11 +277,11 @@ TEST_CASE("Extension KHR_materials_emissive_strength", "[gltf-loader]") { TEST_CASE("Extension KHR_materials_sheen", "[gltf-loader]") { auto sheenChair = sampleModels / "2.0" / "SheenChair" / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(sheenChair / "SheenChair.gltf")); + fastgltf::GltfFileStream jsonData(sheenChair / "SheenChair.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser(fastgltf::Extensions::KHR_materials_sheen | fastgltf::Extensions::KHR_texture_transform); - auto asset = parser.loadGltfJson(&jsonData, sheenChair); + auto asset = parser.loadGltfJson(jsonData, sheenChair); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -296,11 +295,11 @@ TEST_CASE("Extension KHR_materials_sheen", "[gltf-loader]") { TEST_CASE("Extension KHR_materials_unlit", "[gltf-loader]") { auto unlitTest = sampleModels / "2.0" / "UnlitTest" / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(unlitTest / "UnlitTest.gltf")); + fastgltf::GltfFileStream jsonData(unlitTest / "UnlitTest.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser(fastgltf::Extensions::KHR_materials_unlit); - auto asset = parser.loadGltfJson(&jsonData, unlitTest); + auto asset = parser.loadGltfJson(jsonData, unlitTest); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -311,11 +310,11 @@ TEST_CASE("Extension KHR_materials_unlit", "[gltf-loader]") { TEST_CASE("Extension KHR_materials_anisotropy", "[gltf-loader]") { auto carbonFibre = sampleModels / "2.0" / "CarbonFibre" / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(carbonFibre / "CarbonFibre.gltf")); + fastgltf::GltfFileStream jsonData(carbonFibre / "CarbonFibre.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser(fastgltf::Extensions::KHR_materials_anisotropy); - auto asset = parser.loadGltfJson(&jsonData, carbonFibre); + auto asset = parser.loadGltfJson(jsonData, carbonFibre); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -330,11 +329,12 @@ TEST_CASE("Extension KHR_materials_anisotropy", "[gltf-loader]") { TEST_CASE("Extension EXT_mesh_gpu_instancing", "[gltf-loader]") { auto simpleInstancingTest = sampleModels / "2.0" / "SimpleInstancing" / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(simpleInstancingTest / "SimpleInstancing.gltf")); + + fastgltf::GltfFileStream jsonData(simpleInstancingTest / "SimpleInstancing.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser(fastgltf::Extensions::EXT_mesh_gpu_instancing); - auto asset = parser.loadGltfJson(&jsonData, simpleInstancingTest, fastgltf::Options::None, fastgltf::Category::Accessors | fastgltf::Category::Nodes); + auto asset = parser.loadGltfJson(jsonData, simpleInstancingTest, fastgltf::Options::None, fastgltf::Category::Accessors | fastgltf::Category::Nodes); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -351,11 +351,11 @@ TEST_CASE("Extension EXT_mesh_gpu_instancing", "[gltf-loader]") { #if FASTGLTF_ENABLE_DEPRECATED_EXT TEST_CASE("Extension KHR_materials_pbrSpecularGlossiness", "[gltf-loader]") { auto specularGlossinessTest = sampleModels / "2.0" / "SpecGlossVsMetalRough" / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(specularGlossinessTest / "SpecGlossVsMetalRough.gltf")); + fastgltf::GltfFileStream jsonData(specularGlossinessTest / "SpecGlossVsMetalRough.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser(fastgltf::Extensions::KHR_materials_pbrSpecularGlossiness | fastgltf::Extensions::KHR_materials_specular); - auto asset = parser.loadGltfJson(&jsonData, specularGlossinessTest); + auto asset = parser.loadGltfJson(jsonData, specularGlossinessTest); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -402,11 +402,10 @@ TEST_CASE("Extension KHR_materials_dispersion", "[gltf-loader]") { } } ]})"; - fastgltf::GltfDataBuffer jsonData; - jsonData.copyBytes(reinterpret_cast(json.data()), json.size()); + fastgltf::GltfDataBuffer jsonData(reinterpret_cast(json.data()), json.size()); fastgltf::Parser parser(fastgltf::Extensions::KHR_materials_dispersion); - auto asset = parser.loadGltfJson(&jsonData, {}, fastgltf::Options::DontRequireValidAssetMember); + auto asset = parser.loadGltfJson(jsonData, {}, fastgltf::Options::DontRequireValidAssetMember); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -416,11 +415,12 @@ TEST_CASE("Extension KHR_materials_dispersion", "[gltf-loader]") { TEST_CASE("Extension KHR_materials_variant", "[gltf-loader]") { auto velvetSofa = sampleModels / "2.0" / "GlamVelvetSofa" / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(velvetSofa / "GlamVelvetSofa.gltf")); + + fastgltf::GltfFileStream jsonData(velvetSofa / "GlamVelvetSofa.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser(fastgltf::Extensions::KHR_materials_variants | fastgltf::Extensions::KHR_texture_transform); - auto asset = parser.loadGltfJson(&jsonData, velvetSofa, fastgltf::Options::None); + auto asset = parser.loadGltfJson(jsonData, velvetSofa, fastgltf::Options::None); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); diff --git a/tests/glb_tests.cpp b/tests/glb_tests.cpp index 38f3b1cd1..0c1e877a4 100644 --- a/tests/glb_tests.cpp +++ b/tests/glb_tests.cpp @@ -7,13 +7,13 @@ #include "gltf_path.hpp" TEST_CASE("Load basic GLB file", "[gltf-loader]") { - fastgltf::Parser parser; auto folder = sampleModels / "2.0" / "Box" / "glTF-Binary"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(folder / "Box.glb")); + fastgltf::GltfDataBuffer jsonData(folder / "Box.glb"); + // REQUIRE(jsonData.isOpen()); + fastgltf::Parser parser; SECTION("Load basic Box.glb") { - auto asset = parser.loadGltfBinary(&jsonData, folder, fastgltf::Options::None, fastgltf::Category::Buffers); + auto asset = parser.loadGltfBinary(jsonData, folder, fastgltf::Options::None, fastgltf::Category::Buffers); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -28,7 +28,7 @@ TEST_CASE("Load basic GLB file", "[gltf-loader]") { } SECTION("Load basic Box.glb and load buffers") { - auto asset = parser.loadGltfBinary(&jsonData, folder, fastgltf::Options::LoadGLBBuffers, fastgltf::Category::Buffers); + auto asset = parser.loadGltfBinary(jsonData, folder, fastgltf::Options::LoadGLBBuffers, fastgltf::Category::Buffers); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -45,13 +45,12 @@ TEST_CASE("Load basic GLB file", "[gltf-loader]") { std::ifstream file(folder / "Box.glb", std::ios::binary | std::ios::ate); auto length = static_cast(file.tellg()); file.seekg(0, std::ifstream::beg); - std::vector bytes(length + fastgltf::getGltfBufferPadding()); + std::vector bytes(length); file.read(reinterpret_cast(bytes.data()), static_cast(length)); - fastgltf::GltfDataBuffer byteBuffer; - REQUIRE(byteBuffer.fromByteView(bytes.data(), length, length + fastgltf::getGltfBufferPadding())); + fastgltf::GltfDataBuffer byteBuffer(reinterpret_cast(bytes.data()), length); - auto asset = parser.loadGltfBinary(&byteBuffer, folder, fastgltf::Options::LoadGLBBuffers, fastgltf::Category::Buffers); + auto asset = parser.loadGltfBinary(byteBuffer, folder, fastgltf::Options::LoadGLBBuffers, fastgltf::Category::Buffers); REQUIRE(asset.error() == fastgltf::Error::None); } } diff --git a/tests/uri_tests.cpp b/tests/uri_tests.cpp index 8e8dee4be..d3a9fd2bd 100644 --- a/tests/uri_tests.cpp +++ b/tests/uri_tests.cpp @@ -108,11 +108,11 @@ TEST_CASE("Validate URI copying/moving", "[uri-tests]") { TEST_CASE("Validate escaped/percent-encoded URI", "[uri-tests]") { const std::string_view gltfString = R"({"images": [{"uri": "grande_sph\u00E8re.png"}]})"; - fastgltf::GltfDataBuffer dataBuffer; - dataBuffer.copyBytes(reinterpret_cast(gltfString.data()), gltfString.size()); + fastgltf::GltfDataBuffer dataBuffer(reinterpret_cast(gltfString.data()), + gltfString.size()); fastgltf::Parser parser; - auto asset = parser.loadGltfJson(&dataBuffer, "", fastgltf::Options::DontRequireValidAssetMember); + auto asset = parser.loadGltfJson(dataBuffer, "", fastgltf::Options::DontRequireValidAssetMember); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(asset->images.size() == 1); @@ -127,11 +127,12 @@ TEST_CASE("Validate escaped/percent-encoded URI", "[uri-tests]") { TEST_CASE("Test percent-encoded URIs in glTF", "[uri-tests]") { auto boxWithSpaces = sampleModels / "2.0" / "Box With Spaces" / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(boxWithSpaces / "Box With Spaces.gltf")); + + fastgltf::GltfFileStream jsonData(boxWithSpaces / "Box With Spaces.gltf"); + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser; - auto asset = parser.loadGltfJson(&jsonData, boxWithSpaces); + auto asset = parser.loadGltfJson(jsonData, boxWithSpaces); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); diff --git a/tests/write_tests.cpp b/tests/write_tests.cpp index 4731f52bd..fecdc6c1f 100644 --- a/tests/write_tests.cpp +++ b/tests/write_tests.cpp @@ -27,11 +27,11 @@ TEST_CASE("Test simple glTF composition", "[write-tests]") { TEST_CASE("Read glTF, write it, and then read it again and validate", "[write-tests]") { auto cubePath = sampleModels / "2.0" / "Cube" / "glTF"; - auto cubeJsonData = std::make_unique(); - REQUIRE(cubeJsonData->loadFromFile(cubePath / "Cube.gltf")); + fastgltf::GltfFileStream cubeJsonData(cubePath / "Cube.gltf"); + REQUIRE(cubeJsonData.isOpen()); fastgltf::Parser parser; - auto cube = parser.loadGltfJson(cubeJsonData.get(), cubePath); + auto cube = parser.loadGltfJson(cubeJsonData, cubePath); REQUIRE(cube.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(cube.get()) == fastgltf::Error::None); @@ -39,18 +39,17 @@ TEST_CASE("Read glTF, write it, and then read it again and validate", "[write-te auto expected = exporter.writeGltfJson(cube.get()); REQUIRE(expected.error() == fastgltf::Error::None); - fastgltf::GltfDataBuffer cube2JsonData; - cube2JsonData.copyBytes(reinterpret_cast(expected.get().output.data()), - expected.get().output.size()); - auto cube2 = parser.loadGltfJson(&cube2JsonData, cubePath); + fastgltf::GltfDataBuffer exportedJsonData(reinterpret_cast(expected.get().output.data()), + expected.get().output.size()); + auto cube2 = parser.loadGltfJson(exportedJsonData, cubePath); REQUIRE(cube2.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(cube2.get()) == fastgltf::Error::None); } TEST_CASE("Rewrite read glTF with multiple material extensions", "[write-tests]") { auto dishPath = sampleModels / "2.0" / "IridescentDishWithOlives" / "glTF"; - fastgltf::GltfDataBuffer dishJsonData; - REQUIRE(dishJsonData.loadFromFile(dishPath / "IridescentDishWithOlives.gltf")); + fastgltf::GltfFileStream dishJsonData(dishPath / "IridescentDishWithOlives.gltf"); + REQUIRE(dishJsonData.isOpen()); static constexpr auto requiredExtensions = fastgltf::Extensions::KHR_materials_ior | fastgltf::Extensions::KHR_materials_iridescence | @@ -58,7 +57,7 @@ TEST_CASE("Rewrite read glTF with multiple material extensions", "[write-tests]" fastgltf::Extensions::KHR_materials_volume; fastgltf::Parser parser(requiredExtensions); - auto dish = parser.loadGltfJson(&dishJsonData, dishPath); + auto dish = parser.loadGltfJson(dishJsonData, dishPath); REQUIRE(dish.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(dish.get()) == fastgltf::Error::None); @@ -66,10 +65,9 @@ TEST_CASE("Rewrite read glTF with multiple material extensions", "[write-tests]" auto expected = exporter.writeGltfJson(dish.get()); REQUIRE(expected.error() == fastgltf::Error::None); - fastgltf::GltfDataBuffer exportedDishJsonData; - exportedDishJsonData.copyBytes(reinterpret_cast(expected.get().output.data()), - expected.get().output.size()); - auto exportedDish = parser.loadGltfJson(&exportedDishJsonData, dishPath); + fastgltf::GltfDataBuffer exportedDishJsonData(reinterpret_cast(expected.get().output.data()), + expected.get().output.size()); + auto exportedDish = parser.loadGltfJson(exportedDishJsonData, dishPath); REQUIRE(exportedDish.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(exportedDish.get()) == fastgltf::Error::None); } @@ -77,17 +75,17 @@ TEST_CASE("Rewrite read glTF with multiple material extensions", "[write-tests]" TEST_CASE("Try writing a glTF with all buffers and images", "[write-tests]") { auto cubePath = sampleModels / "2.0" / "Cube" / "glTF"; - fastgltf::GltfDataBuffer gltfDataBuffer; - gltfDataBuffer.loadFromFile(cubePath / "Cube.gltf"); + fastgltf::GltfFileStream cubeJson(cubePath / "Cube.gltf"); + REQUIRE(cubeJson.isOpen()); fastgltf::Parser parser; auto options = fastgltf::Options::LoadExternalBuffers | fastgltf::Options::LoadExternalImages; - auto cube = parser.loadGltfJson(&gltfDataBuffer, cubePath, options); + auto cube = parser.loadGltfJson(cubeJson, cubePath, options); REQUIRE(cube.error() == fastgltf::Error::None); // Destroy the directory to make sure that the FileExporter correctly creates directories. auto exportedFolder = path / "export"; - if (std::filesystem::exists(exportedFolder)) { + if (std::filesystem::is_directory(exportedFolder)) { std::error_code ec; std::filesystem::remove_all(exportedFolder, ec); REQUIRE(!ec); @@ -97,17 +95,20 @@ TEST_CASE("Try writing a glTF with all buffers and images", "[write-tests]") { auto error = exporter.writeGltfJson(cube.get(), exportedFolder / "cube.gltf", fastgltf::ExportOptions::PrettyPrintJson); REQUIRE(error == fastgltf::Error::None); + REQUIRE(std::filesystem::exists(exportedFolder / "buffer0.bin")); + REQUIRE(std::filesystem::exists(exportedFolder / "image0.bin")); + REQUIRE(std::filesystem::exists(exportedFolder / "image1.bin")); } TEST_CASE("Try writing a GLB with all buffers and images", "[write-tests]") { auto cubePath = sampleModels / "2.0" / "Cube" / "glTF"; - fastgltf::GltfDataBuffer gltfDataBuffer; - gltfDataBuffer.loadFromFile(cubePath / "Cube.gltf"); + fastgltf::GltfFileStream cubeJson(cubePath / "Cube.gltf"); + REQUIRE(cubeJson.isOpen()); fastgltf::Parser parser; auto options = fastgltf::Options::LoadExternalBuffers | fastgltf::Options::LoadExternalImages; - auto cube = parser.loadGltfJson(&gltfDataBuffer, cubePath, options); + auto cube = parser.loadGltfJson(cubeJson, cubePath, options); REQUIRE(cube.error() == fastgltf::Error::None); // Destroy the directory to make sure that the FileExporter correctly creates directories. @@ -122,6 +123,8 @@ TEST_CASE("Try writing a GLB with all buffers and images", "[write-tests]") { auto exportedPath = exportedFolder / "cube.glb"; auto error = exporter.writeGltfBinary(cube.get(), exportedPath); REQUIRE(error == fastgltf::Error::None); + REQUIRE(std::filesystem::exists(exportedFolder / "image0.bin")); + REQUIRE(std::filesystem::exists(exportedFolder / "image1.bin")); // Make sure the GLB buffer is written std::ifstream glb(exportedPath, std::ios::binary); @@ -183,9 +186,8 @@ TEST_CASE("Test all local models and re-export them", "[write-tests]") { continue; // Parse the glTF - fastgltf::GltfDataBuffer gltfDataBuffer; - gltfDataBuffer.loadFromFile(epath); - auto model = parser.loadGltf(&gltfDataBuffer, epath.parent_path()); + fastgltf::GltfFileStream gltfData(epath); + auto model = parser.loadGltf(gltfData, epath.parent_path()); if (model.error() == fastgltf::Error::UnsupportedVersion || model.error() == fastgltf::Error::UnknownRequiredExtension) continue; // Skip any glTF 1.0 or 0.x files or glTFs with unsupported extensions. @@ -202,9 +204,9 @@ TEST_CASE("Test all local models and re-export them", "[write-tests]") { REQUIRE(simdjson::validate_utf8(exportedJson)); // Parse the re-generated glTF and validate - fastgltf::GltfDataBuffer regeneratedJson; - regeneratedJson.copyBytes(reinterpret_cast(exportedJson.data()), exportedJson.size()); - auto regeneratedModel = parser.loadGltf(®eneratedJson, epath.parent_path()); + fastgltf::GltfDataBuffer regeneratedJson(reinterpret_cast(exportedJson.data()), + exportedJson.size()); + auto regeneratedModel = parser.loadGltf(regeneratedJson, epath.parent_path()); REQUIRE(regeneratedModel.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(regeneratedModel.get()) == fastgltf::Error::None); @@ -217,16 +219,15 @@ TEST_CASE("Test all local models and re-export them", "[write-tests]") { TEST_CASE("Test Unicode exporting", "[write-tests]") { #if FASTGLTF_CPP_20 auto unicodePath = sampleModels / "2.0" / std::filesystem::path(u8"Unicode❤♻Test") / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(unicodePath / std::filesystem::path(u8"Unicode❤♻Test.gltf"))); + fastgltf::GltfFileStream jsonData(unicodePath / std::filesystem::path(u8"Unicode❤♻Test.gltf")); #else auto unicodePath = sampleModels / "2.0" / std::filesystem::u8path(u8"Unicode❤♻Test") / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(unicodePath / std::filesystem::u8path(u8"Unicode❤♻Test.gltf"))); + fastgltf::GltfFileStream jsonData(unicodePath / std::filesystem::u8path(u8"Unicode❤♻Test.gltf")); #endif + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser; - auto asset = parser.loadGltfJson(&jsonData, unicodePath); + auto asset = parser.loadGltfJson(jsonData, unicodePath); REQUIRE(asset.error() == fastgltf::Error::None); fastgltf::Exporter exporter; @@ -237,9 +238,8 @@ TEST_CASE("Test Unicode exporting", "[write-tests]") { auto& exportedJson = exported.get().output; REQUIRE(simdjson::validate_utf8(exportedJson)); - fastgltf::GltfDataBuffer regeneratedJson; - regeneratedJson.copyBytes(reinterpret_cast(exportedJson.data()), exportedJson.size()); - auto reparsed = parser.loadGltfJson(®eneratedJson, unicodePath); + fastgltf::GltfDataBuffer regeneratedJson(reinterpret_cast(exportedJson.data()), exportedJson.size()); + auto reparsed = parser.loadGltfJson(regeneratedJson, unicodePath); REQUIRE(reparsed.error() == fastgltf::Error::None); REQUIRE(!asset->materials.empty()); @@ -254,16 +254,15 @@ TEST_CASE("Test Unicode exporting", "[write-tests]") { TEST_CASE("Test URI normalization and removing backslashes", "[write-tests]") { #if FASTGLTF_CPP_20 auto unicodePath = sampleModels / "2.0" / std::filesystem::path(u8"Unicode❤♻Test") / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(unicodePath / std::filesystem::path(u8"Unicode❤♻Test.gltf"))); + fastgltf::GltfFileStream jsonData(unicodePath / std::filesystem::path(u8"Unicode❤♻Test.gltf")); #else auto unicodePath = sampleModels / "2.0" / std::filesystem::u8path(u8"Unicode❤♻Test") / "glTF"; - fastgltf::GltfDataBuffer jsonData; - REQUIRE(jsonData.loadFromFile(unicodePath / std::filesystem::u8path(u8"Unicode❤♻Test.gltf"))); + fastgltf::GltfFileStream jsonData(unicodePath / std::filesystem::u8path(u8"Unicode❤♻Test.gltf")); #endif + REQUIRE(jsonData.isOpen()); fastgltf::Parser parser; - auto asset = parser.loadGltfJson(&jsonData, unicodePath, fastgltf::Options::LoadExternalImages); + auto asset = parser.loadGltfJson(jsonData, unicodePath, fastgltf::Options::LoadExternalImages); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(asset->images.size() == 1); From 63b5a2b0f3728c78e0e322000d65394e7a6d2484 Mon Sep 17 00:00:00 2001 From: sean Date: Sat, 23 Mar 2024 16:41:54 +0100 Subject: [PATCH 2/9] Change: Use named constructors for GltfDataBuffer --- examples/gl_viewer/gl_viewer.cpp | 6 +-- include/fastgltf/core.hpp | 84 +++++++++++++++++++++++--------- src/fastgltf.cpp | 83 +++++++++++++++---------------- tests/basic_test.cpp | 7 +-- tests/benchmarks.cpp | 36 +++++++++----- tests/extension_tests.cpp | 6 ++- tests/glb_tests.cpp | 16 +++--- tests/uri_tests.cpp | 8 +-- tests/write_tests.cpp | 27 +++++----- 9 files changed, 168 insertions(+), 105 deletions(-) diff --git a/examples/gl_viewer/gl_viewer.cpp b/examples/gl_viewer/gl_viewer.cpp index 997b0da2a..bb2bce9d9 100644 --- a/examples/gl_viewer/gl_viewer.cpp +++ b/examples/gl_viewer/gl_viewer.cpp @@ -353,10 +353,10 @@ bool loadGltf(Viewer* viewer, std::filesystem::path path) { fastgltf::Options::LoadExternalImages | fastgltf::Options::GenerateMeshIndices; - fastgltf::GltfDataBuffer data; - data.loadFromFile(path); + fastgltf::GltfFileStream fileStream(path); + REQUIRE(fileStream.isOpen()); - auto asset = parser.loadGltf(&data, path.parent_path(), gltfOptions); + auto asset = parser.loadGltf(fileStream, path.parent_path(), gltfOptions); if (asset.error() != fastgltf::Error::None) { std::cerr << "Failed to load glTF: " << fastgltf::getErrorMessage(asset.error()) << '\n'; return false; diff --git a/include/fastgltf/core.hpp b/include/fastgltf/core.hpp index 882420a50..f2035bf07 100644 --- a/include/fastgltf/core.hpp +++ b/include/fastgltf/core.hpp @@ -89,6 +89,7 @@ namespace fastgltf { InvalidURI = 11, ///< A URI from a buffer or image failed to be parsed. InvalidFileData = 12, ///< The file data is invalid, or the file type could not be determined. FailedWritingFiles = 13, ///< The exporter failed to write some files (buffers/images) to disk. + FileBufferAllocationFailed = 14, ///< The constructor of GltfDataBuffer failed to allocate a sufficiently large buffer. }; inline std::string_view getErrorName(Error error) { @@ -107,6 +108,7 @@ namespace fastgltf { case Error::InvalidURI: return "InvalidURI"; case Error::InvalidFileData: return "InvalidFileData"; case Error::FailedWritingFiles: return "FailedWritingFiles"; + case Error::FileBufferAllocationFailed: return "FileBufferAllocationFailed"; default: FASTGLTF_UNREACHABLE } } @@ -127,6 +129,7 @@ namespace fastgltf { case Error::InvalidURI: return "A URI from a buffer or image failed to be parsed."; case Error::InvalidFileData: return "The file data is invalid, or the file type could not be determined."; case Error::FailedWritingFiles: return "The exporter failed to write some files (buffers/images) to disk."; + case Error::FileBufferAllocationFailed: return "The constructor of GltfDataBuffer failed to allocate a sufficiently large buffer."; default: FASTGLTF_UNREACHABLE } } @@ -260,9 +263,6 @@ namespace fastgltf { */ DecomposeNodeMatrices = 1 << 5, - // TODO: Remove or keep for any compatibility? - MinimiseJsonBeforeParsing [[deprecated]] = 1 << 6, - /** * Loads all external images into CPU memory. It does not decode any texture data. Complementary * to LoadExternalBuffers. @@ -628,25 +628,59 @@ namespace fastgltf { std::size_t idx = 0; - void allocateAndCopy(const std::byte* bytes); + Error error = Error::None; + + void allocateAndCopy(const std::byte* bytes) noexcept; + + explicit GltfDataBuffer(const std::filesystem::path& path) noexcept; + explicit GltfDataBuffer(const std::byte* bytes, std::size_t count) noexcept; +#if FASTGLTF_CPP_20 + explicit GltfDataBuffer(std::span span) noexcept; +#endif public: - GltfDataBuffer(const std::filesystem::path& path); - GltfDataBuffer(const std::byte* bytes, std::size_t count); + explicit GltfDataBuffer() noexcept = default; + GltfDataBuffer(const GltfDataBuffer& other) = delete; + GltfDataBuffer& operator=(const GltfDataBuffer& other) = delete; + GltfDataBuffer(GltfDataBuffer&& other) noexcept = default; + GltfDataBuffer& operator=(GltfDataBuffer&& other) noexcept = default; + ~GltfDataBuffer() noexcept = default; + + static Expected FromPath(const std::filesystem::path& path) noexcept { + GltfDataBuffer buffer(path); + if (buffer.error != fastgltf::Error::None) { + return buffer.error; + } + return std::move(buffer); + } + + static Expected FromBytes(const std::byte* bytes, std::size_t count) noexcept { + GltfDataBuffer buffer(bytes, count); + if (buffer.error != fastgltf::Error::None) { + return buffer.error; + } + return std::move(buffer); + } #if FASTGLTF_CPP_20 - GltfDataBuffer(std::span span); + static Expected FromSpan(std::span data) noexcept { + GltfDataBuffer buffer(data); + if (buffer.buffer.get() == nullptr) { + return buffer.error; + } + return std::move(buffer); + } #endif void read(void* ptr, std::size_t count) override; - span read(std::size_t count, std::size_t padding) override; + [[nodiscard]] span read(std::size_t count, std::size_t padding) override; void reset() override; - std::size_t bytesRead() override; + [[nodiscard]] std::size_t bytesRead() override; - std::size_t totalSize() override; + [[nodiscard]] std::size_t totalSize() override; [[nodiscard]] explicit operator span() { return span(buffer.get(), dataSize); @@ -660,36 +694,40 @@ namespace fastgltf { std::size_t fileSize; public: - GltfFileStream(const std::filesystem::path& path); + explicit GltfFileStream(const std::filesystem::path& path); + ~GltfFileStream() noexcept = default; - bool isOpen() const; + [[nodiscard]] bool isOpen() const; void read(void* ptr, std::size_t count) override; - span read(std::size_t count, std::size_t padding) override; + [[nodiscard]] span read(std::size_t count, std::size_t padding) override; void reset() override; - std::size_t bytesRead() override; + [[nodiscard]] std::size_t bytesRead() override; - std::size_t totalSize() override; + [[nodiscard]] std::size_t totalSize() override; }; #if defined(__ANDROID__) void setAndroidAssetManager(AAssetManager* assetManager) noexcept; class AndroidGltfDataBuffer : public GltfDataBuffer { + explicit AndroidGltfDataBuffer(const std::filesystem::path& path, std::uint64_t byteOffset); + public: - explicit AndroidGltfDataBuffer() noexcept; + explicit AndroidGltfDataBuffer() noexcept = default; ~AndroidGltfDataBuffer() noexcept = default; - /** - * Loads a file from within an Android APK. - * - * @note This requires a valid AAssetManager to have been specified through fastgltf::setAndroidAssetManager. - */ - bool loadFromAndroidAsset(const std::filesystem::path& path, std::uint64_t byteOffset = 0) noexcept; - }; + static Expected FromAsset(const std::filesystem::path& path, std::uint64_t byteOffset = 0) noexcept { + AndroidGltfDataBuffer buffer(path); + if (buffer.buffer.get() == nullptr) { + return buffer.error; + } + return std::move(buffer); + } + }; #endif /** diff --git a/src/fastgltf.cpp b/src/fastgltf.cpp index 5e6a0a4f9..66d71545c 100644 --- a/src/fastgltf.cpp +++ b/src/fastgltf.cpp @@ -3732,46 +3732,55 @@ fg::Error fg::Parser::parseTextures(simdjson::dom::array& textures, Asset& asset #pragma endregion #pragma region GltfDataBuffer -fg::GltfDataBuffer::GltfDataBuffer(const std::filesystem::path& path) { +fg::GltfDataBuffer::GltfDataBuffer(const fs::path& path) noexcept { std::error_code ec; - dataSize = static_cast(std::filesystem::file_size(path, ec)); + dataSize = static_cast(fs::file_size(path, ec)); if (ec) { - // TODO: Fail? + error = Error::InvalidPath; + return; } // Open the file and determine the size. std::ifstream file(path, std::ios::binary); - if (!file.is_open() || file.bad()) + if (!file.is_open() || file.bad()) { + error = Error::InvalidPath; return; + } allocatedSize = dataSize + simdjson::SIMDJSON_PADDING; - buffer = decltype(buffer)(new std::byte[allocatedSize]); // To mimic std::make_unique_for_overwrite (C++20) - if (!buffer) - return; + buffer = decltype(buffer)(new(std::nothrow) std::byte[allocatedSize]); // To mimic std::make_unique_for_overwrite (C++20) - // Copy the data and fill the padding region with zeros. - file.read(reinterpret_cast(buffer.get()), static_cast(dataSize)); - std::memset(buffer.get() + dataSize, 0, allocatedSize - dataSize); + if (buffer != nullptr) { + // Copy the data and fill the padding region with zeros. + file.read(reinterpret_cast(buffer.get()), static_cast(dataSize)); + std::memset(buffer.get() + dataSize, 0, allocatedSize - dataSize); + } else { + error = Error::FileBufferAllocationFailed; + } } -fg::GltfDataBuffer::GltfDataBuffer(const std::byte *bytes, std::size_t count) { +fg::GltfDataBuffer::GltfDataBuffer(const std::byte *bytes, std::size_t count) noexcept { dataSize = count; allocateAndCopy(bytes); } #if FASTGLTF_CPP_20 -fg::GltfDataBuffer::GltfDataBuffer(std::span span) { +fg::GltfDataBuffer::GltfDataBuffer(std::span span) noexcept { dataSize = span.size_bytes(); allocateAndCopy(span.data()); } #endif -void fg::GltfDataBuffer::allocateAndCopy(const std::byte *bytes) { +void fg::GltfDataBuffer::allocateAndCopy(const std::byte *bytes) noexcept { allocatedSize = dataSize + simdjson::SIMDJSON_PADDING; - buffer = decltype(buffer)(new std::byte[allocatedSize]); + buffer = decltype(buffer)(new(std::nothrow) std::byte[allocatedSize]); - std::memcpy(buffer.get(), bytes, dataSize); - std::memset(buffer.get() + dataSize, 0, allocatedSize - dataSize); + if (buffer != nullptr) { + std::memcpy(buffer.get(), bytes, dataSize); + std::memset(buffer.get() + dataSize, 0, allocatedSize - dataSize); + } else { + error = Error::FileBufferAllocationFailed; + } } void fg::GltfDataBuffer::read(void *ptr, std::size_t count) { @@ -3841,48 +3850,40 @@ void fg::setAndroidAssetManager(AAssetManager* assetManager) noexcept { androidAssetManager = assetManager; } -fg::AndroidGltfDataBuffer::AndroidGltfDataBuffer() noexcept = default; - -bool fg::AndroidGltfDataBuffer::loadFromAndroidAsset(const fs::path& path, std::uint64_t byteOffset) noexcept { +fg::AndroidGltfDataBuffer::AndroidGltfDataBuffer(const fs::path& path, std::uint64_t byteOffset) noexcept { if (androidAssetManager == nullptr) { - return false; + error = Error::InvalidPath; + return; } - using namespace simdjson; - const auto filenameString = path.string(); - auto file = deletable_unique_ptr( AAssetManager_open(androidAssetManager, filenameString.c_str(), AASSET_MODE_BUFFER)); if (file == nullptr) { - return false; + error = Error::InvalidPath; + return; } const auto length = AAsset_getLength(file.get()); if (length == 0) { - return false; + error = Error::InvalidPath; + return; } dataSize = length - byteOffset; allocatedSize = dataSize + simdjson::SIMDJSON_PADDING; - buffer = decltype(buffer)(new std::byte[allocatedSize]); - if (!buffer) { - return false; - } - - bufferPointer = buffer.get(); + buffer = decltype(buffer)(new(std::nothrow) std::byte[allocatedSize]); - if (byteOffset > 0) { - AAsset_seek64(file.get(), byteOffset, SEEK_SET); - } - - AAsset_read(file.get(), bufferPointer, dataSize); - - std::memset(bufferPointer + dataSize, 0, allocatedSize - dataSize); - - filePath = path; + if (buffer == nullptr) { + error = Error::FileBufferAllocationFailed; + } else { + if (byteOffset > 0) + AAsset_seek64(file.get(), byteOffset, SEEK_SET); - return true; + // Copy the data and fill the padding region with zeros. + AAsset_read(file.get(), buffer.get(), dataSize); + std::memset(buffer.get() + dataSize, 0, allocatedSize - dataSize); + } } #endif #pragma endregion diff --git a/tests/basic_test.cpp b/tests/basic_test.cpp index 38c1cebfd..f1e328c7a 100644 --- a/tests/basic_test.cpp +++ b/tests/basic_test.cpp @@ -651,11 +651,12 @@ TEST_CASE("Test extras callback", "[gltf-loader]") { // Update the data buffer auto& string = json.get().output; - fastgltf::GltfDataBuffer reexportedJson(reinterpret_cast(string.data()), - string.size()); + auto reexportedJson = fastgltf::GltfDataBuffer::FromBytes( + reinterpret_cast(string.data()), string.size()); + REQUIRE(reexportedJson.error() == fastgltf::Error::None); nodeNames.clear(); - auto reparsed = parseJson(reexportedJson); + auto reparsed = parseJson(reexportedJson.get()); REQUIRE(reparsed.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(reparsed.get()) == fastgltf::Error::None); diff --git a/tests/benchmarks.cpp b/tests/benchmarks.cpp index 16d95afbe..c4d130735 100644 --- a/tests/benchmarks.cpp +++ b/tests/benchmarks.cpp @@ -103,10 +103,12 @@ TEST_CASE("Benchmark loading of NewSponza", "[gltf-benchmark]") { #endif auto bytes = readFileAsBytes(intelSponza / "NewSponza_Main_glTF_002.gltf"); - fastgltf::GltfDataBuffer jsonData(reinterpret_cast(bytes.data()), bytes.size()); + auto jsonData = fastgltf::GltfDataBuffer::FromBytes( + reinterpret_cast(bytes.data()), bytes.size()); + REQUIRE(jsonData.error() == fastgltf::Error::None); BENCHMARK("Parse NewSponza") { - return parser.loadGltfJson(jsonData, intelSponza, benchmarkOptions); + return parser.loadGltfJson(jsonData.get(), intelSponza, benchmarkOptions); }; #ifdef HAS_TINYGLTF @@ -152,10 +154,12 @@ TEST_CASE("Benchmark base64 decoding from glTF file", "[gltf-benchmark]") { auto cylinderEngine = sampleModels / "2.0" / "2CylinderEngine" / "glTF-Embedded"; auto bytes = readFileAsBytes(cylinderEngine / "2CylinderEngine.gltf"); - fastgltf::GltfDataBuffer jsonData(reinterpret_cast(bytes.data()), bytes.size()); + auto jsonData = fastgltf::GltfDataBuffer::FromBytes( + reinterpret_cast(bytes.data()), bytes.size()); + REQUIRE(jsonData.error() == fastgltf::Error::None); BENCHMARK("Parse 2CylinderEngine and decode base64") { - return parser.loadGltfJson(jsonData, cylinderEngine, benchmarkOptions); + return parser.loadGltfJson(jsonData.get(), cylinderEngine, benchmarkOptions); }; #ifdef HAS_TINYGLTF @@ -205,10 +209,12 @@ TEST_CASE("Benchmark raw JSON parsing", "[gltf-benchmark]") { auto buggyPath = sampleModels / "2.0" / "Buggy" / "glTF"; auto bytes = readFileAsBytes(buggyPath / "Buggy.gltf"); - fastgltf::GltfDataBuffer jsonData(reinterpret_cast(bytes.data()), bytes.size()); + auto jsonData = fastgltf::GltfDataBuffer::FromBytes( + reinterpret_cast(bytes.data()), bytes.size()); + REQUIRE(jsonData.error() == fastgltf::Error::None); BENCHMARK("Parse Buggy.gltf") { - return parser.loadGltfJson(jsonData, buggyPath, benchmarkOptions); + return parser.loadGltfJson(jsonData.get(), buggyPath, benchmarkOptions); }; #ifdef HAS_TINYGLTF @@ -259,10 +265,12 @@ TEST_CASE("Benchmark massive gltf file", "[gltf-benchmark]") { #endif auto bytes = readFileAsBytes(bistroPath / "bistro.gltf"); - fastgltf::GltfDataBuffer jsonData(reinterpret_cast(bytes.data()), bytes.size()); + auto jsonData = fastgltf::GltfDataBuffer::FromBytes( + reinterpret_cast(bytes.data()), bytes.size()); + REQUIRE(jsonData.error() == fastgltf::Error::None); BENCHMARK("Parse Bistro") { - return parser.loadGltfJson(jsonData, bistroPath, benchmarkOptions); + return parser.loadGltfJson(jsonData.get(), bistroPath, benchmarkOptions); }; #ifdef HAS_TINYGLTF @@ -302,7 +310,9 @@ TEST_CASE("Benchmark massive gltf file", "[gltf-benchmark]") { TEST_CASE("Compare parsing performance with minified documents", "[gltf-benchmark]") { auto buggyPath = sampleModels / "2.0" / "Buggy" / "glTF"; auto bytes = readFileAsBytes(buggyPath / "Buggy.gltf"); - fastgltf::GltfDataBuffer jsonData(reinterpret_cast(bytes.data()), bytes.size()); + auto jsonData = fastgltf::GltfDataBuffer::FromBytes( + reinterpret_cast(bytes.data()), bytes.size()); + REQUIRE(jsonData.error() == fastgltf::Error::None); // Create a minified JSON string std::vector minified(bytes.size()); @@ -320,15 +330,17 @@ TEST_CASE("Compare parsing performance with minified documents", "[gltf-benchmar return result; }; - fastgltf::GltfDataBuffer minifiedJsonData(reinterpret_cast(minified.data()), minified.size()); + auto minifiedJsonData = fastgltf::GltfDataBuffer::FromBytes( + reinterpret_cast(bytes.data()), bytes.size()); + REQUIRE(minifiedJsonData.error() == fastgltf::Error::None); fastgltf::Parser parser; BENCHMARK("Parse Buggy.gltf with normal JSON") { - return parser.loadGltfJson(jsonData, buggyPath, benchmarkOptions); + return parser.loadGltfJson(jsonData.get(), buggyPath, benchmarkOptions); }; BENCHMARK("Parse Buggy.gltf with minified JSON") { - return parser.loadGltfJson(minifiedJsonData, buggyPath, benchmarkOptions); + return parser.loadGltfJson(minifiedJsonData.get(), buggyPath, benchmarkOptions); }; } diff --git a/tests/extension_tests.cpp b/tests/extension_tests.cpp index e417c7c9b..5bd54575c 100644 --- a/tests/extension_tests.cpp +++ b/tests/extension_tests.cpp @@ -402,10 +402,12 @@ TEST_CASE("Extension KHR_materials_dispersion", "[gltf-loader]") { } } ]})"; - fastgltf::GltfDataBuffer jsonData(reinterpret_cast(json.data()), json.size()); + auto jsonData = fastgltf::GltfDataBuffer::FromBytes( + reinterpret_cast(json.data()), json.size()); + REQUIRE(jsonData.error() == fastgltf::Error::None); fastgltf::Parser parser(fastgltf::Extensions::KHR_materials_dispersion); - auto asset = parser.loadGltfJson(jsonData, {}, fastgltf::Options::DontRequireValidAssetMember); + auto asset = parser.loadGltfJson(jsonData.get(), {}, fastgltf::Options::DontRequireValidAssetMember); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); diff --git a/tests/glb_tests.cpp b/tests/glb_tests.cpp index 0c1e877a4..69b186eb7 100644 --- a/tests/glb_tests.cpp +++ b/tests/glb_tests.cpp @@ -8,12 +8,12 @@ TEST_CASE("Load basic GLB file", "[gltf-loader]") { auto folder = sampleModels / "2.0" / "Box" / "glTF-Binary"; - fastgltf::GltfDataBuffer jsonData(folder / "Box.glb"); - // REQUIRE(jsonData.isOpen()); + auto jsonData = fastgltf::GltfDataBuffer::FromPath(folder / "Box.glb"); + REQUIRE(jsonData.error() == fastgltf::Error::None); fastgltf::Parser parser; SECTION("Load basic Box.glb") { - auto asset = parser.loadGltfBinary(jsonData, folder, fastgltf::Options::None, fastgltf::Category::Buffers); + auto asset = parser.loadGltfBinary(jsonData.get(), folder, fastgltf::Options::None, fastgltf::Category::Buffers); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -22,13 +22,13 @@ TEST_CASE("Load basic GLB file", "[gltf-loader]") { auto& buffer = asset->buffers.front(); auto* bufferView = std::get_if(&buffer.data); REQUIRE(bufferView != nullptr); - auto jsonSpan = fastgltf::span(jsonData); + auto jsonSpan = fastgltf::span(jsonData.get()); REQUIRE(bufferView->bytes.data() - jsonSpan.data() == 1016); REQUIRE(jsonSpan.size() == 1664); } SECTION("Load basic Box.glb and load buffers") { - auto asset = parser.loadGltfBinary(jsonData, folder, fastgltf::Options::LoadGLBBuffers, fastgltf::Category::Buffers); + auto asset = parser.loadGltfBinary(jsonData.get(), folder, fastgltf::Options::LoadGLBBuffers, fastgltf::Category::Buffers); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -48,9 +48,11 @@ TEST_CASE("Load basic GLB file", "[gltf-loader]") { std::vector bytes(length); file.read(reinterpret_cast(bytes.data()), static_cast(length)); - fastgltf::GltfDataBuffer byteBuffer(reinterpret_cast(bytes.data()), length); + auto byteBuffer = fastgltf::GltfDataBuffer::FromBytes( + reinterpret_cast(bytes.data()), length); + REQUIRE(byteBuffer.error() == fastgltf::Error::None); - auto asset = parser.loadGltfBinary(byteBuffer, folder, fastgltf::Options::LoadGLBBuffers, fastgltf::Category::Buffers); + auto asset = parser.loadGltfBinary(byteBuffer.get(), folder, fastgltf::Options::LoadGLBBuffers, fastgltf::Category::Buffers); REQUIRE(asset.error() == fastgltf::Error::None); } } diff --git a/tests/uri_tests.cpp b/tests/uri_tests.cpp index d3a9fd2bd..ad671e8ac 100644 --- a/tests/uri_tests.cpp +++ b/tests/uri_tests.cpp @@ -108,11 +108,13 @@ TEST_CASE("Validate URI copying/moving", "[uri-tests]") { TEST_CASE("Validate escaped/percent-encoded URI", "[uri-tests]") { const std::string_view gltfString = R"({"images": [{"uri": "grande_sph\u00E8re.png"}]})"; - fastgltf::GltfDataBuffer dataBuffer(reinterpret_cast(gltfString.data()), - gltfString.size()); + auto dataBuffer = fastgltf::GltfDataBuffer::FromBytes( + reinterpret_cast(gltfString.data()), + gltfString.size()); + REQUIRE(dataBuffer.error() == fastgltf::Error::None); fastgltf::Parser parser; - auto asset = parser.loadGltfJson(dataBuffer, "", fastgltf::Options::DontRequireValidAssetMember); + auto asset = parser.loadGltfJson(dataBuffer.get(), "", fastgltf::Options::DontRequireValidAssetMember); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(asset->images.size() == 1); diff --git a/tests/write_tests.cpp b/tests/write_tests.cpp index fecdc6c1f..4bdd1e22d 100644 --- a/tests/write_tests.cpp +++ b/tests/write_tests.cpp @@ -39,9 +39,10 @@ TEST_CASE("Read glTF, write it, and then read it again and validate", "[write-te auto expected = exporter.writeGltfJson(cube.get()); REQUIRE(expected.error() == fastgltf::Error::None); - fastgltf::GltfDataBuffer exportedJsonData(reinterpret_cast(expected.get().output.data()), - expected.get().output.size()); - auto cube2 = parser.loadGltfJson(exportedJsonData, cubePath); + auto exportedJsonData = fastgltf::GltfDataBuffer::FromBytes( + reinterpret_cast(expected.get().output.data()), expected.get().output.size()); + REQUIRE(exportedJsonData.error() == fastgltf::Error::None); + auto cube2 = parser.loadGltfJson(exportedJsonData.get(), cubePath); REQUIRE(cube2.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(cube2.get()) == fastgltf::Error::None); } @@ -65,9 +66,10 @@ TEST_CASE("Rewrite read glTF with multiple material extensions", "[write-tests]" auto expected = exporter.writeGltfJson(dish.get()); REQUIRE(expected.error() == fastgltf::Error::None); - fastgltf::GltfDataBuffer exportedDishJsonData(reinterpret_cast(expected.get().output.data()), - expected.get().output.size()); - auto exportedDish = parser.loadGltfJson(exportedDishJsonData, dishPath); + auto exportedDishJsonData = fastgltf::GltfDataBuffer::FromBytes( + reinterpret_cast(expected.get().output.data()), expected.get().output.size()); + REQUIRE(exportedDishJsonData.error() == fastgltf::Error::None); + auto exportedDish = parser.loadGltfJson(exportedDishJsonData.get(), dishPath); REQUIRE(exportedDish.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(exportedDish.get()) == fastgltf::Error::None); } @@ -204,9 +206,10 @@ TEST_CASE("Test all local models and re-export them", "[write-tests]") { REQUIRE(simdjson::validate_utf8(exportedJson)); // Parse the re-generated glTF and validate - fastgltf::GltfDataBuffer regeneratedJson(reinterpret_cast(exportedJson.data()), - exportedJson.size()); - auto regeneratedModel = parser.loadGltf(regeneratedJson, epath.parent_path()); + auto regeneratedJson = fastgltf::GltfDataBuffer::FromBytes( + reinterpret_cast(exportedJson.data()), exportedJson.size()); + REQUIRE(regeneratedJson.error() == fastgltf::Error::None); + auto regeneratedModel = parser.loadGltf(regeneratedJson.get(), epath.parent_path()); REQUIRE(regeneratedModel.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(regeneratedModel.get()) == fastgltf::Error::None); @@ -238,8 +241,10 @@ TEST_CASE("Test Unicode exporting", "[write-tests]") { auto& exportedJson = exported.get().output; REQUIRE(simdjson::validate_utf8(exportedJson)); - fastgltf::GltfDataBuffer regeneratedJson(reinterpret_cast(exportedJson.data()), exportedJson.size()); - auto reparsed = parser.loadGltfJson(regeneratedJson, unicodePath); + auto regeneratedJson = fastgltf::GltfDataBuffer::FromBytes( + reinterpret_cast(exportedJson.data()), exportedJson.size()); + REQUIRE(regeneratedJson.error() == fastgltf::Error::None); + auto reparsed = parser.loadGltfJson(regeneratedJson.get(), unicodePath); REQUIRE(reparsed.error() == fastgltf::Error::None); REQUIRE(!asset->materials.empty()); From 9f2d1480079d0e5c561a25969c3e9b213f543043 Mon Sep 17 00:00:00 2001 From: sean Date: Sat, 23 Mar 2024 16:44:47 +0100 Subject: [PATCH 3/9] Add: MappedGltfFile --- include/fastgltf/core.hpp | 46 ++++++++++++++++++++ src/fastgltf.cpp | 92 ++++++++++++++++++++++++++++++++++++++- tests/basic_test.cpp | 16 +++++++ 3 files changed, 152 insertions(+), 2 deletions(-) diff --git a/include/fastgltf/core.hpp b/include/fastgltf/core.hpp index f2035bf07..49919af13 100644 --- a/include/fastgltf/core.hpp +++ b/include/fastgltf/core.hpp @@ -621,6 +621,7 @@ namespace fastgltf { }; class GltfDataBuffer : public GltfDataGetter { + protected: std::unique_ptr buffer; std::size_t allocatedSize = 0; @@ -687,6 +688,51 @@ namespace fastgltf { } }; +#if defined(__APPLE__) || defined(__linux__) + /** Maps a glTF file into memory using mmap */ + class MappedGltfFile : public GltfDataGetter { + void* mappedFile = nullptr; + std::uint64_t fileSize = 0; + + std::size_t idx = 0; + + Error error = Error::None; + + explicit MappedGltfFile(const std::filesystem::path& path) noexcept; + + public: + explicit MappedGltfFile() = default; + MappedGltfFile(const MappedGltfFile& other) = delete; + MappedGltfFile& operator=(const MappedGltfFile& other) = delete; + MappedGltfFile(MappedGltfFile&& other) noexcept; + MappedGltfFile& operator=(MappedGltfFile&& other) noexcept; + + static Expected FromPath(const std::filesystem::path& path) noexcept { + MappedGltfFile buffer(path); + if (buffer.error != fastgltf::Error::None) { + return buffer.error; + } + return std::move(buffer); + } + + ~MappedGltfFile() noexcept; + + void read(void* ptr, std::size_t count) override; + + [[nodiscard]] span read(std::size_t count, std::size_t padding) override; + + void reset() override; + + [[nodiscard]] std::size_t bytesRead() override; + + [[nodiscard]] std::size_t totalSize() override; + + [[nodiscard]] explicit operator span() { + return span(static_cast(mappedFile), fileSize); + } + }; +#endif + class GltfFileStream : public GltfDataGetter { std::ifstream fileStream; std::vector buf; diff --git a/src/fastgltf.cpp b/src/fastgltf.cpp index 66d71545c..1076bef76 100644 --- a/src/fastgltf.cpp +++ b/src/fastgltf.cpp @@ -39,6 +39,13 @@ #include #endif +#if defined(__APPLE__) || defined(__linux__) +#include +#include +#include +#include +#endif + #ifdef _MSC_VER #pragma warning(push) #pragma warning(disable : 5030) // attribute 'x' is not recognized @@ -3806,8 +3813,89 @@ std::size_t fg::GltfDataBuffer::totalSize() { return dataSize; } -fg::GltfFileStream::GltfFileStream(const std::filesystem::path &path) : fileStream(path, std::ios::binary) { - fileSize = std::filesystem::file_size(path); +#if defined(__APPLE__) || defined(__linux__) +fg::MappedGltfFile::MappedGltfFile(const fs::path& path) noexcept { + // Open the file + int fd = open(path.c_str(), O_RDONLY, 0); + if (fd == 0) { + // TODO: Cover actual error messages using std::strerror(errno)? + error = Error::InvalidPath; + return; + } + + // Get the file size + struct stat statInfo; + if (fstat(fd, &statInfo) != 0) { + error = Error::InvalidPath; + return; + } + + // Map the file + mappedFile = mmap(nullptr, + statInfo.st_size, + PROT_READ, + MAP_PRIVATE, + fd, + 0); + if (mappedFile != MAP_FAILED) { + fileSize = static_cast(statInfo.st_size); + } else { + std::perror("Mapping file"); + error = Error::FileBufferAllocationFailed; + } + + // The file descriptor is not used for the memory mapped and is not required anymore. + close(fd); +} + +fg::MappedGltfFile::MappedGltfFile(fastgltf::MappedGltfFile &&other) noexcept { + // Make sure that munmap is never called when other gets destructud. + mappedFile = std::exchange(other.mappedFile, MAP_FAILED); + fileSize = other.fileSize; + idx = other.idx; + error = other.error; +} + +fg::MappedGltfFile& fg::MappedGltfFile::operator=(fastgltf::MappedGltfFile &&other) noexcept { + mappedFile = std::exchange(other.mappedFile, MAP_FAILED); + fileSize = other.fileSize; + idx = other.idx; + error = other.error; + return *this; +} + +fg::MappedGltfFile::~MappedGltfFile() noexcept { + if (mappedFile != MAP_FAILED) { + munmap(mappedFile, fileSize); + } +} + +void fg::MappedGltfFile::read(void *ptr, std::size_t count) { + std::memcpy(ptr, static_cast(mappedFile) + idx, count); + idx += count; +} + +fg::span fg::MappedGltfFile::read(std::size_t count, std::size_t padding) { + span sub(static_cast(mappedFile) + idx, count); + idx += count; + return sub; +} + +void fg::MappedGltfFile::reset() { + idx = 0; +} + +std::size_t fg::MappedGltfFile::bytesRead() { + return idx; +} + +std::size_t fg::MappedGltfFile::totalSize() { + return fileSize; +} +#endif + +fg::GltfFileStream::GltfFileStream(const fs::path& path) : fileStream(path, std::ios::binary) { + fileSize = fs::file_size(path); } bool fg::GltfFileStream::isOpen() const { diff --git a/tests/basic_test.cpp b/tests/basic_test.cpp index f1e328c7a..8ea49e7bc 100644 --- a/tests/basic_test.cpp +++ b/tests/basic_test.cpp @@ -666,3 +666,19 @@ TEST_CASE("Test extras callback", "[gltf-loader]") { REQUIRE(nodeNames[2] == "Shoe.obj"); } } + +#if defined(__APPLE__) || defined(__linux__) +TEST_CASE("Test glTF file loading", "[gltf-loader]") { + SECTION("Mapped files") { + auto cubePath = sampleModels / "2.0" / "Cube" / "glTF"; + auto mappedFile = fastgltf::MappedGltfFile::FromPath(cubePath / "Cube.gltf"); + REQUIRE(mappedFile.error() == fastgltf::Error::None); + + REQUIRE(fastgltf::determineGltfFileType(mappedFile.get()) == fastgltf::GltfType::glTF); + + fastgltf::Parser parser; + auto asset = parser.loadGltfJson(mappedFile.get(), cubePath); + REQUIRE(asset.error() == fastgltf::Error::None); + } +} +#endif From 72ed4af6fb3a99d1531934234fb4ea86eac5e665 Mon Sep 17 00:00:00 2001 From: sean Date: Thu, 25 Apr 2024 11:20:40 +0200 Subject: [PATCH 4/9] Change: Always load GLB buffers --- include/fastgltf/core.hpp | 16 ++++++++++-- src/fastgltf.cpp | 52 +++++++++++++++++---------------------- tests/glb_tests.cpp | 8 +++--- 3 files changed, 40 insertions(+), 36 deletions(-) diff --git a/include/fastgltf/core.hpp b/include/fastgltf/core.hpp index 49919af13..411dc8c54 100644 --- a/include/fastgltf/core.hpp +++ b/include/fastgltf/core.hpp @@ -245,7 +245,7 @@ namespace fastgltf { * a byte offset and length into the GLB file, which can be useful when using APIs like * DirectStorage or Metal IO. */ - LoadGLBBuffers = 1 << 3, + LoadGLBBuffers [[deprecated]] = 1 << 3, /** * Loads all external buffers into CPU memory. If disabled, fastgltf will only provide @@ -607,12 +607,23 @@ namespace fastgltf { */ class GltfDataGetter { public: + /** + * The read functions expect the implementation to store an offset from the start + * of the buffer/file to the current position. The parse process will always linearly + * access the memory, meaning it will go through the memory once from start to finish. + */ virtual void read(void* ptr, std::size_t count) = 0; + /** + * Reads a chunk of memory from the current offset, with some amount of padding. + * This padding is necessary for the simdjson parser, but can be initialized to anything. + * The memory pointed to by the span only needs to live until the next call to read(). + */ [[nodiscard]] virtual span read(std::size_t count, std::size_t padding) = 0; /** * Reset is used to put the offset index back to the start of the buffer/file. - * This is only used with determineGltfFileType, as it needs to peek into the beginning of the file. + * This is only necessary for functionality like determineGltfFileType. However, reset() + * will be called at the beginning of every parse process. */ virtual void reset() = 0; @@ -707,6 +718,7 @@ namespace fastgltf { MappedGltfFile(MappedGltfFile&& other) noexcept; MappedGltfFile& operator=(MappedGltfFile&& other) noexcept; + /** Memory maps a file. If this fails, you can check std::strerror for a more exact error. */ static Expected FromPath(const std::filesystem::path& path) noexcept { MappedGltfFile buffer(path); if (buffer.error != fastgltf::Error::None) { diff --git a/src/fastgltf.cpp b/src/fastgltf.cpp index 1076bef76..c64c46c39 100644 --- a/src/fastgltf.cpp +++ b/src/fastgltf.cpp @@ -3824,7 +3824,7 @@ fg::MappedGltfFile::MappedGltfFile(const fs::path& path) noexcept { } // Get the file size - struct stat statInfo; + struct stat statInfo {}; if (fstat(fd, &statInfo) != 0) { error = Error::InvalidPath; return; @@ -3840,7 +3840,6 @@ fg::MappedGltfFile::MappedGltfFile(const fs::path& path) noexcept { if (mappedFile != MAP_FAILED) { fileSize = static_cast(statInfo.st_size); } else { - std::perror("Mapping file"); error = Error::FileBufferAllocationFailed; } @@ -3849,7 +3848,7 @@ fg::MappedGltfFile::MappedGltfFile(const fs::path& path) noexcept { } fg::MappedGltfFile::MappedGltfFile(fastgltf::MappedGltfFile &&other) noexcept { - // Make sure that munmap is never called when other gets destructud. + // Make sure that munmap is never called when other gets destructed. mappedFile = std::exchange(other.mappedFile, MAP_FAILED); fileSize = other.fileSize; idx = other.idx; @@ -4043,11 +4042,12 @@ fg::Expected fg::Parser::loadGltfJson(GltfDataGetter& data, fs::path } #endif + data.reset(); auto jsonSpan = data.read(data.totalSize(), SIMDJSON_PADDING); - simdjson::padded_string_view view(reinterpret_cast(jsonSpan.data()), + padded_string_view view(reinterpret_cast(jsonSpan.data()), data.totalSize(), data.totalSize() + SIMDJSON_PADDING); - simdjson::dom::object root; + dom::object root; if (auto error = jsonParser->parse(view).get(root); error != SUCCESS) FASTGLTF_UNLIKELY { return Error::InvalidJson; } @@ -4066,6 +4066,8 @@ fg::Expected fg::Parser::loadGltfBinary(GltfDataGetter& data, fs::pat return Error::InvalidPath; } + data.reset(); + BinaryGltfHeader header = {}; data.read(&header, sizeof header); if (header.magic != binaryGltfHeaderMagic) { @@ -4108,34 +4110,26 @@ fg::Expected fg::Parser::loadGltfBinary(GltfDataGetter& data, fs::pat return Error::InvalidGLB; } + // TODO: Somehow allow skipping the binary part in the future? if (binaryChunk.chunkLength != 0) { - if (hasBit(options, Options::LoadGLBBuffers)) { - if (config.mapCallback != nullptr) { - auto info = config.mapCallback(binaryChunk.chunkLength, config.userPointer); - if (info.mappedMemory != nullptr) { - data.read(info.mappedMemory, binaryChunk.chunkLength); - if (config.unmapCallback != nullptr) { - config.unmapCallback(&info, config.userPointer); - } - glbBuffer = sources::CustomBuffer{info.customId, MimeType::None}; + if (config.mapCallback != nullptr) { + auto info = config.mapCallback(binaryChunk.chunkLength, config.userPointer); + if (info.mappedMemory != nullptr) { + data.read(info.mappedMemory, binaryChunk.chunkLength); + if (config.unmapCallback != nullptr) { + config.unmapCallback(&info, config.userPointer); } - } else { - StaticVector binaryData(binaryChunk.chunkLength); - data.read(binaryData.data(), binaryChunk.chunkLength); - - sources::Array vectorData = { - std::move(binaryData), - MimeType::GltfBuffer, - }; - glbBuffer = std::move(vectorData); + glbBuffer = sources::CustomBuffer{info.customId, MimeType::None}; } } else { - // TODO: What to do with this? - //const span glbBytes(reinterpret_cast(buffer->bufferPointer + offset), binaryChunk.chunkLength); - sources::ByteView glbByteView = {}; - //glbByteView.bytes = glbBytes; - glbByteView.mimeType = MimeType::GltfBuffer; - glbBuffer = glbByteView; + StaticVector binaryData(binaryChunk.chunkLength); + data.read(binaryData.data(), binaryChunk.chunkLength); + + sources::Array vectorData = { + std::move(binaryData), + MimeType::GltfBuffer, + }; + glbBuffer = std::move(vectorData); } } } diff --git a/tests/glb_tests.cpp b/tests/glb_tests.cpp index 69b186eb7..ced72e407 100644 --- a/tests/glb_tests.cpp +++ b/tests/glb_tests.cpp @@ -20,11 +20,9 @@ TEST_CASE("Load basic GLB file", "[gltf-loader]") { REQUIRE(asset->buffers.size() == 1); auto& buffer = asset->buffers.front(); - auto* bufferView = std::get_if(&buffer.data); - REQUIRE(bufferView != nullptr); - auto jsonSpan = fastgltf::span(jsonData.get()); - REQUIRE(bufferView->bytes.data() - jsonSpan.data() == 1016); - REQUIRE(jsonSpan.size() == 1664); + auto* array = std::get_if(&buffer.data); + REQUIRE(array != nullptr); + REQUIRE(array->bytes.size() == 1664 - 1016); } SECTION("Load basic Box.glb and load buffers") { From 3119b68c7ee7e4c4f0e4ccf1fb5571b7f5950100 Mon Sep 17 00:00:00 2001 From: sean Date: Thu, 25 Apr 2024 11:47:21 +0200 Subject: [PATCH 5/9] Change: Update documentation on GltfDataBuffer changes --- docs/api.rst | 23 ++++++++++++++++++--- docs/guides.rst | 49 ++++++++++++++++++++++++++++++++++++++++++--- docs/overview.rst | 27 ++++++++++++++++--------- tests/glb_tests.cpp | 4 ++-- 4 files changed, 85 insertions(+), 18 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 90836a463..53f676597 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -283,25 +283,41 @@ Expected :undoc-members: -GltfDataBuffer +GltfDataGetter -------------- .. doxygenfunction:: fastgltf::getGltfBufferPadding +.. doxygenclass:: fastgltf::GltfDataGetter + :members: + :undoc-members: + + +GltfDataBuffer +-------------- + .. doxygenclass:: fastgltf::GltfDataBuffer :members: :undoc-members: +GltfFileStream +-------------- + +.. doxygenclass:: fastgltf::GltfFileStream + :members: + :undoc-members: + + Parser ------ +.. doxygenfunction:: fastgltf::determineGltfFileType + .. doxygenclass:: fastgltf::Parser :members: :undoc-members: -.. doxygenfunction:: fastgltf::determineGltfFileType - .. doxygenstruct:: fastgltf::BufferInfo :members: @@ -317,6 +333,7 @@ Exporter :members: :undoc-members: + Utility ======= diff --git a/docs/guides.rst b/docs/guides.rst index 1e72ee5c5..3cb6a6d7e 100644 --- a/docs/guides.rst +++ b/docs/guides.rst @@ -4,6 +4,46 @@ Guides .. contents:: Table of Contents +How to load glTF files for parsing +================================== + +Before you can have the ``fastgltf::Parser`` object parse the glTF, you need to load it into memory. +The ``loadGltf`` functions all take a reference to a ``fastgltf::GltfDataGetter``, which defines a common interface for reading memory. +**fastgltf** provides some implementations for this interface natively, which can be used out of the box. +Most notably, ``fastgltf::GltfDataBuffer`` and ``fastgltf::GltfFileStream``. + +GltfDataBuffer +-------------- + +This basic class essentially holds a buffer with the contents of a glTF. +This buffer can be created and filled using the factory constructors. +These constructors return an ``Expected`` which holds the buffer, as well as an error, if one occurred. +Be sure to always check if any of these functions returned an error. + +.. code:: c++ + + auto gltfFile = fastgltf::GltfDataBuffer::FromPath("./asset.gltf"); + auto gltfData = fastgltf::GltfDataBuffer::FromBytes(bytes.data(), bytes.size()); + +GltfFileStream +-------------- + +``GltfFileStream`` is a basic wrapper around ``std::ifstream``, implementing the ``fastgltf::GltfDataGetter`` interface. +The usage of this class is essentially just a cut down version of the stdlib class. + +.. code:: c++ + + fastgltf::GltfFileStream fileStream("./asset.gltf"); + if (!fileStream.isOpen()) + return false; + +AndroidGltfDataBuffer +--------------------- + +This is essentially the same interface as ``fastgltf::GltfDataBuffer``, but additionally supports loading APK assets. +See :ref:`this section ` for more information. + + How to read glTF extras ======================= @@ -76,6 +116,7 @@ Additionally, ``fastgltf::Exporter`` also supports writing extras: exporter.setExtrasWriteCallback(extrasWriteCallback); auto exported = exporter.writeGltfJson(asset, fastgltf::ExportOptions::None); +.. _android-guide: How to use fastgltf on Android ============================== @@ -94,12 +135,14 @@ The glTF file itself, however, needs to be loaded using a special function: .. code:: c++ - fastgltf::AndroidGltfDataBuffer jsonData; - jsonData.loadFromAndroidAsset(filePath); + auto jsonData = fastgltf::AndroidGltfDataBuffer::FromAsset(filePath); + if (jsonData.error() != fastgltf::Error::None) + return false; .. note:: - Always check the return value from functions related to ``fastgltf::GltfDataBuffer``. + Always check the return value from the factory functions from classes inheriting ``fastgltf::GltfDataGetter``, + since they return an Expected which could possibly contain an error. How to load data from accessors diff --git a/docs/overview.rst b/docs/overview.rst index 827ebf5a9..ff00553ba 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -121,24 +121,29 @@ The following snippet illustrates how to use fastgltf to load a glTF file. #include #include - void load(std::filesystem::path path) { + bool load(std::filesystem::path path) { // Creates a Parser instance. Optimally, you should reuse this across loads, but don't use it // across threads. To enable extensions, you have to pass them into the parser's constructor. fastgltf::Parser parser; - // The GltfDataBuffer class is designed for re-usability of the same JSON string. It contains - // utility functions to load data from a std::filesystem::path, copy from an existing buffer, - // or re-use an already existing allocation. Note that it has to outlive the process of every - // parsing function you call. - fastgltf::GltfDataBuffer data; - data.loadFromFile(path); + // The GltfDataBuffer class contains static factories which create a buffer for holding the + // glTF data. These return Expected, which can be checked if an error occurs. + // The parser accepts any subtype of GltfDataGetter, which defines an interface for reading + // chunks of the glTF file for the Parser to handle. fastgltf provides a few predefined classes + // which inherit from GltfDataGetter, so choose whichever fits your usecase the best. + auto data = fastgltf::GltfDataBuffer::FromPath(path); + if (data.error() != fastgltf::Error::None) { + // The file couldn't be loaded, or the buffer could not be allocated. + return false; + } // This loads the glTF file into the gltf object and parses the JSON. // It automatically detects whether this is a JSON-based or binary glTF. // If you know the type, you can also use loadGltfJson or loadGltfBinary. - auto asset = parser.loadGltf(&data, path.parent_path(), fastgltf::Options::None); + auto asset = parser.loadGltf(data.get(), path.parent_path(), fastgltf::Options::None); if (auto error = asset.error(); error != fastgltf::Error::None) { // Some error occurred while reading the buffer, parsing the JSON, or validating the data. + return false; } // The glTF 2.0 asset is now ready to be used. Simply call asset.get(), asset.get_if() or @@ -153,12 +158,14 @@ The following snippet illustrates how to use fastgltf to load a glTF file. // recommend it in a development environment or when debugging to avoid mishaps. // fastgltf::validate(asset.get()); + + return true; } All the nodes, meshes, buffers, textures, ... can now be accessed through the ``fastgltf::Asset`` type. -References in between objects are done with a single ``size_t``, -which is used to index into the various vectors in the asset. +References in between objects are done with a single ``std::size_t``, which is used to index into the +various vectors in the asset. .. _examples: diff --git a/tests/glb_tests.cpp b/tests/glb_tests.cpp index ced72e407..79b5a2a89 100644 --- a/tests/glb_tests.cpp +++ b/tests/glb_tests.cpp @@ -26,7 +26,7 @@ TEST_CASE("Load basic GLB file", "[gltf-loader]") { } SECTION("Load basic Box.glb and load buffers") { - auto asset = parser.loadGltfBinary(jsonData.get(), folder, fastgltf::Options::LoadGLBBuffers, fastgltf::Category::Buffers); + auto asset = parser.loadGltfBinary(jsonData.get(), folder, fastgltf::Options::None, fastgltf::Category::Buffers); REQUIRE(asset.error() == fastgltf::Error::None); REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); @@ -50,7 +50,7 @@ TEST_CASE("Load basic GLB file", "[gltf-loader]") { reinterpret_cast(bytes.data()), length); REQUIRE(byteBuffer.error() == fastgltf::Error::None); - auto asset = parser.loadGltfBinary(byteBuffer.get(), folder, fastgltf::Options::LoadGLBBuffers, fastgltf::Category::Buffers); + auto asset = parser.loadGltfBinary(byteBuffer.get(), folder, fastgltf::Options::None, fastgltf::Category::Buffers); REQUIRE(asset.error() == fastgltf::Error::None); } } From 154340ae308276d46816230c95697faa9d7ab8a6 Mon Sep 17 00:00:00 2001 From: sean Date: Sat, 27 Apr 2024 15:18:25 +0200 Subject: [PATCH 6/9] Fix: Remove all usages of getGltfBufferPadding, virtual destructors --- docs/api.rst | 2 -- examples/gl_viewer/gl_viewer.cpp | 5 ++++- include/fastgltf/core.hpp | 10 ++++++---- tests/benchmarks.cpp | 4 ---- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 53f676597..cb03a661c 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -286,8 +286,6 @@ Expected GltfDataGetter -------------- -.. doxygenfunction:: fastgltf::getGltfBufferPadding - .. doxygenclass:: fastgltf::GltfDataGetter :members: :undoc-members: diff --git a/examples/gl_viewer/gl_viewer.cpp b/examples/gl_viewer/gl_viewer.cpp index bb2bce9d9..3026ef392 100644 --- a/examples/gl_viewer/gl_viewer.cpp +++ b/examples/gl_viewer/gl_viewer.cpp @@ -354,7 +354,10 @@ bool loadGltf(Viewer* viewer, std::filesystem::path path) { fastgltf::Options::GenerateMeshIndices; fastgltf::GltfFileStream fileStream(path); - REQUIRE(fileStream.isOpen()); + if (!fileStream.isOpen()) { + std::cerr << "Failed to open glTF file" << '\n'; + return false; + } auto asset = parser.loadGltf(fileStream, path.parent_path(), gltfOptions); if (asset.error() != fastgltf::Error::None) { diff --git a/include/fastgltf/core.hpp b/include/fastgltf/core.hpp index 411dc8c54..65681a91c 100644 --- a/include/fastgltf/core.hpp +++ b/include/fastgltf/core.hpp @@ -245,7 +245,7 @@ namespace fastgltf { * a byte offset and length into the GLB file, which can be useful when using APIs like * DirectStorage or Metal IO. */ - LoadGLBBuffers [[deprecated]] = 1 << 3, + LoadGLBBuffers [[deprecated("This is now default behaviour")]] = 1 << 3, /** * Loads all external buffers into CPU memory. If disabled, fastgltf will only provide @@ -607,6 +607,8 @@ namespace fastgltf { */ class GltfDataGetter { public: + virtual ~GltfDataGetter() noexcept = default; + /** * The read functions expect the implementation to store an offset from the start * of the buffer/file to the current position. The parse process will always linearly @@ -656,7 +658,7 @@ namespace fastgltf { GltfDataBuffer& operator=(const GltfDataBuffer& other) = delete; GltfDataBuffer(GltfDataBuffer&& other) noexcept = default; GltfDataBuffer& operator=(GltfDataBuffer&& other) noexcept = default; - ~GltfDataBuffer() noexcept = default; + ~GltfDataBuffer() noexcept override = default; static Expected FromPath(const std::filesystem::path& path) noexcept { GltfDataBuffer buffer(path); @@ -753,7 +755,7 @@ namespace fastgltf { public: explicit GltfFileStream(const std::filesystem::path& path); - ~GltfFileStream() noexcept = default; + ~GltfFileStream() noexcept override = default; [[nodiscard]] bool isOpen() const; @@ -776,7 +778,7 @@ namespace fastgltf { public: explicit AndroidGltfDataBuffer() noexcept = default; - ~AndroidGltfDataBuffer() noexcept = default; + ~AndroidGltfDataBuffer() noexcept override = default; static Expected FromAsset(const std::filesystem::path& path, std::uint64_t byteOffset = 0) noexcept { AndroidGltfDataBuffer buffer(path); diff --git a/tests/benchmarks.cpp b/tests/benchmarks.cpp index c4d130735..48f856ee0 100644 --- a/tests/benchmarks.cpp +++ b/tests/benchmarks.cpp @@ -130,7 +130,6 @@ TEST_CASE("Benchmark loading of NewSponza", "[gltf-benchmark]") { #endif #ifdef HAS_GLTFRS - auto padding = fastgltf::getGltfBufferPadding(); BENCHMARK("Parse NewSponza with gltf-rs") { auto slice = rust::Slice(reinterpret_cast(bytes.data()), bytes.size()); return rust::gltf::run(slice); @@ -183,7 +182,6 @@ TEST_CASE("Benchmark base64 decoding from glTF file", "[gltf-benchmark]") { #endif #ifdef HAS_GLTFRS - auto padding = fastgltf::getGltfBufferPadding(); BENCHMARK("2CylinderEngine with gltf-rs") { auto slice = rust::Slice(reinterpret_cast(bytes.data()), bytes.size()); return rust::gltf::run(slice); @@ -237,7 +235,6 @@ TEST_CASE("Benchmark raw JSON parsing", "[gltf-benchmark]") { #endif #ifdef HAS_GLTFRS - auto padding = fastgltf::getGltfBufferPadding(); BENCHMARK("Parse Buggy.gltf with gltf-rs") { auto slice = rust::Slice(reinterpret_cast(bytes.data()), bytes.size()); return rust::gltf::run(slice); @@ -293,7 +290,6 @@ TEST_CASE("Benchmark massive gltf file", "[gltf-benchmark]") { #endif #ifdef HAS_GLTFRS - auto padding = fastgltf::getGltfBufferPadding(); BENCHMARK("Parse Bistro with gltf-rs") { auto slice = rust::Slice(reinterpret_cast(bytes.data()), bytes.size()); return rust::gltf::run(slice); From f51a0e0db49565774568490a3d3b044262a3c75d Mon Sep 17 00:00:00 2001 From: sean Date: Sat, 27 Apr 2024 15:35:23 +0200 Subject: [PATCH 7/9] Add: Memory-mapped files on Windows This also moves all functions related to reading files into a new file, io.cpp. This is mostly because fastgltf.cpp is becoming quite large, and including Windows.h caused a lot of issues there. --- CMakeLists.txt | 2 +- examples/gl_viewer/gl_viewer.cpp | 8 +- include/fastgltf/core.hpp | 18 +- src/fastgltf.cpp | 342 ----------------------- src/io.cpp | 456 +++++++++++++++++++++++++++++++ tests/basic_test.cpp | 3 +- 6 files changed, 476 insertions(+), 353 deletions(-) create mode 100644 src/io.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cb03d3357..609b876be 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,7 +30,7 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/compiler_flags.cmake) # Create the library target add_library(fastgltf - "src/fastgltf.cpp" "src/base64.cpp" + "src/fastgltf.cpp" "src/base64.cpp" "src/io.cpp" "include/fastgltf/base64.hpp" "include/fastgltf/glm_element_traits.hpp" "include/fastgltf/core.hpp" "include/fastgltf/tools.hpp" "include/fastgltf/types.hpp" "include/fastgltf/util.hpp") add_library(fastgltf::fastgltf ALIAS fastgltf) diff --git a/examples/gl_viewer/gl_viewer.cpp b/examples/gl_viewer/gl_viewer.cpp index 3026ef392..f5c09ad23 100644 --- a/examples/gl_viewer/gl_viewer.cpp +++ b/examples/gl_viewer/gl_viewer.cpp @@ -353,13 +353,13 @@ bool loadGltf(Viewer* viewer, std::filesystem::path path) { fastgltf::Options::LoadExternalImages | fastgltf::Options::GenerateMeshIndices; - fastgltf::GltfFileStream fileStream(path); - if (!fileStream.isOpen()) { - std::cerr << "Failed to open glTF file" << '\n'; + auto gltfFile = fastgltf::MappedGltfFile::FromPath(path); + if (!bool(gltfFile)) { + std::cerr << "Failed to open glTF file: " << fastgltf::getErrorMessage(gltfFile.error()) << '\n'; return false; } - auto asset = parser.loadGltf(fileStream, path.parent_path(), gltfOptions); + auto asset = parser.loadGltf(gltfFile.get(), path.parent_path(), gltfOptions); if (asset.error() != fastgltf::Error::None) { std::cerr << "Failed to load glTF: " << fastgltf::getErrorMessage(asset.error()) << '\n'; return false; diff --git a/include/fastgltf/core.hpp b/include/fastgltf/core.hpp index 65681a91c..c378eefe5 100644 --- a/include/fastgltf/core.hpp +++ b/include/fastgltf/core.hpp @@ -701,10 +701,19 @@ namespace fastgltf { } }; -#if defined(__APPLE__) || defined(__linux__) - /** Maps a glTF file into memory using mmap */ +#if defined(__APPLE__) || defined(__linux__) || defined(_WIN32) +#define FASTGLTF_HAS_MEMORY_MAPPED_FILE 1 + /** + * Memory-maps a file. This uses mmap on macOS and Linux, and MapViewOfFile on Windows, and is not available elsewhere. + * You should check for FASTGLTF_HAS_MEMORY_MAPPED_FILE before using this class. + */ class MappedGltfFile : public GltfDataGetter { - void* mappedFile = nullptr; + void* mappedFile; +#if defined(_WIN32) + // Windows requires us to keep the file handle alive. Win32 HANDLE is a void*. + void* fileHandle = nullptr; + void* fileMapping = nullptr; +#endif std::uint64_t fileSize = 0; std::size_t idx = 0; @@ -719,6 +728,7 @@ namespace fastgltf { MappedGltfFile& operator=(const MappedGltfFile& other) = delete; MappedGltfFile(MappedGltfFile&& other) noexcept; MappedGltfFile& operator=(MappedGltfFile&& other) noexcept; + ~MappedGltfFile() noexcept override; /** Memory maps a file. If this fails, you can check std::strerror for a more exact error. */ static Expected FromPath(const std::filesystem::path& path) noexcept { @@ -729,8 +739,6 @@ namespace fastgltf { return std::move(buffer); } - ~MappedGltfFile() noexcept; - void read(void* ptr, std::size_t count) override; [[nodiscard]] span read(std::size_t count, std::size_t padding) override; diff --git a/src/fastgltf.cpp b/src/fastgltf.cpp index c64c46c39..5c191bf1b 100644 --- a/src/fastgltf.cpp +++ b/src/fastgltf.cpp @@ -39,13 +39,6 @@ #include #endif -#if defined(__APPLE__) || defined(__linux__) -#include -#include -#include -#include -#endif - #ifdef _MSC_VER #pragma warning(push) #pragma warning(disable : 5030) // attribute 'x' is not recognized @@ -74,14 +67,6 @@ namespace fg = fastgltf; namespace fs = std::filesystem; namespace fastgltf { -#if defined(__ANDROID__) - /** - * Global asset manager that can be accessed freely. - * The value of this global should only be set by fastgltf::setAndroidAssetManager. - */ - static AAssetManager* androidAssetManager = nullptr; -#endif - constexpr std::uint32_t binaryGltfHeaderMagic = 0x46546C67; // ASCII for "glTF". constexpr std::uint32_t binaryGltfJsonChunkMagic = 0x4E4F534A; constexpr std::uint32_t binaryGltfDataChunkMagic = 0x004E4942; @@ -715,96 +700,6 @@ fg::Expected fg::Parser::decodeDataUri(URIView& uri) const noexc return { std::move(source) }; } -#if defined(__ANDROID__) -fg::Expected fg::Parser::loadFileFromApk(const fs::path& path) const noexcept { - auto file = deletable_unique_ptr( - AAssetManager_open(androidAssetManager, path.c_str(), AASSET_MODE_BUFFER)); - if (file == nullptr) { - return Error::MissingExternalBuffer; - } - - const auto length = AAsset_getLength(file.get()); - if (length == 0) { - return Error::MissingExternalBuffer; - } - - if (config.mapCallback != nullptr) { - auto info = config.mapCallback(static_cast(length), config.userPointer); - if (info.mappedMemory != nullptr) { - const sources::CustomBuffer customBufferSource = { info.customId, MimeType::None }; - AAsset_read(file.get(), info.mappedMemory, length); - if (config.unmapCallback != nullptr) { - config.unmapCallback(&info, config.userPointer); - } - - return { customBufferSource }; - } - } - - sources::Array arraySource { - StaticVector(length) - }; - AAsset_read(file.get(), arraySource.bytes.data(), length); - - return { std::move(arraySource) }; -} -#endif - -fg::Expected fg::Parser::loadFileFromUri(URIView& uri) const noexcept { - URI decodedUri(uri.path()); // Re-allocate so we can decode potential characters. -#if FASTGLTF_CPP_20 - // JSON strings need always be in UTF-8, so we can safely assume that the URI contains UTF-8 characters. - // This is technically UB... but I'm not sure how to do it otherwise. - std::u8string_view u8path(reinterpret_cast(decodedUri.path().data()), - decodedUri.path().size()); - auto path = directory / fs::path(u8path); -#else - auto path = directory / fs::u8path(decodedUri.path()); -#endif - -#if defined(__ANDROID__) - if (androidAssetManager != nullptr) { - // Try to load external buffers from the APK. If they're not there, fall through to the file case - if (auto androidResult = loadFileFromApk(path); androidResult.error() == Error::None) { - return std::move(androidResult.get()); - } - } -#endif - - // If we were instructed to load external buffers and the files don't exist, we'll return an error. - std::error_code error; - if (!fs::exists(path, error) || error) { - return Error::MissingExternalBuffer; - } - - auto length = static_cast(fs::file_size(path, error)); - if (error) { - return Error::InvalidURI; - } - - std::ifstream file(path, std::ios::binary); - - if (config.mapCallback != nullptr) { - auto info = config.mapCallback(static_cast(length), config.userPointer); - if (info.mappedMemory != nullptr) { - const sources::CustomBuffer customBufferSource = { info.customId }; - file.read(reinterpret_cast(info.mappedMemory), length); - if (config.unmapCallback != nullptr) { - config.unmapCallback(&info, config.userPointer); - } - - return { customBufferSource }; - } - } - - StaticVector data(static_cast(length)); - file.read(reinterpret_cast(data.data()), length); - sources::Array arraySource { - std::move(data), - }; - return { std::move(arraySource) }; -} - void fg::Parser::fillCategories(Category& inputCategories) noexcept { if (inputCategories == Category::All) return; @@ -3738,243 +3633,6 @@ fg::Error fg::Parser::parseTextures(simdjson::dom::array& textures, Asset& asset #pragma endregion -#pragma region GltfDataBuffer -fg::GltfDataBuffer::GltfDataBuffer(const fs::path& path) noexcept { - std::error_code ec; - dataSize = static_cast(fs::file_size(path, ec)); - if (ec) { - error = Error::InvalidPath; - return; - } - - // Open the file and determine the size. - std::ifstream file(path, std::ios::binary); - if (!file.is_open() || file.bad()) { - error = Error::InvalidPath; - return; - } - - allocatedSize = dataSize + simdjson::SIMDJSON_PADDING; - buffer = decltype(buffer)(new(std::nothrow) std::byte[allocatedSize]); // To mimic std::make_unique_for_overwrite (C++20) - - if (buffer != nullptr) { - // Copy the data and fill the padding region with zeros. - file.read(reinterpret_cast(buffer.get()), static_cast(dataSize)); - std::memset(buffer.get() + dataSize, 0, allocatedSize - dataSize); - } else { - error = Error::FileBufferAllocationFailed; - } -} - -fg::GltfDataBuffer::GltfDataBuffer(const std::byte *bytes, std::size_t count) noexcept { - dataSize = count; - allocateAndCopy(bytes); -} - -#if FASTGLTF_CPP_20 -fg::GltfDataBuffer::GltfDataBuffer(std::span span) noexcept { - dataSize = span.size_bytes(); - allocateAndCopy(span.data()); -} -#endif - -void fg::GltfDataBuffer::allocateAndCopy(const std::byte *bytes) noexcept { - allocatedSize = dataSize + simdjson::SIMDJSON_PADDING; - buffer = decltype(buffer)(new(std::nothrow) std::byte[allocatedSize]); - - if (buffer != nullptr) { - std::memcpy(buffer.get(), bytes, dataSize); - std::memset(buffer.get() + dataSize, 0, allocatedSize - dataSize); - } else { - error = Error::FileBufferAllocationFailed; - } -} - -void fg::GltfDataBuffer::read(void *ptr, std::size_t count) { - std::memcpy(ptr, buffer.get() + idx, count); - idx += count; -} - -fg::span fg::GltfDataBuffer::read(std::size_t count, std::size_t padding) { - span sub(buffer.get() + idx, count); - idx += count; - return sub; -} - -void fg::GltfDataBuffer::reset() { - idx = 0; -} - -std::size_t fg::GltfDataBuffer::bytesRead() { - return idx; -} - -std::size_t fg::GltfDataBuffer::totalSize() { - return dataSize; -} - -#if defined(__APPLE__) || defined(__linux__) -fg::MappedGltfFile::MappedGltfFile(const fs::path& path) noexcept { - // Open the file - int fd = open(path.c_str(), O_RDONLY, 0); - if (fd == 0) { - // TODO: Cover actual error messages using std::strerror(errno)? - error = Error::InvalidPath; - return; - } - - // Get the file size - struct stat statInfo {}; - if (fstat(fd, &statInfo) != 0) { - error = Error::InvalidPath; - return; - } - - // Map the file - mappedFile = mmap(nullptr, - statInfo.st_size, - PROT_READ, - MAP_PRIVATE, - fd, - 0); - if (mappedFile != MAP_FAILED) { - fileSize = static_cast(statInfo.st_size); - } else { - error = Error::FileBufferAllocationFailed; - } - - // The file descriptor is not used for the memory mapped and is not required anymore. - close(fd); -} - -fg::MappedGltfFile::MappedGltfFile(fastgltf::MappedGltfFile &&other) noexcept { - // Make sure that munmap is never called when other gets destructed. - mappedFile = std::exchange(other.mappedFile, MAP_FAILED); - fileSize = other.fileSize; - idx = other.idx; - error = other.error; -} - -fg::MappedGltfFile& fg::MappedGltfFile::operator=(fastgltf::MappedGltfFile &&other) noexcept { - mappedFile = std::exchange(other.mappedFile, MAP_FAILED); - fileSize = other.fileSize; - idx = other.idx; - error = other.error; - return *this; -} - -fg::MappedGltfFile::~MappedGltfFile() noexcept { - if (mappedFile != MAP_FAILED) { - munmap(mappedFile, fileSize); - } -} - -void fg::MappedGltfFile::read(void *ptr, std::size_t count) { - std::memcpy(ptr, static_cast(mappedFile) + idx, count); - idx += count; -} - -fg::span fg::MappedGltfFile::read(std::size_t count, std::size_t padding) { - span sub(static_cast(mappedFile) + idx, count); - idx += count; - return sub; -} - -void fg::MappedGltfFile::reset() { - idx = 0; -} - -std::size_t fg::MappedGltfFile::bytesRead() { - return idx; -} - -std::size_t fg::MappedGltfFile::totalSize() { - return fileSize; -} -#endif - -fg::GltfFileStream::GltfFileStream(const fs::path& path) : fileStream(path, std::ios::binary) { - fileSize = fs::file_size(path); -} - -bool fg::GltfFileStream::isOpen() const { - return fileStream.is_open(); -} - -void fg::GltfFileStream::read(void *ptr, std::size_t count) { - fileStream.read( - reinterpret_cast(ptr), - static_cast(count)); -} - -fg::span fg::GltfFileStream::read(std::size_t count, std::size_t padding) { - static_assert(sizeof(decltype(buf)::value_type) == sizeof(std::byte)); - - buf.resize(count + padding); - fileStream.read( - reinterpret_cast(buf.data()), - static_cast(count)); - - return span(reinterpret_cast(buf.data()), buf.size()); -} - -void fg::GltfFileStream::reset() { - fileStream.seekg(0, std::ifstream::beg); -} - -std::size_t fg::GltfFileStream::bytesRead() { - return fileStream.tellg(); -} - -std::size_t fg::GltfFileStream::totalSize() { - return fileSize; -} -#pragma endregion - -#pragma region AndroidGltfDataBuffer -#if defined(__ANDROID__) -void fg::setAndroidAssetManager(AAssetManager* assetManager) noexcept { - androidAssetManager = assetManager; -} - -fg::AndroidGltfDataBuffer::AndroidGltfDataBuffer(const fs::path& path, std::uint64_t byteOffset) noexcept { - if (androidAssetManager == nullptr) { - error = Error::InvalidPath; - return; - } - - const auto filenameString = path.string(); - auto file = deletable_unique_ptr( - AAssetManager_open(androidAssetManager, filenameString.c_str(), AASSET_MODE_BUFFER)); - if (file == nullptr) { - error = Error::InvalidPath; - return; - } - - const auto length = AAsset_getLength(file.get()); - if (length == 0) { - error = Error::InvalidPath; - return; - } - - dataSize = length - byteOffset; - allocatedSize = dataSize + simdjson::SIMDJSON_PADDING; - buffer = decltype(buffer)(new(std::nothrow) std::byte[allocatedSize]); - - if (buffer == nullptr) { - error = Error::FileBufferAllocationFailed; - } else { - if (byteOffset > 0) - AAsset_seek64(file.get(), byteOffset, SEEK_SET); - - // Copy the data and fill the padding region with zeros. - AAsset_read(file.get(), buffer.get(), dataSize); - std::memset(buffer.get() + dataSize, 0, allocatedSize - dataSize); - } -} -#endif -#pragma endregion - #pragma region Parser fastgltf::GltfType fg::determineGltfFileType(GltfDataGetter& data) { // We'll try and read a BinaryGltfHeader from the buffer to see if the magic is correct. diff --git a/src/io.cpp b/src/io.cpp new file mode 100644 index 000000000..03051a0c8 --- /dev/null +++ b/src/io.cpp @@ -0,0 +1,456 @@ +/* + * Copyright (C) 2022 - 2024 spnda + * This file is part of fastgltf . + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include + +#include + +#if defined(__APPLE__) || defined(__linux__) +#include +#include +#include +#include +#elif defined(_WIN32) +#include +#endif + +namespace fs = std::filesystem; +namespace fg = fastgltf; + +#pragma region glTF file loading +fg::GltfDataBuffer::GltfDataBuffer(const fs::path& path) noexcept { + std::error_code ec; + dataSize = static_cast(fs::file_size(path, ec)); + if (ec) { + error = Error::InvalidPath; + return; + } + + // Open the file and determine the size. + std::ifstream file(path, std::ios::binary); + if (!file.is_open() || file.bad()) { + error = Error::InvalidPath; + return; + } + + allocatedSize = dataSize + simdjson::SIMDJSON_PADDING; + buffer = decltype(buffer)(new(std::nothrow) std::byte[allocatedSize]); // To mimic std::make_unique_for_overwrite (C++20) + + if (buffer != nullptr) { + // Copy the data and fill the padding region with zeros. + file.read(reinterpret_cast(buffer.get()), static_cast(dataSize)); + std::memset(buffer.get() + dataSize, 0, allocatedSize - dataSize); + } else { + error = Error::FileBufferAllocationFailed; + } +} + +fg::GltfDataBuffer::GltfDataBuffer(const std::byte *bytes, std::size_t count) noexcept { + dataSize = count; + allocateAndCopy(bytes); +} + +#if FASTGLTF_CPP_20 +fg::GltfDataBuffer::GltfDataBuffer(std::span span) noexcept { + dataSize = span.size_bytes(); + allocateAndCopy(span.data()); +} +#endif + +void fg::GltfDataBuffer::allocateAndCopy(const std::byte *bytes) noexcept { + allocatedSize = dataSize + simdjson::SIMDJSON_PADDING; + buffer = decltype(buffer)(new(std::nothrow) std::byte[allocatedSize]); + + if (buffer != nullptr) { + std::memcpy(buffer.get(), bytes, dataSize); + std::memset(buffer.get() + dataSize, 0, allocatedSize - dataSize); + } else { + error = Error::FileBufferAllocationFailed; + } +} + +void fg::GltfDataBuffer::read(void *ptr, std::size_t count) { + std::memcpy(ptr, buffer.get() + idx, count); + idx += count; +} + +fg::span fg::GltfDataBuffer::read(std::size_t count, std::size_t padding) { + span sub(buffer.get() + idx, count); + idx += count; + return sub; +} + +void fg::GltfDataBuffer::reset() { + idx = 0; +} + +std::size_t fg::GltfDataBuffer::bytesRead() { + return idx; +} + +std::size_t fg::GltfDataBuffer::totalSize() { + return dataSize; +} + +fg::GltfFileStream::GltfFileStream(const fs::path& path) : fileStream(path, std::ios::binary) { + fileSize = fs::file_size(path); +} + +bool fg::GltfFileStream::isOpen() const { + return fileStream.is_open(); +} + +void fg::GltfFileStream::read(void *ptr, std::size_t count) { + fileStream.read( + reinterpret_cast(ptr), + static_cast(count)); +} + +fg::span fg::GltfFileStream::read(std::size_t count, std::size_t padding) { + static_assert(sizeof(decltype(buf)::value_type) == sizeof(std::byte)); + + buf.resize(count + padding); + fileStream.read( + reinterpret_cast(buf.data()), + static_cast(count)); + + return span(reinterpret_cast(buf.data()), buf.size()); +} + +void fg::GltfFileStream::reset() { + fileStream.seekg(0, std::ifstream::beg); +} + +std::size_t fg::GltfFileStream::bytesRead() { + return fileStream.tellg(); +} + +std::size_t fg::GltfFileStream::totalSize() { + return fileSize; +} + +#if defined(FASTGLTF_HAS_MEMORY_MAPPED_FILE) +#if defined(_WIN32) +fg::MappedGltfFile::MappedGltfFile(const fs::path& path) noexcept : mappedFile(nullptr) { + // Create file with FILE_FLAG_SEQUENTIAL_SCAN flag, to match the Parser behaviour. + auto* file = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, + OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, nullptr); + if (file == nullptr) { + error = Error::InvalidPath; + return; + } + fileHandle = file; + + LARGE_INTEGER result; + if (GetFileSizeEx(fileHandle, &result) == FALSE) { + error = Error::InvalidPath; + return; + } + fileSize = static_cast(result.QuadPart);; + + // Create the file mapping + auto* mapping = CreateFileMapping(fileHandle, nullptr, PAGE_READONLY, + 0, 0, nullptr); + if (mapping == nullptr) { + error = Error::FileBufferAllocationFailed; + return; + } + fileMapping = mapping; + + // Map the view + auto* map = MapViewOfFile(fileMapping, FILE_MAP_READ, + 0, 0, fileSize); + if (map == nullptr) { + error = Error::FileBufferAllocationFailed; + return; + } + mappedFile = map; +#else +fg::MappedGltfFile::MappedGltfFile(const fs::path& path) noexcept : mappedFile(MAP_FAILED) { + // Open the file + int fd = open(path.c_str(), O_RDONLY, 0); + if (fd == 0) { + // TODO: Cover actual error messages using std::strerror(errno)? + error = Error::InvalidPath; + return; + } + + // Get the file size + struct stat statInfo {}; + if (fstat(fd, &statInfo) != 0) { + error = Error::InvalidPath; + return; + } + + // Map the file + mappedFile = mmap(nullptr, + statInfo.st_size, + PROT_READ, + MAP_PRIVATE, + fd, + 0); + if (mappedFile != MAP_FAILED) { + fileSize = static_cast(statInfo.st_size); + + // Hint about map access + madvise(mappedFile, fileSize, MADV_SEQUENTIAL); + } else { + error = Error::FileBufferAllocationFailed; + } + + // The file descriptor is not used for the memory mapped and is not required anymore. + close(fd); +#endif +} + +fg::MappedGltfFile::MappedGltfFile(fastgltf::MappedGltfFile &&other) noexcept { +#if defined(_WIN32) + mappedFile = std::exchange(other.mappedFile, nullptr); + fileMapping = std::exchange(other.fileMapping, nullptr); + fileHandle = std::exchange(other.fileHandle, nullptr); +#else + // Make sure that munmap is never called when other gets destructed. + mappedFile = std::exchange(other.mappedFile, MAP_FAILED); +#endif + fileSize = other.fileSize; + idx = other.idx; + error = other.error; +} + +fg::MappedGltfFile& fg::MappedGltfFile::operator=(fastgltf::MappedGltfFile &&other) noexcept { +#if defined(_WIN32) + if (mappedFile != nullptr) { + UnmapViewOfFile(mappedFile); + } + mappedFile = std::exchange(other.mappedFile, nullptr); + + if (fileMapping != nullptr) { + CloseHandle(fileMapping); + } + fileMapping = std::exchange(other.fileMapping, nullptr); + + if (fileHandle != nullptr) { + CloseHandle(fileHandle); + } + fileHandle = std::exchange(other.fileHandle, nullptr); +#else + if (mappedFile != MAP_FAILED) { + munmap(mappedFile, fileSize); + } + mappedFile = std::exchange(other.mappedFile, MAP_FAILED); +#endif + fileSize = other.fileSize; + idx = other.idx; + error = other.error; + return *this; +} + +fg::MappedGltfFile::~MappedGltfFile() noexcept { +#if defined(_WIN32) + if (mappedFile != nullptr) { + UnmapViewOfFile(mappedFile); + } + if (fileMapping != nullptr) { + CloseHandle(fileMapping); + } + if (fileHandle != nullptr) { + CloseHandle(fileHandle); + } +#else + if (mappedFile != MAP_FAILED) { + munmap(mappedFile, fileSize); + } +#endif +} + +void fg::MappedGltfFile::read(void *ptr, std::size_t count) { + std::memcpy(ptr, static_cast(mappedFile) + idx, count); + idx += count; +} + +fg::span fg::MappedGltfFile::read(std::size_t count, std::size_t padding) { + span sub(static_cast(mappedFile) + idx, count); + idx += count; + return sub; +} + +void fg::MappedGltfFile::reset() { + idx = 0; +} + +std::size_t fg::MappedGltfFile::bytesRead() { + return idx; +} + +std::size_t fg::MappedGltfFile::totalSize() { + return fileSize; +} +#endif // FASTGLTF_HAS_MEMORY_MAPPED_FILE + +#pragma region AndroidGltfDataBuffer +#if defined(__ANDROID__) +namespace fastgltf { + /** + * Global asset manager that can be accessed freely. + * The value of this global should only be set by fastgltf::setAndroidAssetManager. + */ + static AAssetManager* androidAssetManager = nullptr; +} + +void fg::setAndroidAssetManager(AAssetManager* assetManager) noexcept { + androidAssetManager = assetManager; +} + +fg::AndroidGltfDataBuffer::AndroidGltfDataBuffer(const fs::path& path, std::uint64_t byteOffset) noexcept { + if (androidAssetManager == nullptr) { + error = Error::InvalidPath; + return; + } + + const auto filenameString = path.string(); + auto file = deletable_unique_ptr( + AAssetManager_open(androidAssetManager, filenameString.c_str(), AASSET_MODE_BUFFER)); + if (file == nullptr) { + error = Error::InvalidPath; + return; + } + + const auto length = AAsset_getLength(file.get()); + if (length == 0) { + error = Error::InvalidPath; + return; + } + + dataSize = length - byteOffset; + allocatedSize = dataSize + simdjson::SIMDJSON_PADDING; + buffer = decltype(buffer)(new(std::nothrow) std::byte[allocatedSize]); + + if (buffer == nullptr) { + error = Error::FileBufferAllocationFailed; + } else { + if (byteOffset > 0) + AAsset_seek64(file.get(), byteOffset, SEEK_SET); + + // Copy the data and fill the padding region with zeros. + AAsset_read(file.get(), buffer.get(), dataSize); + std::memset(buffer.get() + dataSize, 0, allocatedSize - dataSize); + } +} +#endif +#pragma endregion +#pragma endregion + +#pragma region Parser I/O +#if defined(__ANDROID__) +fg::Expected fg::Parser::loadFileFromApk(const fs::path& path) const noexcept { + auto file = deletable_unique_ptr( + AAssetManager_open(androidAssetManager, path.c_str(), AASSET_MODE_BUFFER)); + if (file == nullptr) { + return Error::MissingExternalBuffer; + } + + const auto length = AAsset_getLength(file.get()); + if (length == 0) { + return Error::MissingExternalBuffer; + } + + if (config.mapCallback != nullptr) { + auto info = config.mapCallback(static_cast(length), config.userPointer); + if (info.mappedMemory != nullptr) { + const sources::CustomBuffer customBufferSource = { info.customId, MimeType::None }; + AAsset_read(file.get(), info.mappedMemory, length); + if (config.unmapCallback != nullptr) { + config.unmapCallback(&info, config.userPointer); + } + + return { customBufferSource }; + } + } + + sources::Array arraySource { + StaticVector(length) + }; + AAsset_read(file.get(), arraySource.bytes.data(), length); + + return { std::move(arraySource) }; +} +#endif + +fg::Expected fg::Parser::loadFileFromUri(URIView& uri) const noexcept { + URI decodedUri(uri.path()); // Re-allocate so we can decode potential characters. +#if FASTGLTF_CPP_20 + // JSON strings need always be in UTF-8, so we can safely assume that the URI contains UTF-8 characters. + // This is technically UB... but I'm not sure how to do it otherwise. + std::u8string_view u8path(reinterpret_cast(decodedUri.path().data()), + decodedUri.path().size()); + auto path = directory / fs::path(u8path); +#else + auto path = directory / fs::u8path(decodedUri.path()); +#endif + +#if defined(__ANDROID__) + if (androidAssetManager != nullptr) { + // Try to load external buffers from the APK. If they're not there, fall through to the file case + if (auto androidResult = loadFileFromApk(path); androidResult.error() == Error::None) { + return std::move(androidResult.get()); + } + } +#endif + + // If we were instructed to load external buffers and the files don't exist, we'll return an error. + std::error_code error; + if (!fs::exists(path, error) || error) { + return Error::MissingExternalBuffer; + } + + auto length = static_cast(fs::file_size(path, error)); + if (error) { + return Error::InvalidURI; + } + + std::ifstream file(path, std::ios::binary); + + if (config.mapCallback != nullptr) { + auto info = config.mapCallback(static_cast(length), config.userPointer); + if (info.mappedMemory != nullptr) { + const sources::CustomBuffer customBufferSource = { info.customId }; + file.read(reinterpret_cast(info.mappedMemory), length); + if (config.unmapCallback != nullptr) { + config.unmapCallback(&info, config.userPointer); + } + + return { customBufferSource }; + } + } + + StaticVector data(static_cast(length)); + file.read(reinterpret_cast(data.data()), length); + sources::Array arraySource { + std::move(data), + }; + return { std::move(arraySource) }; +} +#pragma endregion diff --git a/tests/basic_test.cpp b/tests/basic_test.cpp index 8ea49e7bc..deb531b83 100644 --- a/tests/basic_test.cpp +++ b/tests/basic_test.cpp @@ -667,7 +667,7 @@ TEST_CASE("Test extras callback", "[gltf-loader]") { } } -#if defined(__APPLE__) || defined(__linux__) +#if defined(FASTGLTF_HAS_MEMORY_MAPPED_FILE) TEST_CASE("Test glTF file loading", "[gltf-loader]") { SECTION("Mapped files") { auto cubePath = sampleModels / "2.0" / "Cube" / "glTF"; @@ -679,6 +679,7 @@ TEST_CASE("Test glTF file loading", "[gltf-loader]") { fastgltf::Parser parser; auto asset = parser.loadGltfJson(mappedFile.get(), cubePath); REQUIRE(asset.error() == fastgltf::Error::None); + REQUIRE(fastgltf::validate(asset.get()) == fastgltf::Error::None); } } #endif From 046f51f61069adb20c3a6ced827f275ce6a4cbe7 Mon Sep 17 00:00:00 2001 From: sean Date: Sat, 27 Apr 2024 15:39:35 +0200 Subject: [PATCH 8/9] Fix: Include asset_manager.h Android header correctly --- src/fastgltf.cpp | 4 ---- src/io.cpp | 2 ++ 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/fastgltf.cpp b/src/fastgltf.cpp index 5c191bf1b..99b8160d5 100644 --- a/src/fastgltf.cpp +++ b/src/fastgltf.cpp @@ -35,10 +35,6 @@ #include #include -#if defined(__ANDROID__) -#include -#endif - #ifdef _MSC_VER #pragma warning(push) #pragma warning(disable : 5030) // attribute 'x' is not recognized diff --git a/src/io.cpp b/src/io.cpp index 03051a0c8..8534c34c9 100644 --- a/src/io.cpp +++ b/src/io.cpp @@ -312,6 +312,8 @@ std::size_t fg::MappedGltfFile::totalSize() { #pragma region AndroidGltfDataBuffer #if defined(__ANDROID__) +#include + namespace fastgltf { /** * Global asset manager that can be accessed freely. From 3d9165f4796ff7be4a4aa3aa680703bdd763f98b Mon Sep 17 00:00:00 2001 From: sean Date: Sat, 27 Apr 2024 15:52:37 +0200 Subject: [PATCH 9/9] Fix build issues with AndroidGltfDataBuffer --- include/fastgltf/core.hpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/include/fastgltf/core.hpp b/include/fastgltf/core.hpp index c378eefe5..abc76f480 100644 --- a/include/fastgltf/core.hpp +++ b/include/fastgltf/core.hpp @@ -782,14 +782,18 @@ namespace fastgltf { void setAndroidAssetManager(AAssetManager* assetManager) noexcept; class AndroidGltfDataBuffer : public GltfDataBuffer { - explicit AndroidGltfDataBuffer(const std::filesystem::path& path, std::uint64_t byteOffset); + explicit AndroidGltfDataBuffer(const std::filesystem::path& path, std::uint64_t byteOffset) noexcept; public: explicit AndroidGltfDataBuffer() noexcept = default; + AndroidGltfDataBuffer(const AndroidGltfDataBuffer& other) = delete; + AndroidGltfDataBuffer& operator=(const AndroidGltfDataBuffer& other) = delete; + AndroidGltfDataBuffer(AndroidGltfDataBuffer&& other) noexcept = default; + AndroidGltfDataBuffer& operator=(AndroidGltfDataBuffer&& other) noexcept = default; ~AndroidGltfDataBuffer() noexcept override = default; static Expected FromAsset(const std::filesystem::path& path, std::uint64_t byteOffset = 0) noexcept { - AndroidGltfDataBuffer buffer(path); + AndroidGltfDataBuffer buffer(path, byteOffset); if (buffer.buffer.get() == nullptr) { return buffer.error; }