support 64 bit hashes (#27)

* use image.OverrideFormat()

* 1.6.4

* import std

* uninitialised member variable

* debug statement for 64 bit hash testing

* nopetynope

* uint32_t

* support for 64 bit hashes, but rip compute time :/

* refactor weird comments and constructors

* 1.7.0.0 because of 64 bit hashes

* only check crc64 if at least one mod with crc64 is loaded

* do some dumb shit to deal with incorrect bmps somehow (needs work)

* remove image interpretation for bmps

* create TpfConvert.exe to convert tpf files easily

* attempt

* set up vcpkg and install d3dx though that

* missing env variable?

* Win32 architecture

* update ci to tag TpfConvert.exe and d3dx9.dll as well (required to run TpfConvert)

* documentation

.github/workflows/cd.yaml

@@ -29,8 +29,20 @@ jobs:
     - name: Setup MSBuild.exe
       uses: microsoft/[email protected]
 
-    - name: Build CMake Files
-      run: cmake -S . -B build -A Win32
+    - name: Install vcpkg
+      run: |
+        git clone https://github.com/microsoft/vcpkg.git
+        .\vcpkg\bootstrap-vcpkg.bat
+        
+    - name: Set VCPKG_ROOT
+      run: |
+        echo "VCPKG_ROOT=$(Get-Location)\vcpkg" >> $env:GITHUB_ENV
+      shell: powershell
+
+    - name: Build CMake Files with vcpkg toolchain
+      run: cmake --preset=vcpkg
+      env:
+        VCPKG_ROOT: ${{ env.VCPKG_ROOT }}
 
     - name: Build binaries
       run: cmake --build build --config Release
@@ -54,7 +66,7 @@ jobs:
         mode: update
         tag_name: v${{ steps.set_version.outputs.version }}
         release_name: gMod v${{ steps.set_version.outputs.version }}
-        assets: .\bin\Release\gMod.dll
+        assets: .\bin\Release\gMod.dll,.\bin\Release\TpfConvert.exe,.\bin\Release\d3dx9_43.dll
         github_token: ${{ env.GITHUB_TOKEN }}
         replace_assets: true
         body_mrkdwn: ${{ env.Changelog }}

.github/workflows/ci.yaml

@@ -1,4 +1,4 @@
-name:  gMod CI Pipeline
+name: gMod CI Pipeline
 
 on:
   pull_request:
@@ -8,9 +8,8 @@ on:
     branches:
       - dev
   workflow_dispatch:
-
-jobs:
 
+jobs:
   build:
 
     strategy:
@@ -32,8 +31,20 @@ jobs:
     - name: Setup MSBuild.exe
       uses: microsoft/[email protected]
 
-    - name: Build CMake Files
-      run: cmake -S . -B build -A Win32
+    - name: Install vcpkg
+      run: |
+        git clone https://github.com/microsoft/vcpkg.git
+        .\vcpkg\bootstrap-vcpkg.bat
+        
+    - name: Set VCPKG_ROOT
+      run: |
+        echo "VCPKG_ROOT=$(Get-Location)\vcpkg" >> $env:GITHUB_ENV
+      shell: powershell
+
+    - name: Build CMake Files with vcpkg toolchain
+      run: cmake --preset=vcpkg
+      env:
+        VCPKG_ROOT: ${{ env.VCPKG_ROOT }}
 
     - name: Build binaries
       run: cmake --build build --config Release

.gitignore

@@ -23,6 +23,8 @@ bin/
 */Release
 */bin
 */obj
+vcpkg
+vcpkg_installed
 
 # Bloated Windows Databases
 *.sdf

CMakeLists.txt

@@ -16,9 +16,9 @@ if(${CLEAN} MATCHES "ON")
 endif()
 
 set(VERSION_MAJOR 1)
-set(VERSION_MINOR 6)
-set(VERSION_PATCH 3)
-set(VERSION_TWEAK 1)
+set(VERSION_MINOR 7)
+set(VERSION_PATCH 0)
+set(VERSION_TWEAK 0)
 
 set(VERSION_RC "${CMAKE_CURRENT_BINARY_DIR}/version.rc")
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/version.rc.in" "${VERSION_RC}" @ONLY)
@@ -75,3 +75,5 @@ target_compile_definitions(gMod PRIVATE
     DIRECT_INJECTION
     LOG_MESSAGE
     )
+
+add_subdirectory(TpfConvert)

CMakePresets.json

@@ -0,0 +1,14 @@
+{
+  "version": 2,
+  "configurePresets": [
+    {
+      "name": "vcpkg",
+      "generator": "Visual Studio 17 2022",
+	  "architecture": "Win32",
+      "binaryDir": "${sourceDir}/build",
+      "cacheVariables": {
+        "CMAKE_TOOLCHAIN_FILE": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake"
+      }
+    }
+  ]
+}

README.md

@@ -29,8 +29,17 @@ If you would like to use Reshade in combination with gMod, we recommend running
 Requirements:
 - Visual Studio 2022
 - CMake 3.16+, integrated into the Developer Powershell for VS 2022
+- vcpkg, integrated into the Developer Powershell for VS 2022
 
 Compile:
-- cmake -B build
+- cmake --preset=vcpkg
 - cmake --open build
 - compile
+
+**TpfConvert**
+Small utility to convert old .tpf files with invalid images into .zip files with working images.
+Usage:
+- put TpfConvert.exe and d3dx9.dll anywhere
+- create a folder "plugins" in that folder and put your .tpf/.zip files with invalid images there
+- run TpfConvert.exe, My_Texmod.tpf is fixed and copied into My_Texmod_.zip
+- copy My_Texmod_.zip into your GW Launcher plugins folder, delete My_Texmod.tpf from it, if it still exists

TpfConvert/CMakeLists.txt

@@ -0,0 +1,39 @@
+add_executable(TpfConvert)
+
+# Find all .ixx files in the TpfConvert folder
+file(GLOB TPF_SOURCES "src/*.ixx")
+
+# Add the .ixx files to the TpfConvert target
+target_sources(TpfConvert PRIVATE ${TPF_SOURCES})
+
+set_target_properties(TpfConvert PROPERTIES
+    CXX_STANDARD 23
+    CXX_STANDARD_REQUIRED ON
+)
+
+target_compile_definitions(TpfConvert PRIVATE
+    "NOMINMAX"
+    "_WIN32_WINNT=_WIN32_WINNT_WIN7"
+    "WIN32_LEAN_AND_MEAN"
+    "VC_EXTRALEAN"
+)
+
+if(DEFINED ENV{VCPKG_ROOT})
+    message(STATUS "VCPKG_ROOT: $ENV{VCPKG_ROOT}")
+else()
+    message(STATUS "VCPKG_ROOT is not set")
+endif()
+
+if(DEFINED CMAKE_TOOLCHAIN_FILE)
+    message(STATUS "CMAKE_TOOLCHAIN_FILE: ${CMAKE_TOOLCHAIN_FILE}")
+else()
+    message(STATUS "CMAKE_TOOLCHAIN_FILE is not set")
+endif()
+
+find_package(dxsdk-d3dx CONFIG REQUIRED)
+
+target_link_libraries(TpfConvert PRIVATE
+    libzippp
+    Microsoft::D3DX9
+    d3d9
+)

TpfConvert/src/ModfileLoader.ixx

@@ -0,0 +1,179 @@
+export module ModfileLoader;
+
+import std;
+import <libzippp.h>;
+import ModfileLoader.TpfReader;
+
+export using HashType = uint64_t;
+
+export struct TexEntry {
+    std::vector<uint8_t> data{};
+    HashType crc_hash = 0; // hash value
+    std::string ext{};
+};
+
+namespace {
+    HashType GetCrcFromFilename(const std::string& filename) {
+        const static std::regex re(R"(0x[0-9a-f]{4,16})", std::regex::optimize | std::regex::icase);
+        std::smatch match;
+        if (!std::regex_search(filename, match, re)) {
+            return 0;
+        }
+
+        uint64_t crc64_hash = 0;
+        const auto number_str = match.str();
+        try {
+            crc64_hash = std::stoull(number_str, nullptr, 16);
+        }
+        catch (const std::invalid_argument&) {
+            std::print(stderr, "Failed to parse {} as a hash\n", filename);
+            return 0;
+        }
+        catch (const std::out_of_range&) {
+            std::print(stderr, "Out of range while parsing {} as a hash\n", filename);
+            return 0;
+        }
+        return crc64_hash;
+    }
+}
+
+export class ModfileLoader {
+    std::filesystem::path file_name;
+    const std::string TPF_PASSWORD{
+        0x73, 0x2A, 0x63, 0x7D, 0x5F, 0x0A, static_cast<char>(0xA6), static_cast<char>(0xBD),
+        0x7D, 0x65, 0x7E, 0x67, 0x61, 0x2A, 0x7F, 0x7F,
+        0x74, 0x61, 0x67, 0x5B, 0x60, 0x70, 0x45, 0x74,
+        0x5C, 0x22, 0x74, 0x5D, 0x6E, 0x6A, 0x73, 0x41,
+        0x77, 0x6E, 0x46, 0x47, 0x77, 0x49, 0x0C, 0x4B,
+        0x46, 0x6F
+    };
+
+public:
+    ModfileLoader(const std::filesystem::path& fileName);
+
+    std::vector<TexEntry> GetContents() const;
+
+private:
+
+    std::vector<TexEntry> GetTpfContents() const;
+
+    std::vector<TexEntry> GetFileContents() const;
+
+    static void LoadEntries(libzippp::ZipArchive& archive, std::vector<TexEntry>& entries);
+};
+
+ModfileLoader::ModfileLoader(const std::filesystem::path& fileName)
+{
+    file_name = std::filesystem::absolute(fileName);
+}
+
+std::vector<TexEntry> ModfileLoader::GetContents() const
+{
+    try {
+        return file_name.wstring().ends_with(L".tpf") ? GetTpfContents() : GetFileContents();
+    }
+    catch (const std::exception&) {
+        std::print(stderr, "Failed to open mod file: {}\n", file_name.string().c_str());
+    }
+    return {};
+}
+
+std::vector<TexEntry> ModfileLoader::GetTpfContents() const
+{
+    std::vector<TexEntry> entries;
+    auto tpf_reader = TpfReader(file_name);
+    const auto buffer = tpf_reader.ReadToEnd();
+    const auto zip_archive = libzippp::ZipArchive::fromBuffer(buffer.data(), buffer.size(), false, TPF_PASSWORD);
+    if (!zip_archive) {
+    std::print(stderr, "Failed to open tpf file: {} - {} uint8_ts!\n", file_name.string(), buffer.size());
+        return {};
+    }
+    zip_archive->setErrorHandlerCallback(
+        [](const std::string& message, const std::string& strerror, int zip_error_code, int system_error_code) -> void {
+            std::print(stderr, "GetTpfContents: {} {} {} {}\n", message, strerror, zip_error_code, system_error_code);
+        });
+    zip_archive->open();
+    LoadEntries(*zip_archive, entries);
+    zip_archive->close();
+    libzippp::ZipArchive::free(zip_archive);
+
+    return entries;
+}
+
+std::vector<TexEntry> ModfileLoader::GetFileContents() const
+{
+    std::vector<TexEntry> entries;
+
+    libzippp::ZipArchive zip_archive(file_name.string());
+    zip_archive.open();
+    LoadEntries(zip_archive, entries);
+    zip_archive.close();
+
+    return entries;
+}
+
+void ParseSimpleArchive(const libzippp::ZipArchive& archive, std::vector<TexEntry>& entries)
+{
+    for (const auto& entry : archive.getEntries()) {
+        if (!entry.isFile())
+            continue;
+        const auto crc_hash = GetCrcFromFilename(entry.getName());
+        if (!crc_hash)
+            continue;
+        const auto data_ptr = static_cast<uint8_t*>(entry.readAsBinary());
+        const auto size = entry.getSize();
+        std::vector vec(data_ptr, data_ptr + size);
+        std::filesystem::path tex_name(entry.getName());
+        entries.emplace_back(std::move(vec), crc_hash, tex_name.extension().string());
+        delete[] data_ptr;
+    }
+}
+
+void ParseTexmodArchive(std::vector<std::string>& lines, libzippp::ZipArchive& archive, std::vector<TexEntry>& entries)
+{
+    for (const auto& line : lines) {
+        // 0xC57D73F7|GW.EXE_0xC57D73F7.tga\r\n
+        // match[1] | match[2]
+        const static auto address_file_regex = std::regex(R"(^[\\/.]*([^|]+)\|([^\r\n]+))", std::regex::optimize);
+        std::smatch match;
+        if (!std::regex_search(line, match, address_file_regex))
+            continue;
+        const auto address_string = match[1].str();
+        const auto file_path = match[2].str();
+
+        const auto crc_hash = GetCrcFromFilename(address_string);
+        if (!crc_hash)
+            continue;
+
+        const auto entry = archive.getEntry(file_path);
+        if (entry.isNull() || !entry.isFile())
+            continue;
+
+        const auto data_ptr = static_cast<uint8_t*>(entry.readAsBinary());
+        const auto size = static_cast<size_t>(entry.getSize());
+        std::vector vec(data_ptr, data_ptr + size);
+        const auto tex_name = std::filesystem::path(entry.getName());
+        entries.emplace_back(std::move(vec), crc_hash, tex_name.extension().string());
+        delete[] data_ptr;
+    }
+}
+
+void ModfileLoader::LoadEntries(libzippp::ZipArchive& archive, std::vector<TexEntry>& entries)
+{
+    const auto def_file = archive.getEntry("texmod.def");
+    if (def_file.isNull() || !def_file.isFile()) {
+        ParseSimpleArchive(archive, entries);
+    }
+    else {
+        const auto def = def_file.readAsText();
+        std::istringstream iss(def);
+        std::vector<std::string> lines;
+        std::string line;
+
+        while (std::getline(iss, line)) {
+            lines.push_back(line);
+        }
+
+        ParseTexmodArchive(lines, archive, entries);
+    }
+}