From a8918dcda63e1ddc090bf7113cdce85973d38526 Mon Sep 17 00:00:00 2001 From: Asher Glick Date: Sat, 18 Jan 2025 20:11:34 -0600 Subject: [PATCH 1/2] abstracting the file iteration --- xml_converter/src/file_helper.cpp | 95 +++++++++++++++++++++++++++++ xml_converter/src/file_helper.hpp | 13 ++++ xml_converter/src/string_helper.cpp | 7 +++ xml_converter/src/xml_converter.cpp | 47 +++----------- 4 files changed, 123 insertions(+), 39 deletions(-) diff --git a/xml_converter/src/file_helper.cpp b/xml_converter/src/file_helper.cpp index b4020042..b0af5335 100644 --- a/xml_converter/src/file_helper.cpp +++ b/xml_converter/src/file_helper.cpp @@ -1,7 +1,20 @@ #include "file_helper.hpp" +#include +#include + +#include +#include #include +#include +#include +#include +#include #include +#include +#include + +#include "string_helper.hpp" using namespace std; @@ -12,3 +25,85 @@ void copy_file(string path, string new_path) { ofstream outfile(new_path, ios::binary); outfile << infile.rdbuf(); } + +//////////////////////////////////////////////////////////////////////////////// +// MarkerPackFile::MarkerPackFile +// +// Constructor for the MarkerPackFile class +//////////////////////////////////////////////////////////////////////////////// +MarkerPackFile::MarkerPackFile(std::string base, std::string relative_filepath) { + this->base = base; + this->relative_filepath = relative_filepath; +} + +// Temporary compatibility function mimicing the old string filepath +// functionality until it is removed. +const string MarkerPackFile::tmp_get_path() const { + return join_file_paths(this->base, this->relative_filepath); +} + +//////////////////////////////////////////////////////////////////////////////// +// marker_pack_file_comp +// +// Helper function to compare the full file paths of marker pack files so that +// they can be sorted deterministically. +//////////////////////////////////////////////////////////////////////////////// +bool marker_pack_file_comp(const MarkerPackFile& a, const MarkerPackFile& b) { + if (a.base == b.base) { + return lowercase(a.relative_filepath) < lowercase(b.relative_filepath); + } + return lowercase(a.base) < lowercase(b.base); +} + +//////////////////////////////////////////////////////////////////////////////// +// get_files_by_suffix +// +// Gets all the marker pack files with a given `suffix` inside of a marker pack +// filepath `base` +//////////////////////////////////////////////////////////////////////////////// +vector get_files_by_suffix( + const string& base, + const string& suffix, + const string& subpath) { + vector files; + + if (!filesystem::exists(base)) { + cout << "Error: " << base << " does not exist" << endl; + return vector(); + } + // If it is a directory, call open_directory_file_to_read + else if (filesystem::is_directory(base)) { + DIR* dir = opendir(join_file_paths(base, subpath).c_str()); + struct dirent* entry; + while ((entry = readdir(dir)) != NULL) { + string filename = entry->d_name; + if (filename == ".") { + continue; + } + else if (filename == "..") { + continue; + } + + if (entry->d_type == DT_DIR) { + string new_subpath = join_file_paths(subpath, filename); + vector subfiles = get_files_by_suffix(base, suffix, new_subpath); + files.reserve(files.size() + subfiles.size()); + move(begin(subfiles), end(subfiles), back_inserter(files)); + } + else if (has_suffix(filename, suffix)) { + MarkerPackFile new_file(base, join_file_paths(subpath, filename)); + files.push_back(new_file); + } + } + closedir(dir); + } + // else if (filesystem::is_regular_file(file.base)) { + // } + + sort(files.begin(), files.end(), marker_pack_file_comp); + return files; +} + +vector get_files_by_suffix(const string& base, const string& suffix) { + return get_files_by_suffix(base, suffix, ""); +} diff --git a/xml_converter/src/file_helper.hpp b/xml_converter/src/file_helper.hpp index 5487eedb..e4eea73f 100644 --- a/xml_converter/src/file_helper.hpp +++ b/xml_converter/src/file_helper.hpp @@ -1,4 +1,17 @@ #pragma once #include +#include + +class MarkerPackFile { + public: + MarkerPackFile(std::string base, std::string relative_filepath); + + const std::string tmp_get_path() const; + + std::string base; + std::string relative_filepath; +}; void copy_file(std::string path, std::string new_path); + +std::vector get_files_by_suffix(const std::string& base, const std::string& suffix); diff --git a/xml_converter/src/string_helper.cpp b/xml_converter/src/string_helper.cpp index fc498b87..44ac6c85 100644 --- a/xml_converter/src/string_helper.cpp +++ b/xml_converter/src/string_helper.cpp @@ -292,6 +292,13 @@ bool has_suffix(std::string const& fullString, std::string const& ending) { } string join_file_paths(const string& path_a, const string& path_b) { + if (path_a.empty()) { + return path_b; + } + if (path_b.empty()) { + return path_a; + } + string output = path_a; if (!has_suffix(path_a, "/")) { output += "/"; diff --git a/xml_converter/src/xml_converter.cpp b/xml_converter/src/xml_converter.cpp index 6b9197aa..34ba9063 100644 --- a/xml_converter/src/xml_converter.cpp +++ b/xml_converter/src/xml_converter.cpp @@ -32,37 +32,6 @@ using namespace std; -bool filename_comp(string a, string b) { - return lowercase(a) < lowercase(b); -} - -// Searchs for files within a directory with a suffix and returns their relative paths. -vector get_files_by_suffix(string directory, string suffix) { - vector files; - DIR* dir = opendir(directory.c_str()); - struct dirent* entry; - while ((entry = readdir(dir)) != NULL) { - string filename = entry->d_name; - if (filename != "." && filename != "..") { - string path = join_file_paths(directory, filename); - if (entry->d_type == DT_DIR) { - vector subfiles = get_files_by_suffix(path, suffix); - // Default: markerpacks have all xml files in the first directory - for (string subfile : subfiles) { - cout << subfile << " found in subfolder" << endl; - files.push_back(join_file_paths(filename, subfile)); - } - } - else if (has_suffix(filename, suffix)) { - files.push_back(filename); - } - } - } - closedir(dir); - std::sort(files.begin(), files.end(), filename_comp); - return files; -} - map> read_taco_directory( string input_path, map* marker_categories, @@ -73,10 +42,10 @@ map> read_taco_directory( } else if (filesystem::is_directory(input_path)) { string directory_name = filesystem::path(input_path).filename(); - vector xml_files = get_files_by_suffix(input_path, ".xml"); - for (const string& path : xml_files) { - set top_level_category_names = parse_xml_file(join_file_paths(input_path, path), input_path, marker_categories, parsed_pois); - string relative_path = join_file_paths(directory_name, path); + vector xml_files = get_files_by_suffix(input_path, ".xml"); + for (const MarkerPackFile& path : xml_files) { + set top_level_category_names = parse_xml_file(path.tmp_get_path(), input_path, marker_categories, parsed_pois); + string relative_path = join_file_paths(directory_name, path.relative_filepath); for (set::iterator it = top_level_category_names.begin(); it != top_level_category_names.end(); it++) { top_level_category_file_locations[*it].push_back(relative_path); } @@ -95,10 +64,10 @@ map> read_burrito_directory( } else if (filesystem::is_directory(input_path)) { string directory_name = filesystem::path(input_path).filename(); - vector burrito_files = get_files_by_suffix(input_path, ".guildpoint"); - for (const string& path : burrito_files) { - set top_level_category_names = read_protobuf_file(join_file_paths(input_path, path), input_path, marker_categories, parsed_pois); - string relative_path = join_file_paths(directory_name, path); + vector burrito_files = get_files_by_suffix(input_path, ".guildpoint"); + for (const MarkerPackFile& path : burrito_files) { + set top_level_category_names = read_protobuf_file(path.tmp_get_path(), input_path, marker_categories, parsed_pois); + string relative_path = join_file_paths(directory_name, path.relative_filepath); for (set::iterator it = top_level_category_names.begin(); it != top_level_category_names.end(); it++) { top_level_category_file_locations[*it].push_back(relative_path); } From 2a4b3f02fc090572c994eb8a38cf43272b6d8717 Mon Sep 17 00:00:00 2001 From: Asher Glick Date: Sat, 18 Jan 2025 20:15:16 -0600 Subject: [PATCH 2/2] removing unneeded zip header --- xml_converter/src/file_helper.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/xml_converter/src/file_helper.cpp b/xml_converter/src/file_helper.cpp index b0af5335..8115fc23 100644 --- a/xml_converter/src/file_helper.cpp +++ b/xml_converter/src/file_helper.cpp @@ -1,7 +1,6 @@ #include "file_helper.hpp" #include -#include #include #include