Tzx/231112

CHANGELOG.md

@@ -1,3 +1,37 @@
+# 2.37.0
+
+* Speed up tile-join overzooming and make it use less memory, by not including empty child tiles in the enumeration
+
+# 2.36.0
+
+* Make tile-join distrust the source tilesets' metadata maxzoom and minzoom
+* Add a special case in --detect-longitude-wraparound not to wrap around jumps of exactly 360°
+
+# 2.35.0
+
+* Fix a bug in --detect-longitude-wraparound when there are multiple rings
+
+# 2.34.1
+
+* Further improvements to tile-join speed
+
+# 2.34.0
+
+* Improve speed of overzooming in tile-join
+
+# 2.33.0
+
+* Further reduce memory usage of --no-simplification-of-shared-nodes by calculating the list of shared nodes globally using temporary files rather than in memory for each individual tile
+* Make --no-simplification-of-shared-nodes behave for LineStrings as it does for Polygons, preventing simplification only at crossings, convergences, and divergences, not at every point along collinear segments
+
+# 2.32.1
+
+* Reduce memory usage of --no-simplification-of-shared-nodes for polygons
+
+# 2.32.0
+
+* Extend --no-simplification-of-shared-nodes to also simplify shared polygon borders consistently
+
 # 2.31.0
 
 * Fix tile-join crash when trying to join empty tilesets

CMakeLists.txt

@@ -63,6 +63,7 @@ set(SOURCES
     projection.cpp
     read_json.cpp
     serial.cpp
+    sort.cpp
     sqlite3.c
     text.cpp
     tile.cpp

README.md

@@ -492,7 +492,7 @@ the same layer, enclose them in an `all` expression so they will all be evaluate
  * `-ps` or `--no-line-simplification`: Don't simplify lines and polygons
  * `-pS` or `--simplify-only-low-zooms`: Don't simplify lines and polygons at maxzoom (but do simplify at lower zooms)
  * `--simplification-at-maximum-zoom=`_scale_: Use the specified _scale_ at maxzoom instead of the standard simplification scale (which still applies at lower zooms)
- * `-pn` or `--no-simplification-of-shared-nodes`: Don't simplify away nodes that appear in more than one feature or are used multiple times within the same feature, so that the intersection node will not be lost from intersecting roads. (This will not be effective if you also use `--coalesce`.) For polygons, the vertices where polygon rings meet will be detected, and the shared edges between the polygons will be simplified identically if possible. Use this instead of `--detect-shared-borders`.
+ * `-pn` or `--no-simplification-of-shared-nodes`: Don't simplify away nodes at which LineStrings or Polygon rings converge, diverge, or cross. (This will not be effective if you also use `--coalesce`.) In between intersection nodes, LineString segments or polygon edges will be simplified identically in each feature if possible. Use this instead of `--detect-shared-borders`.
  * `-pt` or `--no-tiny-polygon-reduction`: Don't combine the area of very small polygons into small squares that represent their combined area.
  * `-pT` or `--no-tiny-polygon-reduction-at-maximum-zoom`: Combine the area of very small polygons into small squares that represent their combined area only at zoom levels below the maximum.
  * `--tiny-polygon-size=`_size_: Use the specified _size_ for tiny polygons instead of the default 2. Anything above 6 or so will lead to visible artifacts with the default tile detail.

clip.cpp

@@ -1,6 +1,8 @@
+#include <stdlib.h>
 #include <mapbox/geometry/point.hpp>
 #include <mapbox/geometry/multi_polygon.hpp>
 #include <mapbox/geometry/wagyu/wagyu.hpp>
+#include <limits.h>
 #include "geometry.hpp"
 #include "errors.hpp"
 #include "compression.hpp"
@@ -751,11 +753,12 @@ static std::vector<std::pair<double, double>> clip_poly1(std::vector<std::pair<d
 }
 
 std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int ny,
-		     int detail, int buffer, std::set<std::string> const &keep) {
-	mvt_tile tile, outtile;
-	bool was_compressed;
+		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
+		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles) {
+	mvt_tile tile;
 
 	try {
+		bool was_compressed;
 		if (!tile.decode(s, was_compressed)) {
 			fprintf(stderr, "Couldn't parse tile %d/%u/%u\n", oz, ox, oy);
 			exit(EXIT_MVT);
@@ -765,6 +768,14 @@ std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int
 		exit(EXIT_PROTOBUF);
 	}
 
+	return overzoom(tile, oz, ox, oy, nz, nx, ny, detail, buffer, keep, do_compress, next_overzoomed_tiles);
+}
+
+std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int ny,
+		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
+		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles) {
+	mvt_tile outtile;
+
 	for (auto const &layer : tile.layers) {
 		mvt_layer outlayer = mvt_layer();
 
@@ -813,6 +824,23 @@ std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int
 
 			// Clip to output tile
 
+			long long xmin = LLONG_MAX;
+			long long ymin = LLONG_MAX;
+			long long xmax = LLONG_MIN;
+			long long ymax = LLONG_MIN;
+
+			for (auto const &g : geom) {
+				xmin = std::min(xmin, g.x);
+				ymin = std::min(ymin, g.y);
+				xmax = std::max(xmax, g.x);
+				ymax = std::max(ymax, g.y);
+			}
+
+			long long b = outtilesize * buffer / 256;
+			if (xmax < -b || ymax < -b || xmin > outtilesize + b || ymin > outtilesize + b) {
+				continue;
+			}
+
 			if (t == VT_LINE) {
 				geom = clip_lines(geom, nz, buffer);
 			} else if (t == VT_POLYGON) {
@@ -864,10 +892,37 @@ std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int
 		}
 	}
 
+	if (next_overzoomed_tiles != NULL) {
+		// will any child tiles have features in them?
+		// find out recursively from the tile we just made.
+		//
+		// (yes, we should keep them instead of remaking them
+		// later, but that first requires figuring out where to
+		// keep them.)
+
+		if (outtile.layers.size() > 0) {
+			for (size_t x = 0; x < 2; x++) {
+				for (size_t y = 0; y < 2; y++) {
+					std::string child = overzoom(outtile, nz, nx, ny,
+								     nz + 1, nx * 2 + x, ny * 2 + y,
+								     detail, buffer, keep, false, NULL);
+					if (child.size() > 0) {
+						next_overzoomed_tiles->emplace_back(nx * 2 + x, ny * 2 + y);
+					}
+				}
+			}
+		}
+	}
+
 	if (outtile.layers.size() > 0) {
 		std::string pbf = outtile.encode();
+
 		std::string compressed;
-		compress(pbf, compressed, true);
+		if (do_compress) {
+			compress(pbf, compressed, true);
+		} else {
+			compressed = pbf;
+		}
 
 		return compressed;
 	} else {

dirtiles.hpp

@@ -1,6 +1,7 @@
 #include <string>
 #include <vector>
 #include <sys/stat.h>
+#include "mbtiles.hpp"
 
 #ifndef DIRTILES_HPP
 #define DIRTILES_HPP

geometry.cpp

@@ -485,7 +485,7 @@ drawvec impose_tile_boundaries(drawvec &geom, long long extent) {
 	return out;
 }
 
-drawvec simplify_lines(drawvec &geom, int z, int detail, bool mark_tile_bounds, double simplification, size_t retain, drawvec const &shared_nodes) {
+drawvec simplify_lines(drawvec &geom, int z, int tx, int ty, int detail, bool mark_tile_bounds, double simplification, size_t retain, drawvec const &shared_nodes, struct node *shared_nodes_map, size_t nodepos) {
 	int res = 1 << (32 - detail - z);
 	long long area = 1LL << (32 - z);
 
@@ -501,10 +501,35 @@ drawvec simplify_lines(drawvec &geom, int z, int detail, bool mark_tile_bounds,
 		}
 
 		if (prevent[P_SIMPLIFY_SHARED_NODES]) {
+			// This is kind of weird, because we have two lists of shared nodes to look through:
+			// * the drawvec, which is nodes that were introduced during clipping to the tile edge,
+			//   and which are in local tile coordinates
+			// * the shared_nodes_map, which was made globally before tiling began, and which
+			//   is in global quadkey coordinates.
+			// To look through the latter, we need to offset and encode the coordinates
+			// of the feature we are simplifying.
+
 			auto pt = std::lower_bound(shared_nodes.begin(), shared_nodes.end(), geom[i]);
 			if (pt != shared_nodes.end() && *pt == geom[i]) {
 				geom[i].necessary = true;
 			}
+
+			if (nodepos > 0) {
+				// offset to global
+				draw d = geom[i];
+				if (z != 0) {
+					d.x += tx * (1LL << (32 - z));
+					d.y += ty * (1LL << (32 - z));
+				}
+
+				// to quadkey
+				struct node n;
+				n.index = encode_quadkey((unsigned) d.x, (unsigned) d.y);
+
+				if (bsearch(&n, shared_nodes_map, nodepos / sizeof(node), sizeof(node), nodecmp) != NULL) {
+					geom[i].necessary = true;
+				}
+			}
 		}
 	}
 
@@ -661,6 +686,9 @@ drawvec fix_polygon(drawvec &geom) {
 				ring = tmp;
 			}
 
+			// Now we are rotating the ring to make the first/last point
+			// one that would be unlikely to be simplified away.
+
 			// calculate centroid
 			// a + 1 < size() because point 0 is duplicated at the end
 			long long xtotal = 0;

geometry.hpp

@@ -7,6 +7,7 @@
 #include <string>
 #include <sqlite3.h>
 #include <stdio.h>
+#include <mvt.hpp>
 
 #define VT_POINT 1
 #define VT_LINE 2
@@ -48,6 +49,10 @@ struct draw {
 		}
 	}
 
+	bool operator>(draw const &s) const {
+		return s < *this;
+	}
+
 	bool operator==(draw const &s) const {
 		return y == s.y && x == s.x;
 	}
@@ -73,7 +78,7 @@ drawvec clip_lines(drawvec &geom, int z, long long buffer);
 drawvec stairstep(drawvec &geom, int z, int detail);
 bool point_within_tile(long long x, long long y, int z);
 int quick_check(long long *bbox, int z, long long buffer);
-drawvec simplify_lines(drawvec &geom, int z, int detail, bool mark_tile_bounds, double simplification, size_t retain, drawvec const &shared_nodes);
+drawvec simplify_lines(drawvec &geom, int z, int tx, int ty, int detail, bool mark_tile_bounds, double simplification, size_t retain, drawvec const &shared_nodes, struct node *shared_nodes_map, size_t nodepos);
 drawvec reorder_lines(drawvec &geom);
 drawvec fix_polygon(drawvec &geom);
 std::vector<drawvec> chop_polygon(std::vector<drawvec> &geoms);
@@ -93,7 +98,12 @@ void visvalingam(drawvec &ls, size_t start, size_t end, double threshold, size_t
 int pnpoly(const drawvec &vert, size_t start, size_t nvert, long long testx, long long testy);
 double distance_from_line(long long point_x, long long point_y, long long segA_x, long long segA_y, long long segB_x, long long segB_y);
 
+std::string overzoom(mvt_tile tile, int oz, int ox, int oy, int nz, int nx, int ny,
+		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
+		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles);
+
 std::string overzoom(std::string s, int oz, int ox, int oy, int nz, int nx, int ny,
-		     int detail, int buffer, std::set<std::string> const &keep);
+		     int detail, int buffer, std::set<std::string> const &keep, bool do_compress,
+		     std::vector<std::pair<unsigned, unsigned>> *next_overzoomed_tiles);
 
 #endif