Walk: enforce MAX_STAT during ScanDirectory()

As a quick kludge, this commit creates a copy of ScanDirectory() as a
coroutine so we can `co_await resume_stat` (we stillneed the
synchronous version for method Start()).  The coroutine copy can then
enforce `MAX_STAT` to limit the io_uring statx() concurrency; this is
necessary because cache directories with millions of cached files
would cause the kernel to consume gigabytes of memory and exceed our
memory limit.

debian/changelog

@@ -1,6 +1,6 @@
 cm4all-cash (0.5) unstable; urgency=low
 
-  * 
+  * throttle scanning huge directories (reduce memory usage)
 
  --   
 

src/Walk.cxx

@@ -138,12 +138,34 @@ IsSpecialFilename(const char *s) noexcept
 
 inline void
 Walk::ScanDirectory(WalkDirectory &directory, UniqueFileDescriptor &&fd)
+{
+	// TODO eliminate this method, use only CoScanDirectory
+
+	DirectoryReader r{std::move(fd)};
+	while (const char *name = r.Read()) {
+		if (IsSpecialFilename(name))
+			continue;
+
+		auto *item = new StatItem(*this, directory, name);
+		stat.push_back(*item);
+
+		item->Start(uring);
+	}
+}
+
+inline Co::Task<void>
+Walk::CoScanDirectory(WalkDirectory &directory, UniqueFileDescriptor &&fd)
 {
 	DirectoryReader r{std::move(fd)};
 	while (const char *name = r.Read()) {
 		if (IsSpecialFilename(name))
 			continue;
 
+		/* throttle if there are too many concurrent statx
+                   system calls */
+		while (stat.size() > MAX_STAT) [[unlikely]]
+			co_await resume_stat;
+
 		auto *item = new StatItem(*this, directory, name);
 		stat.push_back(*item);
 
@@ -159,7 +181,7 @@ try {
 		*new WalkDirectory(uring, parent, co_await Uring::CoOpen(uring, parent.fd, name.c_str(), O_PATH|O_DIRECTORY, 0)),
 	};
 
-	ScanDirectory(*directory, co_await Uring::CoOpen(uring, directory->fd, ".", O_DIRECTORY, 0));
+	co_await CoScanDirectory(*directory, co_await Uring::CoOpen(uring, directory->fd, ".", O_DIRECTORY, 0));
 } catch (...) {
 	fmt::print(stderr, "Failed to scan directory: {}\n", std::current_exception());
 }

src/Walk.hxx

@@ -75,6 +75,7 @@ private:
 		     FileTime atime, uint_least64_t size);
 
 	void ScanDirectory(WalkDirectory &directory, UniqueFileDescriptor &&fd);
+	Co::Task<void> CoScanDirectory(WalkDirectory &directory, UniqueFileDescriptor &&fd);
 
 	void OnStatCompletion(StatItem &item) noexcept;
 };