Add directory filter callback to traverseDirectory

Evaluate exclusion rules during traversal so we can rule out
entire subtrees without traversing into them instead of enumerating
them exhaustively only to find that every single file inside them
matches some subtree exclusion pattern.

This can be a massive time reduction on large projects that mix
code- with non-code in the same directory tree. On one large
game project, it reduced qgrep update time from nearly 400s down
to 9s.

Signed-off-by: Fabian Giesen <fabian.giesen@epicgames.com>

src/fileutil.cpp

@@ -63,6 +63,11 @@ bool traverseFileNeeded(const char* name)
 	return true;
 }
 
+bool passthroughDirectoryFilter(const char* name)
+{
+	return true;
+}
+
 void joinPaths(std::string& buf, const char* lhs, const char* rhs)
 {
 	buf = lhs;

src/fileutil.hpp

@@ -6,8 +6,9 @@
 
 #include <stdio.h>
 
-bool traverseDirectory(const char* path, const std::function<void (const char* name, uint64_t mtime, uint64_t size)>& callback);
+bool traverseDirectory(const char* path, const std::function<void (const char* name, uint64_t mtime, uint64_t size)>& callback, const std::function<bool (const char* name)>& directory_filter_callback);
 bool traverseFileNeeded(const char* name);
+bool passthroughDirectoryFilter(const char* name);
 
 void createDirectory(const char* path);
 void createPath(const char* path);

src/fileutil_posix.cpp

@@ -21,7 +21,7 @@
 #include <CoreServices/CoreServices.h>
 #endif
 
-static bool traverseDirectoryRec(const char* path, const char* relpath, const std::function<void (const char* name, uint64_t mtime, uint64_t size)>& callback)
+static bool traverseDirectoryRec(const char* path, const char* relpath, const std::function<void (const char* name, uint64_t mtime, uint64_t size)>& callback, const std::function<bool (const char* name)>& directory_filter_callback)
 {
 	int fd = open(path, O_DIRECTORY);
 	DIR* dir = fdopendir(fd);
@@ -65,7 +65,8 @@ static bool traverseDirectoryRec(const char* path, const char* relpath, const st
 
 			if (type == DT_DIR)
 			{
-				traverseDirectoryRec(buf.c_str(), relbuf.c_str(), callback);
+				if (directory_filter_callback(relbuf.c_str()))
+					traverseDirectoryRec(buf.c_str(), relbuf.c_str(), callback, directory_filter_callback);
 			}
 			else if (type == DT_REG)
 			{
@@ -83,9 +84,9 @@ static bool traverseDirectoryRec(const char* path, const char* relpath, const st
 	return true;
 }
 
-bool traverseDirectory(const char* path, const std::function<void (const char* name, uint64_t mtime, uint64_t size)>& callback)
+bool traverseDirectory(const char* path, const std::function<void (const char* name, uint64_t mtime, uint64_t size)>& callback, const std::function<bool(const char* name)>& directory_filter_callback)
 {
-	return traverseDirectoryRec(path, "", callback);
+	return traverseDirectoryRec(path, "", callback, directory_filter_callback);
 }
 
 bool renameFile(const char* oldpath, const char* newpath)

src/fileutil_win.cpp

@@ -41,7 +41,7 @@ static uint64_t combine(uint32_t hi, uint32_t lo)
 	return (static_cast<uint64_t>(hi) << 32) | lo;
 }
 
-static bool traverseDirectoryRec(const wchar_t* path, const char* relpath, const std::function<void (const char* name, uint64_t mtime, uint64_t size)>& callback)
+static bool traverseDirectoryRec(const wchar_t* path, const char* relpath, const std::function<void (const char* name, uint64_t mtime, uint64_t size)>& callback, const std::function<bool (const char* name)>& directory_filter_callback)
 {
 	std::wstring query = path + std::wstring(L"/*");
 
@@ -73,7 +73,8 @@ static bool traverseDirectoryRec(const wchar_t* path, const char* relpath, const
 				buf += '/';
 				buf += data.cFileName;
 
-				traverseDirectoryRec(buf.c_str(), relbuf.c_str(), callback);
+				if (directory_filter_callback(relbuf.c_str()))
+					traverseDirectoryRec(buf.c_str(), relbuf.c_str(), callback, directory_filter_callback);
 			}
 			else
 			{
@@ -91,9 +92,9 @@ static bool traverseDirectoryRec(const wchar_t* path, const char* relpath, const
 	return true;
 }
 
-bool traverseDirectory(const char* path, const std::function<void (const char* name, uint64_t mtime, uint64_t size)>& callback)
+bool traverseDirectory(const char* path, const std::function<void (const char* name, uint64_t mtime, uint64_t size)>& callback, const std::function<bool (const char* name)>& directory_filter_callback)
 {
-	return traverseDirectoryRec(fromUtf8(path).c_str(), "", callback);
+	return traverseDirectoryRec(fromUtf8(path).c_str(), "", callback, directory_filter_callback);
 }
 
 bool renameFile(const char* oldpath, const char* newpath)

src/project.cpp

@@ -86,7 +86,7 @@ static std::vector<std::string> getProjectsByPrefix(const char* prefix)
 			{
 				result.push_back(pprefix + std::string(path, dot));
 			}
-		});
+		}, passthroughDirectoryFilter);
 	}
 
 	return result;
@@ -280,6 +280,23 @@ bool isFileAcceptable(ProjectGroup* group, const char* path)
 	return true;
 }
 
+// It can help enumeration times massively to be able to stop traversing
+// entire subtrees instead of enumerating them entirely only to reject
+// every single file inside them, so run exclusion rules on directory
+// names during traversal.
+bool isDirectoryAcceptable(ProjectGroup* group, const char* path)
+{
+	size_t length = strlen(path);
+
+	for (; group; group = group->parent)
+	{
+		if (group->exclude && group->exclude->search(path, length))
+			return false;
+	}
+
+	return true;
+}
+
 static void getProjectGroupFilesRec(Output* output, ProjectGroup* group, std::vector<FileInfo>& files)
 {
 	for (auto& path: group->files)
@@ -296,13 +313,15 @@ static void getProjectGroupFilesRec(Output* output, ProjectGroup* group, std::ve
 	{
 		std::string buf;
 
-		bool result = traverseDirectory(folder.c_str(), [&](const char* path, uint64_t mtime, uint64_t size) { 
+		bool result = traverseDirectory(folder.c_str(), [&](const char* path, uint64_t mtime, uint64_t size) {
 			if (isFileAcceptable(group, path))
 			{
 				joinPaths(buf, folder.c_str(), path);
 				files.push_back({ buf, mtime, size });
 			}
-		});
+			}, [&](const char* path) {
+				return isDirectoryAcceptable(group, path);
+			});
 
 		if (!result) output->error("Error reading folder %s\n", folder.c_str());
 	}