diff --git a/CMakeLists.txt b/CMakeLists.txt index a8ea43029e07..dd6eec33209f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,17 @@ set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/build/fbcode_builder/CMake" ${CMAKE_MODULE_PATH}) -set(CMAKE_CXX_STANDARD 17) +if(NOT CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + message(STATUS "setting C++ standard to C++${CMAKE_CXX_STANDARD}") +endif() + +# Explicitly enable coroutine support, since GCC does not enable it +# by default when targeting C++17. +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + add_compile_options($<$:-fcoroutines>) +endif() if (WIN32) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DWIN32_LEAN_AND_MEAN -DNOMINMAX -DSTRICT") diff --git a/build/deps/github_hashes/facebook/fbthrift-rev.txt b/build/deps/github_hashes/facebook/fbthrift-rev.txt index 0bae81d110fc..acb44cf511e6 100644 --- a/build/deps/github_hashes/facebook/fbthrift-rev.txt +++ b/build/deps/github_hashes/facebook/fbthrift-rev.txt @@ -1 +1 @@ -Subproject commit 97518207214e89d6db73da63837f4e1dc2b57acf +Subproject commit 4662bb3dd9ff5cc4324e794e29509b8acb1c8a4b diff --git a/build/deps/github_hashes/facebook/folly-rev.txt b/build/deps/github_hashes/facebook/folly-rev.txt index 1c314a7909c2..24cd2b417c39 100644 --- a/build/deps/github_hashes/facebook/folly-rev.txt +++ b/build/deps/github_hashes/facebook/folly-rev.txt @@ -1 +1 @@ -Subproject commit ab576d641d9ae77662e6e54a5db7fbe6d215fa6d +Subproject commit 819b4c4d46a7d6447584e46a7eb0731297622acf diff --git a/build/deps/github_hashes/facebook/wangle-rev.txt b/build/deps/github_hashes/facebook/wangle-rev.txt index 6c13ab940e45..05fc1ac31f3e 100644 --- a/build/deps/github_hashes/facebook/wangle-rev.txt +++ b/build/deps/github_hashes/facebook/wangle-rev.txt @@ -1 +1 @@ -Subproject commit dd5f918c13d1f4c89519cc76edec50e39c0fdc2b +Subproject commit 7beae5da9cc5293dd785dd424b5ad5f7d819c64b diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index ef81d3c3c887..03a5f3c0430d 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -333,6 +333,12 @@ def run_project_cmd(self, args, loader, manifest): cache = cache_module.create_cache() for m in projects: + fetcher = loader.create_fetcher(m) + if isinstance(fetcher, SystemPackageFetcher): + # We are guaranteed that if the fetcher is set to + # SystemPackageFetcher then this item is completely + # satisfied by the appropriate system packages + continue cached_project = CachedProject(cache, loader, m) if cached_project.download(): continue @@ -348,7 +354,6 @@ def run_project_cmd(self, args, loader, manifest): continue # We need to fetch the sources - fetcher = loader.create_fetcher(m) fetcher.update() @@ -923,6 +928,27 @@ def run_project_cmd(self, args, loader, manifest): self.create_builder(loader, manifest).debug(reconfigure=False) +@cmd( + "env", + "print the environment in a shell sourceable format", +) +class EnvCmd(ProjectCmdBase): + def setup_project_cmd_parser(self, parser): + parser.add_argument( + "--os-type", + help="Filter to just this OS type to run", + choices=["linux", "darwin", "windows"], + action="store", + dest="ostype", + default=None, + ) + + def run_project_cmd(self, args, loader, manifest): + if args.ostype: + loader.build_opts.host_type.ostype = args.ostype + self.create_builder(loader, manifest).printenv(reconfigure=False) + + @cmd("generate-github-actions", "generate a GitHub actions configuration") class GenerateGitHubActionsCmd(ProjectCmdBase): RUN_ON_ALL = """ [push, pull_request]""" @@ -995,6 +1021,8 @@ def write_job_for_platform(self, platform, args): # noqa: C901 if build_opts.is_linux(): artifacts = "linux" runs_on = f"ubuntu-{args.ubuntu_version}" + if args.cpu_cores: + runs_on = f"{args.cpu_cores}-core-ubuntu-{args.ubuntu_version}" elif build_opts.is_windows(): artifacts = "windows" runs_on = "windows-2019" @@ -1246,6 +1274,10 @@ def setup_project_cmd_parser(self, parser): parser.add_argument( "--ubuntu-version", default="22.04", help="Version of Ubuntu to use" ) + parser.add_argument( + "--cpu-cores", + help="Number of CPU cores to use (applicable for Linux OS)", + ) parser.add_argument( "--cron", help="Specify that the job runs on a cron schedule instead of on pushes", diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index 6ad9f2859a98..83916fde9472 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -15,8 +15,10 @@ import subprocess import sys import typing +from shlex import quote as shellquote from typing import Optional +from .copytree import simple_copytree from .dyndeps import create_dyn_dep_munger from .envfuncs import add_path_entry, Env, path_search from .fetcher import copy_if_different @@ -157,6 +159,29 @@ def debug(self, reconfigure: bool) -> None: shell = ["powershell.exe"] if sys.platform == "win32" else ["/bin/sh", "-i"] self._run_cmd(shell, cwd=self.build_dir, env=env) + def printenv(self, reconfigure: bool) -> None: + """print the environment in a shell sourcable format""" + reconfigure = self._reconfigure(reconfigure) + self._apply_patchfile() + self._prepare(reconfigure=reconfigure) + env = self._compute_env(env=Env(src={})) + prefix = "export " + sep = ":" + expand = "$" + expandpost = "" + if self.build_opts.is_windows(): + prefix = "SET " + sep = ";" + expand = "%" + expandpost = "%" + for k, v in sorted(env.items()): + existing = os.environ.get(k, None) + if k.endswith("PATH") and existing: + v = shellquote(v) + sep + f"{expand}{k}{expandpost}" + else: + v = shellquote(v) + print("%s%s=%s" % (prefix, k, v)) + def build(self, reconfigure: bool) -> None: print("Building %s..." % self.manifest.name) reconfigure = self._reconfigure(reconfigure) @@ -225,14 +250,16 @@ def _build(self, reconfigure) -> None: system needs to regenerate its rules.""" pass - def _compute_env(self): + def _compute_env(self, env=None) -> Env: + if env is None: + env = self.env # CMAKE_PREFIX_PATH is only respected when passed through the # environment, so we construct an appropriate path to pass down return self.build_opts.compute_env_for_install_dirs( self.loader, self.dep_manifests, self.ctx, - env=self.env, + env=env, manifest=self.manifest, ) @@ -461,6 +488,61 @@ def _build(self, reconfigure) -> None: self._run_cmd(install_cmd, env=env) +class SystemdBuilder(BuilderBase): + # SystemdBuilder assumes that meson build tool has already been installed on + # the machine. + def __init__( + self, + loader, + dep_manifests, + build_opts, + ctx, + manifest, + src_dir, + build_dir, + inst_dir, + ) -> None: + super(SystemdBuilder, self).__init__( + loader, + dep_manifests, + build_opts, + ctx, + manifest, + src_dir, + build_dir, + inst_dir, + ) + + def _build(self, reconfigure) -> None: + env = self._compute_env() + meson = path_search(env, "meson") + if meson is None: + raise Exception("Failed to find Meson") + + # Meson builds typically require setup, compile, and install steps. + # During this setup step we ensure that the static library is built and + # the prefix is empty. + self._run_cmd( + [ + meson, + "setup", + "-Dstatic-libsystemd=true", + "-Dprefix=/", + self.build_dir, + self.src_dir, + ] + ) + + # Compile step needs to satisfy the build directory that was previously + # prepared during setup. + self._run_cmd([meson, "compile", "-C", self.build_dir]) + + # Install step + self._run_cmd( + [meson, "install", "-C", self.build_dir, "--destdir", self.inst_dir] + ) + + class CMakeBuilder(BuilderBase): MANUAL_BUILD_SCRIPT = """\ #!{sys.executable} @@ -1084,9 +1166,14 @@ def _build(self, reconfigure) -> None: perl = typing.cast(str, path_search(env, "perl", "perl")) make_j_args = [] + extra_args = [] if self.build_opts.is_windows(): - make = "nmake.exe" + # jom is compatible with nmake, adds the /j argument for parallel build + make = "jom.exe" + make_j_args = ["/j%s" % self.num_jobs] args = ["VC-WIN64A-masm", "-utf-8"] + # fixes "if multiple CL.EXE write to the same .PDB file, please use /FS" + extra_args = ["/FS"] elif self.build_opts.is_darwin(): make = "make" make_j_args = ["-j%s" % self.num_jobs] @@ -1119,11 +1206,14 @@ def _build(self, reconfigure) -> None: "no-unit-test", "no-tests", ] + + extra_args ) + # show the config produced + self._run_cmd([perl, "configdata.pm", "--dump"], env=env) make_build = [make] + make_j_args - self._run_cmd(make_build) + self._run_cmd(make_build, env=env) make_install = [make, "install_sw", "install_ssldirs"] - self._run_cmd(make_install) + self._run_cmd(make_install, env=env) class Boost(BuilderBase): @@ -1240,7 +1330,7 @@ def build(self, reconfigure: bool) -> None: os.makedirs(dest_parent) if os.path.isdir(full_src): if not os.path.exists(full_dest): - shutil.copytree(full_src, full_dest) + simple_copytree(full_src, full_dest) else: shutil.copyfile(full_src, full_dest) shutil.copymode(full_src, full_dest) @@ -1252,7 +1342,7 @@ def build(self, reconfigure: bool) -> None: os.chmod(full_dest, st.st_mode | stat.S_IXUSR) else: if not os.path.exists(self.inst_dir): - shutil.copytree(self.src_dir, self.inst_dir) + simple_copytree(self.src_dir, self.inst_dir) class SqliteBuilder(BuilderBase): diff --git a/build/fbcode_builder/getdeps/buildopts.py b/build/fbcode_builder/getdeps/buildopts.py index b315c175b8ab..e7e13c22ec7d 100644 --- a/build/fbcode_builder/getdeps/buildopts.py +++ b/build/fbcode_builder/getdeps/buildopts.py @@ -304,12 +304,14 @@ def compute_env_for_install_dirs( is_direct_dep = ( manifest is not None and m.name in manifest.get_dependencies(ctx) ) - self.add_prefix_to_env( - loader.get_project_install_dir(m), - env, - append=False, - is_direct_dep=is_direct_dep, - ) + d = loader.get_project_install_dir(m) + if os.path.exists(d): + self.add_prefix_to_env( + d, + env, + append=False, + is_direct_dep=is_direct_dep, + ) # Linux is always system openssl system_openssl = self.is_linux() @@ -521,7 +523,8 @@ def find_unused_drive_letter(): return available[-1] -def create_subst_path(path: str) -> str: +def map_subst_path(path: str) -> str: + """find a short drive letter mapping for a path""" for _attempt in range(0, 24): drive = find_existing_win32_subst_for_path( path, subst_mapping=list_win32_subst_letters() @@ -542,9 +545,11 @@ def create_subst_path(path: str) -> str: # other processes on the same host, so this may not succeed. try: subprocess.check_call(["subst", "%s:" % available, path]) - return "%s:\\" % available + subst = "%s:\\" % available + print("Mapped scratch dir %s -> %s" % (path, subst), file=sys.stderr) + return subst except Exception: - print("Failed to map %s -> %s" % (available, path)) + print("Failed to map %s -> %s" % (available, path), file=sys.stderr) raise Exception("failed to set up a subst path for %s" % path) @@ -617,10 +622,7 @@ def setup_build_options(args, host_type=None) -> BuildOptions: os.makedirs(scratch_dir) if is_windows(): - subst = create_subst_path(scratch_dir) - print( - "Mapping scratch dir %s -> %s" % (scratch_dir, subst), file=sys.stderr - ) + subst = map_subst_path(scratch_dir) scratch_dir = subst else: if not os.path.exists(scratch_dir): diff --git a/build/fbcode_builder/getdeps/cargo.py b/build/fbcode_builder/getdeps/cargo.py index cae8bf54cac1..5bb2ada85c2e 100644 --- a/build/fbcode_builder/getdeps/cargo.py +++ b/build/fbcode_builder/getdeps/cargo.py @@ -9,9 +9,11 @@ import os import re import shutil +import sys import typing from .builder import BuilderBase +from .copytree import simple_copytree if typing.TYPE_CHECKING: from .buildopts import BuildOptions @@ -78,7 +80,7 @@ def recreate_dir(self, src, dst) -> None: os.remove(dst) else: shutil.rmtree(dst) - shutil.copytree(src, dst) + simple_copytree(src, dst) def cargo_config_file(self): build_source_dir = self.build_dir @@ -97,7 +99,7 @@ def _create_cargo_config(self): if os.path.isfile(cargo_config_file): with open(cargo_config_file, "r") as f: - print(f"Reading {cargo_config_file}") + print(f"Reading {cargo_config_file}", file=sys.stderr) cargo_content = f.read() else: cargo_content = "" @@ -142,7 +144,8 @@ def _create_cargo_config(self): if new_content != cargo_content: with open(cargo_config_file, "w") as f: print( - f"Writing cargo config for {self.manifest.name} to {cargo_config_file}" + f"Writing cargo config for {self.manifest.name} to {cargo_config_file}", + file=sys.stderr, ) f.write(new_content) @@ -270,7 +273,10 @@ def _patchup_workspace(self, dep_to_git) -> None: new_content += "\n".join(config) if new_content != manifest_content: with open(patch_cargo, "w") as f: - print(f"writing patch to {patch_cargo}") + print( + f"writing patch to {patch_cargo}", + file=sys.stderr, + ) f.write(new_content) def _resolve_config(self, dep_to_git) -> typing.Dict[str, typing.Dict[str, str]]: @@ -296,7 +302,8 @@ def _resolve_config(self, dep_to_git) -> typing.Dict[str, typing.Dict[str, str]] if c in crate_source_map and c not in crates_to_patch_path: crates_to_patch_path[c] = crate_source_map[c] print( - f"{self.manifest.name}: Patching crate {c} via virtual manifest in {self.workspace_dir()}" + f"{self.manifest.name}: Patching crate {c} via virtual manifest in {self.workspace_dir()}", + file=sys.stderr, ) if crates_to_patch_path: git_url_to_crates_and_paths[git_url] = crates_to_patch_path @@ -352,7 +359,8 @@ def _resolve_dep_to_git(self): subpath = subpath.replace("/", "\\") crate_path = os.path.join(dep_source_dir, subpath) print( - f"{self.manifest.name}: Mapped crate {crate} to dep {dep} dir {crate_path}" + f"{self.manifest.name}: Mapped crate {crate} to dep {dep} dir {crate_path}", + file=sys.stderr, ) crate_source_map[crate] = crate_path elif dep_cargo_conf: @@ -367,7 +375,8 @@ def _resolve_dep_to_git(self): crate = match.group(1) if crate: print( - f"{self.manifest.name}: Discovered crate {crate} in dep {dep} dir {crate_root}" + f"{self.manifest.name}: Discovered crate {crate} in dep {dep} dir {crate_root}", + file=sys.stderr, ) crate_source_map[crate] = crate_root @@ -414,7 +423,8 @@ def _resolve_dep_to_crates(self, build_source_dir, dep_to_git): for c in crates: if c not in existing_crates: print( - f"Patch {self.manifest.name} uses {dep_name} crate {crates}" + f"Patch {self.manifest.name} uses {dep_name} crate {crates}", + file=sys.stderr, ) existing_crates.add(c) dep_to_crates.setdefault(name, set()).update(existing_crates) diff --git a/build/fbcode_builder/getdeps/copytree.py b/build/fbcode_builder/getdeps/copytree.py index 2297bd3aa80a..6815f74c898e 100644 --- a/build/fbcode_builder/getdeps/copytree.py +++ b/build/fbcode_builder/getdeps/copytree.py @@ -10,6 +10,7 @@ import subprocess from .platform import is_windows +from .runcmd import run_cmd PREFETCHED_DIRS = set() @@ -65,18 +66,34 @@ def prefetch_dir_if_eden(dirpath) -> None: PREFETCHED_DIRS.add(dirpath) -# pyre-fixme[9]: ignore has type `bool`; used as `None`. -def copytree(src_dir, dest_dir, ignore: bool = None): - """Recursively copy the src_dir to the dest_dir, filtering - out entries using the ignore lambda. The behavior of the - ignore lambda must match that described by `shutil.copytree`. - This `copytree` function knows how to prefetch data when - running in an eden repo. - TODO: I'd like to either extend this or add a variant that - uses watchman to mirror src_dir into dest_dir. - """ - prefetch_dir_if_eden(src_dir) - # pyre-fixme[6]: For 3rd param expected - # `Union[typing.Callable[[Union[PathLike[str], str], List[str]], Iterable[str]], - # typing.Callable[[str, List[str]], Iterable[str]], None]` but got `bool`. - return shutil.copytree(src_dir, dest_dir, ignore=ignore) +def simple_copytree(src_dir, dest_dir, symlinks=False): + """A simple version of shutil.copytree() that can delegate to native tools if faster""" + if is_windows(): + os.makedirs(dest_dir, exist_ok=True) + cmd = [ + "robocopy.exe", + src_dir, + dest_dir, + # copy directories, including empty ones + "/E", + # Ignore Extra files in destination + "/XX", + # enable parallel copy + "/MT", + # be quiet + "/NFL", + "/NDL", + "/NJH", + "/NJS", + "/NP", + ] + if symlinks: + cmd.append("/SL") + # robocopy exits with code 1 if it copied ok, hence allow_fail + # https://learn.microsoft.com/en-us/troubleshoot/windows-server/backup-and-storage/return-codes-used-robocopy-utility + exit_code = run_cmd(cmd, allow_fail=True) + if exit_code > 1: + raise subprocess.CalledProcessError(exit_code, cmd) + return dest_dir + else: + return shutil.copytree(src_dir, dest_dir, symlinks=symlinks) diff --git a/build/fbcode_builder/getdeps/expr.py b/build/fbcode_builder/getdeps/expr.py index 0f51521d6581..3b3d2d13d8eb 100644 --- a/build/fbcode_builder/getdeps/expr.py +++ b/build/fbcode_builder/getdeps/expr.py @@ -151,8 +151,10 @@ def top(self): def ident(self) -> str: ident = self.lex.get_token() + # pyre-fixme[6]: For 2nd argument expected `str` but got `Optional[str]`. if not re.match("[a-zA-Z]+", ident): raise Exception("expected identifier found %s" % ident) + # pyre-fixme[7]: Expected `str` but got `Optional[str]`. return ident def parse_not(self) -> NotExpr: diff --git a/build/fbcode_builder/getdeps/fetcher.py b/build/fbcode_builder/getdeps/fetcher.py index 30cff5b7d99b..c769c118d1d3 100644 --- a/build/fbcode_builder/getdeps/fetcher.py +++ b/build/fbcode_builder/getdeps/fetcher.py @@ -155,6 +155,9 @@ def hash(self) -> str: def get_src_dir(self): return self.path + def clean(self) -> None: + pass + class SystemPackageFetcher(object): def __init__(self, build_options, packages) -> None: @@ -363,10 +366,8 @@ def copy_if_different(src_name, dest_name) -> bool: if exc.errno != errno.ENOENT: raise target = os.readlink(src_name) - print("Symlinking %s -> %s" % (dest_name, target)) os.symlink(target, dest_name) else: - print("Copying %s -> %s" % (src_name, dest_name)) shutil.copy2(src_name, dest_name) return True @@ -471,7 +472,7 @@ def st_dev(path): raise Exception( "%s doesn't exist; check your sparse profile!" % dir_to_mirror ) - + update_count = 0 for root, dirs, files in os.walk(dir_to_mirror): dirs[:] = [d for d in dirs if root_dev == st_dev(os.path.join(root, d))] @@ -485,6 +486,13 @@ def st_dev(path): full_file_list.add(target_name) if copy_if_different(full_name, target_name): change_status.record_change(target_name) + if update_count < 10: + print("Updated %s -> %s" % (full_name, target_name)) + elif update_count == 10: + print("...") + update_count += 1 + if update_count: + print("Updated %s for %s" % (update_count, fbsource_subdir)) # Compare the list of previously shipped files; if a file is # in the old list but not the new list then it has been @@ -865,6 +873,7 @@ def update(self) -> ChangeStatus: if not os.path.exists(self.file_name): self._download() + self._verify_hash() if tarfile.is_tarfile(self.file_name): opener = tarfile.open @@ -874,19 +883,20 @@ def update(self) -> ChangeStatus: raise Exception("don't know how to extract %s" % self.file_name) os.makedirs(self.src_dir) print("Extract %s -> %s" % (self.file_name, self.src_dir)) - t = opener(self.file_name) if is_windows(): # Ensure that we don't fall over when dealing with long paths # on windows src = r"\\?\%s" % os.path.normpath(self.src_dir) else: src = self.src_dir - # The `str` here is necessary to ensure that we don't pass a unicode - # object down to tarfile.extractall on python2. When extracting - # the boost tarball it makes some assumptions and tries to convert - # a non-ascii path to ascii and throws. - src = str(src) - t.extractall(src) + + with opener(self.file_name) as t: + # The `str` here is necessary to ensure that we don't pass a unicode + # object down to tarfile.extractall on python2. When extracting + # the boost tarball it makes some assumptions and tries to convert + # a non-ascii path to ascii and throws. + src = str(src) + t.extractall(src) with open(self.hash_file, "w") as f: f.write(self.sha256) diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index 6af5e3a74dc6..5cec33aabba7 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -8,6 +8,7 @@ import configparser import io import os +import sys from typing import List from .builder import ( @@ -21,6 +22,7 @@ NopBuilder, OpenSSLBuilder, SqliteBuilder, + SystemdBuilder, ) from .cargo import CargoBuilder from .expr import parse_expr @@ -391,7 +393,10 @@ def _is_satisfied_by_preinstalled_environment(self, ctx): return False for key in envs: val = os.environ.get(key, None) - print(f"Testing ENV[{key}]: {repr(val)}") + print( + f"Testing ENV[{key}]: {repr(val)}", + file=sys.stderr, + ) if val is None: return False if len(val) == 0: @@ -425,23 +430,27 @@ def _create_fetcher(self, build_options, ctx): # We can use the code from fbsource return ShipitTransformerFetcher(build_options, self.shipit_project) + # If both of these are None, the package can only be coming from + # preinstalled toolchain or system packages + repo_url = self.get_repo_url(ctx) + url = self.get("download", "url", ctx=ctx) + # Can we satisfy this dep with system packages? - if build_options.allow_system_packages: + if (repo_url is None and url is None) or build_options.allow_system_packages: if self._is_satisfied_by_preinstalled_environment(ctx): return PreinstalledNopFetcher() - packages = self.get_required_system_packages(ctx) - package_fetcher = SystemPackageFetcher(build_options, packages) - if package_fetcher.packages_are_installed(): - return package_fetcher + if build_options.host_type.get_package_manager(): + packages = self.get_required_system_packages(ctx) + package_fetcher = SystemPackageFetcher(build_options, packages) + if package_fetcher.packages_are_installed(): + return package_fetcher - repo_url = self.get_repo_url(ctx) if repo_url: rev = self.get("git", "rev") depth = self.get("git", "depth") return GitFetcher(build_options, self, repo_url, rev, depth) - url = self.get("download", "url", ctx=ctx) if url: # We need to defer this import until now to avoid triggering # a cycle when the facebook/__init__.py is loaded. @@ -459,7 +468,8 @@ def _create_fetcher(self, build_options, ctx): ) raise KeyError( - "project %s has no fetcher configuration matching %s" % (self.name, ctx) + "project %s has no fetcher configuration or system packages matching %s" + % (self.name, ctx) ) def create_fetcher(self, build_options, loader, ctx): @@ -657,6 +667,18 @@ def create_builder( # noqa:C901 inst_dir, ) + if builder == "systemd": + return SystemdBuilder( + loader, + dep_manifests, + build_options, + ctx, + self, + src_dir, + build_dir, + inst_dir, + ) + if builder == "cargo": return self.create_cargo_builder( loader, diff --git a/build/fbcode_builder/getdeps/platform.py b/build/fbcode_builder/getdeps/platform.py index 1e021d99235a..5e4acdc44e82 100644 --- a/build/fbcode_builder/getdeps/platform.py +++ b/build/fbcode_builder/getdeps/platform.py @@ -216,7 +216,6 @@ def __init__(self, ostype=None, distro=None, distrovers=None) -> None: ostype = "darwin" elif is_windows(): ostype = "windows" - # pyre-fixme[16]: Module `sys` has no attribute `getwindowsversion`. distrovers = str(sys.getwindowsversion().major) elif sys.platform.startswith("freebsd"): ostype = "freebsd" diff --git a/build/fbcode_builder/getdeps/py_wheel_builder.py b/build/fbcode_builder/getdeps/py_wheel_builder.py index 335d74afd1d3..7db5f2cb0127 100644 --- a/build/fbcode_builder/getdeps/py_wheel_builder.py +++ b/build/fbcode_builder/getdeps/py_wheel_builder.py @@ -95,13 +95,14 @@ # something like the following pip3 command: # pip3 --isolated install --no-cache-dir --no-index --system \ # --target -# pyre-fixme[13] fields initialized in _build class PythonWheelBuilder(BuilderBase): """This Builder can take Python wheel archives and install them as python libraries that can be used by add_fb_python_library()/add_fb_python_executable() CMake rules. """ + # pyre-fixme[13]: Attribute `dist_info_dir` is never initialized. dist_info_dir: str + # pyre-fixme[13]: Attribute `template_format_dict` is never initialized. template_format_dict: Dict[str, str] def _build(self, reconfigure: bool) -> None: diff --git a/build/fbcode_builder/getdeps/runcmd.py b/build/fbcode_builder/getdeps/runcmd.py index e0b9d2b22fd5..c4a9326f9c2b 100644 --- a/build/fbcode_builder/getdeps/runcmd.py +++ b/build/fbcode_builder/getdeps/runcmd.py @@ -10,16 +10,12 @@ import subprocess import sys +from shlex import quote as shellquote + from .envfuncs import Env from .platform import is_windows -try: - from shlex import quote as shellquote -except ImportError: - from pipes import quote as shellquote - - class RunCommandError(Exception): pass diff --git a/build/fbcode_builder/manifests/eden b/build/fbcode_builder/manifests/eden index 168f460e9af7..d7bc911612e3 100644 --- a/build/fbcode_builder/manifests/eden +++ b/build/fbcode_builder/manifests/eden @@ -55,6 +55,10 @@ python # TODO: teach getdeps to compile lmdb on Windows. lmdb +[dependencies.test=on] +# sapling CLI is needed to run the tests +sapling + [shipit.pathmap.fb=on] # for internal builds that use getdeps fbcode/fb303 = fb303 diff --git a/build/fbcode_builder/manifests/fboss b/build/fbcode_builder/manifests/fboss index ce9b36109e22..ec199f83110c 100644 --- a/build/fbcode_builder/manifests/fboss +++ b/build/fbcode_builder/manifests/fboss @@ -39,6 +39,8 @@ CLI11 exprtk nlohmann-json libgpiod +systemd +range-v3 [shipit.pathmap] fbcode/fboss/github = . diff --git a/build/fbcode_builder/manifests/glog b/build/fbcode_builder/manifests/glog index b5d5fa814cc2..2649eaad1805 100644 --- a/build/fbcode_builder/manifests/glog +++ b/build/fbcode_builder/manifests/glog @@ -24,7 +24,8 @@ HAVE_TR1_UNORDERED_SET=OFF [homebrew] glog -[debs] +# on ubuntu glog brings in liblzma-dev, which in turn breaks watchman tests +[debs.not(distro=ubuntu)] libgoogle-glog-dev [rpms.distro=fedora] diff --git a/build/fbcode_builder/manifests/jom b/build/fbcode_builder/manifests/jom new file mode 100644 index 000000000000..effecab67a16 --- /dev/null +++ b/build/fbcode_builder/manifests/jom @@ -0,0 +1,15 @@ +# jom is compatible with MSVC nmake, but adds the /j argment which +# speeds up openssl build a lot +[manifest] +name = jom + +# see https://download.qt.io/official_releases/jom/changelog.txt for latest version +[download.os=windows] +url = https://download.qt.io/official_releases/jom/jom_1_1_4.zip +sha256 = d533c1ef49214229681e90196ed2094691e8c4a0a0bef0b2c901debcb562682b + +[build.os=windows] +builder = nop + +[install.files.os=windows] +. = bin diff --git a/build/fbcode_builder/manifests/libunwind b/build/fbcode_builder/manifests/libunwind index 0a4f03bc8fef..fda19209ad32 100644 --- a/build/fbcode_builder/manifests/libunwind +++ b/build/fbcode_builder/manifests/libunwind @@ -5,7 +5,8 @@ name = libunwind libunwind-devel libunwind -[debs] +# on ubuntu this brings in liblzma-dev, which in turn breaks watchman tests +[debs.not(distro=ubuntu)] libunwind-dev [download] diff --git a/build/fbcode_builder/manifests/openssl b/build/fbcode_builder/manifests/openssl index beef31c9e2e0..ebd680e7e1e6 100644 --- a/build/fbcode_builder/manifests/openssl +++ b/build/fbcode_builder/manifests/openssl @@ -5,7 +5,7 @@ name = openssl libssl-dev [homebrew] -openssl@1.1 +openssl # on homebrew need the matching curl and ca- [rpms] @@ -16,9 +16,11 @@ openssl-libs [pps] openssl -[download] -url = https://www.openssl.org/source/openssl-1.1.1l.tar.gz -sha256 = 0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1 +# no need to download on the systems where we always use the system libs +[download.not(any(os=linux, os=freebsd))] +# match the openssl version packages in ubuntu LTS folly current supports +url = https://www.openssl.org/source/openssl-3.0.15.tar.gz +sha256 = 23c666d0edf20f14249b3d8f0368acaee9ab585b09e1de82107c66e1f3ec9533 # We use the system openssl on these platforms even without --allow-system-packages [build.any(os=linux, os=freebsd)] @@ -26,7 +28,8 @@ builder = nop [build.not(any(os=linux, os=freebsd))] builder = openssl -subdir = openssl-1.1.1l +subdir = openssl-3.0.15 [dependencies.os=windows] +jom perl diff --git a/build/fbcode_builder/manifests/python-click b/build/fbcode_builder/manifests/python-click index ea9a9d2d3dc3..cdf29c4d0c02 100644 --- a/build/fbcode_builder/manifests/python-click +++ b/build/fbcode_builder/manifests/python-click @@ -7,3 +7,9 @@ sha256 = dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc [build] builder = python-wheel + +[rpms] +python3-click + +[debs] +python3-click diff --git a/build/fbcode_builder/manifests/range-v3 b/build/fbcode_builder/manifests/range-v3 index f96403c83f8c..e3778a368a55 100644 --- a/build/fbcode_builder/manifests/range-v3 +++ b/build/fbcode_builder/manifests/range-v3 @@ -9,3 +9,6 @@ sha256 = 376376615dbba43d3bef75aa590931431ecb49eb36d07bb726a19f680c75e20c [build] builder = cmake subdir = range-v3-0.11.0 + +[cmake.defines] +RANGE_V3_EXAMPLES=OFF diff --git a/build/fbcode_builder/manifests/sapling b/build/fbcode_builder/manifests/sapling index c067360ef82b..cff882c67c87 100644 --- a/build/fbcode_builder/manifests/sapling +++ b/build/fbcode_builder/manifests/sapling @@ -60,7 +60,7 @@ fb303 fbthrift rust-shed -[dependencies.test=on] +[dependencies.all(test=on,not(os=darwin))] hexdump [dependencies.not(os=windows)] diff --git a/build/fbcode_builder/manifests/systemd b/build/fbcode_builder/manifests/systemd new file mode 100644 index 000000000000..0fb2a6f5a519 --- /dev/null +++ b/build/fbcode_builder/manifests/systemd @@ -0,0 +1,15 @@ +[manifest] +name = systemd + +[rpms] +systemd +systemd-devel + +[download] +url = https://github.com/systemd/systemd/archive/refs/tags/v256.7.tar.gz +sha256 = 896d76ff65c88f5fd9e42f90d152b0579049158a163431dd77cdc57748b1d7b0 + + +[build] +builder = systemd +subdir = systemd-256.7 diff --git a/build/fbcode_builder/manifests/xz b/build/fbcode_builder/manifests/xz index 0b27ad63cc91..e6c0808ff01f 100644 --- a/build/fbcode_builder/manifests/xz +++ b/build/fbcode_builder/manifests/xz @@ -1,7 +1,8 @@ [manifest] name = xz -[debs] +# ubuntu's package causes watchman's tests to hang +[debs.not(distro=ubuntu)] liblzma-dev [homebrew] diff --git a/eden/fs/service/eden.thrift b/eden/fs/service/eden.thrift index 4c8925773dba..6b0837206dbf 100644 --- a/eden/fs/service/eden.thrift +++ b/eden/fs/service/eden.thrift @@ -8,6 +8,7 @@ include "eden/fs/config/eden_config.thrift" include "fb303/thrift/fb303_core.thrift" include "thrift/annotation/thrift.thrift" +include "thrift/annotation/cpp.thrift" namespace cpp2 facebook.eden namespace java com.facebook.eden.thrift @@ -125,10 +126,11 @@ enum EdenErrorType { } exception EdenError { + @thrift.ExceptionMessage 1: string message; 2: optional i32 errorCode; 3: EdenErrorType errorType; -} (message = 'message') +} exception NoValueForKeyError { 1: string key; @@ -175,6 +177,7 @@ struct PrivHelperInfo { /** * The current running state of an EdenMount. */ +@cpp.EnumType{type = cpp.EnumUnderlyingType.U32} enum MountState { /** * The EdenMount object has been constructed but has not started @@ -232,7 +235,7 @@ enum MountState { * before we have attempted to start the user-space filesystem mount. */ INIT_ERROR = 9, -} (cpp2.enum_type = 'uint32_t') +} struct MountInfo { 1: PathString mountPoint; @@ -361,7 +364,7 @@ enum FileAttributes { */ DIGEST_HASH = 64, /* NEXT_ATTR = 2^x */ -} (cpp2.enum_type = 'uint64_t') +} typedef unsigned64 RequestedAttributes @@ -885,7 +888,7 @@ enum DataFetchOrigin { LOCAL_BACKING_STORE = 8, REMOTE_BACKING_STORE = 16, /* NEXT_WHERE = 2^x */ -} (cpp2.enum_type = 'uint64_t') +} struct DebugGetScmBlobRequest { 1: MountId mountId; @@ -1749,6 +1752,204 @@ union CheckoutProgressInfoResponse { 2: CheckoutNotInProgress noProgress; } +/* + * Structs/Unions for changesSinceV2 API + */ + +/* + * Small change notification returned when invoking changesSinceV2. + * Indicates that a new filesystem entry has been added to the + * given mount point since the provided journal position. + * + * fileType - Dtype of added filesystem entry. + * path - path (vector of bytes) of added filesystem entry. + */ +struct Added { + 1: Dtype fileType; + 3: PathString path; +} + +/* + * Small change notification returned when invoking changesSinceV2. + * Indicates that an existing filesystem entry has been modified within + * the given mount point since the provided journal position. + * + * fileType - Dtype of modified filesystem entry. + * path - path (vector of bytes) of modified filesystem entry. + */ +struct Modified { + 1: Dtype fileType; + 3: PathString path; +} + +/* + * Small change notification returned when invoking changesSinceV2. + * Indicates that an existing filesystem entry has been renamed within + * the given mount point since the provided journal position. + * + * fileType - Dtype of renamed filesystem entry. + * from - path (vector of bytes) the filesystem entry was previously located at. + * to - path (vector of bytes) the filesystem entry was relocated to. + */ +struct Renamed { + 1: Dtype fileType; + 2: PathString from; + 3: PathString to; +} + +/* + * Small change notification returned when invoking changesSinceV2. + * Indicates that an existing filesystem entry has been replaced within + * the given mount point since the provided journal position. + * + * fileType - Dtype of replaced filesystem entry. + * from - path (vector of bytes) the filesystem entry was previously located at. + * to - path (vector of bytes) the filesystem entry was relocated over. + */ +struct Replaced { + 1: Dtype fileType; + 2: PathString from; + 3: PathString to; +} + +/* + * Small change notification returned when invoking changesSinceV2. + * Indicates that an existing filesystem entry has been removed from + * the given mount point since the provided journal position. + * + * fileType - Dtype of removed filesystem entry. + * path - path (vector of bytes) of removed filesystem entry. + */ +struct Removed { + 1: Dtype fileType; + 3: PathString path; +} + +/* + * Change notification returned when invoking changesSinceV2. + * Indicates that the given change is small in impact - affecting + * one or two filesystem entries at most. + */ +union SmallChangeNotification { + 1: Added added; + 2: Modified modified; + 3: Renamed renamed; + 4: Replaced replaced; + 5: Removed removed; +} + +/* + * Large change notification returned when invoking changesSinceV2. + * Indicates that an existing directory has been renamed within + * the given mount point since the provided journal position. + */ +struct DirectoryRenamed { + 1: PathString from; + 2: PathString to; +} + +/* + * Large change notification returned when invoking changesSinceV2. + * Indicates that a commit transition has occurred within the + * given mount point since the provided journal position. + */ +struct CommitTransition { + 1: ThriftRootId from; + 2: ThriftRootId to; +} + +/* + * Large change notification returned when invoking changesSinceV2. + * Indicates that EdenfS was unable to track changes within the given + * mount point since the provided journal poistion. Callers should + * treat all filesystem entries as changed. + */ +enum LostChangesReason { + // Unknown reason. + UNKNOWN = 0, + // The given mount point was remounted (or EdenFS was restarted). + EDENFS_REMOUNTED = 1, + // EdenFS' journal was truncated. + JOURNAL_TRUNCATED = 2, + // There were too many change notifications to report to the caller. + TOO_MANY_CHANGES = 3, +} + +/* + * Large change notification returned when invoking changesSinceV2. + * Indicates that EdenFS was unable to provide the changes to the caller. + */ +struct LostChanges { + 1: LostChangesReason reason; +} + +/* + * Change notification returned when invoking changesSinceV2. + * Indicates that the given change is large in impact - affecting + * an unknown number of filesystem entries. + */ +union LargeChangeNotification { + 1: DirectoryRenamed directoryRenamed; + 2: CommitTransition commitTransition; + 3: LostChanges lostChanges; +} + +/* + * Changed returned when invoking changesSinceV2. + * Contains a change that occured within the given mount point + * since the provided journal position. + */ +union ChangeNotification { + 1: SmallChangeNotification smallChange; + 2: LargeChangeNotification largeChange; +} + +/** + * Return value of the changesSinceV2 API + * + * toPosition - a new journal poistion that indicates the next change + * that will occur in the future. Should be used in the next call to + * changesSinceV2 go get the next list of changes. + * + * changes - a list of all change notifications that have ocurred in + * within the given mount point since the provided journal position. + */ +struct ChangesSinceV2Result { + 1: JournalPosition toPosition; + 2: list changes; +} + +/** + * Argument to changesSinceV2 API + * + * mountPoint - the EdenFS checkout to request changes about. + * + * fromPosition - the journal position used as the starting point to + * request changes since. Typically, fromPosition is the set to the + * toPostiion value returned in ChangesSinceV2Result. However, for + * the initial invocation of changesSinceV2, the caller can obtain + * the current journal position by calling getCurrentJournalPosition. + * + * includeVCSRoots - optional flag indicating the VCS roots should be included + * in the returned results. By default, VCS roots will be excluded from + * results. + * + * includedRoots - optional list of roots to include in results. If not + * provided or an empty list, all roots will be included in results. + * Applied before roots are excluded - see excludedRoots. + * + * excludedRoots - optional ist of roots to exclude from results. If not + * provided or an empty list, no roots will be excluded from results. + * Applied after roots are included - see includedRoots. + */ +struct ChangesSinceV2Params { + 1: PathString mountPoint; + 2: JournalPosition fromPosition; + 3: optional bool includeVCSRoots; + 4: optional list includedRoots; + 5: optional list excludedRoots; +} + service EdenService extends fb303_core.BaseService { list listMounts() throws (1: EdenError ex); void mount(1: MountArgument info) throws (1: EdenError ex); @@ -2089,9 +2290,8 @@ service EdenService extends fb303_core.BaseService { * for optimization and the result not relied on for operations. This command does not * return the list of prefetched files. */ - void prefetchFiles(1: PrefetchParams params) throws (1: EdenError ex) ( - priority = 'BEST_EFFORT', - ); + @thrift.Priority{level = thrift.RpcPriority.BEST_EFFORT} + void prefetchFiles(1: PrefetchParams params) throws (1: EdenError ex); /** * Has the same behavior as globFiles, but should be called in the case of a prefetch. @@ -2189,7 +2389,8 @@ service EdenService extends fb303_core.BaseService { * Returns information about the running process, including pid and command * line. */ - DaemonInfo getDaemonInfo() throws (1: EdenError ex) (priority = 'IMPORTANT'); + @thrift.Priority{level = thrift.RpcPriority.IMPORTANT} + DaemonInfo getDaemonInfo() throws (1: EdenError ex); /** * Returns information about the privhelper process, including accesibility. @@ -2585,6 +2786,17 @@ service EdenService extends fb303_core.BaseService { void ensureMaterialized(1: EnsureMaterializedParams params) throws ( 1: EdenError ex, ); + + /** + * Returns a list of change notifications along with a new journal position for a given mount + * point since a provided journal position. + * + * This does not resolve expensive operations like moving a directory or changing + * commits. Callers must query Sapling to evaluate those potentially expensive operations. + */ + ChangesSinceV2Result changesSinceV2(1: ChangesSinceV2Params params) throws ( + 1: EdenError ex, + ); } // The following were automatically generated and may benefit from renaming. diff --git a/watchman/.clang-tidy b/watchman/.clang-tidy index a761f4c64ae9..130299acf0d8 100644 --- a/watchman/.clang-tidy +++ b/watchman/.clang-tidy @@ -7,8 +7,6 @@ InheritParentConfig: true Checks: ' facebook-hte-PortabilityInclude-gflags/gflags.h, -facebook-hte-PortabilityInclude-gmock/gmock.h, -facebook-hte-PortabilityInclude-gtest/gtest.h, -facebook-hte-RelativeInclude, ' ... diff --git a/watchman/Command.cpp b/watchman/Command.cpp index 4a1524c1b3bf..cef9428e3e70 100644 --- a/watchman/Command.cpp +++ b/watchman/Command.cpp @@ -102,10 +102,10 @@ ResultErrno Command::run( PduBuffer output_pdu_buffer; if (persistent) { for (;;) { - auto res = passPduToStdout( + auto result = passPduToStdout( stream, buffer, output_format, output_pdu_buffer, pretty); - if (res.hasError()) { - return res; + if (result.hasError()) { + return result; } } } else { diff --git a/watchman/Logging.cpp b/watchman/Logging.cpp index e0f717858db2..124fcafb2368 100644 --- a/watchman/Logging.cpp +++ b/watchman/Logging.cpp @@ -33,7 +33,8 @@ namespace { template void write_stderr(const String& str) { w_string_piece piece = str; - ignore_result(::write(STDERR_FILENO, piece.data(), piece.size())); + ignore_result( + folly::fileops::write(STDERR_FILENO, piece.data(), piece.size())); } template @@ -202,7 +203,7 @@ void Log::doLogToStdErr() { for (auto& item : items) { auto& log = json_to_w_string(item->payload.get("log")); - ignore_result(::write(STDERR_FILENO, log.data(), log.size())); + ignore_result(folly::fileops::write(STDERR_FILENO, log.data(), log.size())); auto level = json_to_w_string(item->payload.get("level")); if (level == kFatal) { diff --git a/watchman/Logging.h b/watchman/Logging.h index 5076c8a44d8c..9eb2fd5d2175 100644 --- a/watchman/Logging.h +++ b/watchman/Logging.h @@ -135,7 +135,7 @@ void logf(LogLevel level, fmt::string_view format_str, Args&&... args) { template void logf_stderr(fmt::string_view format_str, Args&&... args) { auto msg = fmt::format(fmt::runtime(format_str), std::forward(args)...); - ignore_result(write(STDERR_FILENO, msg.data(), msg.size())); + ignore_result(folly::fileops::write(STDERR_FILENO, msg.data(), msg.size())); } #ifdef _WIN32 diff --git a/watchman/cli/Cargo.toml b/watchman/cli/Cargo.toml index c181ef7ec635..80a56c17f7f0 100644 --- a/watchman/cli/Cargo.toml +++ b/watchman/cli/Cargo.toml @@ -14,21 +14,21 @@ anyhow = "1.0.86" duct = "0.13.6" jwalk = "0.6" serde = { version = "1.0.185", features = ["derive", "rc"] } -serde_json = { version = "1.0.125", features = ["float_roundtrip", "unbounded_depth"] } +serde_json = { version = "1.0.132", features = ["float_roundtrip", "unbounded_depth"] } structopt = "0.3.26" sysinfo = "0.30.11" tabular = "0.2.0" -tokio = { version = "1.37.0", features = ["full", "test-util", "tracing"] } +tokio = { version = "1.41.0", features = ["full", "test-util", "tracing"] } watchman_client = { version = "0.9.0", path = "../rust/watchman_client" } [target.'cfg(target_os = "linux")'.dependencies] -nix = "0.25" +nix = { version = "0.29.0", features = ["dir", "event", "hostname", "inotify", "ioctl", "mman", "mount", "net", "poll", "ptrace", "reboot", "resource", "sched", "term", "time", "user", "zerocopy"] } [target.'cfg(target_os = "macos")'.dependencies] -nix = "0.25" +nix = { version = "0.29.0", features = ["dir", "event", "hostname", "inotify", "ioctl", "mman", "mount", "net", "poll", "ptrace", "reboot", "resource", "sched", "term", "time", "user", "zerocopy"] } [target.'cfg(unix)'.dependencies] -nix = "0.25" +nix = { version = "0.29.0", features = ["dir", "event", "hostname", "inotify", "ioctl", "mman", "mount", "net", "poll", "ptrace", "reboot", "resource", "sched", "term", "time", "user", "zerocopy"] } [features] default = [] diff --git a/watchman/cmds/query.cpp b/watchman/cmds/query.cpp index 4330e410b7be..789a6508d3f3 100644 --- a/watchman/cmds/query.cpp +++ b/watchman/cmds/query.cpp @@ -19,7 +19,7 @@ using namespace watchman; /* query /root {query} */ static UntypedResponse cmd_query(Client* client, const json_ref& args) { if (json_array_size(args) != 3) { - throw ErrorResponse("wrong number of arguments for 'query'"); + throw ErrorResponse("wrong number of arguments for 'query', expected 3"); } auto root = resolveRoot(client, args); diff --git a/watchman/integration/eden/test_eden_glob_upper_bound.py b/watchman/integration/eden/test_eden_glob_upper_bound.py index 0e7c7710b476..7c78a4c515c8 100644 --- a/watchman/integration/eden/test_eden_glob_upper_bound.py +++ b/watchman/integration/eden/test_eden_glob_upper_bound.py @@ -137,8 +137,10 @@ def test_eden_since_upper_bound_case_insensitive(self) -> None: ["mixedCASE/file1", "MIXEDcase/file2"], ) self.assertGlobUpperBound( - # We can't bound this query with a glob on a case-sensitive FS. - (None if self.isCaseSensitiveMount(root) else {"mixedcase/**"}) + ( + # We can't bound this query with a glob on a case-sensitive FS. + None if self.isCaseSensitiveMount(root) else {"mixedcase/**"} + ) ) def test_eden_since_upper_bound_includedotfiles(self) -> None: diff --git a/watchman/integration/eden/test_eden_suffix.py b/watchman/integration/eden/test_eden_suffix.py new file mode 100644 index 000000000000..021ec98657ca --- /dev/null +++ b/watchman/integration/eden/test_eden_suffix.py @@ -0,0 +1,89 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-unsafe + +from watchman.integration.lib import WatchmanEdenTestCase + + +def populate(repo) -> None: + # We ignore ".hg" here just so some of the tests that list files don't have to + # explicitly filter out the contents of this directory. However, in most situations + # the .hg directory normally should not be ignored. + repo.write_file(".watchmanconfig", '{"ignore_dirs":[".hg"]}') + + # Create multiple nested directories to test the relative_root option + repo.write_file("foo.c", "1\n") + repo.write_file("subdir/bar.c", "1\n") + repo.write_file("subdir/bar.h", "1\n") + repo.write_file("subdir/subdir2/baz.c", "1\n") + repo.write_file("subdir/subdir2/baz.h", "1\n") + repo.write_file("subdir/subdir2/subdir3/baz.c", "1\n") + + repo.commit("initial commit.") + + +class TestEdenQuery(WatchmanEdenTestCase.WatchmanEdenTestCase): + def test_simple_suffix(self) -> None: + root = self.makeEdenMount(populate) + self.watchmanCommand("watch", root) + + # Test each permutation of relative root + relative_roots = ["", "subdir", "subdir/subdir2", "subdir/subdir2/subdir3"] + expected_output = [ + [ + "foo.c", + "subdir/bar.c", + "subdir/subdir2/baz.c", + "subdir/subdir2/subdir3/baz.c", + ], + [ + "bar.c", + "subdir2/baz.c", + "subdir2/subdir3/baz.c", + ], + [ + "baz.c", + "subdir3/baz.c", + ], + [ + "baz.c", + ], + ] + for relative_root, output in zip(relative_roots, expected_output): + # Test Simple suffix eval + self.assertFileListsEqual( + self.watchmanCommand( + "query", + root, + { + "relative_root": relative_root, + "expression": ["allof", ["type", "f"], ["suffix", ["c"]]], + "fields": ["name"], + }, + )["files"], + output, + ) + + # Check that it is the same as normal suffix eval + self.assertFileListsEqual( + self.watchmanCommand( + "query", + root, + { + "relative_root": relative_root, + "expression": [ + "allof", + # Adding a true expression causes watchman to + # evaluate this normally instead of as a simple suffix + ["true"], + ["type", "f"], + ["suffix", ["c"]], + ], + "fields": ["name"], + }, + )["files"], + output, + ) diff --git a/watchman/integration/eden/test_eden_unmount.py b/watchman/integration/eden/test_eden_unmount.py index d440412b3054..8f43acdfbc70 100644 --- a/watchman/integration/eden/test_eden_unmount.py +++ b/watchman/integration/eden/test_eden_unmount.py @@ -19,7 +19,6 @@ def populate(repo): class TestEdenUnmount(WatchmanEdenTestCase.WatchmanEdenTestCase): def test_eden_unmount(self) -> None: - root = self.makeEdenMount(populate) self.watchmanCommand("watch", root) diff --git a/watchman/integration/lib/path_utils.py b/watchman/integration/lib/path_utils.py index 9cff6576929d..bfca3a7af238 100644 --- a/watchman/integration/lib/path_utils.py +++ b/watchman/integration/lib/path_utils.py @@ -15,7 +15,6 @@ if os.name == "nt": def open_file_win(path): - create_file = ctypes.windll.kernel32.CreateFileW c_path = ctypes.create_unicode_buffer(path) diff --git a/watchman/integration/test_dir_move.py b/watchman/integration/test_dir_move.py index 07058c7ca44d..bf620d465ea5 100644 --- a/watchman/integration/test_dir_move.py +++ b/watchman/integration/test_dir_move.py @@ -17,7 +17,6 @@ @WatchmanTestCase.expand_matrix class TestDirMove(WatchmanTestCase.WatchmanTestCase): - # testing this is flaky at best on windows due to latency # and exclusivity of file handles, so skip it. def checkOSApplicability(self) -> None: diff --git a/watchman/integration/test_path_generator.py b/watchman/integration/test_path_generator.py index e5cbd05326ef..6761982e3ff0 100644 --- a/watchman/integration/test_path_generator.py +++ b/watchman/integration/test_path_generator.py @@ -48,7 +48,9 @@ def test_path_generator_case(self) -> None: self.assertFileListsEqual( self.watchmanCommand( - "query", root, {"fields": ["name"], "path": ["foo"]} # not Foo! + "query", + root, + {"fields": ["name"], "path": ["foo"]}, # not Foo! )["files"], [], message="Case insensitive matching not implemented \ @@ -89,7 +91,9 @@ def test_path_generator_relative_root(self) -> None: self.assertFileListsEqual( self.watchmanCommand( - "query", root, {"fields": ["name"], "path": ["foo"]} # not Foo! + "query", + root, + {"fields": ["name"], "path": ["foo"]}, # not Foo! )["files"], [], message="Case insensitive matching not implemented \ diff --git a/watchman/main.cpp b/watchman/main.cpp index 8775fafdcbcb..fc9f496dca8d 100644 --- a/watchman/main.cpp +++ b/watchman/main.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -138,18 +139,19 @@ std::optional detect_starting_command(pid_t ppid) { #endif // redirect std{in,out,err} - int fd = ::open("/dev/null", O_RDONLY); + int fd = folly::fileops::open("/dev/null", O_RDONLY); if (fd != -1) { ignore_result(::dup2(fd, STDIN_FILENO)); - ::close(fd); + folly::fileops::close(fd); } if (logging::log_name != "-") { - fd = open(logging::log_name.c_str(), O_WRONLY | O_APPEND | O_CREAT, 0600); + fd = folly::fileops::open( + logging::log_name.c_str(), O_WRONLY | O_APPEND | O_CREAT, 0600); if (fd != -1) { ignore_result(::dup2(fd, STDOUT_FILENO)); ignore_result(::dup2(fd, STDERR_FILENO)); - ::close(fd); + folly::fileops::close(fd); } } @@ -259,7 +261,7 @@ static void close_random_fds() { } for (max_fd = open_max; max_fd > STDERR_FILENO; --max_fd) { - close(max_fd); + folly::fileops::close(max_fd); } #endif } diff --git a/watchman/node/.flowconfig b/watchman/node/.flowconfig new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/watchman/node/example.js b/watchman/node/example.js index f31aaada1e47..6e5b5025b542 100644 --- a/watchman/node/example.js +++ b/watchman/node/example.js @@ -5,7 +5,7 @@ * LICENSE file in the root directory of this source tree. */ -var watchman = require('fb-watchman'); +var watchman = require('./index.js'); var client = new watchman.Client(); client.on('end', function() { diff --git a/watchman/node/index.js b/watchman/node/index.js index 388bcc699137..e654ec47063e 100644 --- a/watchman/node/index.js +++ b/watchman/node/index.js @@ -3,18 +3,49 @@ * * This source code is licensed under the MIT license found in the * LICENSE file in the root directory of this source tree. + * + * @format + * @flow */ 'use strict'; -var net = require('net'); -var EE = require('events').EventEmitter; -var util = require('util'); -var childProcess = require('child_process'); -var bser = require('bser'); +const bser = require('bser'); +const childProcess = require('child_process'); +const {EventEmitter} = require('events'); +const net = require('net'); // We'll emit the responses to these when they get sent down to us -var unilateralTags = ['subscription', 'log']; +const unilateralTags = ['subscription', 'log']; + +/*:: +interface BunserBuf extends EventEmitter { + append(buf: Buffer): void; +} + +type Options = { + watchmanBinaryPath?: string, + ... +}; + +type Response = { + capabilities?: ?{[key: string]: boolean}, + version: string, + error?: string, + ... +}; + +type CommandCallback = (error: ?Error, result?: Response) => void; + +type Command = { + cb: CommandCallback, + cmd: mixed, +}; +*/ + +class WatchmanError extends Error { + watchmanResponse /*: mixed */; +} /** * @param options An object with the following optional keys: @@ -22,240 +53,331 @@ var unilateralTags = ['subscription', 'log']; * If not provided, the Client locates the binary using the PATH specified * by the node child_process's default env. */ -function Client(options) { - var self = this; - EE.call(this); - - this.watchmanBinaryPath = 'watchman'; - if (options && options.watchmanBinaryPath) { - this.watchmanBinaryPath = options.watchmanBinaryPath.trim(); - }; - this.commands = []; -} -util.inherits(Client, EE); +class Client extends EventEmitter { + bunser /*: ?BunserBuf */; + commands /*: Array */; + connecting /*: boolean */; + currentCommand /*: ?Command */; + socket /*: ?net.Socket */; + watchmanBinaryPath /*: string */; + + constructor(options /*: Options */) { + super(); + + this.watchmanBinaryPath = 'watchman'; + if (options && options.watchmanBinaryPath) { + this.watchmanBinaryPath = options.watchmanBinaryPath.trim(); + } + this.commands = []; + } -module.exports.Client = Client; + // Try to send the next queued command, if any + sendNextCommand() { + if (this.currentCommand) { + // There's a command pending response, don't send this new one yet + return; + } -// Try to send the next queued command, if any -Client.prototype.sendNextCommand = function() { - if (this.currentCommand) { - // There's a command pending response, don't send this new one yet - return; - } + this.currentCommand = this.commands.shift(); + if (!this.currentCommand) { + // No further commands are queued + return; + } - this.currentCommand = this.commands.shift(); - if (!this.currentCommand) { - // No further commands are queued - return; + if (this.socket) { + this.socket.write(bser.dumpToBuffer(this.currentCommand.cmd)); + } else { + this.emit( + 'error', + new Error('socket is null attempting to send command'), + ); + } } - this.socket.write(bser.dumpToBuffer(this.currentCommand.cmd)); -} + cancelCommands(why /*: string*/) { + const error = new Error(why); -Client.prototype.cancelCommands = function(why) { - var error = new Error(why); + // Steal all pending commands before we start cancellation, in + // case something decides to schedule more commands + const cmds = this.commands; + this.commands = []; - // Steal all pending commands before we start cancellation, in - // case something decides to schedule more commands - var cmds = this.commands; - this.commands = []; + if (this.currentCommand) { + cmds.unshift(this.currentCommand); + this.currentCommand = null; + } - if (this.currentCommand) { - cmds.unshift(this.currentCommand); - this.currentCommand = null; + // Synthesize an error condition for any commands that were queued + cmds.forEach(cmd => { + cmd.cb(error); + }); } - // Synthesize an error condition for any commands that were queued - cmds.forEach(function(cmd) { - cmd.cb(error); - }); -} + connect() { + const makeSock = (sockname /*:string*/) => { + // bunser will decode the watchman BSER protocol for us + const bunser = (this.bunser = new bser.BunserBuf()); + // For each decoded line: + bunser.on('value', obj => { + // Figure out if this is a unliteral response or if it is the + // response portion of a request-response sequence. At the time + // of writing, there are only two possible unilateral responses. + let unilateral /*: false | string */ = false; + for (let i = 0; i < unilateralTags.length; i++) { + const tag = unilateralTags[i]; + if (tag in obj) { + unilateral = tag; + } + } -Client.prototype.connect = function() { - var self = this; - - function makeSock(sockname) { - // bunser will decode the watchman BSER protocol for us - self.bunser = new bser.BunserBuf(); - // For each decoded line: - self.bunser.on('value', function(obj) { - // Figure out if this is a unliteral response or if it is the - // response portion of a request-response sequence. At the time - // of writing, there are only two possible unilateral responses. - var unilateral = false; - for (var i = 0; i < unilateralTags.length; i++) { - var tag = unilateralTags[i]; - if (tag in obj) { - unilateral = tag; + if (unilateral) { + this.emit(unilateral, obj); + } else if (this.currentCommand) { + const cmd = this.currentCommand; + this.currentCommand = null; + if ('error' in obj) { + const error = new WatchmanError(obj.error); + error.watchmanResponse = obj; + cmd.cb(error); + } else { + cmd.cb(null, obj); + } } - } - if (unilateral) { - self.emit(unilateral, obj); - } else if (self.currentCommand) { - var cmd = self.currentCommand; - self.currentCommand = null; - if ('error' in obj) { - var error = new Error(obj.error); - error.watchmanResponse = obj; - cmd.cb(error); - } else { - cmd.cb(null, obj); + // See if we can dispatch the next queued command, if any + this.sendNextCommand(); + }); + bunser.on('error', err => { + this.emit('error', err); + }); + + const socket = (this.socket = net.createConnection(sockname)); + socket.on('connect', () => { + this.connecting = false; + this.emit('connect'); + this.sendNextCommand(); + }); + socket.on('error', err => { + this.connecting = false; + this.emit('error', err); + }); + socket.on('data', buf => { + if (this.bunser) { + this.bunser.append(buf); } + }); + socket.on('end', () => { + this.socket = null; + this.bunser = null; + this.cancelCommands('The watchman connection was closed'); + this.emit('end'); + }); + }; + + // triggers will export the sock path to the environment. + // If we're invoked in such a way, we can simply pick up the + // definition from the environment and avoid having to fork off + // a process to figure it out + if (process.env.WATCHMAN_SOCK) { + makeSock(process.env.WATCHMAN_SOCK); + return; + } + + // We need to ask the client binary where to find it. + // This will cause the service to start for us if it isn't + // already running. + const args = ['--no-pretty', 'get-sockname']; + + // We use the more elaborate spawn rather than exec because there + // are some error cases on Windows where process spawning can hang. + // It is desirable to pipe stderr directly to stderr live so that + // we can discover the problem. + let proc = null; + let spawnFailed = false; + + const spawnError = ( + error /*: Error | {message: string, code?: string, errno?: string}*/, + ) => { + if (spawnFailed) { + // For ENOENT, proc 'close' will also trigger with a negative code, + // let's suppress that second error. + return; } + spawnFailed = true; + if (error.code === 'EACCES' || error.errno === 'EACCES') { + error.message = + 'The Watchman CLI is installed but cannot ' + + 'be spawned because of a permission problem'; + } else if (error.code === 'ENOENT' || error.errno === 'ENOENT') { + error.message = + 'Watchman was not found in PATH. See ' + + 'https://facebook.github.io/watchman/docs/install.html ' + + 'for installation instructions'; + } + console.error('Watchman: ', error.message); + this.emit('error', error); + }; - // See if we can dispatch the next queued command, if any - self.sendNextCommand(); - }); - self.bunser.on('error', function(err) { - self.emit('error', err); - }); + try { + proc = childProcess.spawn(this.watchmanBinaryPath, args, { + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true, + }); + } catch (error) { + spawnError(error); + return; + } - self.socket = net.createConnection(sockname); - self.socket.on('connect', function() { - self.connecting = false; - self.emit('connect'); - self.sendNextCommand(); + const stdout = []; + const stderr = []; + proc.stdout.on('data', data => { + stdout.push(data); }); - self.socket.on('error', function(err) { - self.connecting = false; - self.emit('error', err); + proc.stderr.on('data', data => { + data = data.toString('utf8'); + stderr.push(data); + console.error(data); }); - self.socket.on('data', function(buf) { - if (self.bunser) { - self.bunser.append(buf); - } + proc.on('error', error => { + spawnError(error); }); - self.socket.on('end', function() { - self.socket = null; - self.bunser = null; - self.cancelCommands('The watchman connection was closed'); - self.emit('end'); + + proc.on('close', (code, signal) => { + if (code !== 0) { + spawnError( + new Error( + this.watchmanBinaryPath + + ' ' + + args.join(' ') + + ' returned with exit code=' + + code + + ', signal=' + + signal + + ', stderr= ' + + stderr.join(''), + ), + ); + return; + } + try { + const obj = JSON.parse(stdout.join('')); + if ('error' in obj) { + const error = new WatchmanError(obj.error); + error.watchmanResponse = obj; + this.emit('error', error); + return; + } + makeSock(obj.sockname); + } catch (e) { + this.emit('error', e); + } }); } - // triggers will export the sock path to the environment. - // If we're invoked in such a way, we can simply pick up the - // definition from the environment and avoid having to fork off - // a process to figure it out - if (process.env.WATCHMAN_SOCK) { - makeSock(process.env.WATCHMAN_SOCK); - return; - } + command(args /*: mixed*/, done /*: CommandCallback */ = () => {}) { + // Queue up the command + this.commands.push({cmd: args, cb: done}); - // We need to ask the client binary where to find it. - // This will cause the service to start for us if it isn't - // already running. - var args = ['--no-pretty', 'get-sockname']; - - // We use the more elaborate spawn rather than exec because there - // are some error cases on Windows where process spawning can hang. - // It is desirable to pipe stderr directly to stderr live so that - // we can discover the problem. - var proc = null; - var spawnFailed = false; - - function spawnError(error) { - if (spawnFailed) { - // For ENOENT, proc 'close' will also trigger with a negative code, - // let's suppress that second error. + // Establish a connection if we don't already have one + if (!this.socket) { + if (!this.connecting) { + this.connecting = true; + this.connect(); + return; + } return; } - spawnFailed = true; - if (error.code === 'EACCES' || error.errno === 'EACCES') { - error.message = 'The Watchman CLI is installed but cannot ' + - 'be spawned because of a permission problem'; - } else if (error.code === 'ENOENT' || error.errno === 'ENOENT') { - error.message = 'Watchman was not found in PATH. See ' + - 'https://facebook.github.io/watchman/docs/install.html ' + - 'for installation instructions'; - } - console.error('Watchman: ', error.message); - self.emit('error', error); - } - try { - proc = childProcess.spawn(this.watchmanBinaryPath, args, { - stdio: ['ignore', 'pipe', 'pipe'], - windowsHide: true - }); - } catch (error) { - spawnError(error); - return; + // If we're already connected and idle, try sending the command immediately + this.sendNextCommand(); } - var stdout = []; - var stderr = []; - proc.stdout.on('data', function(data) { - stdout.push(data); - }); - proc.stderr.on('data', function(data) { - data = data.toString('utf8'); - stderr.push(data); - console.error(data); - }); - proc.on('error', function(error) { - spawnError(error); - }); - - proc.on('close', function (code, signal) { - if (code !== 0) { - spawnError(new Error( - self.watchmanBinaryPath + ' ' + args.join(' ') + - ' returned with exit code=' + code + ', signal=' + - signal + ', stderr= ' + stderr.join(''))); - return; - } - try { - var obj = JSON.parse(stdout.join('')); - if ('error' in obj) { - var error = new Error(obj.error); - error.watchmanResponse = obj; - self.emit('error', error); - return; + // This is a helper that we expose for testing purposes + _synthesizeCapabilityCheck /*:: */( + resp /*: T */, + optional /*: $ReadOnlyArray */, + required /*: $ReadOnlyArray */, + ) /*: T & { error?: string, ...} */ { + const capabilities /*:{[key: string]: boolean} */ = (resp.capabilities = + {}); + const version = resp.version; + optional.forEach(name => { + capabilities[name] = have_cap(version, name); + }); + required.forEach(name => { + const have = have_cap(version, name); + capabilities[name] = have; + if (!have) { + resp.error = + 'client required capability `' + + name + + '` is not supported by this server'; } - makeSock(obj.sockname); - } catch (e) { - self.emit('error', e); - } - }); -} - -Client.prototype.command = function(args, done) { - done = done || function() {}; + }); + return resp; + } - // Queue up the command - this.commands.push({cmd: args, cb: done}); + capabilityCheck( + caps /*: $ReadOnly<{optional?: $ReadOnlyArray, required?: $ReadOnlyArray, ...}>*/, + done /*: CommandCallback */, + ) { + const optional = caps.optional || []; + const required = caps.required || []; + this.command( + [ + 'version', + { + optional: optional, + required: required, + }, + ], + (error, resp /*: ?Response */) => { + if (error || !resp) { + done(error || new Error('no watchman response')); + return; + } + if (!('capabilities' in resp)) { + // Server doesn't support capabilities, so we need to + // synthesize the results based on the version + resp = this._synthesizeCapabilityCheck(resp, optional, required); + if (resp.error) { + error = new WatchmanError(resp.error); + error.watchmanResponse = resp; + done(error); + return; + } + } + done(null, resp); + }, + ); + } - // Establish a connection if we don't already have one - if (!this.socket) { - if (!this.connecting) { - this.connecting = true; - this.connect(); - return; + // Close the connection to the service + end() { + this.cancelCommands('The client was ended'); + if (this.socket) { + this.socket.end(); + this.socket = null; } - return; + this.bunser = null; } - - // If we're already connected and idle, try sending the command immediately - this.sendNextCommand(); } -var cap_versions = { - "cmd-watch-del-all": "3.1.1", - "cmd-watch-project": "3.1", - "relative_root": "3.3", - "term-dirname": "3.1", - "term-idirname": "3.1", - "wildmatch": "3.7", -} +const cap_versions /*: $ReadOnly<{[key:string]: ?string}>*/ = { + 'cmd-watch-del-all': '3.1.1', + 'cmd-watch-project': '3.1', + relative_root: '3.3', + 'term-dirname': '3.1', + 'term-idirname': '3.1', + wildmatch: '3.7', +}; // Compares a vs b, returns < 0 if a < b, > 0 if b > b, 0 if a == b -function vers_compare(a, b) { - a = a.split('.'); - b = b.split('.'); - for (var i = 0; i < 3; i++) { - var d = parseInt(a[i] || '0') - parseInt(b[i] || '0'); +function vers_compare(aStr /*:string*/, bStr /*:string*/) { + const a = aStr.split('.'); + const b = bStr.split('.'); + for (let i = 0; i < 3; i++) { + const d = parseInt(a[i] || '0') - parseInt(b[i] || '0'); if (d != 0) { return d; } @@ -263,65 +385,11 @@ function vers_compare(a, b) { return 0; // Equal } -function have_cap(vers, name) { - if (name in cap_versions) { +function have_cap(vers /*:string */, name /*:string*/) { + if (cap_versions[name] != null) { return vers_compare(vers, cap_versions[name]) >= 0; } return false; } -// This is a helper that we expose for testing purposes -Client.prototype._synthesizeCapabilityCheck = function( - resp, optional, required) { - resp.capabilities = {} - var version = resp.version; - optional.forEach(function (name) { - resp.capabilities[name] = have_cap(version, name); - }); - required.forEach(function (name) { - var have = have_cap(version, name); - resp.capabilities[name] = have; - if (!have) { - resp.error = 'client required capability `' + name + - '` is not supported by this server'; - } - }); - return resp; -} - -Client.prototype.capabilityCheck = function(caps, done) { - var optional = caps.optional || []; - var required = caps.required || []; - var self = this; - this.command(['version', { - optional: optional, - required: required - }], function (error, resp) { - if (error) { - done(error); - return; - } - if (!('capabilities' in resp)) { - // Server doesn't support capabilities, so we need to - // synthesize the results based on the version - resp = self._synthesizeCapabilityCheck(resp, optional, required); - if (resp.error) { - error = new Error(resp.error); - error.watchmanResponse = resp; - done(error); - return; - } - } - done(null, resp); - }); -} - -// Close the connection to the service -Client.prototype.end = function() { - this.cancelCommands('The client was ended'); - if (this.socket) { - this.socket.end(); - this.socket = null; - } - this.bunser = null; -} +module.exports.Client = Client; diff --git a/watchman/python/pywatchman/__init__.py b/watchman/python/pywatchman/__init__.py index a1a8763e26b1..6c57a2be0e11 100644 --- a/watchman/python/pywatchman/__init__.py +++ b/watchman/python/pywatchman/__init__.py @@ -23,7 +23,6 @@ # Demandimport causes modules to be loaded lazily. Force the load now # so that we can fall back on pybser if bser doesn't exist - # pyre-ignore bser.pdu_info except ImportError: from . import pybser as bser diff --git a/watchman/python/pywatchman/load.py b/watchman/python/pywatchman/load.py index f728f6d600e3..babb8d4115f0 100644 --- a/watchman/python/pywatchman/load.py +++ b/watchman/python/pywatchman/load.py @@ -65,7 +65,6 @@ def load(fp, mutable: bool = True, value_encoding=None, value_errors=None): if read_len < len(header): return None - # pyre-fixme[16]: Module `pywatchman` has no attribute `bser`. total_len = bser.pdu_len(buf) if total_len > len(buf): ctypes.resize(buf, total_len) @@ -75,7 +74,6 @@ def load(fp, mutable: bool = True, value_encoding=None, value_errors=None): if read_len < len(body): raise RuntimeError("bser data ended early") - # pyre-fixme[16]: Module `pywatchman` has no attribute `bser`. return bser.loads( (ctypes.c_char * total_len).from_buffer(buf, 0), mutable, diff --git a/watchman/python/tests/tests.py b/watchman/python/tests/tests.py index d2637bf3f3b5..5b2c4410fcdf 100755 --- a/watchman/python/tests/tests.py +++ b/watchman/python/tests/tests.py @@ -28,14 +28,17 @@ ) +# pyre-fixme[16]: Module `pywatchman` has no attribute `bser`. if os.path.basename(bser.__file__) == "pybser.py": raise Exception( "bser module resolved to pybser! Something is broken in your build. __file__={!r}, sys.path={!r}".format( - bser.__file__, sys.path + # pyre-fixme[16]: Module `pywatchman` has no attribute `bser`. + bser.__file__, + sys.path, ) ) -PILE_OF_POO = "\U0001F4A9" +PILE_OF_POO = "\U0001f4a9" NON_UTF8_STRING = b"\xff\xff\xff" @@ -433,6 +436,7 @@ def t(ex: bytes): try: document = b"\x00\x01\x05" + struct.pack("@i", len(ex)) + ex print("encoded", document) + # pyre-fixme[16]: `TestBSERDump` has no attribute `bser_mod`. self.bser_mod.loads(document) except Exception: # Exceptions are okay - abort is not. diff --git a/watchman/query/QueryExpr.h b/watchman/query/QueryExpr.h index 19a8553dc152..28239d815996 100644 --- a/watchman/query/QueryExpr.h +++ b/watchman/query/QueryExpr.h @@ -43,6 +43,11 @@ enum AggregateOp { AllOf, }; +/** + * Describes which part of a simple suffix expression + */ +enum SimpleSuffixType { Excluded, Suffix, IsSimpleSuffix, Type }; + class QueryExpr { public: virtual ~QueryExpr() = default; @@ -78,6 +83,8 @@ class QueryExpr { virtual std::optional> computeGlobUpperBound( CaseSensitivity) const = 0; + virtual std::vector getSuffixQueryGlobPatterns() const = 0; + enum ReturnOnlyFiles { No, Yes, Unrelated }; /** @@ -86,6 +93,16 @@ class QueryExpr { * method to handle this query. */ virtual ReturnOnlyFiles listOnlyFiles() const = 0; + + /** + * Returns whether this expression is a simple suffix expression, or a part + * of a simple suffix expression. A simple suffix expression is an allof + * expression that contains a single suffix expresssion containing one or more + * suffixes, and a type expresssion that wants files only. The intention for + * this is to allow watchman to more accurately determine what arguments to + * pass to eden's globFiles API. + */ + virtual SimpleSuffixType evaluateSimpleSuffix() const = 0; }; } // namespace watchman diff --git a/watchman/query/base.cpp b/watchman/query/base.cpp index 635ee9e2d6d7..8e0b5b4669b2 100644 --- a/watchman/query/base.cpp +++ b/watchman/query/base.cpp @@ -64,6 +64,14 @@ class NotExpr : public QueryExpr { } return ReturnOnlyFiles::Unrelated; } + + SimpleSuffixType evaluateSimpleSuffix() const override { + return SimpleSuffixType::Excluded; + } + + std::vector getSuffixQueryGlobPatterns() const override { + return std::vector{}; + } }; W_TERM_PARSER(not, NotExpr::parse); @@ -87,6 +95,14 @@ class TrueExpr : public QueryExpr { ReturnOnlyFiles listOnlyFiles() const override { return ReturnOnlyFiles::Unrelated; } + + SimpleSuffixType evaluateSimpleSuffix() const override { + return SimpleSuffixType::Excluded; + } + + std::vector getSuffixQueryGlobPatterns() const override { + return std::vector{}; + } }; W_TERM_PARSER(true, TrueExpr::parse); @@ -110,6 +126,14 @@ class FalseExpr : public QueryExpr { ReturnOnlyFiles listOnlyFiles() const override { return ReturnOnlyFiles::Unrelated; } + + SimpleSuffixType evaluateSimpleSuffix() const override { + return SimpleSuffixType::Excluded; + } + + std::vector getSuffixQueryGlobPatterns() const override { + return std::vector{}; + } }; W_TERM_PARSER(false, FalseExpr::parse); @@ -312,6 +336,35 @@ class ListExpr : public QueryExpr { } return result; } + + SimpleSuffixType evaluateSimpleSuffix() const override { + if (allof) { + std::vector types; + for (auto& subExpr : exprs) { + types.push_back(subExpr->evaluateSimpleSuffix()); + } + if (types.size() == 2) { + if ((types[0] == SimpleSuffixType::Type && + types[1] == SimpleSuffixType::Suffix) || + (types[1] == SimpleSuffixType::Type && + types[0] == SimpleSuffixType::Suffix)) { + return SimpleSuffixType::IsSimpleSuffix; + } + } + } + return SimpleSuffixType::Excluded; + } + + std::vector getSuffixQueryGlobPatterns() const override { + if (allof) { + for (auto& subExpr : exprs) { + if (subExpr->evaluateSimpleSuffix() == SimpleSuffixType::Suffix) { + return subExpr->getSuffixQueryGlobPatterns(); + } + } + } + return std::vector{}; + } }; W_TERM_PARSER(anyof, ListExpr::parseAnyOf); diff --git a/watchman/query/dirname.cpp b/watchman/query/dirname.cpp index 5c92a1e61830..3f22c45a3756 100644 --- a/watchman/query/dirname.cpp +++ b/watchman/query/dirname.cpp @@ -178,6 +178,14 @@ class DirNameExpr : public QueryExpr { ReturnOnlyFiles listOnlyFiles() const override { return ReturnOnlyFiles::Unrelated; } + + SimpleSuffixType evaluateSimpleSuffix() const override { + return SimpleSuffixType::Excluded; + } + + std::vector getSuffixQueryGlobPatterns() const override { + return std::vector{}; + } }; W_TERM_PARSER(dirname, DirNameExpr::parseDirName); diff --git a/watchman/query/empty.cpp b/watchman/query/empty.cpp index cc5c3833b873..a001b147d8a6 100644 --- a/watchman/query/empty.cpp +++ b/watchman/query/empty.cpp @@ -34,6 +34,14 @@ class ExistsExpr : public QueryExpr { ReturnOnlyFiles listOnlyFiles() const override { return ReturnOnlyFiles::Unrelated; } + + SimpleSuffixType evaluateSimpleSuffix() const override { + return SimpleSuffixType::Excluded; + } + + std::vector getSuffixQueryGlobPatterns() const override { + return std::vector{}; + } }; W_TERM_PARSER(exists, ExistsExpr::parse); @@ -79,6 +87,14 @@ class EmptyExpr : public QueryExpr { ReturnOnlyFiles listOnlyFiles() const override { return ReturnOnlyFiles::Unrelated; } + + SimpleSuffixType evaluateSimpleSuffix() const override { + return SimpleSuffixType::Excluded; + } + + std::vector getSuffixQueryGlobPatterns() const override { + return std::vector{}; + } }; W_TERM_PARSER(empty, EmptyExpr::parse); diff --git a/watchman/query/intcompare.cpp b/watchman/query/intcompare.cpp index c89da8f78657..4b13bb9d3c16 100644 --- a/watchman/query/intcompare.cpp +++ b/watchman/query/intcompare.cpp @@ -132,6 +132,14 @@ class SizeExpr : public QueryExpr { ReturnOnlyFiles listOnlyFiles() const override { return ReturnOnlyFiles::Unrelated; } + + SimpleSuffixType evaluateSimpleSuffix() const override { + return SimpleSuffixType::Excluded; + } + + std::vector getSuffixQueryGlobPatterns() const override { + return std::vector{}; + } }; W_TERM_PARSER(size, SizeExpr::parse); diff --git a/watchman/query/match.cpp b/watchman/query/match.cpp index c3c0297cfec9..8257ff69ade7 100644 --- a/watchman/query/match.cpp +++ b/watchman/query/match.cpp @@ -212,6 +212,14 @@ class WildMatchExpr : public QueryExpr { ReturnOnlyFiles listOnlyFiles() const override { return ReturnOnlyFiles::Unrelated; } + + SimpleSuffixType evaluateSimpleSuffix() const override { + return SimpleSuffixType::Excluded; + } + + std::vector getSuffixQueryGlobPatterns() const override { + return std::vector{}; + } }; W_TERM_PARSER(match, WildMatchExpr::parseMatch); W_TERM_PARSER(imatch, WildMatchExpr::parseIMatch); diff --git a/watchman/query/name.cpp b/watchman/query/name.cpp index d1d3802f0f4f..1b43bf64d0da 100644 --- a/watchman/query/name.cpp +++ b/watchman/query/name.cpp @@ -186,6 +186,14 @@ class NameExpr : public QueryExpr { ReturnOnlyFiles listOnlyFiles() const override { return ReturnOnlyFiles::Unrelated; } + + SimpleSuffixType evaluateSimpleSuffix() const override { + return SimpleSuffixType::Excluded; + } + + std::vector getSuffixQueryGlobPatterns() const override { + return std::vector{}; + } }; W_TERM_PARSER(name, NameExpr::parseName); diff --git a/watchman/query/pcre.cpp b/watchman/query/pcre.cpp index 5f22e7ca0cd7..7cfcce9bb9df 100644 --- a/watchman/query/pcre.cpp +++ b/watchman/query/pcre.cpp @@ -157,6 +157,14 @@ class PcreExpr : public QueryExpr { ReturnOnlyFiles listOnlyFiles() const override { return ReturnOnlyFiles::Unrelated; } + + SimpleSuffixType evaluateSimpleSuffix() const override { + return SimpleSuffixType::Excluded; + } + + std::vector getSuffixQueryGlobPatterns() const override { + return std::vector{}; + } }; W_TERM_PARSER(pcre, PcreExpr::parsePcre); W_TERM_PARSER(ipcre, PcreExpr::parseIPcre); diff --git a/watchman/query/since.cpp b/watchman/query/since.cpp index e290a10ffa7b..da725905d8d2 100644 --- a/watchman/query/since.cpp +++ b/watchman/query/since.cpp @@ -170,6 +170,14 @@ class SinceExpr : public QueryExpr { ReturnOnlyFiles listOnlyFiles() const override { return ReturnOnlyFiles::Unrelated; } + + SimpleSuffixType evaluateSimpleSuffix() const override { + return SimpleSuffixType::Excluded; + } + + std::vector getSuffixQueryGlobPatterns() const override { + return std::vector{}; + } }; W_TERM_PARSER(since, SinceExpr::parse); diff --git a/watchman/query/suffix.cpp b/watchman/query/suffix.cpp index f33230b75fb9..c193c654a22e 100644 --- a/watchman/query/suffix.cpp +++ b/watchman/query/suffix.cpp @@ -100,6 +100,19 @@ class SuffixExpr : public QueryExpr { ReturnOnlyFiles listOnlyFiles() const override { return ReturnOnlyFiles::Unrelated; } + + SimpleSuffixType evaluateSimpleSuffix() const override { + return SimpleSuffixType::Suffix; + } + + std::vector getSuffixQueryGlobPatterns() const override { + std::vector patterns; + for (const auto& suffix : suffixSet_) { + patterns.push_back("**/*." + suffix.string()); + } + + return patterns; + } }; W_TERM_PARSER(suffix, SuffixExpr::parse); W_CAP_REG("suffix-set") diff --git a/watchman/query/type.cpp b/watchman/query/type.cpp index d1b1b68addd8..cf87e9f7d601 100644 --- a/watchman/query/type.cpp +++ b/watchman/query/type.cpp @@ -119,6 +119,17 @@ class TypeExpr : public QueryExpr { } return ReturnOnlyFiles::Yes; } + + SimpleSuffixType evaluateSimpleSuffix() const override { + if (arg == 'f') { + return SimpleSuffixType::Type; + } + return SimpleSuffixType::Excluded; + } + + std::vector getSuffixQueryGlobPatterns() const override { + return std::vector{}; + } }; W_TERM_PARSER(type, TypeExpr::parse); diff --git a/watchman/test/SuffixQueryTest.cpp b/watchman/test/SuffixQueryTest.cpp new file mode 100644 index 000000000000..eff657d0115d --- /dev/null +++ b/watchman/test/SuffixQueryTest.cpp @@ -0,0 +1,133 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include "watchman/query/GlobTree.h" +#include "watchman/query/Query.h" +#include "watchman/query/QueryExpr.h" +#include "watchman/query/TermRegistry.h" +#include "watchman/thirdparty/jansson/jansson.h" + +using namespace watchman; +using namespace testing; + +namespace { + +std::optional parse_json(std::string expression_json) { + json_error_t err{}; + auto expression = json_loads(expression_json.c_str(), JSON_DECODE_ANY, &err); + if (!expression.has_value()) { + ADD_FAILURE() << "JSON parse error in fixture: " << err.text << " at " + << err.source << ":" << err.line << ":" << err.column; + return std::nullopt; + } + return expression; +} + +std::optional expr_evaluate_simple_suffix( + std::string expression_json) { + json_error_t err{}; + auto expression = parse_json(expression_json); + if (!expression.has_value()) { + return std::nullopt; + } + Query query; + // Disable automatic parsing of "match" as "imatch", "name" as "iname", etc. + auto expr = watchman::parseQueryExpr(&query, *expression); + return expr->evaluateSimpleSuffix(); +} + +std::optional> expr_get_suffix_glob( + std::string expression_json) { + json_error_t err{}; + auto expression = parse_json(expression_json); + if (!expression.has_value()) { + return std::nullopt; + } + Query query; + // Disable automatic parsing of "match" as "imatch", "name" as "iname", etc. + auto expr = watchman::parseQueryExpr(&query, *expression); + auto rv = expr->getSuffixQueryGlobPatterns(); + std::sort(rv.begin(), rv.end()); + return rv; +} + +} // namespace + +TEST(SuffixQueryTest, false) { + EXPECT_THAT( + expr_evaluate_simple_suffix(R"( ["false"] )"), + Optional(SimpleSuffixType::Excluded)); +} + +TEST(SuffixQueryTest, false_glob) { + EXPECT_THAT( + expr_get_suffix_glob(R"( ["false"] )"), + Optional(std::vector{})); +} + +TEST(SuffixQueryTest, type_d) { + EXPECT_THAT( + expr_evaluate_simple_suffix(R"( ["type", "d"] )"), + Optional(SimpleSuffixType::Excluded)); +} + +TEST(SuffixQueryTest, type_d_glob) { + EXPECT_THAT( + expr_get_suffix_glob(R"( ["type", "d"] )"), + Optional(std::vector{})); +} + +TEST(SuffixQueryTest, type_f) { + EXPECT_THAT( + expr_evaluate_simple_suffix(R"( ["type", "f"] )"), + Optional(SimpleSuffixType::Type)); +} + +TEST(SuffixQueryTest, type_f_glob) { + EXPECT_THAT( + expr_get_suffix_glob(R"( ["type", "f"] )"), + Optional(std::vector{})); +} +TEST(SuffixQueryTest, suffix) { + EXPECT_THAT( + expr_evaluate_simple_suffix(R"( ["suffix", ["a", "f"]] )"), + Optional(SimpleSuffixType::Suffix)); +} + +TEST(SuffixQueryTest, suffix_glob) { + EXPECT_THAT( + expr_get_suffix_glob(R"( ["suffix", ["a", "f"]] )"), + Optional(std::vector{"**/*.a", "**/*.f"})); +} + +TEST(SuffixQueryTest, allof_excl) { + EXPECT_THAT( + expr_evaluate_simple_suffix(R"( ["allof", ["type", "f"], ["exists"]] )"), + Optional(SimpleSuffixType::Excluded)); +} + +TEST(SuffixQueryTest, allof_excl_glob) { + EXPECT_THAT( + expr_get_suffix_glob(R"( ["allof", ["type", "f"], ["exists"]] )"), + Optional(std::vector{})); +} + +TEST(SuffixQueryTest, allof_yes) { + EXPECT_THAT( + expr_evaluate_simple_suffix( + R"( ["allof", ["type", "f"], ["suffix", ["a"]]] )"), + Optional(SimpleSuffixType::IsSimpleSuffix)); +} + +TEST(SuffixQueryTest, allof_yes_glob) { + EXPECT_THAT( + expr_get_suffix_glob(R"( ["allof", ["type", "f"], ["suffix", ["a"]]] )"), + Optional(std::vector{"**/*.a"})); +} diff --git a/watchman/test/async/AsyncWatchmanTestCase.py b/watchman/test/async/AsyncWatchmanTestCase.py index ca63a8c23376..737a2fd0d6e7 100644 --- a/watchman/test/async/AsyncWatchmanTestCase.py +++ b/watchman/test/async/AsyncWatchmanTestCase.py @@ -43,7 +43,6 @@ def touch_relative(self, base, *fname): self.touch(fname, None) def watchman_command(self, *args): - task = asyncio.wait_for(self.client.query(*args), 10) return self.loop.run_until_complete(task) diff --git a/watchman/watcher/eden.cpp b/watchman/watcher/eden.cpp index 1352ec161fe9..e4209a4881cf 100644 --- a/watchman/watcher/eden.cpp +++ b/watchman/watcher/eden.cpp @@ -613,13 +613,18 @@ static std::string escapeGlobSpecialChars(w_string_piece str) { * We need to respect the ignore_dirs configuration setting and * also remove anything that doesn't match the relative_root constraint * in the query. */ -void filterOutPaths(std::vector& files, QueryContext* ctx) { +void filterOutPaths( + std::vector& files, + QueryContext* ctx, + const std::string& relative_root = "") { files.erase( std::remove_if( files.begin(), files.end(), - [ctx](const NameAndDType& item) { - auto full = w_string::pathCat({ctx->root->root_path, item.name}); + [ctx, relative_root](const NameAndDType& item) { + w_string full; + full = w_string::pathCat( + {ctx->root->root_path, relative_root, item.name}); if (!ctx->fileMatchesRelativeRoot(full)) { // Not in the desired area, so filter it out @@ -655,7 +660,8 @@ std::vector globNameAndDType( const std::vector& globPatterns, bool includeDotfiles, bool splitGlobPattern = false, - bool listOnlyFiles = false) { + bool listOnlyFiles = false, + const std::string& relative_root = "") { // TODO(xavierd): Once the config: "eden_split_glob_pattern" is rolled out // everywhere, remove this code. if (splitGlobPattern && globPatterns.size() > 1) { @@ -672,6 +678,7 @@ std::vector globNameAndDType( params.wantDtype() = true; params.listOnlyFiles() = listOnlyFiles; params.sync() = getSyncBehavior(); + params.searchRoot() = relative_root; globFutures.emplace_back( client->semifuture_globFiles(params).via(executor)); @@ -691,6 +698,7 @@ std::vector globNameAndDType( params.wantDtype() = true; params.listOnlyFiles() = listOnlyFiles; params.sync() = getSyncBehavior(); + params.searchRoot() = relative_root; Glob glob; try { @@ -867,7 +875,8 @@ class EdenView final : public QueryableView { const std::vector& globStrings, QueryContext* ctx, bool includeDotfiles, - bool includeDir = true) const { + bool includeDir = true, + const std::string& relative_root = "") const { auto client = getEdenClient(thriftChannel_); bool listOnlyFiles = false; @@ -882,18 +891,19 @@ class EdenView final : public QueryableView { globStrings, includeDotfiles, splitGlobPattern_, - listOnlyFiles); + listOnlyFiles, + relative_root); ctx->edenGlobFilesDurationUs.store( timer.elapsed().count(), std::memory_order_relaxed); // Filter out any ignored files - filterOutPaths(fileInfo, ctx); + filterOutPaths(fileInfo, ctx, relative_root); for (auto& item : fileInfo) { auto file = make_unique( rootPath_, thriftChannel_, - w_string::pathCat({mountPoint_, item.name}), + w_string::pathCat({mountPoint_, relative_root, item.name}), /*ticks=*/nullptr, /*isNew=*/false, item.dtype); @@ -997,8 +1007,23 @@ class EdenView final : public QueryableView { void allFilesGenerator(const Query*, QueryContext* ctx) const override { ctx->generationStarted(); - auto globPatterns = getGlobPatternsForAllFiles(ctx); - executeGlobBasedQuery(globPatterns, ctx, /*includeDotfiles=*/true); + std::string relative_root = ""; + std::vector globPatterns; + bool includeDir = true; + if (isSimpleSuffixQuery(ctx)) { + globPatterns = getSuffixQueryGlobPatterns(ctx); + relative_root = getSuffixQueryRelativeRoot(ctx); + includeDir = false; + } else { + globPatterns = getGlobPatternsForAllFiles(ctx); + } + + executeGlobBasedQuery( + globPatterns, + ctx, + /*includeDotfiles=*/true, + includeDir, + relative_root); } ClockPosition getMostRecentRootNumberAndTickValue() const override { @@ -1191,6 +1216,26 @@ class EdenView final : public QueryableView { return globPatterns; } + bool isSimpleSuffixQuery(QueryContext* ctx) const { + // Checks if this query expression is a simple suffix query. + // A simple suffix query is an allof expression that only contains + // 1. Type = f + // 2. Suffix + if (ctx->query->expr) { + return ctx->query->expr->evaluateSimpleSuffix() == + SimpleSuffixType::IsSimpleSuffix; + } + return false; + } + + std::vector getSuffixQueryGlobPatterns(QueryContext* ctx) const { + return ctx->query->expr->getSuffixQueryGlobPatterns(); + } + + std::string getSuffixQueryRelativeRoot(QueryContext* ctx) const { + return computeRelativePathPiece(ctx).string(); + } + /** * Returns all the files in the watched directory for a fresh instance. * @@ -1203,8 +1248,14 @@ class EdenView final : public QueryableView { // Avoid a full tree walk if we don't need it! return std::vector(); } - - auto globPatterns = getGlobPatternsForAllFiles(ctx); + std::string relative_root = ""; + std::vector globPatterns; + if (isSimpleSuffixQuery(ctx)) { + globPatterns = getSuffixQueryGlobPatterns(ctx); + relative_root = getSuffixQueryRelativeRoot(ctx); + } else { + globPatterns = getGlobPatternsForAllFiles(ctx); + } auto client = getEdenClient(thriftChannel_); return globNameAndDType( @@ -1212,7 +1263,9 @@ class EdenView final : public QueryableView { mountPoint_, std::move(globPatterns), /*includeDotfiles=*/true, - splitGlobPattern_); + splitGlobPattern_, + /*listOnlyFiles=*/false, + relative_root); } struct GetAllChangesSinceResult {