diff --git a/container.py b/container.py index 35b96d5..121d9fc 100644 --- a/container.py +++ b/container.py @@ -15,7 +15,9 @@ @contextlib.contextmanager -def _maybe_open(path: Optional[str], mode: str) -> Iterator[Optional[IO[AnyStr]]]: +def _maybe_open( + path: Optional[str], mode: str +) -> Iterator[Optional[IO[AnyStr]]]: """A contextmanager that can open a file, or return None. This is useful to provide arguments to subprocess.call() and its friends. @@ -39,7 +41,8 @@ def getImageName(ci: bool) -> str: taggedContainerName = f"{imageName}:v1.9.59" if not subprocess.check_output( - ["docker", "image", "ls", "-q", taggedContainerName], universal_newlines=True + ["docker", "image", "ls", "-q", taggedContainerName], + universal_newlines=True, ).strip(): logging.info("Downloading Docker image %s...", taggedContainerName) subprocess.check_call(["docker", "pull", taggedContainerName]) @@ -170,7 +173,8 @@ def run_command( stdoutPath, "wb" ) as stdout: subprocess.run( - ["docker", "exec", "--interactive", self.containerId] + list(args), + ["docker", "exec", "--interactive", self.containerId] + + list(args), stdin=stdin, stdout=stdout, stderr=subprocess.PIPE, diff --git a/generateresources.py b/generateresources.py index e7b1fef..a427afa 100755 --- a/generateresources.py +++ b/generateresources.py @@ -16,7 +16,9 @@ _SUPPORTED_GENERATORS = frozenset(("png", "testplan")) -def _getSolution(p: problems.Problem, *, rootDirectory: str, ci: bool) -> Optional[str]: +def _getSolution( + p: problems.Problem, *, rootDirectory: str, ci: bool +) -> Optional[str]: """Gets the solution for the problem.""" solutions = [ f @@ -36,7 +38,9 @@ def _getSolution(p: problems.Problem, *, rootDirectory: str, ci: bool) -> Option return os.path.join(rootDirectory, p.path, "solutions", solutions[0]) -def _getInputs(p: problems.Problem, *, rootDirectory: str, ci: bool) -> List[str]: +def _getInputs( + p: problems.Problem, *, rootDirectory: str, ci: bool +) -> List[str]: """Gets the list of .in files for the problem.""" inFilenames = [ f @@ -113,10 +117,17 @@ def _generateImages( relativeInFilename = os.path.relpath(inFilename, rootDirectory) outFilename = f"{os.path.splitext(inFilename)[0]}.out" - logging.debug("%-30s: Generating .pngs for %s", p.title, inFilename) + logging.debug( + "%-30s: Generating .pngs for %s", p.title, inFilename + ) dimMatch = re.search(r"\.(\d*)x(\d*)\.in", inFilename) if dimMatch: - dimOpts = ["--height", dimMatch.group(1), "--width", dimMatch.group(2)] + dimOpts = [ + "--height", + dimMatch.group(1), + "--width", + dimMatch.group(2), + ] else: dimOpts = [] @@ -187,10 +198,14 @@ def _main() -> None: help="Consider all problems, instead of only those that have changed", ) parser.add_argument( - "--ci", action="store_true", help="Signal that this is being run from the CI." + "--ci", + action="store_true", + help="Signal that this is being run from the CI.", ) parser.add_argument( - "--force", action="store_true", help="Force re-generating all resources" + "--force", + action="store_true", + help="Force re-generating all resources", ) parser.add_argument( "--jobs", @@ -208,8 +223,12 @@ def _main() -> None: "Generates everything by default." ), ) - parser.add_argument("--verbose", action="store_true", help="Verbose logging") - parser.add_argument("problem_paths", metavar="PROBLEM", type=str, nargs="*") + parser.add_argument( + "--verbose", action="store_true", help="Verbose logging" + ) + parser.add_argument( + "problem_paths", metavar="PROBLEM", type=str, nargs="*" + ) args = parser.parse_args() if args.generate - _SUPPORTED_GENERATORS: @@ -227,7 +246,9 @@ def _main() -> None: rootDirectory = problems.repositoryRoot() - with concurrent.futures.ThreadPoolExecutor(max_workers=args.jobs) as executor: + with concurrent.futures.ThreadPoolExecutor( + max_workers=args.jobs + ) as executor: futures: List[concurrent.futures.Future[bool]] = [] for p in problems.problems( @@ -257,7 +278,8 @@ def _main() -> None: ) if not all( - future.result() for future in concurrent.futures.as_completed(futures) + future.result() + for future in concurrent.futures.as_completed(futures) ): logging.error("Some resources failed to generate") sys.exit(1) diff --git a/problems.py b/problems.py index 07523c8..9e07538 100644 --- a/problems.py +++ b/problems.py @@ -17,11 +17,15 @@ class Problem(NamedTuple): @staticmethod def load(problemPath: str, rootDirectory: str) -> "Problem": """Load a single proble from the path.""" - with open(os.path.join(rootDirectory, problemPath, "settings.json")) as f: + with open( + os.path.join(rootDirectory, problemPath, "settings.json") + ) as f: problemConfig = json.load(f) return Problem( - path=problemPath, title=problemConfig["title"], config=problemConfig + path=problemPath, + title=problemConfig["title"], + config=problemConfig, ) def shouldGenerateOutputs(self, *, rootDirectory: str) -> bool: @@ -47,7 +51,12 @@ def repositoryRoot() -> str: """ return ( subprocess.check_output( - ["git", "rev-parse", "--show-superproject-working-tree", "--show-toplevel"], + [ + "git", + "rev-parse", + "--show-superproject-working-tree", + "--show-toplevel", + ], universal_newlines=True, ) .strip() @@ -80,7 +89,9 @@ def ci_message( location.append(f"col={col}") print( f'::{kind} {",".join(location)}::' - + message.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A"), + + message.replace("%", "%25") + .replace("\r", "%0D") + .replace("\n", "%0A"), file=sys.stderr, flush=True, ) @@ -164,7 +175,9 @@ def problems( logging.warning("Problem %s disabled. Skipping.", problem["title"]) continue configProblems.append( - Problem.load(problemPath=problem["path"], rootDirectory=rootDirectory) + Problem.load( + problemPath=problem["path"], rootDirectory=rootDirectory + ) ) if allProblems: diff --git a/runtests.py b/runtests.py index 4614cf1..758185f 100755 --- a/runtests.py +++ b/runtests.py @@ -40,7 +40,9 @@ def _threadInitializer( ) -> None: """Set the thread affinity mapping for the current thread.""" with lock: - threadAffinityMapping[threading.get_ident()] = len(threadAffinityMapping) + threadAffinityMapping[threading.get_ident()] = len( + threadAffinityMapping + ) def _testProblem( @@ -169,12 +171,16 @@ def _main() -> None: parser = argparse.ArgumentParser("Run tests") parser.add_argument( - "--ci", action="store_true", help="Signal that this is being run from the CI." + "--ci", + action="store_true", + help="Signal that this is being run from the CI.", ) parser.add_argument( "--all", action="store_true", - help=("Consider all problems, instead of " "only those that have changed"), + help=( + "Consider all problems, instead of " "only those that have changed" + ), ) parser.add_argument( "--jobs", @@ -183,7 +189,9 @@ def _main() -> None: default=_availableProcessors(), help="Number of threads to run concurrently", ) - parser.add_argument("--verbose", action="store_true", help="Verbose logging") + parser.add_argument( + "--verbose", action="store_true", help="Verbose logging" + ) parser.add_argument( "--results-directory", default=os.path.join(rootDirectory, "results"), @@ -199,7 +207,9 @@ def _main() -> None: action="store_true", help=("Don't run tests: " "only download the Docker container"), ) - parser.add_argument("problem_paths", metavar="PROBLEM", type=str, nargs="*") + parser.add_argument( + "problem_paths", metavar="PROBLEM", type=str, nargs="*" + ) args = parser.parse_args() logging.basicConfig( @@ -244,7 +254,9 @@ def _main() -> None: ): if not filename.endswith(".out"): continue - os.unlink(os.path.join(rootDirectory, p.path, "cases", filename)) + os.unlink( + os.path.join(rootDirectory, p.path, "cases", filename) + ) futures.append( executor.submit( @@ -304,7 +316,10 @@ def _main() -> None: elif testResult["type"] == "invalid-inputs": testedFile = os.path.normpath( os.path.join( - p.path, "tests", "invalid-inputs", testResult["filename"] + p.path, + "tests", + "invalid-inputs", + testResult["filename"], ) ) expected = {"verdict": "WA"} @@ -335,7 +350,9 @@ def _main() -> None: f"logs at {os.path.relpath(logsDirectory, rootDirectory)}" ) - failureMessages: DefaultDict[str, List[str]] = collections.defaultdict(list) + failureMessages: DefaultDict[ + str, List[str] + ] = collections.defaultdict(list) normalizedScore = decimal.Decimal(got["score"]) round(normalizedScore, 15) * 100 @@ -368,7 +385,9 @@ def _main() -> None: f'{"-"*20}-+-{"-"*20}-+-{"-"*7}-+-{"-"*7}' ) - failureMessages[testedFile].append("\n".join(groupReportTable)) + failureMessages[testedFile].append( + "\n".join(groupReportTable) + ) failedCases = { c["name"] @@ -415,7 +434,8 @@ def _main() -> None: expectedFailure = err.read().strip() else: logging.error( - "Unexpected test result type: " f'{testResult["type"]}' + "Unexpected test result type: " + f'{testResult["type"]}' ) with open( @@ -448,9 +468,13 @@ def _main() -> None: f"{failureMessage}" ) - failureMessages[associatedFile].append(failureMessage) + failureMessages[associatedFile].append( + failureMessage + ) else: - logging.warning("Logs directory %r not found.", logsDirectory) + logging.warning( + "Logs directory %r not found.", logsDirectory + ) for path, messages in failureMessages.items(): problems.error( diff --git a/upload.py b/upload.py index 350faac..797c0cc 100755 --- a/upload.py +++ b/upload.py @@ -18,14 +18,18 @@ def createProblemZip( problemConfig: Mapping[str, Any], problemPath: str, zipPath: str ) -> None: """Creates a problem .zip on the provided path.""" - with zipfile.ZipFile(zipPath, "w", compression=zipfile.ZIP_DEFLATED) as archive: + with zipfile.ZipFile( + zipPath, "w", compression=zipfile.ZIP_DEFLATED + ) as archive: def _addFile(f: str) -> None: logging.debug("writing %s", f) archive.write(f, os.path.relpath(f, problemPath)) def _recursiveAdd(directory: str) -> None: - for root, _, filenames in os.walk(os.path.join(problemPath, directory)): + for root, _, filenames in os.walk( + os.path.join(problemPath, directory) + ): for f in filenames: _addFile(os.path.join(root, f)) @@ -87,10 +91,15 @@ def uploadProblemZip( "validator": validator["name"], "validator_time_limit": validator["limits"]["TimeLimit"], "email_clarifications": misc["email_clarifications"], - "group_score_policy": misc.get("group_score_policy", "sum-if-not-zero"), + "group_score_policy": misc.get( + "group_score_policy", "sum-if-not-zero" + ), } - exists = client.problem.details(problem_alias=alias, check_=False)["status"] == "ok" + exists = ( + client.problem.details(problem_alias=alias, check_=False)["status"] + == "ok" + ) if not exists: if not canCreate: @@ -144,7 +153,9 @@ def uploadProblemZip( if targetAdmins is not None: admins = { - a["username"].lower() for a in allAdmins["admins"] if a["role"] == "admin" + a["username"].lower() + for a in allAdmins["admins"] + if a["role"] == "admin" } desiredAdmins = {admin.lower() for admin in targetAdmins} @@ -161,7 +172,9 @@ def uploadProblemZip( for admin in adminsToRemove: logging.info("Removing problem admin: %s", admin) - client.problem.removeAdmin(problem_alias=alias, usernameOrEmail=admin) + client.problem.removeAdmin( + problem_alias=alias, usernameOrEmail=admin + ) if targetAdminGroups is not None: adminGroups = { @@ -185,7 +198,8 @@ def uploadProblemZip( if "tags" in misc: tags = { - t["name"].lower() for t in client.problem.tags(problem_alias=alias)["tags"] + t["name"].lower() + for t in client.problem.tags(problem_alias=alias)["tags"] } desiredTags = {t.lower() for t in misc["tags"]} @@ -202,7 +216,9 @@ def uploadProblemZip( for tag in tagsToAdd: logging.info("Adding problem tag: %s", tag) client.problem.addTag( - problem_alias=alias, name=tag, public=payload.get("public", False) + problem_alias=alias, + name=tag, + public=payload.get("public", False), ) @@ -236,20 +252,28 @@ def uploadProblem( def _main() -> None: env = os.environ - parser = argparse.ArgumentParser(description="Deploy a problem to omegaUp.") + parser = argparse.ArgumentParser( + description="Deploy a problem to omegaUp." + ) parser.add_argument( - "--ci", action="store_true", help="Signal that this is being run from the CI." + "--ci", + action="store_true", + help="Signal that this is being run from the CI.", ) parser.add_argument( "--all", action="store_true", help="Consider all problems, instead of only those that have changed", ) - parser.add_argument("--verbose", action="store_true", help="Verbose logging") + parser.add_argument( + "--verbose", action="store_true", help="Verbose logging" + ) parser.add_argument( "--url", default="https://omegaup.com", help="URL of the omegaUp host." ) - parser.add_argument("--api-token", type=str, default=env.get("OMEGAUP_API_TOKEN")) + parser.add_argument( + "--api-token", type=str, default=env.get("OMEGAUP_API_TOKEN") + ) parser.add_argument( "-u", "--username", @@ -267,7 +291,10 @@ def _main() -> None: parser.add_argument( "--can-create", action="store_true", - help=("Whether it's allowable to create the " "problem if it does not exist."), + help=( + "Whether it's allowable to create the " + "problem if it does not exist." + ), ) parser.add_argument( "--timeout", @@ -275,7 +302,9 @@ def _main() -> None: default=60, help="Timeout for deploy API call (in seconds)", ) - parser.add_argument("problem_paths", metavar="PROBLEM", type=str, nargs="*") + parser.add_argument( + "problem_paths", metavar="PROBLEM", type=str, nargs="*" + ) args = parser.parse_args() logging.basicConfig(