From effe95d103c69eb6e52dfca667f20840754432fc Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Thu, 12 Sep 2024 19:35:49 +0100 Subject: [PATCH 1/4] binary file via init --- docs/warnet.md | 10 +++ resources/charts/binary-runner/Chart.yaml | 5 ++ resources/charts/binary-runner/Values.yaml | 11 +++ .../charts/binary-runner/templates/NOTES.txt | 1 + .../binary-runner/templates/_helpers.tpl | 60 ++++++++++++++++ .../charts/binary-runner/templates/pod.yaml | 31 ++++++++ src/warnet/constants.py | 1 + src/warnet/control.py | 71 ++++++++++++++++++- src/warnet/main.py | 3 +- 9 files changed, 191 insertions(+), 2 deletions(-) create mode 100644 resources/charts/binary-runner/Chart.yaml create mode 100644 resources/charts/binary-runner/Values.yaml create mode 100644 resources/charts/binary-runner/templates/NOTES.txt create mode 100644 resources/charts/binary-runner/templates/_helpers.tpl create mode 100644 resources/charts/binary-runner/templates/pod.yaml diff --git a/docs/warnet.md b/docs/warnet.md index 707dc8b5b..45a4cb38a 100644 --- a/docs/warnet.md +++ b/docs/warnet.md @@ -73,6 +73,16 @@ options: | scenario_file | Path | yes | | | additional_args | String | | | +### `warnet run-binary` +Run a file in warnet + Pass `-- --help` to get individual scenario help + +options: +| name | type | required | default | +|-----------------|--------|------------|-----------| +| file | Path | yes | | +| additional_args | String | | | + ### `warnet setup` Setup warnet diff --git a/resources/charts/binary-runner/Chart.yaml b/resources/charts/binary-runner/Chart.yaml new file mode 100644 index 000000000..39997e2dc --- /dev/null +++ b/resources/charts/binary-runner/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: bin-runner +description: A Helm chart for the bin-runner Pod +version: 0.1.0 +type: application diff --git a/resources/charts/binary-runner/Values.yaml b/resources/charts/binary-runner/Values.yaml new file mode 100644 index 000000000..89f60b9ef --- /dev/null +++ b/resources/charts/binary-runner/Values.yaml @@ -0,0 +1,11 @@ +--- +nameOverride: "" +fullnameOverride: "" + +pod: + name: bin-runner + namespace: warnet + +podLabels: + app: "warnet" + mission: "binary" diff --git a/resources/charts/binary-runner/templates/NOTES.txt b/resources/charts/binary-runner/templates/NOTES.txt new file mode 100644 index 000000000..4fc607ff7 --- /dev/null +++ b/resources/charts/binary-runner/templates/NOTES.txt @@ -0,0 +1 @@ +Binary executing diff --git a/resources/charts/binary-runner/templates/_helpers.tpl b/resources/charts/binary-runner/templates/_helpers.tpl new file mode 100644 index 000000000..a9360f3b0 --- /dev/null +++ b/resources/charts/binary-runner/templates/_helpers.tpl @@ -0,0 +1,60 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "binary-runner.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "binary-runner.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s" .Release.Name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "binary-runner.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "binary-runner.labels" -}} +helm.sh/chart: {{ include "binary-runner.chart" . }} +{{ include "binary-runner.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Values.podLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "binary-runner.selectorLabels" -}} +app.kubernetes.io/name: {{ include "binary-runner.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "binary-runner.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "binary-runner.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/resources/charts/binary-runner/templates/pod.yaml b/resources/charts/binary-runner/templates/pod.yaml new file mode 100644 index 000000000..7453b48a7 --- /dev/null +++ b/resources/charts/binary-runner/templates/pod.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "binary-runner.fullname" . }} + labels: + {{- include "binary-runner.labels" . | nindent 4 }} + app: {{ include "binary-runner.name" . }} + mission: binary +spec: + restartPolicy: Never + volumes: + - name: shared-data + emptyDir: {} + containers: + - name: {{ .Values.pod.name }}-runner + image: alpine + command: ["/bin/sh", "-c"] + args: + - | + echo "Waiting for binary file..." + while [ ! -f /data/binary ]; do + sleep 1 + done + echo "Binary found!" + chmod +x /data/binary + /data/binary + exit 0 + volumeMounts: + - name: shared-data + mountPath: /data diff --git a/src/warnet/constants.py b/src/warnet/constants.py index 4ce1ef2a5..49f72ab9f 100644 --- a/src/warnet/constants.py +++ b/src/warnet/constants.py @@ -32,6 +32,7 @@ BITCOIN_CHART_LOCATION = str(CHARTS_DIR.joinpath("bitcoincore")) FORK_OBSERVER_CHART = str(CHARTS_DIR.joinpath("fork-observer")) COMMANDER_CHART = str(CHARTS_DIR.joinpath("commander")) +BINARY_CHART = str(CHARTS_DIR.joinpath("binary-runner")) NAMESPACES_CHART_LOCATION = CHARTS_DIR.joinpath("namespaces") FORK_OBSERVER_CHART = str(files("resources.charts").joinpath("fork-observer")) CADDY_CHART = str(files("resources.charts").joinpath("caddy")) diff --git a/src/warnet/control.py b/src/warnet/control.py index 929b2b187..99390d744 100644 --- a/src/warnet/control.py +++ b/src/warnet/control.py @@ -1,6 +1,7 @@ import base64 import json import os +import shlex import subprocess import sys import time @@ -15,7 +16,7 @@ from rich.prompt import Confirm, Prompt from rich.table import Table -from .constants import COMMANDER_CHART, LOGGING_NAMESPACE +from .constants import BINARY_CHART, COMMANDER_CHART, LOGGING_NAMESPACE from .deploy import _port_stop_internal from .k8s import ( get_default_namespace, @@ -233,6 +234,74 @@ def run(scenario_file: str, additional_args: tuple[str]): print(f"Error: {e.stderr}") +@click.command(context_settings={"ignore_unknown_options": True}) +@click.argument("file", type=click.Path(exists=True, file_okay=True, dir_okay=False)) +@click.argument("additional_args", nargs=-1, type=click.UNPROCESSED) +def run_binary(file: str, additional_args: tuple[str]): + """ + Run a file in warnet + Pass `-- --help` to get individual scenario help + """ + file_path = Path(file).resolve() + file_name = file_path.stem + + name = f"binary-{file_name.replace('_', '')}-{int(time.time())}" + namespace = get_default_namespace() + + try: + # Construct Helm command + helm_command = [ + "helm", + "upgrade", + "--install", + "--namespace", + namespace, + "--set", + f"fullnameOverride={name}", + "--set", + f"pod.name={name}", + ] + + # Add additional arguments + if additional_args: + helm_command.extend(["--set", f"args={' '.join(additional_args)}"]) + if "--help" in additional_args or "-h" in additional_args: + return subprocess.run([sys.executable, file_path, "--help"]) + + helm_command.extend([name, BINARY_CHART]) + + # Execute Helm command to start the pod + result = subprocess.run(helm_command, check=True, capture_output=True, text=True) + + # Wait for the pod to be ready + wait_command = [ + "kubectl", + "wait", + "--for=condition=PodReadyToStartContainers", + "pod", + "--namespace", + namespace, + "--timeout=30s", + name, + ] + subprocess.run(wait_command, check=True) + + # Copy the binary into the container using k8s + command = f"kubectl cp {file_path} -n {namespace} {name}:/data/binary -c {name}-runner" + subprocess.run(shlex.split(command)) + + if result.returncode == 0: + print(f"Successfully started binary: {file_name}") + print(f"Pod name: {name}") + else: + print(f"Failed to start binary: {file_name}") + print(f"Error: {result.stderr}") + + except subprocess.CalledProcessError as e: + print(f"Failed to start binary: {file_name}") + print(f"Error: {e.stderr}") + + @click.command() @click.argument("pod_name", type=str, default="") @click.option("--follow", "-f", is_flag=True, default=False, help="Follow logs") diff --git a/src/warnet/main.py b/src/warnet/main.py index 76893575c..1bf91addb 100644 --- a/src/warnet/main.py +++ b/src/warnet/main.py @@ -2,7 +2,7 @@ from .admin import admin from .bitcoin import bitcoin -from .control import down, logs, run, snapshot, stop +from .control import down, logs, run, run_binary, snapshot, stop from .dashboard import dashboard from .deploy import deploy from .graph import create, graph @@ -29,6 +29,7 @@ def cli(): cli.add_command(logs) cli.add_command(new) cli.add_command(run) +cli.add_command(run_binary) cli.add_command(setup) cli.add_command(snapshot) cli.add_command(status) From 5c996827f0d5777518dd26529709789ca4ebf62f Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Thu, 12 Sep 2024 21:04:42 +0100 Subject: [PATCH 2/4] add binary to status --- src/warnet/status.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/warnet/status.py b/src/warnet/status.py index 1093fe7df..3bae461c8 100644 --- a/src/warnet/status.py +++ b/src/warnet/status.py @@ -15,6 +15,7 @@ def status(): tanks = _get_tank_status() scenarios = _get_deployed_scenarios() + binaries = _get_active_binaries() # Create a unified table table = Table(title="Warnet Status", show_header=True, header_style="bold magenta") @@ -40,6 +41,20 @@ def status(): else: table.add_row("Scenario", "No active scenarios", "") + # Add a separator if there are both tanks or scenarios and binaries + if (tanks or scenarios) and binaries: + table.add_row("", "", "") + + # Add binaries to the table + active_binaries = 0 + if binaries: + for binary in binaries: + table.add_row("Binary", binary["name"], binary["status"]) + if binary["status"] == "running" or binary["status"] == "pending": + active_binaries += 1 + else: + table.add_row("Binaries", "No active binaries", "") + # Create a panel to wrap the table panel = Panel( table, @@ -56,6 +71,7 @@ def status(): summary = Text() summary.append(f"\nTotal Tanks: {len(tanks)}", style="bold cyan") summary.append(f" | Active Scenarios: {active}", style="bold green") + summary.append(f" | Active Binaries: {active_binaries}", style="bold red") console.print(summary) _connected(end="\r") @@ -68,3 +84,8 @@ def _get_tank_status(): def _get_deployed_scenarios(): commanders = get_mission("commander") return [{"name": c.metadata.name, "status": c.status.phase.lower()} for c in commanders] + + +def _get_active_binaries(): + binaries = get_mission("binary") + return [{"name": b.metadata.name, "status": b.status.phase.lower()} for b in binaries] From 42977129f3fb263e232e60eb5b5b96cdd5aca9a6 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Thu, 12 Sep 2024 20:49:18 +0100 Subject: [PATCH 3/4] test for binary file --- .github/workflows/test.yml | 1 + test/binary_test.py | 68 +++++++++++++++++++++++ test/data/small_2_node/network.yaml | 5 ++ test/data/small_2_node/node-defaults.yaml | 4 ++ 4 files changed, 78 insertions(+) create mode 100755 test/binary_test.py create mode 100644 test/data/small_2_node/network.yaml create mode 100644 test/data/small_2_node/node-defaults.yaml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c717bddd4..e51dee5e8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -69,6 +69,7 @@ jobs: - services_test.py - signet_test.py - scenarios_test.py + - binary_test.py steps: - uses: actions/checkout@v4 - uses: azure/setup-helm@v4.2.0 diff --git a/test/binary_test.py b/test/binary_test.py new file mode 100755 index 000000000..9e84c0b21 --- /dev/null +++ b/test/binary_test.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +import json +import os +import tempfile +import time +from pathlib import Path + +from test_base import TestBase + +from warnet.k8s import get_default_namespace +from warnet.process import run_command + + +class BinaryTest(TestBase): + TEST_STRING = "Hello, World!" + + def __init__(self): + super().__init__() + self.network_dir = Path(os.path.dirname(__file__)) / "data" / "small_2_node" + + def run_test(self): + try: + self.setup_network() + self.test_generic_binary() + finally: + self.cleanup() + + def setup_network(self): + self.log.info("Setting up network") + self.log.info(self.warnet(f"deploy {self.network_dir}")) + self.wait_for_all_tanks_status(target="running") + self.wait_for_all_edges() + + def test_generic_binary(self): + self.log.info("Launching binary") + temp_file_content = f"""#!/usr/bin/env sh +echo {self.TEST_STRING}""" + + with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".sh") as temp_file: + temp_file.write(temp_file_content) + temp_file_path = temp_file.name + + os.chmod(temp_file_path, 0o755) + self.warnet(f"run-binary {temp_file_path}") + + # Get the commander pod name + pods = run_command(f"kubectl get pods -n {get_default_namespace()} -o json") + pods = json.loads(pods) + pod_list = [item["metadata"]["name"] for item in pods["items"]] + binary_pod = next((pod for pod in pod_list if pod.startswith("binary")), None) + if binary_pod is None: + raise ValueError("No pod found starting with 'binary'") + self.log.info(f"Got pod: {binary_pod}") + + def g_log(): + logs = self.warnet(f"logs {binary_pod}") + return self.TEST_STRING in logs + + time.sleep(5) + self.wait_for_predicate(g_log, timeout=60, interval=5) + + os.unlink(temp_file_path) + + +if __name__ == "__main__": + test = BinaryTest() + test.run_test() diff --git a/test/data/small_2_node/network.yaml b/test/data/small_2_node/network.yaml new file mode 100644 index 000000000..3d51aef3f --- /dev/null +++ b/test/data/small_2_node/network.yaml @@ -0,0 +1,5 @@ +nodes: + - name: tank-0000 + connect: + - tank-0001 + - name: tank-0001 diff --git a/test/data/small_2_node/node-defaults.yaml b/test/data/small_2_node/node-defaults.yaml new file mode 100644 index 000000000..7e021cad1 --- /dev/null +++ b/test/data/small_2_node/node-defaults.yaml @@ -0,0 +1,4 @@ +image: + repository: bitcoindevproject/bitcoin + pullPolicy: IfNotPresent + tag: "27.0" From f0b94d2dc86256573833b58cdb70a08e9a77b1b7 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Fri, 13 Sep 2024 08:44:17 +0100 Subject: [PATCH 4/4] enable stop for binaries --- src/warnet/control.py | 57 ++++++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/src/warnet/control.py b/src/warnet/control.py index 99390d744..5a9e2a62a 100644 --- a/src/warnet/control.py +++ b/src/warnet/control.py @@ -25,33 +25,38 @@ snapshot_bitcoin_datadir, ) from .process import run_command, stream_command +from .status import _get_active_binaries, _get_deployed_scenarios console = Console() @click.command() -@click.argument("scenario_name", required=False) -def stop(scenario_name): - """Stop a running scenario or all scenarios""" - active_scenarios = [sc.metadata.name for sc in get_mission("commander")] +@click.argument("name", required=False) +def stop(name): + """Stop one or all running scenarios or binaries""" + all_running = [c["name"] for c in _get_deployed_scenarios()] + [ + b["name"] for b in _get_active_binaries() + ] - if not active_scenarios: - console.print("[bold red]No active scenarios found.[/bold red]") + if not all_running: + console.print("[bold red]No active scenarios or binaries found.[/bold red]") return - if not scenario_name: - table = Table(title="Active Scenarios", show_header=True, header_style="bold magenta") + if not name: + table = Table( + title="Active Scenarios & binaries", show_header=True, header_style="bold magenta" + ) table.add_column("Number", style="cyan", justify="right") - table.add_column("Scenario Name", style="green") + table.add_column("Name", style="green") - for idx, name in enumerate(active_scenarios, 1): + for idx, name in enumerate(all_running, 1): table.add_row(str(idx), name) console.print(table) - choices = [str(i) for i in range(1, len(active_scenarios) + 1)] + ["a", "q"] + choices = [str(i) for i in range(1, len(all_running) + 1)] + ["a", "q"] choice = Prompt.ask( - "[bold yellow]Enter the number of the scenario to stop, 'a' to stop all, or 'q' to quit[/bold yellow]", + "[bold yellow]Enter the number you want to stop, 'a' to stop all, or 'q' to quit[/bold yellow]", choices=choices, show_choices=False, ) @@ -61,18 +66,18 @@ def stop(scenario_name): return elif choice == "a": if Confirm.ask("[bold red]Are you sure you want to stop all scenarios?[/bold red]"): - stop_all_scenarios(active_scenarios) + stop_all_scenarios(all_running) else: console.print("[bold blue]Operation cancelled.[/bold blue]") return - scenario_name = active_scenarios[int(choice) - 1] + name = all_running[int(choice) - 1] - if scenario_name not in active_scenarios: - console.print(f"[bold red]No active scenario found with name: {scenario_name}[/bold red]") + if name not in all_running: + console.print(f"[bold red]No active scenario or binary found with name: {name}[/bold red]") return - stop_scenario(scenario_name) + stop_scenario(name) def stop_scenario(scenario_name): @@ -103,6 +108,24 @@ def stop_all_scenarios(scenarios): console.print("[bold green]All scenarios have been stopped.[/bold green]") +def list_active_scenarios(): + """List all active scenarios""" + active_scenarios = [c["name"] for c in _get_deployed_scenarios()] + if not active_scenarios: + print("No active scenarios found.") + return + + console = Console() + table = Table(title="Active Scenarios", show_header=True, header_style="bold magenta") + table.add_column("Name", style="cyan") + table.add_column("Status", style="green") + + for scenario in active_scenarios: + table.add_row(scenario, "deployed") + + console.print(table) + + @click.command() def down(): """Bring down a running warnet quickly"""