From ca2124aae76788b446b6849932c092f0b71bd9d7 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Fri, 4 Oct 2024 15:37:46 +0100 Subject: [PATCH 1/2] Openstack backend + tests --- .gitignore | 1 + docker-compose.yaml | 1 + docs/development/concepts.md | 1 + poetry.lock | 207 +++++++++++++++++++++- pyproject.toml | 1 + runner_manager/backend/openstack.py | 101 +++++++++++ runner_manager/models/backend.py | 50 ++++++ runner_manager/models/runner_group.py | 10 +- tests/unit/backend/test_openstack.py | 236 ++++++++++++++++++++++++++ tests/unit/models/test_settings.py | 27 ++- 10 files changed, 632 insertions(+), 3 deletions(-) create mode 100644 runner_manager/backend/openstack.py create mode 100644 tests/unit/backend/test_openstack.py diff --git a/.gitignore b/.gitignore index 1574dea4..c3be2b7f 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ coverage.xml config.yaml .env .reports +clouds.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml index b752bf16..7ccb6148 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -27,6 +27,7 @@ services: volumes: - ./runner_manager:/app/runner_manager - ./config.yaml:/app/config.yaml + - ./clouds.yaml:/app/clouds.yaml - /var/run/docker.sock:/var/run/docker.sock webhook: profiles: diff --git a/docs/development/concepts.md b/docs/development/concepts.md index bd620d6d..5b070fac 100644 --- a/docs/development/concepts.md +++ b/docs/development/concepts.md @@ -37,6 +37,7 @@ for hosting the runners. The following backends will be supported: - GCP. - AWS. +- Openstack. - VMware vSphere. - Docker (For local functional testing). - FakeBackend (For local unit testing). diff --git a/poetry.lock b/poetry.lock index b3ab8cb5..7098e387 100644 --- a/poetry.lock +++ b/poetry.lock @@ -948,6 +948,17 @@ ssh = ["bcrypt (>=3.1.5)"] test = ["certifi", "cryptography-vectors (==43.0.1)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] test-randomorder = ["pytest-randomly"] +[[package]] +name = "decorator" +version = "5.1.1" +description = "Decorators for Humans" +optional = false +python-versions = ">=3.5" +files = [ + {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"}, + {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, +] + [[package]] name = "docker" version = "7.1.0" @@ -970,6 +981,24 @@ docs = ["myst-parser (==0.18.0)", "sphinx (==5.1.1)"] ssh = ["paramiko (>=2.4.3)"] websockets = ["websocket-client (>=1.3.0)"] +[[package]] +name = "dogpile-cache" +version = "1.3.2" +description = "A caching front-end based on the Dogpile lock." +optional = false +python-versions = ">=3.8" +files = [ + {file = "dogpile.cache-1.3.2-py3-none-any.whl", hash = "sha256:c59250e23ddb4c03259c315c3b03d18b0658ec4f30ee665b39b91faf6401ef41"}, + {file = "dogpile.cache-1.3.2.tar.gz", hash = "sha256:4f71dc0333ad351c9c6f704f5ba2a37bf51c6eed0437d1adf56e075959afe63b"}, +] + +[package.dependencies] +decorator = ">=4.0.0" +stevedore = ">=3.0.0" + +[package.extras] +pifpaf = ["pifpaf (>=2.5.0)", "setuptools"] + [[package]] name = "fastapi" version = "0.114.0" @@ -1517,6 +1546,17 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "iso8601" +version = "2.1.0" +description = "Simple module to parse ISO 8601 dates" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "iso8601-2.1.0-py3-none-any.whl", hash = "sha256:aac4145c4dcb66ad8b648a02830f5e2ff6c24af20f4f482689be402db2429242"}, + {file = "iso8601-2.1.0.tar.gz", hash = "sha256:6b1d3829ee8921c4301998c909f7829fa9ed3cbdac0d3b16af2d743aed1ba8df"}, +] + [[package]] name = "isort" version = "5.13.2" @@ -1559,6 +1599,31 @@ files = [ {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, ] +[[package]] +name = "jsonpatch" +version = "1.33" +description = "Apply JSON-Patches (RFC 6902)" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +files = [ + {file = "jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade"}, + {file = "jsonpatch-1.33.tar.gz", hash = "sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c"}, +] + +[package.dependencies] +jsonpointer = ">=1.9" + +[[package]] +name = "jsonpointer" +version = "2.4" +description = "Identify specific nodes in a JSON document (RFC 6901)" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +files = [ + {file = "jsonpointer-2.4-py2.py3-none-any.whl", hash = "sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a"}, + {file = "jsonpointer-2.4.tar.gz", hash = "sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88"}, +] + [[package]] name = "jsonschema" version = "4.21.1" @@ -1594,6 +1659,31 @@ files = [ [package.dependencies] referencing = ">=0.31.0" +[[package]] +name = "keystoneauth1" +version = "5.8.0" +description = "Authentication Library for OpenStack Identity" +optional = false +python-versions = ">=3.8" +files = [ + {file = "keystoneauth1-5.8.0-py3-none-any.whl", hash = "sha256:e69dff80c509ab64d4de4494658d914e81f26af720828dc584ceee74ecd666d9"}, + {file = "keystoneauth1-5.8.0.tar.gz", hash = "sha256:3157c212e121164de64d63e5ef7e1daad2bd3649a68de1e971b76877019ef1c4"}, +] + +[package.dependencies] +iso8601 = ">=0.1.11" +os-service-types = ">=1.2.0" +pbr = ">=2.0.0" +requests = ">=2.14.2" +stevedore = ">=1.20.0" + +[package.extras] +betamax = ["betamax (>=0.7.0)", "fixtures (>=3.0.0)", "mock (>=2.0.0)"] +kerberos = ["requests-kerberos (>=0.8.0)"] +oauth1 = ["oauthlib (>=0.6.2)"] +saml2 = ["lxml (>=4.2.0)"] +test = ["PyYAML (>=3.12)", "bandit (>=1.7.6,<1.8.0)", "betamax (>=0.7.0)", "coverage (>=4.0)", "fixtures (>=3.0.0)", "flake8-docstrings (>=1.7.0,<1.8.0)", "flake8-import-order (>=0.18.2,<0.19.0)", "hacking (>=6.1.0,<6.2.0)", "lxml (>=4.2.0)", "oauthlib (>=0.6.2)", "oslo.config (>=5.2.0)", "oslo.utils (>=3.33.0)", "oslotest (>=3.2.0)", "requests-kerberos (>=0.8.0)", "requests-mock (>=1.2.0)", "stestr (>=1.0.0)", "testresources (>=2.0.0)", "testtools (>=2.2.0)"] + [[package]] name = "markdown" version = "3.5.2" @@ -1811,6 +1901,45 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +[[package]] +name = "netifaces" +version = "0.11.0" +description = "Portable network interface information." +optional = false +python-versions = "*" +files = [ + {file = "netifaces-0.11.0-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eb4813b77d5df99903af4757ce980a98c4d702bbcb81f32a0b305a1537bdf0b1"}, + {file = "netifaces-0.11.0-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:5f9ca13babe4d845e400921973f6165a4c2f9f3379c7abfc7478160e25d196a4"}, + {file = "netifaces-0.11.0-cp27-cp27m-win32.whl", hash = "sha256:7dbb71ea26d304e78ccccf6faccef71bb27ea35e259fb883cfd7fd7b4f17ecb1"}, + {file = "netifaces-0.11.0-cp27-cp27m-win_amd64.whl", hash = "sha256:0f6133ac02521270d9f7c490f0c8c60638ff4aec8338efeff10a1b51506abe85"}, + {file = "netifaces-0.11.0-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:08e3f102a59f9eaef70948340aeb6c89bd09734e0dca0f3b82720305729f63ea"}, + {file = "netifaces-0.11.0-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c03fb2d4ef4e393f2e6ffc6376410a22a3544f164b336b3a355226653e5efd89"}, + {file = "netifaces-0.11.0-cp34-cp34m-win32.whl", hash = "sha256:73ff21559675150d31deea8f1f8d7e9a9a7e4688732a94d71327082f517fc6b4"}, + {file = "netifaces-0.11.0-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:815eafdf8b8f2e61370afc6add6194bd5a7252ae44c667e96c4c1ecf418811e4"}, + {file = "netifaces-0.11.0-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:50721858c935a76b83dd0dd1ab472cad0a3ef540a1408057624604002fcfb45b"}, + {file = "netifaces-0.11.0-cp35-cp35m-win32.whl", hash = "sha256:c9a3a47cd3aaeb71e93e681d9816c56406ed755b9442e981b07e3618fb71d2ac"}, + {file = "netifaces-0.11.0-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:aab1dbfdc55086c789f0eb37affccf47b895b98d490738b81f3b2360100426be"}, + {file = "netifaces-0.11.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c37a1ca83825bc6f54dddf5277e9c65dec2f1b4d0ba44b8fd42bc30c91aa6ea1"}, + {file = "netifaces-0.11.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:28f4bf3a1361ab3ed93c5ef360c8b7d4a4ae060176a3529e72e5e4ffc4afd8b0"}, + {file = "netifaces-0.11.0-cp36-cp36m-win32.whl", hash = "sha256:2650beee182fed66617e18474b943e72e52f10a24dc8cac1db36c41ee9c041b7"}, + {file = "netifaces-0.11.0-cp36-cp36m-win_amd64.whl", hash = "sha256:cb925e1ca024d6f9b4f9b01d83215fd00fe69d095d0255ff3f64bffda74025c8"}, + {file = "netifaces-0.11.0-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:84e4d2e6973eccc52778735befc01638498781ce0e39aa2044ccfd2385c03246"}, + {file = "netifaces-0.11.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:18917fbbdcb2d4f897153c5ddbb56b31fa6dd7c3fa9608b7e3c3a663df8206b5"}, + {file = "netifaces-0.11.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:48324183af7f1bc44f5f197f3dad54a809ad1ef0c78baee2c88f16a5de02c4c9"}, + {file = "netifaces-0.11.0-cp37-cp37m-win32.whl", hash = "sha256:8f7da24eab0d4184715d96208b38d373fd15c37b0dafb74756c638bd619ba150"}, + {file = "netifaces-0.11.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2479bb4bb50968089a7c045f24d120f37026d7e802ec134c4490eae994c729b5"}, + {file = "netifaces-0.11.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:3ecb3f37c31d5d51d2a4d935cfa81c9bc956687c6f5237021b36d6fdc2815b2c"}, + {file = "netifaces-0.11.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:96c0fe9696398253f93482c84814f0e7290eee0bfec11563bd07d80d701280c3"}, + {file = "netifaces-0.11.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c92ff9ac7c2282009fe0dcb67ee3cd17978cffbe0c8f4b471c00fe4325c9b4d4"}, + {file = "netifaces-0.11.0-cp38-cp38-win32.whl", hash = "sha256:d07b01c51b0b6ceb0f09fc48ec58debd99d2c8430b09e56651addeaf5de48048"}, + {file = "netifaces-0.11.0-cp38-cp38-win_amd64.whl", hash = "sha256:469fc61034f3daf095e02f9f1bbac07927b826c76b745207287bc594884cfd05"}, + {file = "netifaces-0.11.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:5be83986100ed1fdfa78f11ccff9e4757297735ac17391b95e17e74335c2047d"}, + {file = "netifaces-0.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:54ff6624eb95b8a07e79aa8817288659af174e954cca24cdb0daeeddfc03c4ff"}, + {file = "netifaces-0.11.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:841aa21110a20dc1621e3dd9f922c64ca64dd1eb213c47267a2c324d823f6c8f"}, + {file = "netifaces-0.11.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:e76c7f351e0444721e85f975ae92718e21c1f361bda946d60a214061de1f00a1"}, + {file = "netifaces-0.11.0.tar.gz", hash = "sha256:043a79146eb2907edf439899f262b3dfe41717d34124298ed281139a8b93ca32"}, +] + [[package]] name = "nodeenv" version = "1.8.0" @@ -1825,6 +1954,46 @@ files = [ [package.dependencies] setuptools = "*" +[[package]] +name = "openstacksdk" +version = "3.1.0" +description = "An SDK for building applications to work with OpenStack" +optional = false +python-versions = ">=3.7" +files = [ + {file = "openstacksdk-3.1.0-py3-none-any.whl", hash = "sha256:4ce8339b87633f3cae52be4a2e9d1c784fba52f3d60422b594a594b185c63bf3"}, + {file = "openstacksdk-3.1.0.tar.gz", hash = "sha256:707f15d7ec074ab2434b9a5e1984facab00f822d272f4c2a5de0d22a09eb79cd"}, +] + +[package.dependencies] +cryptography = ">=2.7" +decorator = ">=4.4.1" +"dogpile.cache" = ">=0.6.5" +iso8601 = ">=0.1.11" +jmespath = ">=0.9.0" +jsonpatch = ">=1.16,<1.20 || >1.20" +keystoneauth1 = ">=3.18.0" +netifaces = ">=0.10.4" +os-service-types = ">=1.7.0" +pbr = ">=2.0.0,<2.1.0 || >2.1.0" +platformdirs = ">=3" +PyYAML = ">=3.13" +requestsexceptions = ">=1.2.0" + +[[package]] +name = "os-service-types" +version = "1.7.0" +description = "Python library for consuming OpenStack sevice-types-authority data" +optional = false +python-versions = "*" +files = [ + {file = "os-service-types-1.7.0.tar.gz", hash = "sha256:31800299a82239363995b91f1ebf9106ac7758542a1e4ef6dc737a5932878c6c"}, + {file = "os_service_types-1.7.0-py2.py3-none-any.whl", hash = "sha256:0505c72205690910077fb72b88f2a1f07533c8d39f2fe75b29583481764965d6"}, +] + +[package.dependencies] +pbr = ">=2.0.0,<2.1.0 || >2.1.0" + [[package]] name = "packaging" version = "23.2" @@ -1857,6 +2026,17 @@ files = [ {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, ] +[[package]] +name = "pbr" +version = "6.0.0" +description = "Python Build Reasonableness" +optional = false +python-versions = ">=2.6" +files = [ + {file = "pbr-6.0.0-py2.py3-none-any.whl", hash = "sha256:4a7317d5e3b17a3dccb6a8cfe67dab65b20551404c52c8ed41279fa4f0cb4cda"}, + {file = "pbr-6.0.0.tar.gz", hash = "sha256:d1377122a5a00e2f940ee482999518efe16d745d423a670c27773dfbc3c9a7d9"}, +] + [[package]] name = "platformdirs" version = "4.2.0" @@ -2506,6 +2686,17 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "requestsexceptions" +version = "1.4.0" +description = "Import exceptions from potentially bundled packages in requests." +optional = false +python-versions = "*" +files = [ + {file = "requestsexceptions-1.4.0-py2.py3-none-any.whl", hash = "sha256:3083d872b6e07dc5c323563ef37671d992214ad9a32b0ca4a3d7f5500bf38ce3"}, + {file = "requestsexceptions-1.4.0.tar.gz", hash = "sha256:b095cbc77618f066d459a02b137b020c37da9f46d9b057704019c9f77dba3065"}, +] + [[package]] name = "rpds-py" version = "0.18.0" @@ -2770,6 +2961,20 @@ anyio = ">=3.4.0,<5" [package.extras] full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.7)", "pyyaml"] +[[package]] +name = "stevedore" +version = "5.2.0" +description = "Manage dynamic plugins for Python applications" +optional = false +python-versions = ">=3.8" +files = [ + {file = "stevedore-5.2.0-py3-none-any.whl", hash = "sha256:1c15d95766ca0569cad14cb6272d4d31dae66b011a929d7c18219c176ea1b5c9"}, + {file = "stevedore-5.2.0.tar.gz", hash = "sha256:46b93ca40e1114cea93d738a6c1e365396981bb6bb78c27045b7587c9473544d"}, +] + +[package.dependencies] +pbr = ">=2.0.0,<2.1.0 || >2.1.0" + [[package]] name = "types-awscrt" version = "0.20.5" @@ -3206,4 +3411,4 @@ dev = ["doc8", "flake8", "flake8-import-order", "rstcheck[sphinx]", "sphinx"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "61ee5fe793ec73a6c734621cc27fe04cb7c1f4d0dfd1d654f7e7752e3006f129" +content-hash = "59eec33eb722465701c5030652a640040e56f0ff5d03fda4b2263203b11248aa" diff --git a/pyproject.toml b/pyproject.toml index 9db7a813..c16de0eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ botocore = "^1.34.162" boto3-stubs = { extras = ["ec2"], version = "^1.35.14" } githubkit = { extras = ["auth-app"], version = "^0.11.8" } rq-scheduler = "^0.13.1" +openstacksdk = "^3.1.0" pyvmomi = "^8.0.3.0.1" vapi-runtime = { url = "https://raw.githubusercontent.com/vmware/vsphere-automation-sdk-python/v8.0.1.0/lib/vapi-runtime/vapi_runtime-2.40.0-py2.py3-none-any.whl" } vcenter-bindings = { url = "https://raw.githubusercontent.com/vmware/vsphere-automation-sdk-python/v8.0.1.0/lib/vcenter-bindings/vcenter_bindings-4.1.0-py2.py3-none-any.whl" } diff --git a/runner_manager/backend/openstack.py b/runner_manager/backend/openstack.py new file mode 100644 index 00000000..491005a6 --- /dev/null +++ b/runner_manager/backend/openstack.py @@ -0,0 +1,101 @@ +from typing import List, Literal, Optional + +import openstack +from githubkit.versions.latest.webhooks import WorkflowJobEvent +from openstack.compute.v2.server import Server +from openstack.connection import Connection +from openstack.exceptions import SDKException +from pydantic import Field +from redis_om import NotFoundError + +from runner_manager.backend.base import BaseBackend +from runner_manager.logging import log +from runner_manager.models.backend import ( + Backends, + OpenstackConfig, + OpenstackInstance, + OpenstackInstanceConfig, +) +from runner_manager.models.runner import Runner + + +class OpenstackBackend(BaseBackend): + name: Literal[Backends.openstack] = Field(default=Backends.openstack) + config: OpenstackConfig = OpenstackConfig() + instance_config: OpenstackInstanceConfig = OpenstackInstanceConfig() + + @property + def client(self) -> Connection: + return openstack.connect( + cloud=self.config.cloud, region_name=self.config.region_name + ) + + def create(self, runner: Runner): + """Create a runner""" + instance_resource: OpenstackInstance = self.instance_config.configure_instance( + runner + ) + try: + instance = self.client.create_server(**instance_resource) + runner.instance_id = instance.id + except Exception as e: + log.error(e) + raise e + return super().create(runner) + + def delete(self, runner: Runner): + """Delete a runner""" + if runner.instance_id: + try: + self.client.delete_server(name_or_id=runner.instance_id) + except SDKException as e: + log.error(e) + raise e + return super().delete(runner) + + def list(self) -> List[Runner]: + try: + servers: List[Server] = list( + self.client.compute.servers( + tags={ + "manager": str(self.manager), + "runner_group": str(self.runner_group), + } + ) + ) + except Exception as e: + log.error(e) + raise e + runners: List[Runner] = [] + for instance in servers: + instance_id = instance.id + name = instance.name + try: + runner = Runner.find( + Runner.instance_id == instance_id, + ).first() + except NotFoundError: + runner = Runner( + name=name, + instance_id=instance_id, + runner_group_name=self.runner_group, + busy=False, + created_at=instance.launched_at, + ) + runners.append(runner) + return runners + + def update( + self, runner: Runner, webhook: Optional[WorkflowJobEvent] = None + ) -> Runner: + """Update a runner""" + if runner.instance_id: + try: + self.client.set_server_metadata( + runner.instance_id, + {"status": runner.status, "busy": str(runner.busy)}, + ) + except Exception as e: + log.error(e) + raise e + return super().update(runner, webhook) diff --git a/runner_manager/models/backend.py b/runner_manager/models/backend.py index ce75bda7..29c7e714 100644 --- a/runner_manager/models/backend.py +++ b/runner_manager/models/backend.py @@ -29,6 +29,7 @@ class Backends(str, Enum): docker = "docker" gcloud = "gcloud" aws = "aws" + openstack = "openstack" vsphere = "vsphere" @@ -224,6 +225,55 @@ def configure_instance(self, runner: Runner) -> AwsInstance: ) +OpenstackInstance = TypedDict( + "OpenstackInstance", + { + "name": str, + "image": str, + "flavor": str, + "volume_size": int, + "userdata": str, + "meta": dict[str, str], + "network": str, + }, +) + + +class OpenstackConfig(BackendConfig): + cloud: Optional[str] = "" + region_name: Optional[str] = "" + + +class OpenstackInstanceConfig(InstanceConfig): + image: str = "" + flavor: str = "" + volume_size: int = 20 + network: str = "" + meta: dict[str, str] = {} + + def configure_instance(self, runner: Runner) -> OpenstackInstance: + """Configure Instance""" + meta: dict[str, str] = { + "Name": runner.name, + "manager": runner.manager if runner.manager else "", + "runner_group": runner.runner_group_name, + "busy": str(runner.busy), + "status": runner.status, + } + self.meta.update(meta) + userdata = self.template_startup(runner) + + return OpenstackInstance( + name=runner.name, + image=self.image, + flavor=self.flavor, + network=self.network, + volume_size=self.volume_size, + userdata=userdata, + meta=self.meta, + ) + + class VsphereConfig(BackendConfig): """Configuration for vSphere backend.""" diff --git a/runner_manager/models/runner_group.py b/runner_manager/models/runner_group.py index acaf5b67..bda351b3 100644 --- a/runner_manager/models/runner_group.py +++ b/runner_manager/models/runner_group.py @@ -18,6 +18,7 @@ from runner_manager.backend.base import BaseBackend from runner_manager.backend.docker import DockerBackend from runner_manager.backend.gcloud import GCPBackend +from runner_manager.backend.openstack import OpenstackBackend from runner_manager.backend.vsphere import VsphereBackend from runner_manager.clients.github import GitHub from runner_manager.clients.github import RunnerGroup as GitHubRunnerGroup @@ -50,7 +51,14 @@ class BaseRunnerGroup(PydanticBaseModel): job_completed_script: Optional[str] = "" backend: Annotated[ - Union[BaseBackend, DockerBackend, GCPBackend, AWSBackend, VsphereBackend], + Union[ + BaseBackend, + DockerBackend, + GCPBackend, + AWSBackend, + VsphereBackend, + OpenstackBackend, + ], PydanticField(..., discriminator="name"), ] diff --git a/tests/unit/backend/test_openstack.py b/tests/unit/backend/test_openstack.py new file mode 100644 index 00000000..e53013c5 --- /dev/null +++ b/tests/unit/backend/test_openstack.py @@ -0,0 +1,236 @@ +import os +import uuid +from inspect import signature +from time import sleep +from typing import Callable + +from openstack.compute.v2 import _proxy, server +from openstack.connection import Connection +from openstack.proxy import Proxy +from pytest import fixture, raises +from redis_om import NotFoundError + +from runner_manager.backend.openstack import OpenstackBackend, OpenstackInstance +from runner_manager.models.backend import ( + Backends, + OpenstackConfig, + OpenstackInstanceConfig, +) +from runner_manager.models.runner import Runner, RunnerLabel +from runner_manager.models.runner_group import RunnerGroup + + +def compare_signatures(func1: Callable, func2: Callable): + return signature(func1) == signature(func2) + + +class MockCompute(Proxy): + _instances: dict[str, server.Server] = {} + + def __init__(self): + pass + + def servers(self, details=True, all_projects=False, **query): + for instance in self._instances.values(): + yield instance + + def create_server(self, **attrs): + new_server = MockServer(**attrs) + self._instances[str(new_server.id)] = new_server + return new_server + + def delete_server(self, server, ignore_missing=True, force=False): + return self._instances.pop(server) + + def set_server_metadata(self, server, **metadata): + self._instances[server].set_metadata(session=None, metadata=metadata) + + +class MockConnection: + compute: MockCompute = MockCompute() + + def create_server( + self, + name, + image=None, + flavor=None, + auto_ip=True, + ips=None, + ip_pool=None, + root_volume=None, + terminate_volume=False, + wait=False, + timeout=180, + reuse_ips=True, + network=None, + boot_from_volume=False, + volume_size="50", + boot_volume=None, + volumes=None, + nat_destination=None, + group=None, + **kwargs, + ): + return self.compute.create_server( + name=name, + image=image, + flavor=flavor, + volume_size=volume_size, + network=network, + **kwargs, + ) + + def delete_server( + self, name_or_id, wait=False, timeout=180, delete_ips=False, delete_ip_retry=1 + ): + return self.compute.delete_server(name_or_id) + + def set_server_metadata(self, name_or_id, metadata): + self.compute.set_server_metadata(name_or_id, **metadata) + + def get_server_by_id(self, id): + return self.compute._instances.get(id, None) + + +class MockServer(server.Server): + name: str = "" + image: str = "" + flavor: str = "" + volume_size: str = "20" + userdata: str = "" + meta: dict[str, str] = {} + network: str = "" + status: str = "ACTIVE" + + def __init__( + self, + name: str, + image, + flavor, + volume_size: str, + network, + userdata: str = "", + meta: dict[str, str] = {}, + *args, + **kwargs, + ): + super().__init__(*args, **kwargs) + self.name = name + self.image = image + self.flavor = flavor + self.volume_size = volume_size + self.userdata = userdata + self.meta = meta + self.network = network + self.id = str(uuid.uuid4()) + + def set_metadata(self, session, metadata, *args, **kwargs) -> None: + self.meta.update(metadata) + + +@fixture() +def openstack_group(settings) -> RunnerGroup: + runner_group: RunnerGroup = RunnerGroup( + id=1, + name="test", + organization="octo-org", + manager=settings.name, + backend=OpenstackBackend( + name=Backends.openstack, + config=OpenstackConfig(cloud=os.getenv("OPENSTACK_CLOUD")), + manager=settings.name, + instance_config=OpenstackInstanceConfig( + image="182c6fd0-775c-4dh3-9be3-d14c95add0d3", + network="c4d017da-c7e8-4019-bdb1-3d426d638c93", + ), + ), + labels=[ + "label", + ], + ) + return runner_group + + +@fixture() +def openstack_runner(runner: Runner, openstack_group: RunnerGroup) -> Runner: + openstack_group.backend.delete(runner) + return runner + + +def use_live_openstack(): + if os.getenv("USE_LIVE_OPENSTACK") == "true": + return True + return False + + +def test_openstack_instance_config(runner: Runner): + instance_config = OpenstackInstanceConfig( + network="8175b591-j79a-4846-9g97-43cb083a93b5", + meta={"test": "test"}, + ) + instance: OpenstackInstance = instance_config.configure_instance(runner) + assert instance["image"] == instance_config.image + assert instance["flavor"] == instance_config.flavor + assert instance["volume_size"] == instance_config.volume_size + assert instance["network"] == instance_config.network + meta = instance["meta"] + assert "Name" in meta + assert "test" in meta + + +def test_create_delete(openstack_group, openstack_runner, monkeypatch): + if not use_live_openstack(): + monkeypatch.setattr(OpenstackBackend, "client", MockConnection()) + runner = openstack_group.backend.create(openstack_runner) + assert runner.instance_id is not None + assert runner.backend == "openstack" + assert Runner.find(Runner.instance_id == runner.instance_id).first() == runner + openstack_group.backend.delete(runner) + with raises(NotFoundError): + Runner.find(Runner.instance_id == runner.instance_id).first() + + +def test_list(openstack_group, openstack_runner, monkeypatch): + if not use_live_openstack(): + monkeypatch.setattr(OpenstackBackend, "client", MockConnection()) + runner = openstack_group.backend.create(openstack_runner) + runners = openstack_group.backend.list() + assert runner in runners + openstack_group.backend.delete(runner) + with raises(NotFoundError): + openstack_group.backend.get(runner.instance_id) + + +def test_update(openstack_group, openstack_runner, monkeypatch): + if not use_live_openstack(): + monkeypatch.setattr(OpenstackBackend, "client", MockConnection()) + runner = openstack_group.backend.create(openstack_runner) + runner.labels = [RunnerLabel(name="test", type="custom")] + instance: server.Server | None = openstack_group.backend.client.get_server_by_id( + runner.instance_id + ) + while instance and str(instance.status) == "BUILD": + instance = openstack_group.backend.client.get_server_by_id(runner.instance_id) + sleep(1) + openstack_group.backend.update(runner) + assert runner.labels == [RunnerLabel(name="test", type="custom")] + openstack_group.backend.delete(runner) + with raises(NotFoundError): + openstack_group.backend.get(runner.instance_id) + + +def test_mock_signatures(): + assert compare_signatures(Connection.create_server, MockConnection.create_server) + assert compare_signatures(Connection.delete_server, MockConnection.delete_server) + assert compare_signatures( + Connection.set_server_metadata, MockConnection.set_server_metadata + ) + assert compare_signatures( + Connection.get_server_by_id, MockConnection.get_server_by_id + ) + assert compare_signatures(_proxy.Proxy.servers, MockCompute.servers) + assert compare_signatures(_proxy.Proxy.create_server, MockCompute.create_server) + assert compare_signatures(_proxy.Proxy.delete_server, MockCompute.delete_server) + assert compare_signatures( + _proxy.Proxy.set_server_metadata, MockCompute.set_server_metadata + ) diff --git a/tests/unit/models/test_settings.py b/tests/unit/models/test_settings.py index 4c5785c5..55176ed8 100644 --- a/tests/unit/models/test_settings.py +++ b/tests/unit/models/test_settings.py @@ -30,7 +30,20 @@ def yaml_data(): }, "organization": "octo-org", "labels": ["label"], - } + }, + { + "name": "test-openstack", + "backend": { + "name": "openstack", + "config": { + "cloud": "test-cloud-01", + "region_name": "test-region-01", + }, + "instance_config": {}, + }, + "organization": "octo-org", + "labels": ["label"], + }, ], } @@ -81,6 +94,18 @@ def test_redhat_credentials(config_file, monkeypatch): ) +def test_openstack_credentials_yaml(config_file, yaml_data): + settings = Settings() + assert ( + settings.runner_groups[1].backend.config.cloud + == yaml_data["runner_groups"][1]["backend"]["config"]["cloud"] + ) + assert ( + settings.runner_groups[1].backend.config.region_name + == yaml_data["runner_groups"][1]["backend"]["config"]["region_name"] + ) + + def test_env_file(): os.environ["REDIS_OM_URL"] = "redis://localhost:6379/0" os.environ["GITHUB_BASE_URL"] = "https://github.com" From e03667091baafbe5637d98f6bc208d942012c797 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Fri, 4 Oct 2024 17:17:17 +0100 Subject: [PATCH 2/2] Use full Python image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b8e5cee2..99a6c023 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.11 ARG VERSION=dev