From 5095fedb145d50c9f351a9ce53c7e1799986e17d Mon Sep 17 00:00:00 2001 From: Sara Walton Date: Thu, 11 Jul 2024 15:49:10 -0600 Subject: [PATCH 1/4] adding readthedocs --- rtd/.readthedocs.yaml | 22 ++ rtd/README.rst | 46 ++++ rtd/docs/Makefile | 20 ++ rtd/docs/make.bat | 35 +++ rtd/docs/requirements.txt | 2 + rtd/docs/source/conf.py | 50 ++++ rtd/docs/source/deployment/index.rst | 9 + rtd/docs/source/deployment/test.rst | 4 + rtd/docs/source/index.rst | 67 ++++++ rtd/docs/source/maestro-index.rst | 8 + rtd/docs/source/maestro-quickstart.rst | 215 ++++++++++++++++++ rtd/docs/source/maestro/index.rst | 8 + .../source/maestro/maestro-quickstart.rst | 214 +++++++++++++++++ rtd/paper.lock | 1 + rtd/pyproject.toml | 8 + rtd/update-branches.py | 39 ++++ 16 files changed, 748 insertions(+) create mode 100644 rtd/.readthedocs.yaml create mode 100644 rtd/README.rst create mode 100644 rtd/docs/Makefile create mode 100644 rtd/docs/make.bat create mode 100644 rtd/docs/requirements.txt create mode 100644 rtd/docs/source/conf.py create mode 100644 rtd/docs/source/deployment/index.rst create mode 100644 rtd/docs/source/deployment/test.rst create mode 100644 rtd/docs/source/index.rst create mode 100644 rtd/docs/source/maestro-index.rst create mode 100644 rtd/docs/source/maestro-quickstart.rst create mode 100644 rtd/docs/source/maestro/index.rst create mode 100644 rtd/docs/source/maestro/maestro-quickstart.rst create mode 100644 rtd/paper.lock create mode 100644 rtd/pyproject.toml create mode 100644 rtd/update-branches.py diff --git a/rtd/.readthedocs.yaml b/rtd/.readthedocs.yaml new file mode 100644 index 0000000..26efcab --- /dev/null +++ b/rtd/.readthedocs.yaml @@ -0,0 +1,22 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: rtd/docs/source/conf.py + +# We recommend specifying your dependencies to enable reproducible builds: +# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: rtd/docs/requirements.txt diff --git a/rtd/README.rst b/rtd/README.rst new file mode 100644 index 0000000..4fba89a --- /dev/null +++ b/rtd/README.rst @@ -0,0 +1,46 @@ +OVIS-HPC Maestro Documentation +######################## + +This repository hosts all Maestro related documentation such as how-to tutorials, getting started with Maestro, docker-hub links, API's and much more. Documentation webpage can be found in the `Maestro readthedocs webpage `_ + +Contributing to ReadTheDocs +############################ +Instructions and documentation on how to use ReadTheDocs can be found here: +`readthedocs Help Guide `_ + + +* Clone the repository: + +.. code-block:: RST + + > git clone git@github.com:/ovis-docs.git + +* Add any existing file name(s) you will be editing to paper.lock + +.. code-block:: RST + + > vi paper.lock + + | mm/dd | + +* Make necessary changes, update paper.lock file and push to repo. + +.. code-block:: RST + + > vi paper.lock + + ## remove line + > git add + > git commit -m "add message" + > git push + +Adding A New File +****************** +For any new RST files created, please include them in rtd/docs/src/index.rst under their corresponding sections. All RST files not included in index.rst will not populate on the offical webpage (e.g. readthedocs). + +Paper Lock +************ +This is for claiming any sections you are working on so there is no overlap. +Please USE paper.lock to indicate if you are editing an existing RST file. + + diff --git a/rtd/docs/Makefile b/rtd/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/rtd/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/rtd/docs/make.bat b/rtd/docs/make.bat new file mode 100644 index 0000000..9534b01 --- /dev/null +++ b/rtd/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/rtd/docs/requirements.txt b/rtd/docs/requirements.txt new file mode 100644 index 0000000..8893b57 --- /dev/null +++ b/rtd/docs/requirements.txt @@ -0,0 +1,2 @@ +# compatable with the newest version of Sphinx (v7.2.1) +sphinx_rtd_theme==1.3.0rc1 diff --git a/rtd/docs/source/conf.py b/rtd/docs/source/conf.py new file mode 100644 index 0000000..40df176 --- /dev/null +++ b/rtd/docs/source/conf.py @@ -0,0 +1,50 @@ +# Configuration file for the Sphinx documentation builder. + +# -- Project information + +project = 'OVIS-HPC' +copyright = '2024, Sandia National Laboratories and Open Grid Computing, Inc.' +author = 'SNL/OGC' + +release = '0.1' +version = '0.1.0' + +# -- General configuration + +extensions = [ + 'sphinx.ext.duration', + 'sphinx.ext.doctest', + 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'sphinx.ext.intersphinx', +] + +intersphinx_mapping = { + 'python': ('https://docs.python.org/3/', None), + 'sphinx': ('https://www.sphinx-doc.org/en/master/', None), + + # Link to the "apis" of the "hpc-ovis" project and subprojects + "ovis-hpc": ("https://ovis-hpc.readthedocs.io/en/latest/", None), + "sos": ("https://ovis-hpc.readthedocs.io/projects/sos/en/latest/", None), + "maestro": ("https://ovis-hpc.readthedocs.io/projects/maestro/en/latest/", None), + "baler": ("https://ovis-hpc.readthedocs.io/projects/baler/en/latest/", None), + "ldms": ("https://ovis-hpc.readthedocs.io/projects/ldms/en/latest/", None), + +} +intersphinx_disabled_domains = ['std'] +intersphinx_disabled_reftypes = ["*"] + +templates_path = ['_templates'] + +# -- Options for HTML output + +html_theme = 'sphinx_rtd_theme' +html_static_path = ['static'] +html_logo = "https://github.com/ovis-hpc/readthedocs/blob/main/docs/source/images/ovis-logo.png?raw=true" +html_theme_options = { + 'logo_only': True, + 'display_version': False, +} + +# -- Options for EPUB output +epub_show_urls = 'footnote' diff --git a/rtd/docs/source/deployment/index.rst b/rtd/docs/source/deployment/index.rst new file mode 100644 index 0000000..44b3ca1 --- /dev/null +++ b/rtd/docs/source/deployment/index.rst @@ -0,0 +1,9 @@ +Maestro Deployment +=============== +This section covers how to deploy and test Maestro + +.. toctree:: + :maxdepth: 2 + + test + diff --git a/rtd/docs/source/deployment/test.rst b/rtd/docs/source/deployment/test.rst new file mode 100644 index 0000000..9d51e9b --- /dev/null +++ b/rtd/docs/source/deployment/test.rst @@ -0,0 +1,4 @@ +Github +====== + +Documentation for this is currently under development. diff --git a/rtd/docs/source/index.rst b/rtd/docs/source/index.rst new file mode 100644 index 0000000..8e52833 --- /dev/null +++ b/rtd/docs/source/index.rst @@ -0,0 +1,67 @@ +.. Copyright 2023 Sandia National Laboratories, LLC + (c.f. AUTHORS, NOTICE.LLNS, COPYING) + + SPDX-License-Identifier: (LGPL-3.0) + +.. Flux documentation master file, created by + sphinx-quickstart on Fri Jan 10 15:11:07 2020. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome To OVIS-HPC Documentation! +==================================== +.. image:: https://github.com/ovis-hpc/readthedocs/blob/main/docs/source/images/ovis-logo.png?raw=true + :width: 225 + :height: 250 + :align: center + +**OVIS** is a modular system for HPC data collection, transport, storage, analysis, visualization, and log message exploration. The Lightweight Distributed Metric Service (**LDMS**) is a scalable low-overhead, low-latency framework for collection, movement, and storage of metric/event data on distributed computer systems. + +.. toctree:: + :maxdepth: 2 + :caption: OVIS and Group Activity + + About Ovis + LDMS Users Group Conference (LDMSCON) + LDSM Users Group + OVIS Publications + +.. toctree:: + :maxdepth: 4 + :caption: OVIS Components + + LDMS + SOS + maestro-index + Baler + ASF + +.. toctree:: + :maxdepth: 6 + :caption: Deployment + + LDMS + SOS + deployment/index + Baler + ASF + + +Other Projects +==================================== + +`ldms `_ +`ovis-publications `_ +`maestro `_ +`sos `_ +`baler `_ + + + + + + + + + + diff --git a/rtd/docs/source/maestro-index.rst b/rtd/docs/source/maestro-index.rst new file mode 100644 index 0000000..26f6b18 --- /dev/null +++ b/rtd/docs/source/maestro-index.rst @@ -0,0 +1,8 @@ +Maestro +==================================== +.. toctree:: + :maxdepth: 2 + :caption: Introduction To Maestro + + maestro-quickstart + diff --git a/rtd/docs/source/maestro-quickstart.rst b/rtd/docs/source/maestro-quickstart.rst new file mode 100644 index 0000000..cc08744 --- /dev/null +++ b/rtd/docs/source/maestro-quickstart.rst @@ -0,0 +1,215 @@ +Maestro Quick Start +######################### + +Introduction +******************** +LDMS Monitoring Cluster Load Balancing Service + +Maestro is a Python3 implementation of a service that load balances a cluster configuration across a number of configured Aggregators. + +Aggregators are configured in groups, and groups are organized into a hierarchy. The lowest level of the hierarchy (level 1) communicate with the sampler daemons; 2nd level aggregators monitor 1st level aggregators, 3rd level aggregators monitor 2nd level aggregators and so on. + +There are multiple goals of the maestro daemon: + +* Simplify the monitoring configuration of a large cluster +* Create a more resilient and responsive monitoring infrastructure +* Monitor the health and performance of aggregators in the cluster +* Manage aggregator failure by rebalancing the distributed configuration across the remaining aggregators in a group + +In this current release, Maestro does not start and start ldmsd daemons, however, this feature is planned for the future. +Install / Uninstall + +.. code-block:: console + + $ pip3 install --upgrade --prefix /opt/ovis . + # This assumed that LDMS is installed with /opt/ovis prefix. + # Use `upgrade` option so that the existing installation will get upgrade + # from the source tree. Otherwise, the existing files/directories were left + # untouched. + + # to uninstall + $ pip3 uninstall maestro + +Dependencies +************ +etcd3 can be downlaoded via pip3, or from source at: https://github.com/etcd-io/etcd + +Configuration Generation +************************** + +A ldms cluster's configuration can be generated with the maestro_ctrl command. The current implementation of the generator will generate v4 ldmsd configuration files for each group in the ldmsd cluster config. The generator will automatically "balance" samplers across aggregators, if sampler's are loaded by multiple aggregators, effectively creating a unique configuration file for each aggregator in a group. The command to generate config files is listed below: maestro_ctrl --cluster config/etcd.yaml --ldms_config config/ldms-config.yaml --prefix --generate-config-path +Configuration Management + +A cluster's configuration is mangaged in a distributed RAFT based key-value store called etcd. The etcd service is queried using the python etcd3 client interface. + +There are two configuration files consumed by maestro: + +* A file that defines the etcd cluster configuration +* A file that defines the LDMS Cluster Configuration + +There are two principle commands, maestro and maestro_ctrl. maestro will run the load balancing daemon, as well as start/configure ldmsd's. maestro_ctrl parses a yaml ldms cluster configuration file, and loads it into a etcd key/value store. Both commands are demonstrated below: + +.. code-block:: console + + maestro --cluster config/etcd.yaml --prefix orion + maestro_ctrl --cluster config/etcd.yaml --ldms_config config/orion.yaml --prefix orion + +Sampler interval's and offsets can be configured during runtime, by updating the ldmsd yaml configuration file, and running the maestro_ctrl command to update the etcd cluster with the new configuration. Maestro will detect updates to sampler intervals and offsets, and make the proper configuration updates to the running samplers. +ETCD Cluster Configuration + +Maestro consumes configuration files in YAML format. + +Here's an example of an etcd cluster configuration: + +.. code-block:: console + + cluster: voltrino + members: + - host: 10.128.0.7 + port: 2379 + members: + - host: 10.128.0.8 + port: 2379 + members: + - host: 10.128.0.9 + port: 2379 + +LDMS Cluster Configuration +********************** + +The primary configuration groups are daemons - defines LDMS daemons, their hosts, ports, and endpoints aggregators - defines aggregator configuration ldmsd's samplers - defines sampler configuration for ldmsd's stores - defines the various stores for aggregators + +Example LDMS Configuration + +.. code-block:: console + + daemons: + - names : &samplers "sampler-[1-10]" + hosts : &node-hosts "node-[1-10]" + endpoints : + - names : &sampler-endpoints "node-[1-10]-[10002]" + ports : &sampler-ports "[10002]" + maestro_comm : True + xprt : sock + auth : + name : ovis1 # The authentication domain name + plugin : ovis # The plugin type + conf : /opt/ovis/maestro/secret.conf + + - names : &sampler-rdma-endpoints "node-[1-10]-10002/rdma" + ports : *sampler-ports + maestro_comm : False + xprt : rdma + auth : + name : munge1 + plugin : munge + + - names : &l1-agg "l1-aggs-[11-14]" + hosts : &l1-agg-hosts "node-[11-14]" + endpoints : + - names : &l1-agg-endpoints "node-[11-14]-[10101]" + ports : &agg-ports "[10101]" + maestro_comm : True + xprt : sock + auth : + name : munge1 + plugin : munge + + - names : &l2-agg "l2-agg" + hosts : &l2-agg-host "node-15" + endpoints : + - names : &l2-agg-endpoints "node-[15]" + ports : "[10104]" + maestro_comm : True + xprt : sock + auth : + name : munge1 + plugin : munge + + aggregators: + - daemons : *l1-agg + peers : + - endpoints : *sampler-endpoints + reconnect : 20s + type : active + updaters : + - mode : pull + interval : "1.0s" + offset : "0ms" + sets : + - regex : .* + field : inst + + - daemons : *l2-agg + peers : + - endpoints : *l1-agg-endpoints + reconnect : 20s + type : active + updaters : + - mode : pull + interval : "1.0s" + offset : "0ms" + sets : + - regex : .* + field : inst + + samplers: + - daemons : *samplers + config : + - name : meminfo # Variables can be specific to plugin + interval : "1.0s" # Used when starting the sampler plugin + offset : "0ms" + perm : "0777" + + - name : vmstat + interval : "1.0s" + offset : "0ms" + perm : "0777" + + - name : procstat + interval : "1.0s" + offset : "0ms" + perm : "0777" + + stores: + - name : sos-meminfo + daemons : *l2-agg + container : ldms_data + schema : meminfo + flush : 10s + plugin : + name : store_sos + config : { path : /DATA } + + - name : sos-vmstat + daemons : *l2-agg + container : ldms_data + schema : vmstat + flush : 10s + plugin : + name : store_sos + config : { path : /DATA } + + - name : sos-procstat + daemons : *l2-agg + container : ldms_data + schema : procstat + flush : 10s + plugin : + name : store_sos + config : { path : /DATA } + + - name : csv + daemons : *l2-agg + container : ldms_data + schema : meminfo + plugin : + name : store_csv + config : + path : /DATA/csv/ + altheader : 0 + typeheader : 1 + create_uid : 3031 + create_gid : 3031 + + diff --git a/rtd/docs/source/maestro/index.rst b/rtd/docs/source/maestro/index.rst new file mode 100644 index 0000000..8c5f200 --- /dev/null +++ b/rtd/docs/source/maestro/index.rst @@ -0,0 +1,8 @@ +Maestro +==== +About Maestro + +.. toctree:: + :maxdepth: 2 + + maestro-quickstart diff --git a/rtd/docs/source/maestro/maestro-quickstart.rst b/rtd/docs/source/maestro/maestro-quickstart.rst new file mode 100644 index 0000000..b7e8f47 --- /dev/null +++ b/rtd/docs/source/maestro/maestro-quickstart.rst @@ -0,0 +1,214 @@ +Maestro Quick Start +######################### + +Introduction +******************** +LDMS Monitoring Cluster Load Balancing Service + +Maestro is a Python3 implementation of a service that load balances a cluster configuration across a number of configured Aggregators. + +Aggregators are configured in groups, and groups are organized into a hierarchy. The lowest level of the hierarchy (level 1) communicate with the sampler daemons; 2nd level aggregators monitor 1st level aggregators, 3rd level aggregators monitor 2nd level aggregators and so on. + +There are multiple goals of the maestro daemon: + +* Simplify the monitoring configuration of a large cluster +* Create a more resilient and responsive monitoring infrastructure +* Monitor the health and performance of aggregators in the cluster +* Manage aggregator failure by rebalancing the distributed configuration across the remaining aggregators in a group + +In this current release, Maestro does not start and start ldmsd daemons, however, this feature is planned for the future. +Install / Uninstall + +.. code-block:: console + + $ pip3 install --upgrade --prefix /opt/ovis . + # This assumed that LDMS is installed with /opt/ovis prefix. + # Use `upgrade` option so that the existing installation will get upgrade + # from the source tree. Otherwise, the existing files/directories were left + # untouched. + + # to uninstall + $ pip3 uninstall maestro + +Dependencies +************ +etcd3 can be downlaoded via pip3, or from source at: https://github.com/etcd-io/etcd + +Configuration Generation +************************** + +A ldms cluster's configuration can be generated with the maestro_ctrl command. The current implementation of the generator will generate v4 ldmsd configuration files for each group in the ldmsd cluster config. The generator will automatically "balance" samplers across aggregators, if sampler's are loaded by multiple aggregators, effectively creating a unique configuration file for each aggregator in a group. The command to generate config files is listed below: maestro_ctrl --cluster config/etcd.yaml --ldms_config config/ldms-config.yaml --prefix --generate-config-path +Configuration Management + +A cluster's configuration is mangaged in a distributed RAFT based key-value store called etcd. The etcd service is queried using the python etcd3 client interface. + +There are two configuration files consumed by maestro: + +* A file that defines the etcd cluster configuration +* A file that defines the LDMS Cluster Configuration + +There are two principle commands, maestro and maestro_ctrl. maestro will run the load balancing daemon, as well as start/configure ldmsd's. maestro_ctrl parses a yaml ldms cluster configuration file, and loads it into a etcd key/value store. Both commands are demonstrated below: + +.. code-block:: console + + maestro --cluster config/etcd.yaml --prefix orion + maestro_ctrl --cluster config/etcd.yaml --ldms_config config/orion.yaml --prefix orion + +Sampler interval's and offsets can be configured during runtime, by updating the ldmsd yaml configuration file, and running the maestro_ctrl command to update the etcd cluster with the new configuration. Maestro will detect updates to sampler intervals and offsets, and make the proper configuration updates to the running samplers. +ETCD Cluster Configuration + +Maestro consumes configuration files in YAML format. + +Here's an example of an etcd cluster configuration: + +.. code-block:: console + + cluster: voltrino + members: + - host: 10.128.0.7 + port: 2379 + members: + - host: 10.128.0.8 + port: 2379 + members: + - host: 10.128.0.9 + port: 2379 + +LDMS Cluster Configuration +********************** + +The primary configuration groups are daemons - defines LDMS daemons, their hosts, ports, and endpoints aggregators - defines aggregator configuration ldmsd's samplers - defines sampler configuration for ldmsd's stores - defines the various stores for aggregators + +Example LDMS Configuration + +.. code-block:: console + + daemons: + - names : &samplers "sampler-[1-10]" + hosts : &node-hosts "node-[1-10]" + endpoints : + - names : &sampler-endpoints "node-[1-10]-[10002]" + ports : &sampler-ports "[10002]" + maestro_comm : True + xprt : sock + auth : + name : ovis1 # The authentication domain name + plugin : ovis # The plugin type + conf : /opt/ovis/maestro/secret.conf + + - names : &sampler-rdma-endpoints "node-[1-10]-10002/rdma" + ports : *sampler-ports + maestro_comm : False + xprt : rdma + auth : + name : munge1 + plugin : munge + + - names : &l1-agg "l1-aggs-[11-14]" + hosts : &l1-agg-hosts "node-[11-14]" + endpoints : + - names : &l1-agg-endpoints "node-[11-14]-[10101]" + ports : &agg-ports "[10101]" + maestro_comm : True + xprt : sock + auth : + name : munge1 + plugin : munge + + - names : &l2-agg "l2-agg" + hosts : &l2-agg-host "node-15" + endpoints : + - names : &l2-agg-endpoints "node-[15]" + ports : "[10104]" + maestro_comm : True + xprt : sock + auth : + name : munge1 + plugin : munge + + aggregators: + - daemons : *l1-agg + peers : + - endpoints : *sampler-endpoints + reconnect : 20s + type : active + updaters : + - mode : pull + interval : "1.0s" + offset : "0ms" + sets : + - regex : .* + field : inst + + - daemons : *l2-agg + peers : + - endpoints : *l1-agg-endpoints + reconnect : 20s + type : active + updaters : + - mode : pull + interval : "1.0s" + offset : "0ms" + sets : + - regex : .* + field : inst + + samplers: + - daemons : *samplers + config : + - name : meminfo # Variables can be specific to plugin + interval : "1.0s" # Used when starting the sampler plugin + offset : "0ms" + perm : "0777" + + - name : vmstat + interval : "1.0s" + offset : "0ms" + perm : "0777" + + - name : procstat + interval : "1.0s" + offset : "0ms" + perm : "0777" + + stores: + - name : sos-meminfo + daemons : *l2-agg + container : ldms_data + schema : meminfo + flush : 10s + plugin : + name : store_sos + config : { path : /DATA } + + - name : sos-vmstat + daemons : *l2-agg + container : ldms_data + schema : vmstat + flush : 10s + plugin : + name : store_sos + config : { path : /DATA } + + - name : sos-procstat + daemons : *l2-agg + container : ldms_data + schema : procstat + flush : 10s + plugin : + name : store_sos + config : { path : /DATA } + + - name : csv + daemons : *l2-agg + container : ldms_data + schema : meminfo + plugin : + name : store_csv + config : + path : /DATA/csv/ + altheader : 0 + typeheader : 1 + create_uid : 3031 + create_gid : 3031 + diff --git a/rtd/paper.lock b/rtd/paper.lock new file mode 100644 index 0000000..eb8b6d1 --- /dev/null +++ b/rtd/paper.lock @@ -0,0 +1 @@ +Name | Date | File(s) diff --git a/rtd/pyproject.toml b/rtd/pyproject.toml new file mode 100644 index 0000000..82cfe1d --- /dev/null +++ b/rtd/pyproject.toml @@ -0,0 +1,8 @@ +[build-system] +requires = ["flit_core >=3.2,<4"] +build-backend = "flit_core.buildapi" + +[project] +name = "maestro" +authors = [{name = "maestro", email = "ldms@sandia.gov"}] +dynamic = ["version", "description"] diff --git a/rtd/update-branches.py b/rtd/update-branches.py new file mode 100644 index 0000000..a29ce03 --- /dev/null +++ b/rtd/update-branches.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +""" +Updates all other branches to the latest master + +Since this repository is structured such that different branches build with +different themes, the result is a lot of branches that can fall behind master. +This script simply updates all branches to master. It is meant to be run by +somebody with push access to "origin". + + ./update-branches.py +""" + +import os +import subprocess +import sys + + +path_to_conf = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'docs') +sys.path.append(path_to_conf) + +from conf import branch_to_theme_mapping # noqa + + +branches = branch_to_theme_mapping.keys() + +for branch in branches: + if branch == 'master': + continue + + print(u'*' * 77) + print(u'Syncing branch {} to master...'.format(branch)) + subprocess.check_output(['git', 'checkout', branch]) + subprocess.check_output(['git', 'merge', '--ff-only', 'master']) + subprocess.check_output(['git', 'push', 'origin', branch]) + print(u'*' * 77) + print(u'\n') + +print(u'Returning to master branch...') +subprocess.check_output(['git', 'checkout', 'master']) From 635bf6913dcec2349356f0c5710ac441d57cf315 Mon Sep 17 00:00:00 2001 From: Sara Walton Date: Tue, 16 Jul 2024 14:50:25 -0600 Subject: [PATCH 2/4] removing white spaces --- rtd/README.rst | 6 +++--- rtd/docs/source/conf.py | 2 +- rtd/docs/source/index.rst | 4 ++-- rtd/docs/source/maestro-quickstart.rst | 30 +++++++++++++------------- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/rtd/README.rst b/rtd/README.rst index 4fba89a..e952283 100644 --- a/rtd/README.rst +++ b/rtd/README.rst @@ -33,14 +33,14 @@ Instructions and documentation on how to use ReadTheDocs can be found here: > git add > git commit -m "add message" > git push - -Adding A New File + +Adding A New File ****************** For any new RST files created, please include them in rtd/docs/src/index.rst under their corresponding sections. All RST files not included in index.rst will not populate on the offical webpage (e.g. readthedocs). Paper Lock ************ This is for claiming any sections you are working on so there is no overlap. -Please USE paper.lock to indicate if you are editing an existing RST file. +Please USE paper.lock to indicate if you are editing an existing RST file. diff --git a/rtd/docs/source/conf.py b/rtd/docs/source/conf.py index 40df176..7de1dba 100644 --- a/rtd/docs/source/conf.py +++ b/rtd/docs/source/conf.py @@ -22,7 +22,7 @@ intersphinx_mapping = { 'python': ('https://docs.python.org/3/', None), 'sphinx': ('https://www.sphinx-doc.org/en/master/', None), - + # Link to the "apis" of the "hpc-ovis" project and subprojects "ovis-hpc": ("https://ovis-hpc.readthedocs.io/en/latest/", None), "sos": ("https://ovis-hpc.readthedocs.io/projects/sos/en/latest/", None), diff --git a/rtd/docs/source/index.rst b/rtd/docs/source/index.rst index 8e52833..cbdd4e1 100644 --- a/rtd/docs/source/index.rst +++ b/rtd/docs/source/index.rst @@ -46,7 +46,7 @@ Welcome To OVIS-HPC Documentation! Baler ASF - + Other Projects ==================================== @@ -56,7 +56,7 @@ Other Projects `sos `_ `baler `_ - + diff --git a/rtd/docs/source/maestro-quickstart.rst b/rtd/docs/source/maestro-quickstart.rst index cc08744..2cd90e5 100644 --- a/rtd/docs/source/maestro-quickstart.rst +++ b/rtd/docs/source/maestro-quickstart.rst @@ -26,7 +26,7 @@ Install / Uninstall # Use `upgrade` option so that the existing installation will get upgrade # from the source tree. Otherwise, the existing files/directories were left # untouched. - + # to uninstall $ pip3 uninstall maestro @@ -74,7 +74,7 @@ Here's an example of an etcd cluster configuration: - host: 10.128.0.9 port: 2379 -LDMS Cluster Configuration +LDMS Cluster Configuration ********************** The primary configuration groups are daemons - defines LDMS daemons, their hosts, ports, and endpoints aggregators - defines aggregator configuration ldmsd's samplers - defines sampler configuration for ldmsd's stores - defines the various stores for aggregators @@ -95,7 +95,7 @@ Example LDMS Configuration name : ovis1 # The authentication domain name plugin : ovis # The plugin type conf : /opt/ovis/maestro/secret.conf - + - names : &sampler-rdma-endpoints "node-[1-10]-10002/rdma" ports : *sampler-ports maestro_comm : False @@ -103,7 +103,7 @@ Example LDMS Configuration auth : name : munge1 plugin : munge - + - names : &l1-agg "l1-aggs-[11-14]" hosts : &l1-agg-hosts "node-[11-14]" endpoints : @@ -114,7 +114,7 @@ Example LDMS Configuration auth : name : munge1 plugin : munge - + - names : &l2-agg "l2-agg" hosts : &l2-agg-host "node-15" endpoints : @@ -125,7 +125,7 @@ Example LDMS Configuration auth : name : munge1 plugin : munge - + aggregators: - daemons : *l1-agg peers : @@ -139,7 +139,7 @@ Example LDMS Configuration sets : - regex : .* field : inst - + - daemons : *l2-agg peers : - endpoints : *l1-agg-endpoints @@ -152,7 +152,7 @@ Example LDMS Configuration sets : - regex : .* field : inst - + samplers: - daemons : *samplers config : @@ -160,17 +160,17 @@ Example LDMS Configuration interval : "1.0s" # Used when starting the sampler plugin offset : "0ms" perm : "0777" - + - name : vmstat interval : "1.0s" offset : "0ms" perm : "0777" - + - name : procstat interval : "1.0s" offset : "0ms" perm : "0777" - + stores: - name : sos-meminfo daemons : *l2-agg @@ -180,7 +180,7 @@ Example LDMS Configuration plugin : name : store_sos config : { path : /DATA } - + - name : sos-vmstat daemons : *l2-agg container : ldms_data @@ -189,7 +189,7 @@ Example LDMS Configuration plugin : name : store_sos config : { path : /DATA } - + - name : sos-procstat daemons : *l2-agg container : ldms_data @@ -198,7 +198,7 @@ Example LDMS Configuration plugin : name : store_sos config : { path : /DATA } - + - name : csv daemons : *l2-agg container : ldms_data @@ -211,5 +211,5 @@ Example LDMS Configuration typeheader : 1 create_uid : 3031 create_gid : 3031 - + From a72587ecf6b4e43ce4a70b7d3bdcd7782e25f7b7 Mon Sep 17 00:00:00 2001 From: Sara Walton Date: Mon, 26 Aug 2024 13:52:02 -0600 Subject: [PATCH 3/4] fixing whitespace --- .../source/maestro/maestro-quickstart.rst | 30 +++++++------- rtd/update-branches.py | 39 ------------------- 2 files changed, 15 insertions(+), 54 deletions(-) delete mode 100644 rtd/update-branches.py diff --git a/rtd/docs/source/maestro/maestro-quickstart.rst b/rtd/docs/source/maestro/maestro-quickstart.rst index b7e8f47..7df9908 100644 --- a/rtd/docs/source/maestro/maestro-quickstart.rst +++ b/rtd/docs/source/maestro/maestro-quickstart.rst @@ -26,7 +26,7 @@ Install / Uninstall # Use `upgrade` option so that the existing installation will get upgrade # from the source tree. Otherwise, the existing files/directories were left # untouched. - + # to uninstall $ pip3 uninstall maestro @@ -74,7 +74,7 @@ Here's an example of an etcd cluster configuration: - host: 10.128.0.9 port: 2379 -LDMS Cluster Configuration +LDMS Cluster Configuration ********************** The primary configuration groups are daemons - defines LDMS daemons, their hosts, ports, and endpoints aggregators - defines aggregator configuration ldmsd's samplers - defines sampler configuration for ldmsd's stores - defines the various stores for aggregators @@ -95,7 +95,7 @@ Example LDMS Configuration name : ovis1 # The authentication domain name plugin : ovis # The plugin type conf : /opt/ovis/maestro/secret.conf - + - names : &sampler-rdma-endpoints "node-[1-10]-10002/rdma" ports : *sampler-ports maestro_comm : False @@ -103,7 +103,7 @@ Example LDMS Configuration auth : name : munge1 plugin : munge - + - names : &l1-agg "l1-aggs-[11-14]" hosts : &l1-agg-hosts "node-[11-14]" endpoints : @@ -114,7 +114,7 @@ Example LDMS Configuration auth : name : munge1 plugin : munge - + - names : &l2-agg "l2-agg" hosts : &l2-agg-host "node-15" endpoints : @@ -125,7 +125,7 @@ Example LDMS Configuration auth : name : munge1 plugin : munge - + aggregators: - daemons : *l1-agg peers : @@ -139,7 +139,7 @@ Example LDMS Configuration sets : - regex : .* field : inst - + - daemons : *l2-agg peers : - endpoints : *l1-agg-endpoints @@ -152,7 +152,7 @@ Example LDMS Configuration sets : - regex : .* field : inst - + samplers: - daemons : *samplers config : @@ -160,17 +160,17 @@ Example LDMS Configuration interval : "1.0s" # Used when starting the sampler plugin offset : "0ms" perm : "0777" - + - name : vmstat interval : "1.0s" offset : "0ms" perm : "0777" - + - name : procstat interval : "1.0s" offset : "0ms" perm : "0777" - + stores: - name : sos-meminfo daemons : *l2-agg @@ -180,7 +180,7 @@ Example LDMS Configuration plugin : name : store_sos config : { path : /DATA } - + - name : sos-vmstat daemons : *l2-agg container : ldms_data @@ -189,7 +189,7 @@ Example LDMS Configuration plugin : name : store_sos config : { path : /DATA } - + - name : sos-procstat daemons : *l2-agg container : ldms_data @@ -198,7 +198,7 @@ Example LDMS Configuration plugin : name : store_sos config : { path : /DATA } - + - name : csv daemons : *l2-agg container : ldms_data @@ -211,4 +211,4 @@ Example LDMS Configuration typeheader : 1 create_uid : 3031 create_gid : 3031 - + diff --git a/rtd/update-branches.py b/rtd/update-branches.py deleted file mode 100644 index a29ce03..0000000 --- a/rtd/update-branches.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python -""" -Updates all other branches to the latest master - -Since this repository is structured such that different branches build with -different themes, the result is a lot of branches that can fall behind master. -This script simply updates all branches to master. It is meant to be run by -somebody with push access to "origin". - - ./update-branches.py -""" - -import os -import subprocess -import sys - - -path_to_conf = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'docs') -sys.path.append(path_to_conf) - -from conf import branch_to_theme_mapping # noqa - - -branches = branch_to_theme_mapping.keys() - -for branch in branches: - if branch == 'master': - continue - - print(u'*' * 77) - print(u'Syncing branch {} to master...'.format(branch)) - subprocess.check_output(['git', 'checkout', branch]) - subprocess.check_output(['git', 'merge', '--ff-only', 'master']) - subprocess.check_output(['git', 'push', 'origin', branch]) - print(u'*' * 77) - print(u'\n') - -print(u'Returning to master branch...') -subprocess.check_output(['git', 'checkout', 'master']) From 213596a187ecc2e0ee84cc9fe367c28ba2d48cd8 Mon Sep 17 00:00:00 2001 From: Sara Walton Date: Wed, 15 Jan 2025 13:57:31 -0700 Subject: [PATCH 4/4] updating sos --- .readthedocs.yaml | 32 +++++++++++++++++ {rtd/docs => docs}/Makefile | 0 {rtd => docs}/README.rst | 0 {rtd/docs/source => docs}/conf.py | 0 .../docs/source => docs}/deployment/index.rst | 0 {rtd/docs/source => docs}/deployment/test.rst | 0 {rtd/docs/source => docs}/index.rst | 0 {rtd/docs/source => docs}/maestro-index.rst | 0 .../source => docs}/maestro-quickstart.rst | 0 {rtd/docs/source => docs}/maestro/index.rst | 0 .../maestro/maestro-quickstart.rst | 0 {rtd/docs => docs}/requirements.txt | 0 rtd/.readthedocs.yaml | 22 ------------ rtd/docs/make.bat | 35 ------------------- rtd/paper.lock | 1 - rtd/pyproject.toml | 8 ----- 16 files changed, 32 insertions(+), 66 deletions(-) create mode 100644 .readthedocs.yaml rename {rtd/docs => docs}/Makefile (100%) rename {rtd => docs}/README.rst (100%) rename {rtd/docs/source => docs}/conf.py (100%) rename {rtd/docs/source => docs}/deployment/index.rst (100%) rename {rtd/docs/source => docs}/deployment/test.rst (100%) rename {rtd/docs/source => docs}/index.rst (100%) rename {rtd/docs/source => docs}/maestro-index.rst (100%) rename {rtd/docs/source => docs}/maestro-quickstart.rst (100%) rename {rtd/docs/source => docs}/maestro/index.rst (100%) rename {rtd/docs/source => docs}/maestro/maestro-quickstart.rst (100%) rename {rtd/docs => docs}/requirements.txt (100%) delete mode 100644 rtd/.readthedocs.yaml delete mode 100644 rtd/docs/make.bat delete mode 100644 rtd/paper.lock delete mode 100644 rtd/pyproject.toml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..511a6ca --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,32 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-24.04 + tools: + python: "3.13" + # You can also specify other tool versions: + # nodejs: "23" + # rust: "1.82" + # golang: "1.23" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/conf.py + +# Optionally build your docs in additional formats such as PDF and ePub +# formats: +# - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/requirements.txt diff --git a/rtd/docs/Makefile b/docs/Makefile similarity index 100% rename from rtd/docs/Makefile rename to docs/Makefile diff --git a/rtd/README.rst b/docs/README.rst similarity index 100% rename from rtd/README.rst rename to docs/README.rst diff --git a/rtd/docs/source/conf.py b/docs/conf.py similarity index 100% rename from rtd/docs/source/conf.py rename to docs/conf.py diff --git a/rtd/docs/source/deployment/index.rst b/docs/deployment/index.rst similarity index 100% rename from rtd/docs/source/deployment/index.rst rename to docs/deployment/index.rst diff --git a/rtd/docs/source/deployment/test.rst b/docs/deployment/test.rst similarity index 100% rename from rtd/docs/source/deployment/test.rst rename to docs/deployment/test.rst diff --git a/rtd/docs/source/index.rst b/docs/index.rst similarity index 100% rename from rtd/docs/source/index.rst rename to docs/index.rst diff --git a/rtd/docs/source/maestro-index.rst b/docs/maestro-index.rst similarity index 100% rename from rtd/docs/source/maestro-index.rst rename to docs/maestro-index.rst diff --git a/rtd/docs/source/maestro-quickstart.rst b/docs/maestro-quickstart.rst similarity index 100% rename from rtd/docs/source/maestro-quickstart.rst rename to docs/maestro-quickstart.rst diff --git a/rtd/docs/source/maestro/index.rst b/docs/maestro/index.rst similarity index 100% rename from rtd/docs/source/maestro/index.rst rename to docs/maestro/index.rst diff --git a/rtd/docs/source/maestro/maestro-quickstart.rst b/docs/maestro/maestro-quickstart.rst similarity index 100% rename from rtd/docs/source/maestro/maestro-quickstart.rst rename to docs/maestro/maestro-quickstart.rst diff --git a/rtd/docs/requirements.txt b/docs/requirements.txt similarity index 100% rename from rtd/docs/requirements.txt rename to docs/requirements.txt diff --git a/rtd/.readthedocs.yaml b/rtd/.readthedocs.yaml deleted file mode 100644 index 26efcab..0000000 --- a/rtd/.readthedocs.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# .readthedocs.yaml -# Read the Docs configuration file -# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details - -# Required -version: 2 - -# Set the version of Python and other tools you might need -build: - os: ubuntu-22.04 - tools: - python: "3.11" - -# Build documentation in the docs/ directory with Sphinx -sphinx: - configuration: rtd/docs/source/conf.py - -# We recommend specifying your dependencies to enable reproducible builds: -# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html -python: - install: - - requirements: rtd/docs/requirements.txt diff --git a/rtd/docs/make.bat b/rtd/docs/make.bat deleted file mode 100644 index 9534b01..0000000 --- a/rtd/docs/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=source -set BUILDDIR=build - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd diff --git a/rtd/paper.lock b/rtd/paper.lock deleted file mode 100644 index eb8b6d1..0000000 --- a/rtd/paper.lock +++ /dev/null @@ -1 +0,0 @@ -Name | Date | File(s) diff --git a/rtd/pyproject.toml b/rtd/pyproject.toml deleted file mode 100644 index 82cfe1d..0000000 --- a/rtd/pyproject.toml +++ /dev/null @@ -1,8 +0,0 @@ -[build-system] -requires = ["flit_core >=3.2,<4"] -build-backend = "flit_core.buildapi" - -[project] -name = "maestro" -authors = [{name = "maestro", email = "ldms@sandia.gov"}] -dynamic = ["version", "description"]