Automatically Generate Repository Git Page (#391)

* add git page action

* Add scripts to create dot and png files for workflows

* Add scripts to add png link to md

* Add mkdocs.yml file

* changed png link to be relative path instead of absolute.

* Added script to add header to wdl markdown, and remove task info from wdl workflow markdown

* edited wdlviz.py to add the option to name the output file by a specified name, edited create_wdl_visual.py to use wdl basename for dot file

---------

Co-authored-by: bshifaw <[email protected]>

.github/workflows/git_page.yml

@@ -0,0 +1,50 @@
+name: Build and deploy site to GitHub Pages
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  github-pages:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+
+      - name: Cache node_modules
+        uses: actions/cache@v3
+        with:
+          path: "**/node_modules"
+          key: ${{ runner.os }}-modules-${{ env.cache-name }}-${{ hashFiles('**/yarn.lock') }}
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install Requirements
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install wdldoc~=1.7.0
+          python3 -m pip install mkdocs~=1.4.2
+      - name: Create Site Homepage
+        run: |
+          cp README.md docs/index.md && sed -i.bak 's/\.\/docs/\.\//g' docs/index.md
+      - name: Build Docs
+        run: |
+          wdldoc --debug -o docs ./wdl/pipelines ./wdl/tasks
+          
+          python3 ./scripts/git_page/md_post_process.py --debug --md_dir ./docs
+          
+          python3 ./scripts/git_page/create_wdl_visual.py --debug ./wdl/pipelines ./wdl/tasks --output_path ./docs/dot
+          
+          python3 ./scripts/git_page/add_dot_link_to_md.py --md_dir ./docs/workflows --dot_dir ./docs/dot
+          
+          mkdocs build
+      - name: Deploy
+        uses: JamesIves/github-pages-deploy-action@releases/v4
+        with:
+          BRANCH: gh-pages
+          FOLDER: ./site

mkdocs.yml

@@ -0,0 +1,12 @@
+site_name: "Broad Institute Long Read Pipelines"
+repo_url: "https://github.com/broadinstitute/long-read-pipelines"
+
+site_description: "This repository contains pipelines for processing of long read data from PacBio and/or Oxford Nanopore platforms."
+site_author: "Broad Institute"
+copyright: "Copyright (c) 2023, Broad Institute"
+
+theme: readthedocs
+
+markdown_extensions:
+    - def_list
+

scripts/git_page/add_dot_link_to_md.py

@@ -0,0 +1,118 @@
+import errno
+
+import os
+import argparse
+from pathlib import Path
+import logging
+
+import utility as util
+
+Logger = logging.getLogger(__name__)
+logging.basicConfig()
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Add dot header and dot.png link to workflow markdown files"
+    )
+    parser.add_argument(
+        "--md_dir", metavar="MARKDOWN_DIRECTORY", nargs='+', required=True,
+        help="One of or more directories containing markdown "
+             "documentation files for workflow"
+    )
+    parser.add_argument(
+        "--dot_dir", metavar="DOT_DIRECTORY", nargs='+', required=True,
+        help="One of or more directories containing dot.png "
+             "files for workflows"
+    )
+    parser.add_argument("--debug", action="store_true", help="verbose logging")
+
+    args = parser.parse_args()
+
+    util.set_logging_level(args)
+
+    md_dir_names = args.md_dir
+    dot_png_dir_names = args.dot_dir
+
+    dot_png_paths = get_all_dot_png_in_directory(dot_png_dir_names)
+
+    if len(dot_png_paths) == 0:
+        Logger.error(f'Could not find any dot.png files in {dot_png_dir_names}!')
+        raise FileNotFoundError(
+            errno.ENOENT, os.strerror(errno.ENOENT), dot_png_paths)
+
+    for md_dir_name in md_dir_names:
+
+        Logger.debug(f'Processing {md_dir_name}...')
+
+        md_dir = util.get_absolute_path(md_dir_name)
+
+        md_paths = util.get_all_files_with_extension(directory=md_dir, ext='md')
+
+        for md in md_paths:
+            Logger.debug(f'Processing {md}...')
+            append_png_to_md(md_path=md, dot_paths=dot_png_paths)
+
+    print("Done!")
+
+
+def append_png_to_md(md_path: str, dot_paths: list) -> None:
+    """
+    Append dot.png link to markdown file
+
+    @param md_path: path to markdown file
+    @param dot_paths: list of paths to dot.png files
+    @return:
+    """
+
+    md_basename = util.get_basename(md_path)
+
+    dot_png_matches = []
+
+    for dot_path in dot_paths:
+        if md_basename == util.get_basename(dot_path):
+            dot_png_matches.append(dot_path)
+
+    if len(dot_png_matches) == 1:
+        relative_d_p_path = get_relative_path(dot_png_matches[0], md_path)
+        with open(md_path, 'a') as md_file:
+            md_file.write(f'### Dot Diagram\n')
+            md_file.write(f'![{md_basename}](../{relative_d_p_path})\n')
+    elif len(dot_png_matches) > 1:
+        Logger.error(f'Found more than one dot.png for {md_basename}!')
+
+
+def get_all_dot_png_in_directory(directory_names: list) -> list:
+    """
+    Get all dot.png files in a list of directories
+    @param directory_names: list of directories to search
+    @return: list of paths to dot.png files
+    """
+
+    dot_png_paths = []
+    for dot_png_dir_name in directory_names:
+        dot_png_dir = util.get_absolute_path(dot_png_dir_name)
+
+        dot_png_paths = (
+                dot_png_paths + util.get_all_files_with_extension(
+                    directory=dot_png_dir, ext='dot.png'
+                )
+        )
+    return dot_png_paths
+
+
+def get_relative_path(path, relative_to_path) -> Path:
+    """
+    Get the relative path for a path
+    @param relative_to_path:  Path to get relative path to
+    @param path: Path to get relative path for
+    @return:
+    """
+    p = Path(path)
+    parent_dir = Path(relative_to_path).parent
+    return p.relative_to(parent_dir.parent)
+
+
+if __name__ == "__main__":
+    main()
+

scripts/git_page/create_wdl_visual.py

@@ -0,0 +1,75 @@
+import argparse
+import logging
+from pathlib import Path
+
+import utility as util
+import wdlviz
+import WDL
+
+Logger = logging.getLogger(__name__)
+logging.basicConfig()
+
+
+def main():
+    # read command-line arguments
+    parser = argparse.ArgumentParser(
+        description="Visualize a WDL workflow using miniwdl and graphviz"
+    )
+    parser.add_argument(
+        "wdl_dir", metavar="WDL_DIRECTORY", nargs='+',
+        help="One of or more directories containing WDL workflow files"
+    )
+    parser.add_argument("--output_path", help="Path to output directory")
+    parser.add_argument("--debug", action="store_true", help="verbose logging")
+
+    args = parser.parse_args()
+
+    util.set_logging_level(args)
+
+    dir_names = args.wdl_dir
+    output_path = str(util.get_absolute_path(args.output_path)) if args.output_path else None
+
+    for dir_name in dir_names:
+
+        Logger.debug(f'Processing {dir_name}...')
+
+        wdl_dir = util.get_absolute_path(dir_name)
+
+        wdl_paths = util.get_all_files_with_extension(directory=wdl_dir, ext='wdl')
+
+        for wdl in wdl_paths:
+            Logger.debug(f'Processing {wdl}...')
+
+            if WDL.load(wdl).workflow:  # check if wdl is a workflow
+                run_wdlviz(wdl_path=wdl, output_path=output_path)
+            else:
+                Logger.debug(f"{wdl} is not a WDL workflow (does not contain a "
+                             "workflow block). Skipping...")
+
+
+def run_wdlviz(
+    wdl_path: str, use_basename: bool = True, output_path: str = None
+) -> None:
+    """
+    Run wdlviz on a WDL file to create dot and png files
+    @param use_basename: name output files using the basename of the WDL file
+    @param wdl_path: path to WDL file
+    @param output_path: path to output directory
+    @return:
+    """
+    wdlviz_args = [wdl_path]
+
+    if output_path is not None:
+        wdlviz_args.append("--output_path")
+        wdlviz_args.append(output_path)
+    if use_basename:
+        wdlviz_args.append("--dot-basename")
+        wdlviz_args.append(Path(wdl_path).stem)
+
+    Logger.debug(f'Running wdlviz with args: {wdlviz_args}...')
+    wdlviz.main(args=wdlviz_args)
+
+
+if __name__ == "__main__":
+    main()
+