Parallelize pyodide based docs (#314)

holoviz-dev · Sep 12, 2024 · 6e7a375 · 6e7a375
1 parent 67c9a8a
commit 6e7a375
Show file tree

Hide file tree

Showing 6 changed files with 228 additions and 111 deletions.
diff --git a/nbsite/__main__.py b/nbsite/__main__.py
@@ -43,13 +43,13 @@ def main(args=None):
 
     build_parser = subparsers.add_parser("build", help=inspect.getdoc(build))
     build_parser.add_argument('--what',type=str,help='type of output to generate',default='html')
-
     build_parser.add_argument('--project-name', type=str, help='name of project', default='')
     build_parser.add_argument('--org',type=str,help='github organization',default='')
     build_parser.add_argument('--host',type=str,help='host to use when generating notebook links',default='GitHub')
     build_parser.add_argument('--repo',type=str,help='name of repo',default='')
     build_parser.add_argument('--branch',type=str,help='branch to point to in notebook links',default='main')
     build_parser.add_argument('--binder',type=str,help='where to place binder link',choices=['bottom', 'top', 'both', 'none'], default='none')
+    build_parser.add_argument('--disable-parallel',action=argparse.BooleanOptionalAction,help='whether to disable building the docs in parallel')
 
     build_parser.add_argument('--output',type=str,help='where to place output',default="builtdocs")
     _add_common_args(build_parser,'--project-root','--doc','--examples', '--overwrite')

diff --git a/nbsite/cmd.py b/nbsite/cmd.py
@@ -47,14 +47,6 @@ def fix_links(output, inspect_links):
         args.append( "--inspect-links")
     subprocess.check_call(args)
 
-def _pyodide_enabled(confpy):
-    import runpy
-    current_dir = os.getcwd()
-    os.chdir(os.path.dirname(confpy))
-    file = runpy.run_path(confpy)
-    os.chdir(current_dir)
-    return 'nbsite.pyodide' in file["extensions"]
-
 def build(what='html',
           output='builtdocs',
           project_root='',
@@ -65,6 +57,7 @@ def build(what='html',
           branch='main',
           org='',
           binder='none',
+          disable_parallel=False,
           examples='examples',
           examples_assets='assets',
           clean_dry_run=False,
@@ -75,17 +68,18 @@ def build(what='html',
 
     Usually this is run after `nbsite scaffold`
     """
-    env={'PROJECT_NAME':project_name,
-         'PROJECT_ROOT':project_root if project_root!='' else os.getcwd(),
-         'HOST':host,
-         'REPO':repo,
-         'BRANCH':branch,
-         'ORG':org,
-         'EXAMPLES':examples,
-         'DOC':doc,
-         'EXAMPLES_ASSETS':examples_assets,
-         'BINDER':binder
-         }
+    env = {
+        'PROJECT_NAME':project_name,
+        'PROJECT_ROOT':project_root if project_root!='' else os.getcwd(),
+        'HOST':host,
+        'REPO':repo,
+        'BRANCH':branch,
+        'ORG':org,
+        'EXAMPLES':examples,
+        'DOC':doc,
+        'EXAMPLES_ASSETS':examples_assets,
+        'BINDER':binder
+    }
     merged_env = dict(os.environ, **env)
     none_vals = {k:v for k,v in merged_env.items() if v is None}
     if none_vals:
@@ -96,11 +90,8 @@ def build(what='html',
         for path in glob.glob(os.path.join(paths['doc'], '**', '*.ipynb'), recursive=True):
             print('Removing evaluated notebook from {}'.format(path))
             os.remove(path)
-    if _pyodide_enabled(os.path.join(paths['doc'], 'conf.py')):
-        # Currently pyodide does not work with -j auto
-        cmd = ["sphinx-build", "-b", what, paths['doc'], output]
-    else:
-        cmd = ["sphinx-build", "-j", "auto", "-b", what, paths['doc'], output]
+    extras = [] if disable_parallel else ["-j", "auto"]
+    cmd = ["sphinx-build", "-b", what, paths['doc'], output] + extras
     subprocess.check_call(cmd, env=merged_env)
     print('Copying json blobs (used for holomaps) from {} to {}'.format(paths['doc'], output))
     copy_files(paths['doc'], output, '**/*.json')

diff --git a/nbsite/pyodide/ServiceWorker.js b/nbsite/pyodide/ServiceWorker.js
@@ -51,7 +51,9 @@ self.addEventListener('fetch', (e) => {
       throw Error(`[Service Worker] Fetching resource ${e.request.url} failed with response: ${response.status}`);
     }
     console.log(`[Service Worker] Caching new resource: ${e.request.url}`);
-    cache.put(e.request, response.clone());
+    if (e.request.mode !== 'no-cors') {
+      cache.put(e.request, response.clone());
+    }
     return response;
   })());
 });