feat: support multiple npm registries and tokens via .npmrc (#503)

* feat: support multiple npm registries and tokens via npm_auth

* docs: add docstring to _get_npm_auth

* refactor: move default registry url to utils.bzl

* refactor: make package url available to npm_import

* refactor: make get_npm_auth unit testable

* test: add get_npm_auth unit tests

* fix: readd fallback download_url value to npm_import

* chore: add @aspect-build/a npm package from private GitHub registry to e2e/npm_translate_lock_auth

* refactor: return npm tokens, registries and scopes in get_npm_auth

* fix: build tarball url for custom registries

* fix: update npm_repositories.bzl in e2e/rules_foo

* fix: add trailing / to npm registry url

Co-authored-by: Greg Magolan <[email protected]>

.github/workflows/ci.yaml

@@ -83,6 +83,7 @@ jobs:
                   # Bazelisk will download bazel to here
                   XDG_CACHE_HOME: ~/.cache/bazel-repo
                   ASPECT_NPM_AUTH_TOKEN: ${{ secrets.ASPECT_NPM_AUTH_TOKEN }}
+                  ASPECT_GH_PACKAGES_AUTH_TOKEN: ${{ secrets.ASPECT_GH_PACKAGES_AUTH_TOKEN }}
               working-directory: ${{ matrix.folder }}
               # Don't run e2e/npm_translate_lock_auth unless there is an auth token secret; this won't
               # be set on forks
@@ -96,6 +97,7 @@ jobs:
                   # Bazelisk will download bazel to here
                   XDG_CACHE_HOME: ~/.cache/bazel-repo
                   ASPECT_NPM_AUTH_TOKEN: ${{ secrets.ASPECT_NPM_AUTH_TOKEN }}
+                  ASPECT_GH_PACKAGES_AUTH_TOKEN: ${{ secrets.ASPECT_GH_PACKAGES_AUTH_TOKEN }}
               working-directory: ${{ matrix.folder }}
               # Coverage does not work properly with RBE. See: bazelbuild/bazel#4685
               if: ${{ matrix.config == 'local' }}

e2e/npm_translate_lock_auth/.npmrc

@@ -1,3 +1,9 @@
 hoist=false
+
+# @aspect-build private GitHub registry
+@aspect-build:registry=https://npm.pkg.github.com/
+//npm.pkg.github.com/:_authToken=${ASPECT_GH_PACKAGES_AUTH_TOKEN}
+
+# @aspect-priv-npm private npm registry
 @aspect-priv-npm:registry=https://registry.npmjs.org/
 //registry.npmjs.org/:_authToken=${ASPECT_NPM_AUTH_TOKEN}

e2e/npm_translate_lock_auth/package.json

@@ -1,6 +1,7 @@
 {
     "private": true,
     "dependencies": {
+        "@aspect-build/a": "1.0.0",
         "@aspect-priv-npm/a": "1.0.0"
     }
 }

npm/private/npm_import.bzl

@@ -346,11 +346,8 @@ _PACKAGE_JSON_BZL_FILENAME = "package_json.bzl"
 def _impl(rctx):
     # scoped packages contain a slash in the name, which doesn't appear in the later part of the URL
     package_name_no_scope = rctx.attr.package.rsplit("/", 1)[-1]
-    download_url = rctx.attr.url if rctx.attr.url else "https://registry.npmjs.org/{0}/-/{1}-{2}.tgz".format(
-        rctx.attr.package,
-        package_name_no_scope,
-        utils.strip_peer_dep_version(rctx.attr.version),
-    )
+    download_url = rctx.attr.url if rctx.attr.url else utils.npm_registry_download_url(rctx.attr.package, rctx.attr.version)
+
     auth = {
         download_url: {
             "type": "pattern",

npm/private/npm_translate_lock.bzl

@@ -43,7 +43,8 @@ _NPM_IMPORT_TMPL = \
         link_packages = {link_packages},
         package = "{package}",
         version = "{version}",
-        lifecycle_hooks_no_sandbox = {lifecycle_hooks_no_sandbox},{maybe_integrity}{maybe_url}{maybe_deps}{maybe_transitive_closure}{maybe_patches}{maybe_patch_args}{maybe_run_lifecycle_hooks}{maybe_custom_postinstall}{maybe_lifecycle_hooks_env}{maybe_lifecycle_hooks_execution_requirements}{maybe_bins}{maybe_npm_auth}
+        url = "{url}",
+        lifecycle_hooks_no_sandbox = {lifecycle_hooks_no_sandbox},{maybe_integrity}{maybe_deps}{maybe_transitive_closure}{maybe_patches}{maybe_patch_args}{maybe_run_lifecycle_hooks}{maybe_custom_postinstall}{maybe_lifecycle_hooks_env}{maybe_lifecycle_hooks_execution_requirements}{maybe_bins}{maybe_npm_auth}
     )
 """
 
@@ -132,35 +133,87 @@ def _gather_values_from_matching_names(keyed_lists, *names):
                 result.append(v)
     return result
 
-def _get_npm_auth(rctx):
-    _NPM_TOKEN_KEY = "//registry.npmjs.org/:_authtoken"
-    token = None
+def get_npm_auth(npmrc, npmrc_path, environ):
+    """Parses npm tokens, registries and scopes from `.npmrc`.
+
+    - creates a token by registry dict: {registry: token}
+    - creates a registry by scope dict: {scope: registry}
+
+    For example:
+        Given the following `.npmrc`:
+
+        ```
+        @myorg:registry=https://somewhere-else.com/myorg
+        @another:registry=https://somewhere-else.com/another
+        ; would apply only to @myorg
+        //somewhere-else.com/myorg/:_authToken=MYTOKEN1
+        ; would apply only to @another
+        //somewhere-else.com/another/:_authToken=MYTOKEN2
+        ```
+
+        `get_npm_auth(rctx)` creates the following dict:
+
+        ```starlark
+        tokens = {
+            "somewhere-else.com/myorg": "MYTOKEN1",
+            "somewhere-else.com/another": "MYTOKEN2",
+        }
+        registries = {
+                "@myorg": "somewhere-else.com/myorg",
+                "@another": "somewhere-else.com/another",
+        }
+        auth = (tokens, registries)
+        ```
 
-    # Read token from npmrc label
-    if rctx.attr.npmrc:
-        npmrc_path = rctx.path(rctx.attr.npmrc)
-        npmrc = parse_ini(rctx.read(npmrc_path))
-        if _NPM_TOKEN_KEY in npmrc:
-            # Parse environment variable from config
-            token = npmrc[_NPM_TOKEN_KEY]
+    Args:
+        npmrc: The `.npmrc` file.
+        npmrc_path: The file path to `.npmrc`.
+        environ: A map of environment variables with their values.
+
+    Returns:
+        A tuple with a tokens dict and a registries dict.
+    """
+
+    _NPM_TOKEN_KEY = ":_authtoken"
+    _NPM_PKG_SCOPE_KEY = ":registry"
+    tokens = {}
+    registries = {}
+
+    for (k, v) in npmrc.items():
+        if k.find(_NPM_TOKEN_KEY) != -1:
+            # //somewhere-else.com/myorg/:_authToken=MYTOKEN1
+            # registry: somewhere-else.com/myorg
+            # token: MYTOKEN1
+            registry = k.removeprefix("//").removesuffix("/{}".format(_NPM_TOKEN_KEY))
+            token = v
 
             # A token can be a reference to an environment variable
             if token.startswith("$"):
                 # ${NPM_TOKEN} -> NPM_TOKEN
                 # $NPM_TOKEN -> NPM_TOKEN
                 token = token.removeprefix("$").removeprefix("{").removesuffix("}")
-                if token in rctx.os.environ.keys() and rctx.os.environ[token]:
-                    token = rctx.os.environ[token]
+                if token in environ.keys() and environ[token]:
+                    token = environ[token]
                 else:
                     print("""\
 WARNING: Issue while reading "{npmrc}". Failed to replace env in config: ${{{token}}}
 """.format(
                         npmrc = npmrc_path,
                         token = token,
                     ))
-    return token
+            tokens[registry] = token
+
+        if k.find(_NPM_PKG_SCOPE_KEY) != -1:
+            # @myorg:registry=https://somewhere-else.com/myorg
+            # scope: @myorg
+            # registry: somewhere-else.com/myorg
+            scope = k.removesuffix(_NPM_PKG_SCOPE_KEY)
+            registry = v.split("//", 1)[-1]
+            registries[scope] = registry
 
-def _gen_npm_imports(lockfile, root_package, attr):
+    return (tokens, registries)
+
+def _gen_npm_imports(lockfile, root_package, attr, registries = {}):
     "Converts packages from the lockfile to a struct of attributes for npm_import"
 
     if attr.prod and attr.dev:
@@ -277,37 +330,15 @@ def _gen_npm_imports(lockfile, root_package, attr):
         url = None
         if tarball:
             if _is_url(tarball):
-                if registry and tarball.startswith("https://registry.npmjs.org/"):
-                    url = registry + tarball[len("https://registry.npmjs.org/"):]
+                if registry and tarball.startswith(utils.npm_registry_url):
+                    url = registry + tarball[len(utils.npm_registry_url):]
                 else:
                     url = tarball
             else:
-                # pnpm 6.x may omit the registry component from the tarball value when it is configured
-                # via an .npmrc registry setting for the package. If there is a registry value, then use
-                # that as the prefix. If there isn't then prefix with the default npm registry value and
-                # suggest upgrading to a newer version pnpm.
                 if not registry:
-                    registry = "https://registry.npmjs.org/"
-
-                    # buildifier: disable=print
-                    if attr.warn_on_unqualified_tarball_url:
-                        print("""
-
-====================================================================================================
-WARNING: The pnpm lockfile package entry for {} ({})
-does not contain a fully qualified tarball URL or a registry setting to indicate which registry to
-use. Prefixing tarball url `{}`
-with the default npm registry url `https://registry.npmjs.org/`.
-
-If you are using an older version of pnpm such as 6.x, upgrading to 7.x or newer and
-re-generating the lockfile should generate a fully qualified tarball URL for this package.
-
-To disable this warning, set `warn_on_unqualified_tarball_url` to False in your
-`npm_translate_lock` repository rule.
-====================================================================================================
-
-""".format(name, version, tarball))
-                url = registry + tarball
+                    (scope, _) = utils.parse_package_name(name)
+                    registry = "https://{}".format(registries[scope]) if scope in registries else utils.npm_registry_url
+                url = "{0}/{1}".format(registry.removesuffix("/"), tarball)
 
         result.append(struct(
             custom_postinstall = custom_postinstall,
@@ -437,7 +468,14 @@ def _impl(rctx):
     root_package = None
     link_workspace = None
     lockfile_description = None
-    npm_auth = _get_npm_auth(rctx)
+    npm_tokens = {}
+    npm_registries = {}
+
+    # Read tokens from npmrc label
+    if rctx.attr.npmrc:
+        npmrc_path = rctx.path(rctx.attr.npmrc)
+        npmrc = parse_ini(rctx.read(npmrc_path))
+        (npm_tokens, npm_registries) = get_npm_auth(npmrc, npmrc_path, rctx.os.environ)
 
     _validate_attrs(rctx)
 
@@ -561,7 +599,7 @@ or disable this check by setting 'verify_node_modules_ignored = None' in `npm_tr
     defs_bzl_header = generated_by_lines + ["""# buildifier: disable=bzl-visibility
 load("@aspect_rules_js//js:defs.bzl", _js_library = "js_library")"""]
 
-    npm_imports = _gen_npm_imports(lockfile, root_package, rctx.attr)
+    npm_imports = _gen_npm_imports(lockfile, root_package, rctx.attr, npm_registries)
 
     fp_links = {}
     rctx_files = {
@@ -738,15 +776,13 @@ load("@aspect_rules_js//npm/private:npm_package_store.bzl", _npm_package_store =
     # check all links and fail if there are duplicates which can happen with public hoisting
     _check_for_conflicting_public_links(npm_imports, rctx.attr.public_hoist_packages)
 
-    maybe_npm_auth = ("""
-        npm_auth = "%s",""" % npm_auth) if npm_auth else ""
     stores_bzl = []
     links_bzl = {}
     for (i, _import) in enumerate(npm_imports):
+        url = _import.url if _import.url else utils.npm_registry_download_url(_import.package, _import.version, npm_registries)
+
         maybe_integrity = """
         integrity = "%s",""" % _import.integrity if _import.integrity else ""
-        maybe_url = """
-        url = "%s",""" % _import.url if _import.url else ""
         maybe_deps = ("""
         deps = %s,""" % starlark_codegen_utils.to_dict_attr(_import.deps, 2)) if len(_import.deps) > 0 else ""
         maybe_transitive_closure = ("""
@@ -766,6 +802,17 @@ load("@aspect_rules_js//npm/private:npm_package_store.bzl", _npm_package_store =
         maybe_bins = ("""
         bins = %s,""" % starlark_codegen_utils.to_dict_attr(_import.bins, 2)) if len(_import.bins) > 0 else ""
 
+        _registry = url.split("//", 1)[-1]
+        npm_token = None
+        match_len = 0
+        for (auth_registry, auth_token) in npm_tokens.items():
+            if _registry.startswith(auth_registry) and len(auth_registry) > match_len:
+                npm_token = auth_token
+                match_len = len(auth_registry)
+
+        maybe_npm_auth = ("""
+        npm_auth = "%s",""" % npm_token) if npm_token else ""
+
         repositories_bzl.append(_NPM_IMPORT_TMPL.format(
             link_packages = starlark_codegen_utils.to_dict_attr(_import.link_packages, 2, quote_value = False),
             link_workspace = link_workspace,
@@ -779,7 +826,7 @@ load("@aspect_rules_js//npm/private:npm_package_store.bzl", _npm_package_store =
             maybe_lifecycle_hooks_execution_requirements = maybe_lifecycle_hooks_execution_requirements,
             lifecycle_hooks_no_sandbox = rctx.attr.lifecycle_hooks_no_sandbox,
             maybe_transitive_closure = maybe_transitive_closure,
-            maybe_url = maybe_url,
+            url = url,
             maybe_bins = maybe_bins,
             name = _import.name,
             package = _import.package,

npm/private/test/BUILD.bazel

@@ -7,6 +7,7 @@ load(":utils_tests.bzl", "utils_tests")
 load(":ini_test.bzl", "ini_tests")
 load(":pkg_glob_tests.bzl", "pkg_glob_tests")
 load(":generated_pkg_json_test.bzl", "generated_pkg_json_test")
+load(":npm_auth_test.bzl", "npm_auth_test_suite")
 
 # gazelle:exclude *_checked.bzl
 
@@ -27,6 +28,8 @@ yaml_tests(name = "test_yaml")
 
 generated_pkg_json_test(name = "test_generated_pkg_json")
 
+npm_auth_test_suite()
+
 write_source_files(
     name = "write_npm_translate_lock",
     files = {