diff --git a/.spec-docs.json b/.spec-docs.json new file mode 100644 index 000000000..54ec52f63 --- /dev/null +++ b/.spec-docs.json @@ -0,0 +1,20 @@ +{ + "docsRoot": "docs", + "defaultBranch": "master", + "branchPathBase": "preview", + "redocTheme": "ga4gh", + "buildPages": [ + { + "apiSpecPath": "openapi/data_repository_service.openapi.yaml", + "htmlOutfile": "index.html", + "yamlOutfile": "openapi.yaml", + "jsonOutfile": "openapi.json" + }, + { + "apiSpecPath": "pages/more-background-on-compact-identifiers/openapi.yaml", + "htmlOutfile": "more-background-on-compact-identifiers.html", + "yamlOutfile": "more-background-on-compact-identifiers.yaml", + "jsonOutfile": "more-background-on-compact-identifiers.json" + } + ] +} diff --git a/.travis.yml b/.travis.yml index 9462f1abd..d09ae5484 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,3 @@ -dist: trusty -language: python -python: -- '2.7' -- '3.6' - # Specifying `branches.only = ['master']` can cause tagged builds to # not deploy. See travis-ci/travis-ci#2498 and travis-ci/travis-ci#1675. # We can fix this by only build master and branches/tags that follow the @@ -18,40 +12,17 @@ branches: - "/^release\\/\\d+\\.\\d+(\\.\\d+)?$/" - "/^release\\/(drs-)?\\d+\\.\\d+(\\.\\d+)?$/" -stages: - - linting -# - test - # The deploy stage only has `deploy` actions which will only run on commits to master - # (and not pull requests). That said, we still explicitly skip this stage because - # if we don't, Travis will only stop execution once the build stage has already been - # set up. If we skip the deploy stage for pull requests at the stage level, we can - # shave off a couple minutes of runtime. - - name: deploy - if: type != pull_request && branch = "master" - jobs: include: - # If the linting stage fails, then none of the other stages will run. - - stage: linting - python: '3.6' - script: - # Travis will install requirements.txt by default - - flake8 --select=E121,E123,E126,E226,E24,E704,W503,W504 --ignore=E501 app.py tests - - # the build process from James Eddy adapted from the WES repo - - stage: build_pages - language: java - jdk: oraclejdk8 - before_install: - - chmod +x gradlew - - chmod +x scripts/fetchpages.sh - - chmod +x scripts/stagepages.sh + - stage: build_docs + language: node_js + node_js: + - "12" + before_script: + - npm install -g @redocly/openapi-cli && npm install -g redoc-cli + - npm install -g @ga4gh/gh-openapi-docs@0.2.2-rc3 script: - - "./scripts/fetchpages.sh" - - "./gradlew installSwagger buildSwagger asciidoctor" - - "./scripts/stagepages.sh" - before_deploy: - - "mv misc/.gitignore .gitignore" + - gh-openapi-docs deploy: provider: pages skip_cleanup: true @@ -59,28 +30,3 @@ jobs: github-token: $GITHUB_TOKEN on: all_branches: true - - # Deploy to PyPI on every tagged commit - - stage: deploy - python: '3.6' - script: ignore - before_install: ignore - deploy: - - provider: pypi - on: - tags: true - branch: master - python: '3.6' - repo: ga4gh/data-repository-service-schemas - user: david4096 - password: - secure: LlQn8ZBAb5ekujHnoDrmzrmXaM6TpyzByNHPH4FTbbdnJ8lkDPb/ZhYvdmqrOvXPQg81/IoYKlIvP7fY9kc3oGUJ2IXhcPFqiw8njsRE5Qaebp+YppQO7C3IWGlHoZtXNtC608ZSA4x0oneNeNy+Y8KYnqKbmOlbuvrYRlNYfe9/8z7yLPH8wdmp0GyvbViedr3p7PXhtQVUKAgPpgjffZnSA7P/Y6AdkvjHHv2xMAzWP/QmOFWZNxUXjg0miR0K7eGFeGBNMM/+QsVXrGOu/TCtPtJ4JXyD86nzrZUbsOluyAblxwGlrv05se5ImVhR210OC5zvSW2902y/lxCw5uek+xg4/tcSA1ckshxLeu02GfDygCktMUtqtKVIZ+qvU7H4dEQ6Jnz9yBvZW5M6V94Ew3wBFy0RB5I9k3MMQY21FdynIUEZzBgJbOChCbmlIDT1varBHvWBiwg8EwPOVuJt1CsOoptJxUsoJND4tAOPIvXMNI17qGJ+VWAVMVNn7cVUuhEeGXwQF4urrkFBA7WIYOp6O9R8Ipg6WnQdxVdnqb3NsEc19SRdFXQ82SYibKfIZxjpdmYVgKzTYsJGMhfG6fTw9D4JABhggfgShsnByrFtbbkn/9g64jXDOjwPLeRXwXYZe6ZV6M69PDWdo0o326Qq/OHBG5eU7z2plNI= - -before_install: -- python setup.py sdist -- pip install dist/ga4gh_drs_schemas-*.tar.gz -- npm install -g swagger2openapi -script: -- make schemas -#- nosetests python/ -- ga4gh_drs_client diff --git a/DOCSBUILD.md b/DOCSBUILD.md index d1a16f17f..728e06768 100644 --- a/DOCSBUILD.md +++ b/DOCSBUILD.md @@ -1,346 +1,25 @@ # Documentation Build Process -This doc (from James Eddy) describes the build process for Swagger UI and static docs (in Travis CI) and subsequent deployment to GitHub Pages (cc @briandoconnor, @denis-yuen, @david4096, @natanlao). +This doc outlines the build process for OpenAPI and HTML docs (in Travis CI) and subsequent deployment to GitHub Pages. -These instructions are based on the current configuration for the [**Workflow Execution Service (WES) API schema repo**](https://github.com/ga4gh/workflow-execution-service-schemas), which includes: +These instructions are based on best practices for using `gh-openapi-docs`, which includes: + **When code is merged into the `master` branch of this repository**, artifacts are created and hosted at the following paths: - + ga4gh.github.io/[repo]/swagger-ui/ — Swagger UI for the API spec - + ga4gh.github.io/[repo]/docs/ — reference docs for the API - + ga4gh.github.io/[repo]/swagger.json — API spec in JSON format - + ga4gh.github.io/[repo]/swagger.yaml — API spec in YAML format + + ga4gh.github.io/[repo]/docs/ — reference docs for the API + + ga4gh.github.io/[repo]/openapi.json — API spec in JSON format + + ga4gh.github.io/[repo]/openapi.yaml — API spec in YAML format + **For non-`master` branches**, reviewers can preview documentation and other pages under "ga4gh.github.io/[repo]/preview/[branch-name]/" - + swagger-ui/ — Swagger UI preview for current version of [branch-name] + docs/ — docs preview for current version of [branch-name] - + swagger.json — spec (JSON) preview for current version of [branch-name] - + swagger.yaml — spec (YAML) preview for current version of [branch-name] + + openapi.json — spec (JSON) preview for current version of [branch-name] + + openapi.yaml — spec (YAML) preview for current version of [branch-name] + When changes are pushed to branches on a fork of the main repo (and the user has set up Travis for their forked repo), the same path apply but should be relative to "[user-or-org].github.io/[repo]/". -+ `README.md` and `CONTRIBUTING.md` updated with all of the above links - -+ `README.md` badges indicating Travis CI build status and Swagger/OpenAPI validation status - - --- -## Reference docs with `asciidoctor` and `swagger2markup` - -Uses the swagger2markup gradle plugin and asciidoctor to (1) automatically generate human-readable asciidoc files from the contract-first OpenAPI (swagger) yaml spec; and (2) incorporate -manual content to build an overall document in HTML and PDF formats. - -You'll need gradle installed to test locally. - -
- -Steps - -### Set up directory - -I started with the setup used in [**this template**](https://github.com/Swagger2Markup/swagger2markup-gradle-project-template) and copied over files for the Swagger2Markup [**gradle plugin**](http://swagger2markup.github.io/swagger2markup/1.3.1/#_gradle_plugin). - -**Note:** the choice of directory structure used here was my own, and is somewhat arbitrary. You can reorganize however you like, but you'll need to keep track of paths across various scripts and config files. - -```terminal -. # top level repo directory, e.g., 'workflow-execution-service-schemas/' -├── build.gradle -├── gradle -│   └── wrapper -│   ├── gradle-wrapper.jar -│   └── gradle-wrapper.properties -└── gradlew -``` - -### Update `gradle.settings` - -Change root project name (to the name of your repo's project): -```groovy -rootProject.name = 'workflow-execution-service-schemas' -``` - -### Update `build.gradle` - -Add `asciiDocDir` to `ext`: -```groovy -ext { - asciiDocDir = file("docs/asciidoc") - asciiDocOutputDir = file("docs/asciidoc/swagger2markup") -} -``` - -Update paths in `convertSwagger2markup`: -```groovy -convertSwagger2markup { - swaggerInput file("openapi/workflow_execution_service.swagger.yaml").getAbsolutePath() - outputDir asciiDocOutputDir - config = ['swagger2markup.markupLanguage' : 'ASCIIDOC', - 'swagger2markup.extensions.dynamicDefinitions.contentPath' : file('docs/asciidoc/swagger2markup/definitions').absolutePath, - 'swagger2markup.extensions.dynamicOverview.contentPath' : file('docs/asciidoc/swagger2markup/overview').absolutePath, - 'swagger2markup.extensions.dynamicPaths.contentPath' : file('docs/asciidoc/swagger2markup/paths').absolutePath, - 'swagger2markup.extensions.dynamicSecurity.contentPath' : file('docs/asciidoc/swagger2markup/security').absolutePath] -} -``` - -Add `sourceDir` and `outputDir` to `asciidoctor`: -```groovy -asciidoctor { - dependsOn convertSwagger2markup - sourceDir asciiDocDir - outputDir file("docs") - sources { - include 'index.adoc' - } - backends = ['html5', 'pdf'] - attributes = [ - doctype: 'book', - toc: 'left', - toclevels: '3', - numbered: '', - sectlinks: '', - sectanchors: '', - hardbreaks: '', - generated: asciiDocOutputDir - ] -} -``` - -Update paths in `watch`: -```groovy -watch { - asciidoc { - files fileTree('docs/asciidoc') - tasks 'asciidoctor' - } -} -``` - -### Generate AsciiDoc docs - -Run `./gradlew convertSwagger2markup` to convert swagger YAML to AsciiDoc files and initialize the `docs` folder: -```terminal -. -└── docs -    └── asciidoc -       └── swagger2markup -       ├── definitions.adoc -       ├── overview.adoc -       ├── paths.adoc -       └── security.adoc -``` - -### Add `index.adoc` and `front_matter.adoc` - -The [index file](https://github.com/ga4gh/workflow-execution-service-schemas/blob/master/docs/asciidoc/index.adoc) allows you to control the order in which pages are built for HTML and PDF docs; it looks like this: -```adoc -include::{generated}/overview.adoc[] -include::front_matter.adoc[] -include::{generated}/paths.adoc[] -include::{generated}/definitions.adoc[] -``` - -The ["front matter" file](https://github.com/ga4gh/workflow-execution-service-schemas/blob/master/docs/asciidoc/front_matter.adoc) is where you can add any manual content that you want to integrate with the -generated docs. This content needs to be composed using AsciiDoc (`.adoc`) format: - -```adoc -== Section header - -Some summary text. - -Features: - -* feature 1 -* feature 2 - -== Another section header - -More text... -``` - -### Build reference docs - -Run `./gradlew asciidoctor` to test. Check `docs/asciidoc/html5/index.html` to see the generated HTML report or `docs/asciidoc/pdf/index.pdf` to see the generated PDF report. - -```terminal -. -└── docs -    ├── README.md -    ├── asciidoc -    │   ├── front_matter.adoc -    │   ├── index.adoc -    │   └── swagger2markup -    │   ├── definitions.adoc -    │   ├── overview.adoc -    │   ├── paths.adoc -    │   └── security.adoc -    ├── html5 -    │   └── index.html -    └── pdf -    └── index.pdf -``` - -You can also add a `README.md` to the `docs` folder with a link to where generated docs will be hosted: -```md -View the full [Reference Documentation](https://ga4gh.github.io/workflow-execution-service-schemas/docs/) for the Workflow Execution Service API. -``` - -
- -## Swagger UI - -I initially used the node package [**generator-openapi-repo**](https://github.com/Rebilly/generator-openapi-repo) to set up Swagger UI stuff for the repo. However, I found that the generator code did way more than I needed, and some other stuff that I couldn't control. I cut out some of the excess pieces and rewrote the associated scripts to minimize the need for random JS code. - -Also, with help from @coverbeck, I updated the gradle plugin to install and provide an interface to the Swagger UI node components (eliminating the need for `gulpfile.js`). - -
- -Steps - -### Set up directory - -```terminal -. -├── gulpfile.js # deprecated; need to remove -├── package.json -└── scripts -    ├── buildui.js # deprecated; need to remove -    ├── fetchpages.sh -    └── stagepages.sh -``` - -### Add/edit `package.json` - -You should be able to copy the contents of [`package.json`](https://github.com/ga4gh/workflow-execution-service-schemas/blob/master/package.json) from the WES repo to get started. Update `name` and `version` to match the information for your repo. - -### Edit `stagepages.sh` - -This script builds Swagger UI and sets up various elements in their target locations for deployment to GitHub pages. The path to the swagger YAML is hardcoded in a couple lines, so you'll need to change that. - -```shell -#!/usr/bin/env bash - -set -e -set -v - -if [ "$TRAVIS_BRANCH" == "master" ]; then - cp docs/html5/index.html docs/ - cp openapi/workflow_execution_service.swagger.yaml ./swagger.yaml - cp -R web_deploy/* . -elif [ "$TRAVIS_BRANCH" != "gh-pages" ]; then - branch=$(echo "$TRAVIS_BRANCH" | awk '{print tolower($0)}') - branchpath="preview/$branch" - mkdir -p "$branchpath/docs" - cp docs/html5/index.html "$branchpath/docs/" - cp openapi/workflow_execution_service.swagger.yaml "$branchpath/swagger.yaml" - cp -R web_deploy/* "$branchpath/" -fi -``` - -
- -### Tweak .gitignore - -You need to remove the following line from .gitignore otherwise -the javascript libraries from swagger won't be copied during -gh-pages deploy: - - lib/ - - -## Configure Travis CI for repo - -This last step should be pretty straightforward — even though it was the hardest and most time consuming to troubleshoot. :) Here, you'll extend the `.travis.yml` config to run additional steps (after your API spec code has been built and tested) to build, set up, and deploy docs and Swagger elements. - -
- -Steps - -### Create/add GitHub token - -Follow [instructions](https://docs.travis-ci.com/user/deployment/pages/#setting-the-github-token) from Travis CI docs. - -### Update `travis.yml` - -If you already have a build/test/deply job configured in Travis, you can separate this as a separate stage in `jobs/include` — it's OK for different stages to use different environments. I believe you could also use `matrix` here, but this seems to work. - -```yaml -jobs: - include: - - stage: test - language: python - python: - - '2.7' - before_install: - - sudo apt-get update -qq - - pip install . --process-dependency-links - - pip install -r python/dev-requirements.txt - script: - - nosetests python/ - - flake8 python - - ga4gh_wes_client - deploy: - ... - - - stage: build_pages - ... -``` - -Add docs/swagger build commands for Java-based stage: - -**Note:** the `fetchpages.sh` step here effectively acts to retrieve the current state of the `gh-pages` branch and store it to be re-pushed along with the newly generated pages — rather than overwritten. - -```yaml -jobs: - include: - - stage: test - ... - - - stage: build_pages - language: java - jdk: oraclejdk8 - before_install: - - chmod +x gradlew - - chmod +x scripts/fetchpages.sh - - chmod +x scripts/stagepages.sh - script: - - "./scripts/fetchpages.sh" - - "./gradlew installSwagger buildSwagger asciidoctor" - - "./scripts/stagepages.sh" -``` - -Add deploy instructions for GitHub pages: - -**Note:** It is important that all of your build/deploy steps for docs and Swagger elements use the same language for the build environment (and preferably part of the same job/stage). Travis *does not* cache information between jobs of different languages, and so pushing to `gh-pages` without missing or overwriting something from a previous job gets really complicated. - -```yaml -jobs: - include: - - stage: test - ... - - - stage: build_pages - language: java - jdk: oraclejdk8 - before_install: - - chmod +x gradlew - - chmod +x scripts/fetchpages.sh - - chmod +x scripts/stagepages.sh - script: - - "./scripts/fetchpages.sh" - - "./gradlew installSwagger buildSwagger asciidoctor" - - "./scripts/stagepages.sh" - deploy: - provider: pages - skip-cleanup: true - github-token: $GITHUB_TOKEN - on: - all_branches: true -``` - -Push to your repo and cross your fingers... - -
+## Reference docs with `gh-openapi-docs` -## Update README links/badges +This repo uses the `gh-openapi-docs` ([Github](https://github.com/ga4gh/gh-openapi-docs), [npm](https://www.npmjs.com/package/@ga4gh/gh-openapi-docs)) command to automatically generate human-readable HTML pages from the OpenAPI specification. -For ideas on how to set up references to docs and Swagger elements in your main `README.md`, refer to the [**README**](https://github.com/ga4gh/workflow-execution-service-schemas/blob/master/README.md) for the WES API repo. +You'll need `nodejs`, `npm`, `openapi-cli`, `redoc-cli`, and `gh-openapi-docs` installed on your machine to build documentation locally. See the [README](https://github.com/ga4gh/gh-openapi-docs) for instructions on installing the `gh-openapi-docs` tool in general, and [.travis.yml](./.travis.yml) for how the tool has been specifically installed for DRS. \ No newline at end of file diff --git a/Makefile b/Makefile deleted file mode 100644 index 6764ed83a..000000000 --- a/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -SWAGGER_PATH = openapi/data_repository_service.swagger.yaml -OPENAPI3_PATH = openapi/data_repository_service.openapi.yaml -SMARTAPI_PATH = openapi/data_repository_service.smartapi.yaml -SMARTAPI_PART_PATH = openapi/data_repository_service.smartapi.yaml.part -SWAGGER2OPENAPI_PATH = swagger2openapi - -$(OPENAPI3_PATH) : $(SWAGGER_PATH) - $(SWAGGER2OPENAPI_PATH) -y $(SWAGGER_PATH) --outfile $(OPENAPI3_PATH) - -$(SMARTAPI_PATH) : $(OPENAPI3_PATH) - python merge_yaml.py $(OPENAPI3_PATH) $(SMARTAPI_PART_PATH) > $(SMARTAPI_PATH) - -schemas : $(OPENAPI3_PATH) $(SMARTAPI_PATH) - true - -.PHONY: schemas diff --git a/README.md b/README.md index 66536cc3a..0f4e7249a 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,6 @@ `develop` branch status: [![Build Status](https://travis-ci.org/ga4gh/data-repository-service-schemas.svg?branch=develop)](https://travis-ci.org/ga4gh/data-repository-service-schemas?branch=develop) Swagger Validator [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1405753.svg)](https://doi.org/10.5281/zenodo.1405753) - - - The [Global Alliance for Genomics and Health](http://genomicsandhealth.org/) (GA4GH) is an international coalition, formed to enable the sharing of genomic and clinical data. # About the GA4GH Cloud Work Stream @@ -28,14 +23,14 @@ For more information see our HTML documentation links in the table below. # API Definition -| **Branch** | **Reference Documentation** | **[OpenAPI YAML description](openapi/data_repository_service.swagger.yaml)** | +| **Branch** | **Reference Documentation** | Swagger Editor | | --- | --- | --- | -| **master**: The current release | [HTML](https://ga4gh.github.io/data-repository-service-schemas/docs/) | [Swagger](https://ga4gh.github.io/data-repository-service-schemas/swagger-ui/#/DataRepositoryService/) | -| **develop**: the stable development branch, into which feature branches are merged | [HTML](https://ga4gh.github.io/data-repository-service-schemas/preview/develop/docs/) | [Swagger](https://ga4gh.github.io/data-repository-service-schemas/preview/develop/swagger-ui/#/DataRepositoryService/) | -| **release 1.1.0**: The 1.1.0 release of DRS that includes *no* API changes only documentation changes. This introduces a new URI convention using compact identifiers along with clear directions on how to use identifiers.org/n2t.net to resolve them. | [HTML](https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.1.0/docs/) | [Swagger](https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.1.0/swagger-ui/#/DataRepositoryService/) | -| **release 1.0.0**: The 1.0.0 release of DRS that is now an approved GA4GH standard | [HTML](https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.0.0/docs/) | [Swagger](https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.0.0/swagger-ui/#/DataRepositoryService/) | -| **release 0.1**: Simplifying DRS to core functionality | [HTML](https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-0.1.0/docs/) | [Swagger](https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-0.1.0/swagger-ui/#/DataRepositoryService/) | -| **release 0.0.1**: The initial DRS after the rename from DOS | [HTML](https://ga4gh.github.io/data-repository-service-schemas/preview/release/0.0.1/docs/) | [Swagger](https://ga4gh.github.io/data-repository-service-schemas/preview/release/0.0.1/swagger-ui/#/DataRepositoryService/) | +| **develop**: the stable development branch, into which feature branches are merged | [HTML](https://ga4gh.github.io/data-repository-service-schemas/preview/develop/docs/) | [Swagger Editor](https://editor.swagger.io?url=https://ga4gh.github.io/data-repository-service-schemas/preview/develop/openapi.yaml) | +| **release 1.2.0**: The 1.2.0 release of DRS adds the standardized `/service-info` endpoint, and 2 `POST` endpoints that are functionally equivalent to the current `GET` endpoints, but they enable the submission of large passport tokens in the request body. | [HTML](https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.2.0/docs/) | [Swagger Editor](https://editor.swagger.io?url=https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.2.0/openapi.yaml) | +| **release 1.1.0**: The 1.1.0 release of DRS that includes *no* API changes only documentation changes. This introduces a new URI convention using compact identifiers along with clear directions on how to use identifiers.org/n2t.net to resolve them. | [HTML](https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.1.0/docs/) | [Swagger Editor](https://editor.swagger.io?url=https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.1.0/swagger.yaml) | +| **release 1.0.0**: The 1.0.0 release of DRS that is now an approved GA4GH standard | [HTML](https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.0.0/docs/) | [Swagger Editor](https://editor.swagger.io?url=https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.0.0/swagger.yaml) | +| **release 0.1**: Simplifying DRS to core functionality | [HTML](https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-0.1.0/docs/) | [Swagger Editor](https://editor.swagger.io?url=https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-0.1.0/swagger.yaml) | +| **release 0.0.1**: The initial DRS after the rename from DOS | [HTML](https://ga4gh.github.io/data-repository-service-schemas/preview/release/0.0.1/docs/) | [Swagger Editor](https://editor.swagger.io?url=https://ga4gh.github.io/data-repository-service-schemas/preview/release/0.0.1/swagger.yaml) | To monitor development work on various branches, add 'preview/\' to the master URLs above (e.g., 'https://ga4gh.github.io/data-repository-service-schemas/preview/\/docs'). diff --git a/build.gradle b/build.gradle deleted file mode 100644 index 26956fa81..000000000 --- a/build.gradle +++ /dev/null @@ -1,106 +0,0 @@ -buildscript { - repositories { - jcenter() - mavenCentral() - maven { url 'http://oss.jfrog.org/artifactory/oss-snapshot-local/' } - maven { - url "https://plugins.gradle.org/m2/" - } - //mavenLocal() - } - dependencies { - classpath 'org.asciidoctor:asciidoctor-gradle-plugin:1.5.3' - classpath 'io.github.swagger2markup:swagger2markup-gradle-plugin:1.3.1' - classpath "io.github.swagger2markup:swagger2markup:1.3.1" - classpath "io.github.swagger2markup:swagger2markup-import-files-ext:1.3.1" - classpath "com.bluepapa32:gradle-watch-plugin:0.1.5" - classpath "org.kordamp.gradle:livereload-gradle-plugin:0.2.1" - classpath "com.moowork.gradle:gradle-node-plugin:1.2.0" - } -} - -apply plugin: 'org.asciidoctor.convert' -apply plugin: 'com.bluepapa32.watch' -apply plugin: 'org.kordamp.gradle.livereload' -apply plugin: 'io.github.swagger2markup' -apply plugin: 'com.moowork.node' -node { - version = '8.9.0' - download = true -} - -group 'io.github.swagger2markup' -version '1.3.1' - -repositories { - jcenter() - mavenCentral() -} - -ext { - asciiDocDir = file("docs/asciidoc") - asciiDocOutputDir = file("docs/asciidoc/swagger2markup") -} - -convertSwagger2markup { - swaggerInput file("openapi/data_repository_service.swagger.yaml").getAbsolutePath() - outputDir asciiDocOutputDir - config = ['swagger2markup.markupLanguage' : 'ASCIIDOC', - 'swagger2markup.extensions.dynamicDefinitions.contentPath' : file('docs/asciidoc/swagger2markup/definitions').absolutePath, - 'swagger2markup.extensions.dynamicOverview.contentPath' : file('docs/asciidoc/swagger2markup/overview').absolutePath, - 'swagger2markup.extensions.dynamicPaths.contentPath' : file('docs/asciidoc/swagger2markup/paths').absolutePath, - 'swagger2markup.extensions.dynamicSecurity.contentPath' : file('docs/asciidoc/swagger2markup/security').absolutePath] -} - -asciidoctorj { - version = '1.5.5' -} - -asciidoctor { - dependsOn convertSwagger2markup - sourceDir asciiDocDir - outputDir file("docs") - sources { - include 'index.adoc', 'more_background_on_compact_identifiers.adoc' - } - backends = ['html5', 'pdf'] - attributes = [ - doctype: 'book', - toc: 'left', - toclevels: '3', - numbered: '', - sectlinks: '', - sectanchors: '', - hardbreaks: '', - generated: asciiDocOutputDir - ] -} - -dependencies { - // add converters and extensions using `asciidoctor` configuration - asciidoctor 'org.asciidoctor:asciidoctorj-pdf:1.5.0-alpha.15' -} - -watch { - asciidoc { - files fileTree('docs/asciidoc') - tasks 'asciidoctor' - } -} - -liveReload { - docRoot asciidoctor.outputDir.canonicalPath -} - - -task wrapper(type: Wrapper) { - gradleVersion = '3.5' -} - -task installSwagger(type: NpmTask) { - npmCommand = ["install"] -} - -task buildSwagger(type: NpmTask) { - npmCommand = ["run", "build"] -} diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 9936ec133..000000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -SPHINXPROJ = DataObjectService -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index 7437cc984..000000000 --- a/docs/README.md +++ /dev/null @@ -1 +0,0 @@ -View the main [reference documentation page](https://ga4gh.github.io/data-repository-service-schemas/docs/) which is generated from these templates for the Data Repository Service. diff --git a/docs/asciidoc/back_matter.adoc b/docs/asciidoc/back_matter.adoc deleted file mode 100644 index 882871c51..000000000 --- a/docs/asciidoc/back_matter.adoc +++ /dev/null @@ -1,124 +0,0 @@ -== Appendix: Motivation - -[cols="40a,60a"] -|=== -|Data sharing requires portable data, consistent with the FAIR data principles (findable, accessible, interoperable, reusable). Today’s researchers and clinicians are surrounded by potentially useful data, but often need bespoke tools and processes to work with each dataset. Today’s data publishers don’t have a reliable way to make their data useful to all (and only) the people they choose. And today’s data controllers are tasked with implementing standard controls of non-standard mechanisms for data access. -|image::figure1.png[] -_Figure 1: there’s an ocean of data, with many different tools to drink from it, but no guarantee that any tool will work with any subset of the data_ -|=== - -[cols="40a,60a"] -|=== -|We need a standard way for data producers to make their data available to data consumers, that supports the control needs of the former and the access needs of the latter. And we need it to be interoperable, so anyone who builds access tools and systems can be confident they'll work with all the data out there, and anyone who publishes data can be confident it will work with all the tools out there. -|image::figure2.png[] -_Figure 2: by defining a standard Data Repository API, and adapting tools to use it, every data publisher can now make their data useful to every data consumer_ -|=== - - -[cols="75a,25a"] -|=== - -|We envision a world where: - -* there are many many **data consumers**, working in research and in care, who can use the tools of their choice to access any and all data that they have permission to see -* there are many **data access tools** and platforms, supporting discovery, visualization, analysis, and collaboration -* there are many **data repositories**, each with their own policies and characteristics, which can be accessed by a variety of tools -* there are many **data publishing tools** and platforms, supporting a variety of data lifecycles and formats -* there are many many **data producers**, generating data of all types, who can use the tools of their choice to make their data as widely available as is appropriate - -|image::figure3.png[] -_Figure 3: a standard Data Repository API enables an ecosystem of data producers and consumers_ -|=== - -This spec defines a standard **Data Repository Service (DRS) API** (“the yellow box”), to enable that ecosystem of data producers and consumers. Our goal is that the only thing data consumers need to know about a data repo is _"here's the DRS endpoint to access it"_, and the only thing data publishers need to know to tap into the world of consumption tools is _"here's how to tell it where my DRS endpoint lives"_. - -=== Federation - -The world's biomedical data is controlled by groups with very different policies and restrictions on where their data lives and how it can be accessed. A primary purpose of DRS is to support unified access to disparate and distributed data. (As opposed to the alternative centralized model of "let's just bring all the data into one single data repository”, which would be technically easier but is no more realistic than “let’s just bring all the websites into one single web host”.) - -In a DRS-enabled world, tool builders don’t have to worry about where the data their tools operate on lives -- they can count on DRS to give them access. And tool users only need to know which DRS server is managing the data they need, and whether they have permission to access it; they don’t have to worry about how to physically get access to, or (worse) make a copy of the data. For example, if I have appropriate permissions, I can run a pooled analysis where I run a single tool across data managed by different DRS servers, potentially in different locations. - -== Appendix: Background Notes on DRS URIs - -=== Design Motivation - -DRS URIs are aligned with the https://www.nature.com/articles/sdata201618[FAIR data principles] and the https://doi.org/10.1038/sdata.2018.2[Joint Declaration of Data Citation Principles] -- both hostname-based and compact identifier-based URIs provide globally unique, machine-resolvable, persistent identifiers for data. - -* We require all URIs to begin with `drs://` as a signal to humans and systems consuming these URIs that the response they will ultimately receive, after transforming the URI to a fetchable URL, will be a DRS JSON packet. This signal differentiates DRS URIs from the wide variety of other entities (HTML documents, PDFs, ontology notes, etc.) that can be represented by compact identifiers. -* We support hostname-based URIs because of their simplicity and efficiency for server and client implementers. -* We support compact identifier-based URIs, and the meta-resolver services of identifiers.org and n2t.net (Name-to-Thing), because of the wide adoption of compact identifiers in the research community. as detailed by https://doi.org/10.1038/sdata.2018.29[Wimalaratne et al (2018)] in "Uniform resolution of compact identifiers for biomedical data." - -== Appendix: Compact Identifier-Based URIs - -.Note: Identifiers.org/n2t.net API Changes -**** -The examples below show the current API interactions with https://n2t.net/e/compact_ids.html[n2t.net] and https://docs.identifiers.org/[identifiers.org] which may change over time. Please refer to the documentation from each site for the most up-to-date information. We will make best efforts to keep the DRS specification current but DRS clients MUST maintain their ability to use either the identifiers.org or n2t.net APIs to resolve compact identifier-based DRS URIs. -**** - -=== Registering a DRS Server on a Meta-Resolver - -See the documentation on the https://n2t.net/e/compact_ids.html[n2t.net] and https://docs.identifiers.org/[identifiers.org] meta-resolvers for adding your own compact identifier type and registering your DRS server as a resolver. You can register new prefixes (or mirrors by adding resource provider codes) for free using a simple online form. For more information see link:more_background_on_compact_identifiers[More Background on Compact Identifiers]. - -=== Calling Meta-Resolver APIs for Compact Identifier-Based DRS URIs - -Clients resolving Compact Identifier-based URIs need to convert a prefix (e.g. “drs.42”) into an URL pattern. They can do so by calling either the identifiers.org or the n2t.net API, since the two meta-resolvers keep their mapping databases in sync. - -==== Calling the identifiers.org API as a Client - -It takes two API calls to get the URL pattern. - -(i) The client makes a GET request to identifiers.org to find information about the prefix: - - GET https://registry.api.identifiers.org/restApi/namespaces/search/findByPrefix?prefix=drs.42 - -This request returns a JSON structure including various URLs containing an embedded namespace id, such as: - - "namespace" : { - "href":"https://registry.api.identifiers.org/restApi/namespaces/1234" - } - -(ii) The client extracts the namespace id (in this example 1234), and uses it to make a second GET request to identifiers.org to find information about the namespace: - - GET https://registry.api.identifiers.org/restApi/resources/search/findAllByNamespaceId?id=1234 - -This request returns a JSON structure including an urlPattern field, whose value is an URL pattern containing a `${id}` parameter, such as: - - "urlPattern" : "https://drs.myexample.org/ga4gh/drs/v1/objects/{$id}" - -==== Calling the n2t.net API as a Client - -It takes one API call to get the URL pattern. - -The client makes a GET request to n2t.net to find information about the namespace. (Note the trailing colon.) - - GET https://n2t.net/drs.42: - -This request returns a text structure including a redirect field, whose value is an URL pattern containing a `$id` parameter, such as: - - redirect: https://drs.myexample.org/ga4gh/drs/v1/objects/$id - -=== Caching with Compact Identifiers - -Identifiers.org/n2t.net compact identifier resolver records do not change frequently. This reality is useful for caching resolver records and their URL patterns for performance reasons. Builders of systems that use compact identifier-based DRS URIs should cache prefix resolver records from identifiers.org/n2t.net and occasionally refresh the records (such as every 24 hours). This approach will reduce the burden on these community services since we anticipate many DRS URIs will be regularly resolved in workflow systems. Alternatively, system builders may decide to directly mirror the registries themselves, instructions are provided on the identifiers.org/n2t.net websites. - -=== Security with Compact Identifiers - -As mentioned earlier, identifiers.org/n2t.net performs some basic verification of new prefixes and provider code mirror registrations on their sites. However, builders of systems that consume and resolve DRS URIs may have certain security compliance requirements and regulations that prohibit relying on an external site for resolving compact identifiers. In this case, systems under these security and compliance constraints may wish to whitelist certain compact identifier resolvers and/or vet records from identifiers.org/n2t.net before enabling in their systems. - -=== Accession Encoding to Valid DRS IDs - -The compact identifier format used by identifiers.org/n2t.net does not percent-encode reserved URI characters but, instead, relies on the first ":" character to separate prefix from accession. Since these accessions can contain any characters, and characters like "/" will interfere with DRS API calls, you _must_ percent encode the accessions extracted from DRS compact identifier-based URIs when using as DRS IDs in subsequent DRS GET requests. An easy way for a DRS client to handle this is to get the initial DRS object JSON response from whatever redirects the compact identifier resolves to, then look for the `self_uri` in the JSON, which will give you the correctly percent-encoded DRS ID for subsequent DRS API calls such as the `access` method. - -=== Additional Examples - -For additional examples, see the document link:more_background_on_compact_identifiers[More Background on Compact Identifiers]. - -== Appendix: Hostname-Based URIs - -=== Encoding DRS IDs - -In hostname-based DRS URIs, the ID is always percent-encoded to ensure special characters do not interfere with subsequent DRS endpoint calls. As such, ":" is not allowed in the URI and is a convenient way of differentiating from a compact identifier-based DRS URI. Also, if a given DRS service implementation uses compact identifier accessions as their DRS IDs, they must be percent encoded before using them as DRS IDs in hostname-based DRS URIs and subsequent GET requests to a DRS service endpoint. - -=== Future DRS Versions and Service Registry/Info - -In the future, as new major versions of DRS are released, a DRS server might support multiple API versions on different URL paths. At that point we expect to add support for https://github.com/ga4gh-discovery/ga4gh-service-registry[service-registry] and https://github.com/ga4gh-discovery/ga4gh-service-info[service-info] endpoints to the API, and to update the URI resolution logic to describe how to use those endpoints when translating hostname-based DRS URIs to URLs. diff --git a/docs/asciidoc/front_matter.adoc b/docs/asciidoc/front_matter.adoc deleted file mode 100644 index 492772213..000000000 --- a/docs/asciidoc/front_matter.adoc +++ /dev/null @@ -1,148 +0,0 @@ -== Introduction - -The Data Repository Service (DRS) API provides a generic interface to data repositories so data consumers, including workflow systems, can access data objects in a single, standard way regardless of where they are stored and how they are managed. The primary functionality of DRS is to map a logical ID to a means for physically retrieving the data represented by the ID. The sections below describe the characteristics of those IDs, the types of data supported, how they can be pointed to using URIs, and how clients can use these URIs to ultimately make successful DRS API requests. This document also describes the DRS API in detail and provides information on the specific endpoints, request formats, and responses. This specification is intended for developers of DRS-compatible services and of clients that will call these DRS services. - -The key words MUST, MUST NOT, REQUIRED, SHALL, SHALL NOT, SHOULD, SHOULD NOT, RECOMMENDED, MAY, and OPTIONAL in this document are to be interpreted as described in https://tools.ietf.org/html/rfc2119[RFC 2119]. - -== DRS API Principles - -=== DRS IDs - -Each implementation of DRS can choose its own id scheme, as long as it follows these guidelines: - -* DRS IDs are strings made up of uppercase and lowercase letters, decimal digits, hypen, period, underscore and tilde [A-Za-z0-9.-_~]. See https://tools.ietf.org/html/rfc3986#section-2.3[RFC 3986 § 2.3]. -* DRS IDs can contain other characters, but they MUST be encoded into valid DRS IDs whenever they are used in API calls. This is because non-encoded IDs may interfere with the interpretation of the `objects/{id}/access` endpoint. To overcome this limitation use percent-encoding of the ID, see https://tools.ietf.org/html/rfc3986#section-2.4[RFC 3986 § 2.4] -* One DRS ID MUST always return the same object data (or, in the case of a collection, the same set of objects). This constraint aids with reproducibility. -* DRS implementations MAY have more than one ID that maps to the same object. -* DRS version 1.x does NOT support semantics around multiple versions of an object. (For example, there’s no notion of “get latest version” or “list all versions”.) Individual implementations MAY choose an ID scheme that includes version hints. - - -=== DRS URIs - -For convenience, including when passing content references to a https://github.com/ga4gh/workflow-execution-service-schemas[WES server], we define a https://en.wikipedia.org/wiki/Uniform_Resource_Identifier#Generic_syntax[URI scheme] for DRS-accessible content. This section documents the syntax of DRS URIs, and the rules clients follow for translating a DRS URI into a URL that they use for making the DRS API calls described in this spec. - -There are two styles of DRS URIs, Hostname-based and Compact Identifier-based, both using the `drs://` URI scheme. DRS servers may choose either style when exposing references to their content;. DRS clients MUST support resolving both styles. - -TIP: See <<_appendix_background_notes_on_drs_uris>> for more information on our design motivations for DRS URIs. - -==== Hostname-based DRS URIs - -Hostname-based DRS URIs are simpler than compact identifier-based URIs. They contain the DRS server name and the DRS ID only and can be converted directly into a fetchable URL based on a simple rule. They take the form: - - drs:/// - -DRS URIs of this form mean _"you can fetch the content with DRS id from the DRS server at "_. -For example, here are the client resolution steps if the URI is: - - drs://drs.example.org/314159 - -1) The client parses the string to extract the hostname of “drs.example.org” and the id of “314159”. -2) The client makes a GET request to the DRS server, using the standard DRS URL syntax: - - GET https://drs.example.org/ga4gh/drs/v1/objects/314159 - -The protocol is always https and the port is always the standard 443 SSL port. It is invalid to include a different port in a DRS hostname-based URI. - -TIP: See the <<_appendix_hostname_based_uris>> for information on how hostname-based DRS URI resolution to URLs is likely to change in the future, when the DRS v2 major release happens. - -==== Compact Identifier-based DRS URIs - -Compact Identifier-based DRS URIs use resolver registry services (specifically, https://identifiers.org/[identifiers.org] and https://n2t.net/[n2t.net (Name-To-Thing)]) to provide a layer of indirection between the DRS URI and the DRS server name -- the actual DNS name of the DRS server isn’t present in the URI. This approach is based on the Joint Declaration of Data Citation Principles as detailed by https://doi.org/10.1038/sdata.2018.29[Wimalaratne et al (2018)]. - -For more information, see the document link:more_background_on_compact_identifiers[More Background on Compact Identifiers]. - -Compact Identifiers take the form: - - drs://[provider_code/]namespace:accession - -Together, provider code and the namespace are referred to as the _prefix_. The provider code is optional and is used by identifiers.org/n2t.net for compact identifier resolver mirrors. Both the `provider_code` and `namespace` disallow spaces or punctuation, only lowercase alphanumerical characters, underscores and dots are allowed (e.g. [A-Za-z0-9._]). - -TIP: See the <<_appendix_compact_identifier_based_uris>> for more background on Compact Identifiers and resolver registry services like identifiers.org/n2t.net (aka meta-resolvers), how to register prefixes, possible caching strategies, and security considerations. - -===== For DRS Servers - -If your DRS implementation will issue DRS URIs based on _your own_ compact identifiers, you MUST first register a new prefix with identifiers.org (which is automatically mirrored to n2t.net). You will also need to include a provider resolver resource in this registration which links the prefix to your DRS server, so that DRS clients can get sufficient information to make a successful DRS GET request. For clarity, we recommend you choose a namespace beginning with `drs.`. - -===== For DRS Clients - -A DRS client parses the DRS URI compact identifier components to extract the prefix and the accession, and then uses meta-resolver APIs to locate the actual DRS server. For example, here are the client resolution steps if the URI is: - - drs://drs.42:314159 - -1) The client parses the string to extract the prefix of `drs.42` and the accession of `314159`, using the first occurrence of a colon (":") character after the initial `drs://` as a delimiter. (The colon character is not allowed in a Hostname-based DRS URI, making it easy to tell them apart.) - -2) The client makes API calls to a meta-resolver to look up the URL pattern for the namespace. (See <<_calling_meta_resolver_apis_for_compact_identifier_based_drs_uris>> for details.) The URL pattern is a string containing a `{$id}` parameter, such as: - - https://drs.myexample.org/ga4gh/drs/v1/objects/{$id} - -3) The client generates a DRS URL from the URL template by replacing {$id} with the accession it extracted in step 1. It then makes a GET request to the DRS server: - - GET https://drs.myexample.org/ga4gh/drs/v1/objects/314159 - -4) The client follows any HTTP redirects returned in step 3, in case the resolver goes through an extra layer of redirection. - -For performance reasons, DRS clients SHOULD cache the URL pattern returned in step 2, with a suggested 24 hour cache life. - -==== Choosing a URI Style - -DRS servers can choose to issue either hostname-based or compact identifier-based DRS URIs, and can be confident that compliant DRS clients will support both. DRS clients *must* be able to accommodate both URI types. Tradeoffs that DRS server builders, and third parties who need to cite DRS objects in datasets, workflows or elsewhere, may want to consider include: - -.Table Choosing a URI Style -|=== -| |Hostname-based |Compact Identifier-based - -|URI Durability -|URIs are valid for as long as the server operator maintains ownership of the published DNS address. (They can of course point that address at different physical serving infrastructure as often as they’d like.) -|URIs are valid for as long as the server operator maintains ownership of the published compact identifier resolver namespace. (They also depend on the meta-resolvers like identifiers.org/n2t.net remaining operational, which is intended to be essentially forever.) - -|Client Efficiency -|URIs require minimal client logic, and no network requests, to resolve. -|URIs require small client logic, and 1-2 cacheable network requests, to resolve. - -|Security -|Servers have full control over their own security practices. -|Server operators, in addition to maintaining their own security practices, should confirm they are comfortable with the resolver registry security practices, including protection against denial of service and namespace-hijacking attacks. (See the <<_appendix_compact_identifier_based_uris>> for more information on resolver registry security.) - -|=== - -=== DRS Datatypes - -DRS v1 supports two types of content: - -* a _blob_ is like a file -- it's a single blob of bytes, represented by a `DrsObject` without a `contents` array -* a _bundle_ is like a folder -- it's a collection of other DRS content (either blobs or bundles), represented by a `DrsObject` with a `contents` array - -=== Read-only - -DRS v1 is a read-only API. We expect that each implementation will define its own mechanisms and interfaces (graphical and/or programmatic) for adding and updating data. - -=== Standards - -The DRS API specification is written in OpenAPI and embodies a RESTful service philosophy. It uses JSON in requests and responses and standard HTTPS on port 443 for information transport. - -== Authorization & Authentication - -=== Making DRS Requests - -The DRS implementation is responsible for defining and enforcing an authorization policy that determines which users are allowed to make which requests. GA4GH recommends that DRS implementations use an OAuth 2.0 https://oauth.net/2/bearer-tokens/[bearer token], although they can choose other mechanisms if appropriate. - -=== Fetching DRS Objects - -The DRS API allows implementers to support a variety of different content access policies, depending on what `AccessMethod` records they return: - -* public content: -** server provides an `access_url` with a `url` and no `headers` -** caller fetches the object bytes without providing any auth info -* private content that requires the caller to have out-of-band auth knowledge (e.g. service account credentials): -** server provides an `access_url` with a `url` and no `headers` -** caller fetches the object bytes, passing the auth info they obtained out-of-band -* private content that requires the caller to pass an Authorization token: -** server provides an `access_url` with a `url` and `headers` -** caller fetches the object bytes, passing auth info via the specified header(s) -* private content that uses an expensive-to-generate auth mechanism (e.g. a signed URL): -** server provides an `access_id` -** caller passes the `access_id` to the `/access` endpoint -** server provides an `access_url` with the generated mechanism (e.g. a signed URL in the `url` field) -** caller fetches the object bytes from the `url` (passing auth info from the specified headers, if any) - -DRS implementers should ensure their solutions restrict access to targets as much as possible, detect attempts to exploit through log monitoring, and they are prepared to take action if an exploit in their DRS implementation is detected. diff --git a/docs/asciidoc/index.adoc b/docs/asciidoc/index.adoc deleted file mode 100644 index bfcac0c61..000000000 --- a/docs/asciidoc/index.adoc +++ /dev/null @@ -1,5 +0,0 @@ -include::{generated}/overview.adoc[] -include::front_matter.adoc[] -include::{generated}/paths.adoc[] -include::{generated}/definitions.adoc[] -include::back_matter.adoc[] diff --git a/docs/asciidoc/more_background_on_compact_identifiers.adoc b/docs/asciidoc/more_background_on_compact_identifiers.adoc deleted file mode 100644 index 1eb2bdcf1..000000000 --- a/docs/asciidoc/more_background_on_compact_identifiers.adoc +++ /dev/null @@ -1 +0,0 @@ -include::more_background_on_compact_identifiers_content.adoc[] diff --git a/docs/asciidoc/more_background_on_compact_identifiers_content.adoc b/docs/asciidoc/more_background_on_compact_identifiers_content.adoc deleted file mode 100644 index 2d29c72a1..000000000 --- a/docs/asciidoc/more_background_on_compact_identifiers_content.adoc +++ /dev/null @@ -1,128 +0,0 @@ -== About - -This document contains more examples of resolving compact identifier-based DRS URIs than we could fit in the DRS specification or appendix. It's provided here for your reference as a supplement to the specification. - -== Background on Compact Identifier-Based URIs - -Compact identifiers refer to locally-unique persistent identifiers that have been namespaced to provide global uniqueness. See https://www.biorxiv.org/content/10.1101/101279v3["Uniform resolution of compact identifiers for biomedical data"] for an excellent introduction to this topic. By using compact identifiers in DRS URIs, along with a resolver registry (identifiers.org/n2t.net), systems can identify the current resolver when they need to translate a DRS URI into a fetchable URL. This allows a project to issue compact identifiers in DRS URIs and not be concerned if the project name or DRS hostname changes in the future, the current resolver can always be found through the identifiers.org/n2t.net registries. Together the identifiers.org/n2t.net systems support the resolver lookup for over 700 compact identifiers formats used in the research community, making it possible for a DRS server to use any of these as DRS IDs (or to register a new compact identifier type and resolver service of their own). - -We use a DRS URI scheme rather than https://en.wikipedia.org/wiki/CURIE[Compact URIs (CURIEs)] directly since we feel that systems consuming DRS objects will be able to better differentiate a DRS URI. CURIEs are widely used in the research community and we feel the fact that they can point to a wide variety of entities (HTML documents, PDFs, identities in data models, etc) makes it more difficult for systems to unambiguously identify entities as DRS objects. - -Still, to make compact identifiers work in DRS URIs we leverage the CURIE format used by identifiers.org/n2t.net. Compact identifiers have the form: - - prefix:accession - -The prefix can be divided into a `provider_code` (optional) and `namespace`. The `accession` here is an Ark, DOI, Data GUID, or another issuers's local ID for the object being pointed to: - - [provider_code/]namespace:accession - -Both the `provider_code` and `namespace` disallow spaces or punctuation, only lowercase alphanumerical characters, underscores and dots are allowed. - -https://n2t.net/e/compact_ids.html[Examples] include (from n2t.net): - - PDB:2gc4 - Taxon:9606 - DOI:10.5281/ZENODO.1289856 - ark:/47881/m6g15z54 - IGSN:SSH000SUA - -TIP: DRS URIs using compact identifiers with resolvers registered in identifiers.org/n2t.net can be distinguished from the hostname-based DRS URIs below based on the required ":" which is not allowed in hostname-based URI. - - -See the documentation on https://n2t.net/e/compact_ids.html[n2t.net] and https://docs.identifiers.org/[identifiers.org] for much more information on the compact identifiers used there and details about the resolution process. - -== Registering a DRS Server on a Meta-Resolver - -See the documentation on the https://n2t.net/e/compact_ids.html[n2t.net] and https://docs.identifiers.org/[identifiers.org] meta-resolvers for adding your own compact identifier type and registering your DRS server as a resolver. You can register new prefixes (or mirrors by adding resource provider codes) for free using a simple online form. - -Keep in mind, while anyone can register prefixes, the identifiers.org/n2t.net sites do basic hand curation to verify new prefix and resource (provider code) requests. See those sites for more details on their security practices. For more information see - -Starting with the prefix for our new compact identifier, let's register the namespace `mydrsprefix` on identifiers.org/n2t.net and use 5-digit numeric IDs as our accessions. We will then link this to the DRS server at `https://mydrs.server.org/ga4gh/drs/v1/` by filling in the provider details. Here's what that the registration for our new namespace looks like on https://registry.identifiers.org/prefixregistrationrequest[identifiers.org]: - -image::prefix_register_1.png[] - -image::prefix_register_2.png[] - -== Example DRS Client Compact Identifier-Based URI Resolution Process - Existing Compact Identifier Provider - -A DRS client identifies the a DRS URI compact identifier components using the first occurance of "/" (optional) and ":" characters. These are not allowed inside the provider_code (optional) or the namespace. The ":" character is not allowed in a Hostname-based DRS URI, providing a convenient mechanism to differentiate them. Once the provider_code (optional) and namespace are extracted from a DRS compact identifier-based URI, a client can use services on identifiers.org to identify available resolvers. - -_Let's look at a specific example DRS compact identifier-based URI that uses DOIs, a popular compact identifier, and walk through the process that a client would use to resolve it. Keep in mind, the resolution process is the same from the client perspective if a given DRS server is using an existing compact identifier type (DOIs, ARKs, Data GUIDs) or creating their own compact identifier type for their DRS server and registering it on identifiers.org/n2t.net._ - -Starting with the DRS URI: - -[source,bash] ----- -drs://doi:10.5072/FK2805660V ----- - -with a namespace of "doi", the following GET request will return information about the namespace: - - GET https://registry.api.identifiers.org/restApi/namespaces/search/findByPrefix?prefix=doi - -This information then points to resolvers for the "doi" namespace. This "doi" namespace was assigned a namespace ID of 75 by identifiers.org. This "id" has nothing to do with compact identifier accessions (which are used in the URL pattern as `{$id}` below) or DRS IDs. This namespace ID (75 below) is purely an identifiers.org internal ID for use with their APIs: - - GET https://registry.api.identifiers.org/restApi/resources/search/findAllByNamespaceId?id=75 - -This returns enough information to, ultimately, identify one or more resolvers and each have a URL pattern that, for DRS-supporting systems, provides a URL template for making a successful DRS GET request. For example, the DOI urlPattern is: - - urlPattern: "https://doi.org/{$id}" - -And the `{$id}` here refers to the accession from the compact identifier (in this example the accession is `10.5072/FK2805660V`). If applicable, a provide code can be supplied in the above requests to specify a particular mirror if there are multiple resolvers for this namespace. In the case of DOIs, you only get a single resolver. - -Given this information you now know you can make a GET on the URL: - - GET https://doi.org/10.5072/FK2805660V - -_The URL above is valid for a DOI object but it is not actually a DRS server! Instead, it redirects to a DRS server through a series of HTTPS redirects. This is likely to be common when working with existing compact identifiers like DOIs or ARKs. Regardless, the redirect should eventually lead to a DRS URL that percent-encodes the accession as a DRS ID in a DRS object API call. For a **hypothetical** example, here's what a redirect to a DRS API URL might ultimately look. A client doesn't have to do anything other than follow the HTTPS redirects. The link between the DOI resolver on doi.org and the DRS server URL below is the result of the DRS server registering their data objects with a DOI issuer._ - - GET https://drs.example.org/ga4gh/drs/v1/objects/10.5072%2FFK2805660V - -IDs in DRS hostname-based URIs/URLs are always percent-encoded to eliminate ambiguity even though the DRS compact identifier-based URIs and the identifier.orgs API do not percent-encode accessions. This was done in order to 1) follow the CURIE conventions of identifiers.org/n2t.net for compact identifier-based DRS URIs and 2) to aid in readability for users who understand they are working with compact identifiers. **The general rule of thumb, when using a compact identifier accession as a DRS ID in a DRS API call, make sure to percent-encode it. An easy way for a DRS client to handle this is to get the initial DRS object JSON response from whatever redirects the compact identifier resolves to, then look for the `self_uri` in the JSON, which will give you the correctly percent-encoded DRS ID for subsequent DRS API calls such as the `access` method.** - - -== Example DRS Client Compact Identifier-Based URI Resolution Process - Registering a new Compact Identifier for Your DRS Server - -See the documentation on https://n2t.net/e/compact_ids.html[n2t.net] and https://docs.identifiers.org/[identifiers.org] for adding your own compact identifier type and registering your DRS server as a resolver. We document this in more detail in the link:index.html#_registering_a_drs_server_on_a_meta_resolver[main specification document]. - -Now the question is how does a client resolve your newly registered compact identifier for your DRS server? _It turns out, whether specific to a DRS implementation or using existing compact identifiers like ARKs or DOIs, the DRS client resolution process for compact identifier-based URIs is exactly the same._ We briefly run through process below for a new compact identifier as an example but, again, a client will not need to do anything different from the resolution process documented in "DRS Client Compact Identifier-Based URI Resolution Process - Existing Compact Identifier Provider". - -Now we can issue DRS URI for our data objects like: - -[source,bash] ----- -drs://mydrsprefix:12345 ----- - -This is a little simpler than working with DOIs or other existing compact identifier issuers out there since we can create our own IDs and not have to allocate them through a third-party service (see "Issuing Existing Compact Identifiers for Use with Your DRS Server" below). - -With a namespace of "mydrsprefix", the following GET request will return information about the namespace: - - GET https://registry.api.identifiers.org/restApi/namespaces/search/findByPrefix?prefix=mydrsprefix - -_Of course, this is a hypothetical example so the actual API call won't work but you can see the GET request is identical to "DRS Client Compact Identifier-Based URI Resolution Process - Existing Compact Identifier Provider"._ - -This information then points to resolvers for the "mydrsprefix" namespace. Hypothetically, this "mydrsprefix" namespace was assigned a namespace ID of 1829 by identifiers.org. This "id" has nothing to do with compact identifier accessions (which are used in the URL pattern as `{$id}` below) or DRS IDs. This namespace ID (1829 below) is purely an identifiers.org internal ID for use with their APIs: - - GET https://registry.api.identifiers.org/restApi/resources/search/findAllByNamespaceId?id=1829 - -_Like the previous GET request this URL won't work but you can see the GET request is identical to "DRS Client Compact Identifier-Based URI Resolution Process - Existing Compact Identifier Provider"._ - -This returns enough information to, ultimately, identify one or more resolvers and each have a URL pattern that, for DRS-supporting systems, provides a URL template for making a successful DRS GET request. For example, the "mydrsprefix" urlPattern is: - - urlPattern: "https://mydrs.server.org/ga4gh/drs/v1/objects/{$id}" - -And the `{$id}` here refers to the accession from the compact identifier (in this example the accession is `12345`). If applicable, a provide code can be supplied in the above requests to specify a particular mirror if there are multiple resolvers for this namespace. - -Given this information you now know you can make a GET on the URL: - - GET https://mydrs.server.org/ga4gh/drs/v1/objects/12345 - -So, compared to using a third party service like DOIs and ARKs, this would be a direct pointer to a DRS server. However, just as with "DRS Client Compact Identifier-Based URI Resolution Process - Existing Compact Identifier Provider", the client should always be prepared to follow HTTPS redirects. - -_To summarize, a client resolving a custom compact identifier registered for a single DRS server is actually the same as resolving using a third-party compact identifier service like ARKs or DOIs with a DRS server, just make sure to follow redirects in all cases._ - -.Note: Issuing Existing Compact Identifiers for Use with Your DRS Server -**** -See the documentation on https://n2t.net/e/compact_ids.html[n2t.net] and https://docs.identifiers.org/[identifiers.org] for information about all the compact identifiers that are supported. You can choose to use an existing compact identifier provider for your DRS server, as we did in the example above using DOIs ("DRS Client Compact Identifier-Based URI Resolution Process - Existing Compact Identifier Provider"). Just keep in mind, each provider will have their own approach for generating compact identifiers and associating them with a DRS data object URL. Some compact identifier providers, like DOIs, provide a method whereby you can register in their network and get your own prefix, allowing you to mint your own accessions. Other services, like the University of California's https://ezid.cdlib.org/[EZID] service, provide accounts and a mechanism to mint accessions centrally for each of your data objects. For experimentation we recommend you take a look at the EZID website that allows you to create DOIs and ARKs and associate them with your data object URLs on your DRS server for testing purposes. -**** diff --git a/docs/figure1.png b/docs/figure1.png deleted file mode 100644 index 0f03c1041..000000000 Binary files a/docs/figure1.png and /dev/null differ diff --git a/docs/figure2.png b/docs/figure2.png deleted file mode 100644 index 1b82112e2..000000000 Binary files a/docs/figure2.png and /dev/null differ diff --git a/docs/figure3.png b/docs/figure3.png deleted file mode 100644 index 062f0b74b..000000000 Binary files a/docs/figure3.png and /dev/null differ diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index 2c96581e7..000000000 --- a/docs/make.bat +++ /dev/null @@ -1,36 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=source -set BUILDDIR=build -set SPHINXPROJ=DataObjectService - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% - -:end -popd diff --git a/docs/prefix_register_1.png b/docs/prefix_register_1.png deleted file mode 100644 index ac4902b9d..000000000 Binary files a/docs/prefix_register_1.png and /dev/null differ diff --git a/docs/prefix_register_2.png b/docs/prefix_register_2.png deleted file mode 100644 index ec6a2e3a6..000000000 Binary files a/docs/prefix_register_2.png and /dev/null differ diff --git a/docs/source/client.rst b/docs/source/client.rst deleted file mode 100644 index 1e3b20692..000000000 --- a/docs/source/client.rst +++ /dev/null @@ -1,5 +0,0 @@ -DOS Python HTTP Client -!!!!!!!!!!!!!!!!!!!!!! - -.. automodule:: ga4gh.dos.client - :members: diff --git a/docs/source/conf.py b/docs/source/conf.py deleted file mode 100644 index 39bb833de..000000000 --- a/docs/source/conf.py +++ /dev/null @@ -1,184 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Data Object Service documentation build configuration file, created by -# sphinx-quickstart on Wed Apr 18 18:56:56 2018. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -import os -import sys -sys.path.insert(0, os.path.abspath('../../python')) -from ga4gh.dos import __version__ - - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.intersphinx', - 'sphinx.ext.coverage', - 'sphinx.ext.viewcode', - 'sphinx.ext.githubpages'] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'Data Object Service' -copyright = u'2018, David Steinberg' -author = u'David Steinberg' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = __version__ -# The full version, including alpha/beta/rc tags. -release = __version__ - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path -exclude_patterns = [] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'alabaster' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - - -# -- Options for HTMLHelp output ------------------------------------------ - -# Output file base name for HTML help builder. -htmlhelp_basename = 'DataObjectServicedoc' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'DataObjectService.tex', u'Data Object Service Documentation', - u'David Steinberg', 'manual'), -] - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'dataobjectservice', u'Data Object Service Documentation', - [author], 1) -] - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'DataObjectService', u'Data Object Service Documentation', - author, 'DataObjectService', 'One line description of project.', - 'Miscellaneous'), -] - - -# -- Options for Epub output ---------------------------------------------- - -# Bibliographic Dublin Core info. -epub_title = project -epub_author = author -epub_publisher = author -epub_copyright = copyright - -# The unique identifier of the text. This can be a ISBN number -# or the project homepage. -# -# epub_identifier = '' - -# A unique identification for the text. -# -# epub_uid = '' - -# A list of files that should not be packed into the epub file. -epub_exclude_files = ['search.html'] - - -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/': None} diff --git a/docs/source/contributors.rst b/docs/source/contributors.rst deleted file mode 100644 index b373633bf..000000000 --- a/docs/source/contributors.rst +++ /dev/null @@ -1,82 +0,0 @@ -.. highlight:: console - -Contributor's Guide -=================== - -Installing ----------- - -To install for development, install from source (and be sure to install the -development requirements as well):: - - $ git clone https://github.com/ga4gh/data-object-service-schemas.git - $ cd data-object-service-schemas - $ python setup.py develop - $ pip install -r requirements.txt - -Documentation -------------- - -We use Sphinx for our documentation. You can generate an HTML build like so:: - - $ cd docs/ - $ make html - -You'll find the built documentation in ``docs/build/``. - -Tests ------ - -To run tests:: - - $ nosetests python/ - -The Travis test suite also tests for PEP8 compliance (checking for all errors -except line length):: - - $ flake8 --select=E121,E123,E126,E226,E24,E704,W503,W504 --ignore=E501 python/ - -Schema architecture -------------------- - -The canonical, authoritative schema is located at ``openapi/data_object_service.swagger.yaml``. All schema changes -must be made to the Swagger schema, and all other specifications (e.g. SmartAPI, OpenAPI 3) are derived from it. - -Building documents -****************** - -To generate the OpenAPI 3 and SmartAPI descriptions, install -`swagger2openapi `_ then run:: - - $ make schemas - - -Releases --------- - -New versions are released when :py:data:`ga4gh.dos.__version__` is incremented, -a commit is tagged (either through a release or manually), and the tagged branch -builds successfully on Travis. When both conditions are met, Travis will -`automatically upload `_ -the distribution to PyPI. - -If :py:data:`ga4gh.dos.__version__` is not incremented in a new release, the -build may appear to complete successfully, but the package will not be uploaded -to PyPI as the distribution will be interpreted as a duplicate release and thus -refused. - -The process above is currently managed by `david4096 `_. -To transfer this responsibility, ownership of the PyPI package must be transferred -to a new account, and their details added to ``.travis.yml`` as described above. - -Note that this repository will not become compliant with Semantic Versioning -until version 1.0 - until then, the API should be considered unstable. - -Documentation is updated independently of this release cycle. - -Code contributions ------------------- - -We welcome code contributions! Feel free to fork the repository and submit a -pull request. Please refer to this `contribution guide `_ -for guidance as to how you should submit changes. diff --git a/docs/source/implementations.rst b/docs/source/implementations.rst deleted file mode 100644 index 293025327..000000000 --- a/docs/source/implementations.rst +++ /dev/null @@ -1,48 +0,0 @@ -.. highlight:: python - -Tools for DOS Implementations -============================= - -The :mod:`ga4gh.dos` package contains some utilities that can help you -develop a compliant DOS resolver. - -Dynamic ``/swagger.json`` with Chalice --------------------------------------- - -If you're using Chalice, you can expose a subset of the Data Object Service -schema using :func:`ga4gh.dos.schema.from_chalice_routes`:: - - from chalice import Chalice - app = Chalice(...) - - @app.route('/swagger.json') - def swagger(): - return ga4gh.dos.schema.from_chalice_routes(app.routes) - -With the above code, a GET request to ``/swagger.json`` will return a schema -in the Swagger / OpenAPI 2 format that correctly lists only the endpoints that -are exposed by your app. - -If you have a different ``basePath``, you can also specify that:: - - @app.route('/swagger.json') - def swagger(): - return ga4gh.dos.schema.from_chalice_routes(app.routes, base_path='/api') - -Compliance testing ------------------- - -This package contains a testing suite -(:class:`~ga4gh.dos.test.integration.AbstractComplianceTest`) -that streamlines testing implementations of the Data Object Service -for compliance with the DOS schema. - -This test suite is meant to supplement, and not replace, an existing -test suite. It does not: - -* test authentication -* test health of the service(s) underpinning an implementation -* test any endpoints not defined in the Data Object Service schema - -.. autoclass:: ga4gh.dos.test.compliance.AbstractComplianceTest - :members: _make_request diff --git a/docs/source/index.rst b/docs/source/index.rst deleted file mode 100644 index eb975fc0d..000000000 --- a/docs/source/index.rst +++ /dev/null @@ -1,36 +0,0 @@ -.. Data Object Service documentation master file, created by - sphinx-quickstart on Wed Apr 18 18:56:56 2018. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Data Object Service Schemas -=========================== - -Welcome to the documentation for the Data Object Service Schemas! These schemas -present an easy-to-implement interface for publishing and accessing data in -heterogeneous storage environments. It also includes a demonstration client and -server to make creating your own DOS implementation easy! - -.. toctree:: - :maxdepth: 2 - - intro - quickstart - server - client - implementations - contributors - - -Data Object Service Schemas is licensed under the Apache 2.0 license. See `LICENSE`_ -for more info. - -.. _LICENSE: https://github.com/ga4gh/data-object-service-schemas/blob/master/LICENSE - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - diff --git a/docs/source/intro.rst b/docs/source/intro.rst deleted file mode 100644 index 2807974cc..000000000 --- a/docs/source/intro.rst +++ /dev/null @@ -1,99 +0,0 @@ -Schemas for the Data Object Service (DOS) API -============================================= - -The `Global Alliance for Genomics and -Health `_ is an international coalition -formed to enable the sharing of genomic and clinical data. This -collaborative consortium takes place primarily via GitHub and public -meetings. - -Cloud Workstream ----------------- - -The `Data Working Group `_ concentrates on data -representation, storage, and analysis, including working with platform -development partners and industry leaders to develop standards that will -facilitate interoperability. The Cloud Workstream is an informal, -multi-vendor working group focused on standards for exchanging -Docker-based tools and CWL/WDL workflows, execution of Docker-based -tools and workflows on clouds, and abstract access to cloud object -stores. - -What is DOS? ------------- - -This proposal for a DOS release is based on the schema work of Brian W. -and others from OHSU along with work by UCSC. It also is informed by -existing object storage systems such as: - -- `GNOS`_ (as used by `PCAWG`_) -- ICGC Storage (`as used to store data on S3`_, see `overture-stack/score`_) -- `Human Cell Atlas Storage`_ (see `HumanCellAtlas/data-store`_) -- `NCI GDC Storage`_ -- `Keep by Curoverse`_ (see `curoverse/arvados`_) - -The goal of DOS is to create a generic API on top of these and other -projects, so workflow systems can access data in the same way regardless -of project. - -.. _GNOS: http://annaisystems.com/ -.. _PCAWG: https://dcc.icgc.org/pcawg -.. _as used to store data on S3: https://dcc.icgc.org/icgc-in-the-cloud/aws -.. _overture-stack/score: https://github.com/overture-stack/score -.. _Human Cell Atlas Storage: https://dss.staging.data.humancellatlas.org/ -.. _HumanCellAtlas/data-store: https://github.com/HumanCellAtlas/data-store -.. _NCI GDC Storage: https://gdc.cancer.gov -.. _Keep by Curoverse: https://arvados.org/ -.. _curoverse/arvados: https://github.com/curoverse/arvados - -Key features ------------- - -Data Object management -^^^^^^^^^^^^^^^^^^^^^^ - -This section of the API focuses on how to read and write Data Objects to -cloud environments and how to join them together as Data Bundles. Data -Bundles are simply a flat collection of one or more files. This section -of the API enables: - -- create/update/delete a file -- create/update/delete a Data Bundle -- register UUIDs with these entities (an optionally track versions of - each) -- generate signed URLs and/or cloud specific object storage paths and - temporary credentials - -Data Object queries -^^^^^^^^^^^^^^^^^^^ - -A key feature of this API beyond creating/modifying/deletion files is -the ability to find Data Objects across cloud environments and -implementations of DOS. This section of the API allows users to query by -Data Bundle or file UUIDs which returns information about where these -Data Objects are available. This response will typically be used to find -the same file or Data Bundle located across multiple cloud environments. - -Implementations ---------------- - -There are currently a few experimental implementations that use some -version of these schemas. - -- `DOS Connect `_ - observes cloud and local storage systems and broadcasts their changes - to a service that presents DOS endpoints. -- `DOS Downloader `_ is a - mechanism for downloading Data Objects from DOS URLs. -- `dos-gdc-lambda `_ - presents data from the GDC public REST API using the Data Object - Service. -- `dos-signpost-lambda `_ - presents data from a signpost instance using the Data Object Service. - -More information ----------------- - -- `Global Alliance for Genomics and - Health `__ -- `GA4GH Cloud Workstream `__ diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst deleted file mode 100644 index b1e2c29de..000000000 --- a/docs/source/quickstart.rst +++ /dev/null @@ -1,44 +0,0 @@ -.. highlight:: console - -Quickstart -========== - -Installing ----------- - -Installing is quick and easy. First, it's always good practice to -work in a virtualenv:: - - $ virtualenv venv - $ source venv/bin/activate - -Then, install from PyPI:: - - $ pip install ga4gh-drs-schemas - -Or, to install from source:: - - $ git clone https://github.com/ga4gh/data-object-service-schemas.git - $ cd data-object-service-schemas - $ python setup.py install - -Running the client and server ------------------------------ - -There's a handy command line hook for the server:: - - $ ga4gh_drs_server - -and for the client:: - - $ ga4gh_drs_demo - -(The client doesn't do anything yet but will soon.) - -Further reading ---------------- - -* `gdc_notebook.ipynb `_ - outlines examples of how to access data with this tool. -* `demo.py `_ - demonstrates basic CRUD functionality implemented by this package. diff --git a/docs/source/server.rst b/docs/source/server.rst deleted file mode 100644 index 859a87e34..000000000 --- a/docs/source/server.rst +++ /dev/null @@ -1,8 +0,0 @@ -Data Object Service Demonstration Server -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -.. automodule:: ga4gh.dos.server - :members: - -.. automodule:: ga4gh.dos.controllers - :members: diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index 1eb19f159..000000000 Binary files a/gradle/wrapper/gradle-wrapper.jar and /dev/null differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties deleted file mode 100644 index fc700a5dc..000000000 --- a/gradle/wrapper/gradle-wrapper.properties +++ /dev/null @@ -1,6 +0,0 @@ -#Thu Jul 20 10:52:24 CEST 2017 -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-3.5-bin.zip diff --git a/gradlew b/gradlew deleted file mode 100755 index 4453ccea3..000000000 --- a/gradlew +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env sh - -############################################################################## -## -## Gradle start up script for UN*X -## -############################################################################## - -# Attempt to set APP_HOME -# Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi -done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null - -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" - -warn ( ) { - echo "$*" -} - -die ( ) { - echo - echo "$*" - echo - exit 1 -} - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi -fi - -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi - -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi - # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" - fi - i=$((i+1)) - done - case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac -fi - -# Escape application args -save ( ) { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=$(save "$@") - -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" - -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" -fi - -exec "$JAVACMD" "$@" diff --git a/merge_yaml.py b/merge_yaml.py deleted file mode 100644 index e59992d9c..000000000 --- a/merge_yaml.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- -"""$ python merge_yaml.py openapi.yaml part.yaml > out.yaml""" -import sys - -import yaml - - -def merge(parent, child): - """Update the `child` dictionary such that it inherits keys - from the `parent` dict.""" - for k in parent: - if k in child and isinstance(child[k], dict) and isinstance(parent[k], dict): - merge(parent[k], child[k]) - else: - child[k] = parent[k] - - -if __name__ == '__main__': - _, parent, child = sys.argv - - with open(parent, 'r') as f: - parent = yaml.load(f) - - with open(child, 'r') as f: - child = yaml.load(f) - - merge(parent, child) - sys.stdout.write(yaml.dump(child, default_flow_style=False)) - diff --git a/misc/.gitignore b/misc/.gitignore deleted file mode 100644 index be92bf24d..000000000 --- a/misc/.gitignore +++ /dev/null @@ -1,119 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -# lib/ -lib64/ -parts/ -sdist/ -var/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*,cover -.hypothesis/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/build/ - -# PyBuilder -target/ - -# IPython Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# dotenv -.env - -# virtualenv -venv/ -ENV/ - -# Spyder project settings -.spyderproject - -# Rope project settings -.ropeproject - -# swagger in the python directory -python/ga4gh/dos/data_objects_service.swagger.json - -# Documentation build system artifacts -.gradle/ -docs/html5/ -docs/pdf/ -docs/asciidoc/swagger2markup/ -spec/ -web_deploy/ -node_modules/ - -# Non-pages content -docs/asciidoc/ -python -setup.py -requirements.txt -gradle -build.gradle -settings.gradle -gradlew -scripts -misc -openapi -package.json -package-lock.json -Makefile -merge_yaml.py -.travis.yml \ No newline at end of file diff --git a/openapi/components/parameters/AccessId.yaml b/openapi/components/parameters/AccessId.yaml new file mode 100644 index 000000000..ae773e24d --- /dev/null +++ b/openapi/components/parameters/AccessId.yaml @@ -0,0 +1,6 @@ +in: path +name: access_id +required: true +description: An `access_id` from the `access_methods` list of a `DrsObject` +schema: + type: string diff --git a/openapi/components/parameters/Expand.yaml b/openapi/components/parameters/Expand.yaml new file mode 100644 index 000000000..e6cfd1f33 --- /dev/null +++ b/openapi/components/parameters/Expand.yaml @@ -0,0 +1,17 @@ +in: query +name: expand +schema: + type: boolean +example: false +description: >- + If false and the object_id refers to a bundle, then the ContentsObject array + contains only those objects directly contained in the bundle. That is, if the + bundle contains other bundles, those other bundles are not recursively + included in the result. + + If true and the object_id refers to a bundle, then the entire set of objects + in the bundle is expanded. That is, if the bundle contains aother bundles, + then those other bundles are recursively expanded and included in the result. + Recursion continues through the entire sub-tree of the bundle. + + If the object_id refers to a blob, then the query parameter is ignored. diff --git a/openapi/components/parameters/ObjectId.yaml b/openapi/components/parameters/ObjectId.yaml new file mode 100644 index 000000000..263ad9a2b --- /dev/null +++ b/openapi/components/parameters/ObjectId.yaml @@ -0,0 +1,6 @@ +in: path +name: object_id +required: true +description: '`DrsObject` identifier' +schema: + type: string diff --git a/openapi/components/requestBodies/Passports.yaml b/openapi/components/requestBodies/Passports.yaml new file mode 100644 index 000000000..507789511 --- /dev/null +++ b/openapi/components/requestBodies/Passports.yaml @@ -0,0 +1,12 @@ +required: true +content: + application/json: + schema: + type: object + properties: + passports: + type: array + items: + type: string + example: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJnYTRnaF9wYXNzcG9ydF92MSI6W119.JJ5rN0ktP0qwyZmIPpxmF_p7JsxAZH6L6brUxtad3CM + description: the encoded JWT GA4GH Passport that contains embedded Visas. The overall JWT is signed as are the individual Passport Visas. \ No newline at end of file diff --git a/openapi/components/requestBodies/PostObjectBody.yaml b/openapi/components/requestBodies/PostObjectBody.yaml new file mode 100644 index 000000000..0a15e8bd8 --- /dev/null +++ b/openapi/components/requestBodies/PostObjectBody.yaml @@ -0,0 +1,19 @@ +required: true +content: + application/json: + schema: + type: object + properties: + expand: + type: + $ref: '#/components/parameters/Expand/schema/type' + example: + $ref: '#/components/parameters/Expand/example' + description: + $ref: '#/components/parameters/Expand/description' + passports: + type: array + items: + type: string + example: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJnYTRnaF9wYXNzcG9ydF92MSI6W119.JJ5rN0ktP0qwyZmIPpxmF_p7JsxAZH6L6brUxtad3CM + description: the encoded JWT GA4GH Passport that contains embedded Visas. The overall JWT is signed as are the individual Passport Visas. diff --git a/openapi/components/responses/200OkAccess.yaml b/openapi/components/responses/200OkAccess.yaml new file mode 100644 index 000000000..264bc58c7 --- /dev/null +++ b/openapi/components/responses/200OkAccess.yaml @@ -0,0 +1,5 @@ +description: The `AccessURL` was found successfully +content: + application/json: + schema: + $ref: '../schemas/AccessURL.yaml' \ No newline at end of file diff --git a/openapi/components/responses/200OkDrsObject.yaml b/openapi/components/responses/200OkDrsObject.yaml new file mode 100644 index 000000000..e8f990ed8 --- /dev/null +++ b/openapi/components/responses/200OkDrsObject.yaml @@ -0,0 +1,5 @@ +description: The `DrsObject` was found successfully +content: + application/json: + schema: + $ref: '../schemas/DrsObject.yaml' \ No newline at end of file diff --git a/openapi/components/responses/200ServiceInfo.yaml b/openapi/components/responses/200ServiceInfo.yaml new file mode 100644 index 000000000..ecb826665 --- /dev/null +++ b/openapi/components/responses/200ServiceInfo.yaml @@ -0,0 +1,7 @@ +description: Retrieve info about the DRS service +content: + application/json: + schema: + allOf: + - '$ref': https://raw.githubusercontent.com/ga4gh-discovery/ga4gh-service-info/v1.0.0/service-info.yaml#/components/schemas/Service + - '$ref': '../schemas/DrsService.yaml' diff --git a/openapi/components/responses/202Accepted.yaml b/openapi/components/responses/202Accepted.yaml new file mode 100644 index 000000000..9f73af951 --- /dev/null +++ b/openapi/components/responses/202Accepted.yaml @@ -0,0 +1,15 @@ +description: > + The operation is delayed and will continue asynchronously. + The client should retry this same request after the delay specified by Retry-After header. +headers: + Retry-After: + description: > + Delay in seconds. The client should retry this same request after waiting for this duration. + To simplify client response processing, this must be an integral relative time in seconds. + This value SHOULD represent the minimum duration the client should wait before attempting + the operation again with a reasonable expectation of success. When it is not feasible + for the server to determine the actual expected delay, the server may return a + brief, fixed value instead. + schema: + type: integer + format: int64 \ No newline at end of file diff --git a/openapi/components/responses/400BadRequest.yaml b/openapi/components/responses/400BadRequest.yaml new file mode 100644 index 000000000..79dbd0db9 --- /dev/null +++ b/openapi/components/responses/400BadRequest.yaml @@ -0,0 +1,5 @@ +description: The request is malformed. +content: + application/json: + schema: + $ref: '../schemas/Error.yaml' \ No newline at end of file diff --git a/openapi/components/responses/401Unauthorized.yaml b/openapi/components/responses/401Unauthorized.yaml new file mode 100644 index 000000000..c3f4f5d30 --- /dev/null +++ b/openapi/components/responses/401Unauthorized.yaml @@ -0,0 +1,5 @@ +description: The request is unauthorized. +content: + application/json: + schema: + $ref: '../schemas/Error.yaml' \ No newline at end of file diff --git a/openapi/components/responses/403Forbidden.yaml b/openapi/components/responses/403Forbidden.yaml new file mode 100644 index 000000000..9392a3cfa --- /dev/null +++ b/openapi/components/responses/403Forbidden.yaml @@ -0,0 +1,5 @@ +description: The requester is not authorized to perform this action. +content: + application/json: + schema: + $ref: '../schemas/Error.yaml' \ No newline at end of file diff --git a/openapi/components/responses/404NotFoundAccess.yaml b/openapi/components/responses/404NotFoundAccess.yaml new file mode 100644 index 000000000..e1c68e2f6 --- /dev/null +++ b/openapi/components/responses/404NotFoundAccess.yaml @@ -0,0 +1,5 @@ +description: The requested `AccessURL` wasn't found. +content: + application/json: + schema: + $ref: '../schemas/Error.yaml' \ No newline at end of file diff --git a/openapi/components/responses/404NotFoundDrsObject.yaml b/openapi/components/responses/404NotFoundDrsObject.yaml new file mode 100644 index 000000000..4160dd769 --- /dev/null +++ b/openapi/components/responses/404NotFoundDrsObject.yaml @@ -0,0 +1,5 @@ +description: The requested `DrsObject` wasn't found. +content: + application/json: + schema: + $ref: '../schemas/Error.yaml' \ No newline at end of file diff --git a/openapi/components/responses/500InternalServerError.yaml b/openapi/components/responses/500InternalServerError.yaml new file mode 100644 index 000000000..16e9fc5d2 --- /dev/null +++ b/openapi/components/responses/500InternalServerError.yaml @@ -0,0 +1,5 @@ +description: An unexpected error occurred. +content: + application/json: + schema: + $ref: '../schemas/Error.yaml' \ No newline at end of file diff --git a/openapi/components/schemas/AccessMethod.yaml b/openapi/components/schemas/AccessMethod.yaml new file mode 100644 index 000000000..caadff539 --- /dev/null +++ b/openapi/components/schemas/AccessMethod.yaml @@ -0,0 +1,32 @@ +type: object +required: + - type +properties: + type: + type: string + enum: + - s3 + - gs + - ftp + - gsiftp + - globus + - htsget + - https + - file + description: Type of the access method. + access_url: + $ref: './AccessURL.yaml' + description: >- + An `AccessURL` that can be used to fetch the actual object bytes. + Note that at least one of `access_url` and `access_id` must be provided. + access_id: + type: string + description: >- + An arbitrary string to be passed to the `/access` method to get an `AccessURL`. + This string must be unique within the scope of a single object. + Note that at least one of `access_url` and `access_id` must be provided. + region: + type: string + description: >- + Name of the region in the cloud service provider that the object belongs to. + example: us-east-1 \ No newline at end of file diff --git a/openapi/components/schemas/AccessURL.yaml b/openapi/components/schemas/AccessURL.yaml new file mode 100644 index 000000000..b60b9616c --- /dev/null +++ b/openapi/components/schemas/AccessURL.yaml @@ -0,0 +1,15 @@ +type: object +required: + - url +properties: + url: + type: string + description: A fully resolvable URL that can be used to fetch the actual object bytes. + headers: + type: array + items: + type: string + description: >- + An optional list of headers to include in the HTTP request to `url`. + These headers can be used to provide auth tokens required to fetch the object bytes. + example: 'Authorization: Basic Z2E0Z2g6ZHJz' \ No newline at end of file diff --git a/openapi/components/schemas/Checksum.yaml b/openapi/components/schemas/Checksum.yaml new file mode 100644 index 000000000..8b025e5d4 --- /dev/null +++ b/openapi/components/schemas/Checksum.yaml @@ -0,0 +1,20 @@ +type: object +required: + - checksum + - type +properties: + checksum: + type: string + description: The hex-string encoded checksum for the data + type: + type: string + description: >- + The digest method used to create the checksum. + + The value (e.g. `sha-256`) SHOULD be listed as `Hash Name String` in the https://www.iana.org/assignments/named-information/named-information.xhtml#hash-alg[IANA Named Information Hash Algorithm Registry]. + Other values MAY be used, as long as implementors are aware of the issues discussed in https://tools.ietf.org/html/rfc6920#section-9.4[RFC6920]. + + GA4GH may provide more explicit guidance for use of non-IANA-registered algorithms in the future. + Until then, if implementors do choose such an algorithm (e.g. because it's implemented by their storage provider), they SHOULD use an existing + standard `type` value such as `md5`, `etag`, `crc32c`, `trunc512`, or `sha1`. + example: sha-256 \ No newline at end of file diff --git a/openapi/components/schemas/ContentsObject.yaml b/openapi/components/schemas/ContentsObject.yaml new file mode 100644 index 000000000..4d91faca2 --- /dev/null +++ b/openapi/components/schemas/ContentsObject.yaml @@ -0,0 +1,35 @@ +type: object +required: + - name +properties: + name: + type: string + description: >- + A name declared by the bundle author that must be + used when materialising this object, + overriding any name directly associated with the object itself. + The name must be unique with the containing bundle. + This string is made up of uppercase and lowercase letters, decimal digits, hypen, period, and underscore [A-Za-z0-9.-_]. See http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap03.html#tag_03_282[portable filenames]. + id: + type: string + description: >- + A DRS identifier of a `DrsObject` (either a single blob or a nested bundle). + If this ContentsObject is an object within a nested bundle, then the id is + optional. Otherwise, the id is required. + drs_uri: + type: array + description: >- + A list of full DRS identifier URI paths + that may be used to obtain the object. + These URIs may be external to this DRS instance. + example: drs://drs.example.org/314159 + items: + type: string + contents: + type: array + description: >- + If this ContentsObject describes a nested bundle and the caller specified + "?expand=true" on the request, then this contents array must be present and + describe the objects within the nested bundle. + items: + $ref: './ContentsObject.yaml' \ No newline at end of file diff --git a/openapi/components/schemas/DrsObject.yaml b/openapi/components/schemas/DrsObject.yaml new file mode 100644 index 000000000..291b97155 --- /dev/null +++ b/openapi/components/schemas/DrsObject.yaml @@ -0,0 +1,113 @@ +type: object +required: + - id + - self_uri + - size + - created_time + - checksums +properties: + id: + type: string + description: An identifier unique to this `DrsObject` + name: + type: string + description: |- + A string that can be used to name a `DrsObject`. + This string is made up of uppercase and lowercase letters, decimal digits, hypen, period, and underscore [A-Za-z0-9.-_]. See http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap03.html#tag_03_282[portable filenames]. + self_uri: + type: string + description: |- + A drs:// hostname-based URI, as defined in the DRS documentation, that tells clients how to access this object. + The intent of this field is to make DRS objects self-contained, and therefore easier for clients to store and pass around. For example, if you arrive at this DRS JSON by resolving a compact identifier-based DRS URI, the `self_uri` presents you with a hostname and properly encoded DRS ID for use in subsequent `access` endpoint calls. + example: + drs://drs.example.org/314159 + size: + type: integer + format: int64 + description: |- + For blobs, the blob size in bytes. + For bundles, the cumulative size, in bytes, of items in the `contents` field. + created_time: + type: string + format: date-time + description: |- + Timestamp of content creation in RFC3339. + (This is the creation time of the underlying content, not of the JSON object.) + updated_time: + type: string + format: date-time + description: >- + Timestamp of content update in RFC3339, identical to `created_time` in systems + that do not support updates. + (This is the update time of the underlying content, not of the JSON object.) + version: + type: string + description: >- + A string representing a version. + + (Some systems may use checksum, a RFC3339 timestamp, or an incrementing version number.) + mime_type: + type: string + description: A string providing the mime-type of the `DrsObject`. + example: + application/json + checksums: + type: array + minItems: 1 + items: + $ref: './Checksum.yaml' + description: >- + The checksum of the `DrsObject`. At least one checksum must be provided. + + For blobs, the checksum is computed over the bytes in the blob. + + For bundles, the checksum is computed over a sorted concatenation of the + checksums of its top-level contained objects (not recursive, names not included). + The list of checksums is sorted alphabetically (hex-code) before concatenation + and a further checksum is performed on the concatenated checksum value. + + For example, if a bundle contains blobs with the following checksums: + + md5(blob1) = 72794b6d + + md5(blob2) = 5e089d29 + + Then the checksum of the bundle is: + + md5( concat( sort( md5(blob1), md5(blob2) ) ) ) + + = md5( concat( sort( 72794b6d, 5e089d29 ) ) ) + + = md5( concat( 5e089d29, 72794b6d ) ) + + = md5( 5e089d2972794b6d ) + + = f7a29a04 + access_methods: + type: array + minItems: 1 + items: + $ref: './AccessMethod.yaml' + description: |- + The list of access methods that can be used to fetch the `DrsObject`. + Required for single blobs; optional for bundles. + contents: + type: array + description: >- + If not set, this `DrsObject` is a single blob. + + If set, this `DrsObject` is a bundle containing the listed `ContentsObject` s (some of which may be further nested). + items: + $ref: './ContentsObject.yaml' + description: + type: string + description: A human readable description of the `DrsObject`. + aliases: + type: array + items: + type: string + description: >- + A list of strings that can be used to find other metadata + about this `DrsObject` from external metadata sources. These + aliases can be used to represent secondary + accession numbers or external GUIDs. diff --git a/openapi/components/schemas/DrsService.yaml b/openapi/components/schemas/DrsService.yaml new file mode 100644 index 000000000..5424d8849 --- /dev/null +++ b/openapi/components/schemas/DrsService.yaml @@ -0,0 +1,13 @@ +type: object +required: + - type +properties: + type: + type: object + required: + - artifact + properties: + artifact: + type: string + enum: [drs] + example: drs diff --git a/openapi/components/schemas/Error.yaml b/openapi/components/schemas/Error.yaml new file mode 100644 index 000000000..59a1e0099 --- /dev/null +++ b/openapi/components/schemas/Error.yaml @@ -0,0 +1,9 @@ +type: object +description: An object that can optionally include information about the error. +properties: + msg: + type: string + description: A detailed error message. + status_code: + type: integer + description: The integer representing the HTTP status code (e.g. 200, 404). diff --git a/openapi/data_repository_service.openapi.yaml b/openapi/data_repository_service.openapi.yaml new file mode 100644 index 000000000..778f82bb3 --- /dev/null +++ b/openapi/data_repository_service.openapi.yaml @@ -0,0 +1,136 @@ +openapi: 3.0.3 +info: + title: Data Repository Service + version: 1.2.0 + x-logo: + url: 'https://www.ga4gh.org/wp-content/themes/ga4gh-theme/gfx/GA-logo-horizontal-tag-RGB.svg' + termsOfService: 'https://www.ga4gh.org/terms-and-conditions/' + contact: + name: GA4GH Cloud Work Stream + email: ga4gh-cloud@ga4gh.org + license: + name: Apache 2.0 + url: 'https://raw.githubusercontent.com/ga4gh/data-repository-service-schemas/master/LICENSE' +servers: + - url: https://{serverURL}/ga4gh/drs/v1 + variables: + serverURL: + default: drs.example.org + description: > + DRS server endpoints MUST be prefixed by the '/ga4gh/drs/v1' endpoint + path +security: + - {} + - BasicAuth: [] + - BearerAuth: [] +tags: + # Overview + - name: Introduction + description: + $ref: ./tags/Introduction.md + - name: DRS API Principles + description: + $ref: ./tags/DrsApiPrinciples.md + - name: Authorization & Authentication + description: + $ref: ./tags/Auth.md + + # Operations + - name: Objects + - name: Service Info + + # Models + - name: AccessMethodModel + x-displayName: AccessMethod + description: | + + - name: AccessURLModel + x-displayName: AccessURL + description: | + + - name: ChecksumModel + x-displayName: Checksum + description: | + + - name: ContentsObjectModel + x-displayName: ContentsObject + description: | + + - name: DrsObjectModel + x-displayName: DrsObject + description: | + + - name: ErrorModel + x-displayName: Error + description: | + + + # Appendices + - name: Motivation + description: + $ref: './tags/Motivation.md' + - name: Background Notes on DRS URIs + description: + $ref: './tags/BackgroundNotesOnDRSURIs.md' + - name: Compact Identifier-Based URIs + description: + $ref: './tags/CompactIdentifierBasedURIs.md' + - name: Hostname-Based URIs + description: + $ref: './tags/HostnameBasedURIs.md' + - name: GA4GH Service Registry + description: + $ref: './tags/ServiceRegistry.md' +x-tagGroups: + - name: Overview + tags: + - Introduction + - DRS API Principles + - Authorization & Authentication + - name: Operations + tags: + - Objects + - Service Info + - name: Models + tags: + - AccessMethodModel + - AccessURLModel + - ChecksumModel + - ContentsObjectModel + - DrsObjectModel + - ErrorModel + - name: Appendices + tags: + - Motivation + - Background Notes on DRS URIs + - Compact Identifier-Based URIs + - Hostname-Based URIs + - GA4GH Service Registry +paths: + /service-info: + $ref: ./paths/service-info.yaml + /objects/{object_id}: + $ref: ./paths/objects@{object_id}.yaml + /objects/{object_id}/access/{access_id}: + $ref: ./paths/objects@{object_id}@access@{access_id}.yaml +components: + securitySchemes: + BasicAuth: + type: http + scheme: basic + description: | + A valid authorization token must be passed in the 'Authorization' header, + e.g. "Basic ${token_string}" + BearerAuth: + type: http + scheme: bearer + description: + A valid authorization token must be passed in the 'Authorization' header, + e.g. "Bearer ${token_string}" + PassportAuth: + type: http + scheme: bearer + x-in: body + bearerFormat: JWT + description: + A valid GA4GH Passport must be passed in the body of an HTTP POST request as a tokens[] array. diff --git a/openapi/data_repository_service.smartapi.yaml.part b/openapi/data_repository_service.smartapi.yaml.part deleted file mode 100644 index 04d8306a1..000000000 --- a/openapi/data_repository_service.smartapi.yaml.part +++ /dev/null @@ -1,23 +0,0 @@ -# To generate the SmartAPI file, we use swagger2openapi to generate -# data_repository_service.openapi.yaml, then merge that file with this file -# to create data_repository_service.smartapi.yaml. -# (data_repository_service.smartapi.yaml should not be edited directly.) -servers: - - url: https://{host}:{port}/{basePath} - description: The production API server - variables: - host: - default: drs.example.org - description: The host the service is serving from. - port: - enum: - - '443' - default: '443' - basePath: - # The default base includes a version in the path. - default: ga4gh/drs/v1 -info: - x-implementationLanguage: en - termsOfService: https://www.ga4gh.org/policies/termsandconditions.html -tags: - - name: NIHdatacommons diff --git a/openapi/data_repository_service.swagger.yaml b/openapi/data_repository_service.swagger.yaml deleted file mode 100644 index 2e023d37b..000000000 --- a/openapi/data_repository_service.swagger.yaml +++ /dev/null @@ -1,406 +0,0 @@ -swagger: '2.0' -basePath: '/ga4gh/drs/v1' -info: - title: Data Repository Service - version: 1.1.0 - description: 'https://github.com/ga4gh/data-repository-service-schemas' - termsOfService: 'https://www.ga4gh.org/terms-and-conditions/' - contact: - name: GA4GH Cloud Work Stream - email: ga4gh-cloud@ga4gh.org - license: - name: Apache 2.0 - url: 'https://raw.githubusercontent.com/ga4gh/data-repository-service-schemas/master/LICENSE' -schemes: - - https -consumes: - - application/json -produces: - - application/json -security: - - {} - - authToken: [] -paths: - '/objects/{object_id}': - get: - summary: Get info about a `DrsObject`. - description: >- - Returns object metadata, and a list of access methods that can be used to fetch object bytes. - operationId: GetObject - responses: - '200': - description: The `DrsObject` was found successfully. - schema: - $ref: '#/definitions/DrsObject' - '202': - description: > - The operation is delayed and will continue asynchronously. - The client should retry this same request after the delay specified by Retry-After header. - headers: - Retry-After: - description: > - Delay in seconds. The client should retry this same request after waiting for this duration. - To simplify client response processing, this must be an integral relative time in seconds. - This value SHOULD represent the minimum duration the client should wait before attempting - the operation again with a reasonable expectation of success. When it is not feasible - for the server to determine the actual expected delay, the server may return a - brief, fixed value instead. - type: integer - format: int64 - '400': - description: The request is malformed. - schema: - $ref: '#/definitions/Error' - '401': - description: The request is unauthorized. - schema: - $ref: '#/definitions/Error' - '403': - description: The requester is not authorized to perform this action. - schema: - $ref: '#/definitions/Error' - '404': - description: The requested `DrsObject` wasn't found - schema: - $ref: '#/definitions/Error' - '500': - description: An unexpected error occurred. - schema: - $ref: '#/definitions/Error' - parameters: - - name: object_id - in: path - required: true - type: string - - in: query - name: expand - type: boolean - default: false - description: >- - If false and the object_id refers to a bundle, then the ContentsObject array - contains only those objects directly contained in the bundle. That is, if the - bundle contains other bundles, those other bundles are not recursively - included in the result. - - If true and the object_id refers to a bundle, then the entire set of objects - in the bundle is expanded. That is, if the bundle contains aother bundles, - then those other bundles are recursively expanded and included in the result. - Recursion continues through the entire sub-tree of the bundle. - - If the object_id refers to a blob, then the query parameter is ignored. - tags: - - DataRepositoryService - x-swagger-router-controller: ga4gh.drs.server - '/objects/{object_id}/access/{access_id}': - get: - summary: Get a URL for fetching bytes. - description: >- - Returns a URL that can be used to fetch the bytes of a `DrsObject`. - - - This method only needs to be called when using an `AccessMethod` that contains an `access_id` - (e.g., for servers that use signed URLs for fetching object bytes). - operationId: GetAccessURL - responses: - '200': - description: The access URL was found successfully. - schema: - $ref: '#/definitions/AccessURL' - '202': - description: > - The operation is delayed and will continue asynchronously. - The client should retry this same request after the delay specified by Retry-After header. - headers: - Retry-After: - description: > - Delay in seconds. The client should retry this same request after waiting for this duration. - To simplify client response processing, this must be an integral relative time in seconds. - This value SHOULD represent the minimum duration the client should wait before attempting - the operation again with a reasonable expectation of success. When it is not feasible - for the server to determine the actual expected delay, the server may return a - brief, fixed value instead. - type: integer - format: int64 - '400': - description: The request is malformed. - schema: - $ref: '#/definitions/Error' - '401': - description: The request is unauthorized. - schema: - $ref: '#/definitions/Error' - '404': - description: The requested access URL wasn't found - schema: - $ref: '#/definitions/Error' - '403': - description: The requester is not authorized to perform this action. - schema: - $ref: '#/definitions/Error' - '500': - description: An unexpected error occurred. - schema: - $ref: '#/definitions/Error' - parameters: - - name: object_id - in: path - required: true - type: string - description: An `id` of a `DrsObject` - - name: access_id - in: path - required: true - type: string - description: An `access_id` from the `access_methods` list of a `DrsObject` - tags: - - DataRepositoryService - x-swagger-router-controller: ga4gh.drs.server -securityDefinitions: - authToken: - description: | - A valid authorization token must be passed in the 'Authorization' header. - Example syntax for using 'Authorization' header : - Bearer: token_string - type: apiKey - name: Authorization - in: header -definitions: - Checksum: - type: object - required: ['checksum', 'type'] - properties: - checksum: - type: string - description: 'The hex-string encoded checksum for the data' - type: - type: string - description: >- - The digest method used to create the checksum. - - - The value (e.g. `sha-256`) SHOULD be listed as `Hash Name String` in the https://www.iana.org/assignments/named-information/named-information.xhtml#hash-alg[IANA Named Information Hash Algorithm Registry]. - Other values MAY be used, as long as implementors are aware of the issues discussed in https://tools.ietf.org/html/rfc6920#section-9.4[RFC6920]. - - - GA4GH may provide more explicit guidance for use of non-IANA-registered algorithms in the future. - Until then, if implementors do choose such an algorithm (e.g. because it's implemented by their storage provider), they SHOULD use an existing - standard `type` value such as `md5`, `etag`, `crc32c`, `trunc512`, or `sha1`. - example: - sha-256 - DrsObject: - type: object - required: ['id', 'self_uri', 'size', 'created_time', 'checksums'] - properties: - id: - type: string - description: |- - An identifier unique to this `DrsObject`. - name: - type: string - description: |- - A string that can be used to name a `DrsObject`. - This string is made up of uppercase and lowercase letters, decimal digits, hypen, period, and underscore [A-Za-z0-9.-_]. See http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap03.html#tag_03_282[portable filenames]. - self_uri: - type: string - description: |- - A drs:// hostname-based URI, as defined in the DRS documentation, that tells clients how to access this object. - The intent of this field is to make DRS objects self-contained, and therefore easier for clients to store and pass around. For example, if you arrive at this DRS JSON by resolving a compact identifier-based DRS URI, the `self_uri` presents you with a hostname and properly encoded DRS ID for use in subsequent `access` endpoint calls. - example: - drs://drs.example.org/314159 - size: - type: integer - format: int64 - description: |- - For blobs, the blob size in bytes. - For bundles, the cumulative size, in bytes, of items in the `contents` field. - created_time: - type: string - format: date-time - description: |- - Timestamp of content creation in RFC3339. - (This is the creation time of the underlying content, not of the JSON object.) - updated_time: - type: string - format: date-time - description: >- - Timestamp of content update in RFC3339, identical to `created_time` in systems - that do not support updates. - (This is the update time of the underlying content, not of the JSON object.) - version: - type: string - description: >- - A string representing a version. - - (Some systems may use checksum, a RFC3339 timestamp, or an incrementing version number.) - mime_type: - type: string - description: |- - A string providing the mime-type of the `DrsObject`. - example: - application/json - checksums: - type: array - minItems: 1 - items: - $ref: '#/definitions/Checksum' - description: >- - The checksum of the `DrsObject`. At least one checksum must be provided. - - For blobs, the checksum is computed over the bytes in the blob. - - - For bundles, the checksum is computed over a sorted concatenation of the - checksums of its top-level contained objects (not recursive, names not included). - The list of checksums is sorted alphabetically (hex-code) before concatenation - and a further checksum is performed on the concatenated checksum value. - - - For example, if a bundle contains blobs with the following checksums: - - md5(blob1) = 72794b6d - - md5(blob2) = 5e089d29 - - - Then the checksum of the bundle is: - - md5( concat( sort( md5(blob1), md5(blob2) ) ) ) - - = md5( concat( sort( 72794b6d, 5e089d29 ) ) ) - - = md5( concat( 5e089d29, 72794b6d ) ) - - = md5( 5e089d2972794b6d ) - - = f7a29a04 - access_methods: - type: array - minItems: 1 - items: - $ref: '#/definitions/AccessMethod' - description: |- - The list of access methods that can be used to fetch the `DrsObject`. - Required for single blobs; optional for bundles. - contents: - type: array - description: >- - If not set, this `DrsObject` is a single blob. - - If set, this `DrsObject` is a bundle containing the listed `ContentsObject` s (some of which may be further nested). - items: - $ref: '#/definitions/ContentsObject' - description: - type: string - description: |- - A human readable description of the `DrsObject`. - aliases: - type: array - items: - type: string - description: >- - A list of strings that can be used to find other metadata - about this `DrsObject` from external metadata sources. These - aliases can be used to represent secondary - accession numbers or external GUIDs. - AccessURL: - type: object - required: ['url'] - properties: - url: - type: string - description: A fully resolvable URL that can be used to fetch the actual object bytes. - headers: - type: array - items: - type: string - description: >- - An optional list of headers to include in the HTTP request to `url`. - These headers can be used to provide auth tokens required to fetch the object bytes. - example: - Authorization: Basic Z2E0Z2g6ZHJz - AccessMethod: - type: object - required: - - type - properties: - type: - type: string - enum: - - s3 - - gs - - ftp - - gsiftp - - globus - - htsget - - https - - file - description: >- - Type of the access method. - access_url: - $ref: '#/definitions/AccessURL' - description: >- - An `AccessURL` that can be used to fetch the actual object bytes. - Note that at least one of `access_url` and `access_id` must be provided. - access_id: - type: string - description: >- - An arbitrary string to be passed to the `/access` method to get an `AccessURL`. - This string must be unique within the scope of a single object. - Note that at least one of `access_url` and `access_id` must be provided. - region: - type: string - description: >- - Name of the region in the cloud service provider that the object belongs to. - example: - us-east-1 - Error: - description: - An object that can optionally include information about the error. - type: object - properties: - msg: - type: string - description: A detailed error message. - status_code: - type: integer - description: The integer representing the HTTP status code (e.g. 200, 404). - ContentsObject: - type: object - properties: - name: - type: string - description: >- - A name declared by the bundle author that must be - used when materialising this object, - overriding any name directly associated with the object itself. - The name must be unique with the containing bundle. - This string is made up of uppercase and lowercase letters, decimal digits, hypen, period, and underscore [A-Za-z0-9.-_]. See http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap03.html#tag_03_282[portable filenames]. - id: - type: string - description: >- - A DRS identifier of a `DrsObject` (either a single blob or a nested bundle). - If this ContentsObject is an object within a nested bundle, then the id is - optional. Otherwise, the id is required. - drs_uri: - type: array - description: >- - A list of full DRS identifier URI paths - that may be used to obtain the object. - These URIs may be external to this DRS instance. - example: - drs://drs.example.org/314159 - items: - type: string - contents: - type: array - description: >- - If this ContentsObject describes a nested bundle and the caller specified - "?expand=true" on the request, then this contents array must be present and - describe the objects within the nested bundle. - items: - $ref: '#/definitions/ContentsObject' - - required: - - name -tags: - - name: DataRepositoryService diff --git a/openapi/paths/objects@{object_id}.yaml b/openapi/paths/objects@{object_id}.yaml new file mode 100644 index 000000000..d4aea6400 --- /dev/null +++ b/openapi/paths/objects@{object_id}.yaml @@ -0,0 +1,58 @@ +get: + summary: Get info about a DrsObject. + description: >- + Returns object metadata, and a list of access methods that can be used to fetch object bytes. + operationId: GetObject + parameters: + - $ref: '../components/parameters/ObjectId.yaml' + - $ref: '../components/parameters/Expand.yaml' + responses: + 200: + $ref: '../components/responses/200OkDrsObject.yaml' + 202: + $ref: '../components/responses/202Accepted.yaml' + 400: + $ref: '../components/responses/400BadRequest.yaml' + 401: + $ref: '../components/responses/401Unauthorized.yaml' + 403: + $ref: '../components/responses/403Forbidden.yaml' + 404: + $ref: '../components/responses/404NotFoundDrsObject.yaml' + 500: + $ref: '../components/responses/500InternalServerError.yaml' + tags: + - Objects + x-swagger-router-controller: ga4gh.drs.server + +post: + summary: Get info about a DrsObject through POST'ing a Passport. + description: >- + Returns object metadata, and a list of access methods that can be used to fetch object bytes. + + Method is a POST to accomodate a JWT GA4GH Passport sent in the formData in order to authorize access. + operationId: PostObject + security: + - PassportAuth: [] + responses: + 200: + $ref: '../components/responses/200OkAccess.yaml' + 202: + $ref: '../components/responses/202Accepted.yaml' + 400: + $ref: '../components/responses/400BadRequest.yaml' + 401: + $ref: '../components/responses/401Unauthorized.yaml' + 403: + $ref: '../components/responses/403Forbidden.yaml' + 404: + $ref: '../components/responses/404NotFoundAccess.yaml' + 500: + $ref: '../components/responses/500InternalServerError.yaml' + tags: + - Objects + x-swagger-router-controller: ga4gh.drs.server + parameters: + - $ref: '../components/parameters/ObjectId.yaml' + requestBody: + $ref: '../components/requestBodies/PostObjectBody.yaml' diff --git a/openapi/paths/objects@{object_id}@access@{access_id}.yaml b/openapi/paths/objects@{object_id}@access@{access_id}.yaml new file mode 100644 index 000000000..d884c3af5 --- /dev/null +++ b/openapi/paths/objects@{object_id}@access@{access_id}.yaml @@ -0,0 +1,64 @@ +get: + summary: Get a URL for fetching bytes + description: >- + Returns a URL that can be used to fetch the bytes of a `DrsObject`. + + This method only needs to be called when using an `AccessMethod` that contains an `access_id` + (e.g., for servers that use signed URLs for fetching object bytes). + operationId: GetAccessURL + responses: + 200: + $ref: '../components/responses/200OkAccess.yaml' + 202: + $ref: '../components/responses/202Accepted.yaml' + 400: + $ref: '../components/responses/400BadRequest.yaml' + 401: + $ref: '../components/responses/401Unauthorized.yaml' + 403: + $ref: '../components/responses/403Forbidden.yaml' + 404: + $ref: '../components/responses/404NotFoundAccess.yaml' + 500: + $ref: '../components/responses/500InternalServerError.yaml' + tags: + - Objects + x-swagger-router-controller: ga4gh.drs.server + parameters: + - $ref: '../components/parameters/ObjectId.yaml' + - $ref: '../components/parameters/AccessId.yaml' +post: + summary: Get a URL for fetching bytes through POST'ing a Passport + description: >- + Returns a URL that can be used to fetch the bytes of a `DrsObject`. + + This method only needs to be called when using an `AccessMethod` that contains an `access_id` + (e.g., for servers that use signed URLs for fetching object bytes). + + Method is a POST to accomodate a JWT GA4GH Passport sent in the formData in order to authorize access. + operationId: PostAccessURL + security: + - PassportAuth: [] + responses: + 200: + $ref: '../components/responses/200OkAccess.yaml' + 202: + $ref: '../components/responses/202Accepted.yaml' + 400: + $ref: '../components/responses/400BadRequest.yaml' + 401: + $ref: '../components/responses/401Unauthorized.yaml' + 403: + $ref: '../components/responses/403Forbidden.yaml' + 404: + $ref: '../components/responses/404NotFoundAccess.yaml' + 500: + $ref: '../components/responses/500InternalServerError.yaml' + tags: + - Objects + x-swagger-router-controller: ga4gh.drs.server + parameters: + - $ref: '../components/parameters/ObjectId.yaml' + - $ref: '../components/parameters/AccessId.yaml' + requestBody: + $ref: '../components/requestBodies/Passports.yaml' diff --git a/openapi/paths/service-info.yaml b/openapi/paths/service-info.yaml new file mode 100644 index 000000000..8f2860da1 --- /dev/null +++ b/openapi/paths/service-info.yaml @@ -0,0 +1,39 @@ +get: + summary: Retrieve information about this service + description: |- + Returns information about the DRS service + + Extends the + [v1.0.0 GA4GH Service Info specification](https://github.com/ga4gh-discovery/ga4gh-service-info) + as the standardized format for GA4GH web services to self-describe. + + According to the + [service-info type registry](https://github.com/ga4gh/TASC/blob/master/service-info/ga4gh-service-info.json) + maintained by the [Technical Alignment Sub Committee (TASC)](https://github.com/ga4gh/TASC), + a DRS service MUST have: + * a `type.group` value of `org.ga4gh` + * a `type.artifact` value of `drs` + + e.g. + ``` + { + "id": "com.example.drs", + "description": "Serves data according to DRS specification", + ... + "type": { + "group": "org.ga4gh", + "artifact": "drs" + } + ... + } + ``` + + See the [Service Registry Appendix](#tag/GA4GH-Service-Registry) for more information on how to register a DRS service with a service registry. + operationId: GetServiceInfo + responses: + 200: + $ref: '../components/responses/200ServiceInfo.yaml' + 500: + $ref: '../components/responses/500InternalServerError.yaml' + tags: + - Service Info diff --git a/openapi/tags/Auth.md b/openapi/tags/Auth.md new file mode 100644 index 000000000..40fdf5e12 --- /dev/null +++ b/openapi/tags/Auth.md @@ -0,0 +1,54 @@ +## Making DRS Requests + +The DRS implementation is responsible for defining and enforcing an authorization policy that determines which users are allowed to make which requests. GA4GH recommends that DRS implementations use an OAuth 2.0 [bearer token](https://oauth.net/2/bearer-tokens/) or a [GA4GH Passport](https://github.com/ga4gh-duri/ga4gh-duri.github.io/tree/master/researcher_ids), although they can choose other mechanisms if appropriate. + +## Fetching DRS Objects + +The DRS API allows implementers to support a variety of different content access policies, depending on what `AccessMethod` records they return. Implementers have a choice to make the +GET /objects/{object_id} and GET /objects/{object_id}/access/{access_id} calls open or requiring a Basic, Bearer, or Passport token (Passport requiring a POST). The following describes the +various access approaches following a successful GET/POST /objects/{object_id} request in order to them obtain access to the bytes for a given object ID/access ID: + +* public content: + * server provides an `access_url` with a `url` and no `headers` + * caller fetches the object bytes without providing any auth info +* private content that requires the caller to have out-of-band auth knowledge (e.g. service account credentials): + * server provides an `access_url` with a `url` and no `headers` + * caller fetches the object bytes, passing the auth info they obtained out-of-band +* private content that requires the caller to pass an Authorization token: + * server provides an `access_url` with a `url` and `headers` + * caller fetches the object bytes, passing auth info via the specified header(s) +* private content that uses an expensive-to-generate auth mechanism (e.g. a signed URL): + * server provides an `access_id` + * caller passes the `access_id` to the `/access` endpoint + * server provides an `access_url` with the generated mechanism (e.g. a signed URL in the `url` field) + * caller fetches the object bytes from the `url` (passing auth info from the specified headers, if any) + +In the approaches above [GA4GH Passports](https://github.com/ga4gh-duri/ga4gh-duri.github.io/tree/master/researcher_ids) are not mentioned and that is on purpose. A DRS server may return a Bearer token or other platform-specific token in a header in response to a valid Bearer token or GA4GH Passport (Option 3 above). But it is not the responsibility of a DRS server to return a Passport, that is the responsibility of a Passport Broker and outside the scope of DRS. + +DRS implementers should ensure their solutions restrict access to targets as much as possible, detect attempts to exploit through log monitoring, and they are prepared to take action if an exploit in their DRS implementation is detected. + +## Authentication + +### BasicAuth + +A valid authorization token must be passed in the 'Authorization' header, e.g. "Basic ${token_string}" + +| Security Scheme Type | HTTP | +|----------------------|------| +| **HTTP Authorization Scheme** | basic | + +### BearerAuth + +A valid authorization token must be passed in the 'Authorization' header, e.g. "Bearer ${token_string}" + +| Security Scheme Type | HTTP | +|----------------------|------| +| **HTTP Authorization Scheme** | bearer | + +### PassportAuth + +A valid authorization [GA4GH Passport](https://github.com/ga4gh-duri/ga4gh-duri.github.io/tree/master/researcher_ids) token must be passed in the body of a POST request + +| Security Scheme Type | HTTP | +|----------------------|------| +| **HTTP POST** | tokens[] | diff --git a/openapi/tags/BackgroundNotesOnDRSURIs.md b/openapi/tags/BackgroundNotesOnDRSURIs.md new file mode 100644 index 000000000..416e0c8ac --- /dev/null +++ b/openapi/tags/BackgroundNotesOnDRSURIs.md @@ -0,0 +1,7 @@ +## Design Motivation + +DRS URIs are aligned with the [FAIR data principles](https://www.nature.com/articles/sdata201618) and the [Joint Declaration of Data Citation Principles](https://www.nature.com/articles/sdata20182) — both hostname-based and compact identifier-based URIs provide globally unique, machine-resolvable, persistent identifiers for data. + +* We require all URIs to begin with `drs://` as a signal to humans and systems consuming these URIs that the response they will ultimately receive, after transforming the URI to a fetchable URL, will be a DRS JSON packet. This signal differentiates DRS URIs from the wide variety of other entities (HTML documents, PDFs, ontology notes, etc.) that can be represented by compact identifiers. +* We support hostname-based URIs because of their simplicity and efficiency for server and client implementers. +* We support compact identifier-based URIs, and the meta-resolver services of identifiers.org and n2t.net (Name-to-Thing), because of the wide adoption of compact identifiers in the research community. as detailed by [Wimalaratne et al (2018)](https://www.nature.com/articles/sdata201829) in "Uniform resolution of compact identifiers for biomedical data." diff --git a/openapi/tags/CompactIdentifierBasedURIs.md b/openapi/tags/CompactIdentifierBasedURIs.md new file mode 100644 index 000000000..3fc2ce0f2 --- /dev/null +++ b/openapi/tags/CompactIdentifierBasedURIs.md @@ -0,0 +1,73 @@ +**Note: Identifiers.org/n2t.net API Changes** + +The examples below show the current API interactions with [n2t.net](https://n2t.net/e/compact_ids.html) and [identifiers.org](https://docs.identifiers.org/) which may change over time. Please refer to the documentation from each site for the most up-to-date information. We will make best efforts to keep the DRS specification current but DRS clients MUST maintain their ability to use either the identifiers.org or n2t.net APIs to resolve compact identifier-based DRS URIs. + +## Registering a DRS Server on a Meta-Resolver + +See the documentation on the [n2t.net](https://n2t.net/e/compact_ids.html) and [identifiers.org](https://docs.identifiers.org/) meta-resolvers for adding your own compact identifier type and registering your DRS server as a resolver. You can register new prefixes (or mirrors by adding resource provider codes) for free using a simple online form. For more information see [More Background on Compact Identifiers](./more-background-on-compact-identifiers.html). + +## Calling Meta-Resolver APIs for Compact Identifier-Based DRS URIs + +Clients resolving Compact Identifier-based URIs need to convert a prefix (e.g. “drs.42”) into an URL pattern. They can do so by calling either the identifiers.org or the n2t.net API, since the two meta-resolvers keep their mapping databases in sync. + +### Calling the identifiers.org API as a Client + +It takes two API calls to get the URL pattern. + +1. The client makes a GET request to identifiers.org to find information about the prefix: + +``` +GET https://registry.api.identifiers.org/restApi/namespaces/search/findByPrefix?prefix=drs.42 +``` + +This request returns a JSON structure including various URLs containing an embedded namespace id, such as: + +``` +"namespace" : { + "href":"https://registry.api.identifiers.org/restApi/namespaces/1234" +} +``` + +2. The client extracts the namespace id (in this example 1234), and uses it to make a second GET request to identifiers.org to find information about the namespace: + +``` +GET https://registry.api.identifiers.org/restApi/resources/search/findAllByNamespaceId?id=1234 +``` + +This request returns a JSON structure including an urlPattern field, whose value is an URL pattern containing a ${id} parameter, such as: + +``` +"urlPattern" : "https://drs.myexample.org/ga4gh/drs/v1/objects/{$id}" +``` + +### Calling the n2t.net API as a Client + +It takes one API call to get the URL pattern. + +The client makes a GET request to n2t.net to find information about the namespace. (Note the trailing colon.) + +``` +GET https://n2t.net/drs.42: +``` + +This request returns a text structure including a redirect field, whose value is an URL pattern containing a `$id` parameter, such as: + +``` +redirect: https://drs.myexample.org/ga4gh/drs/v1/objects/$id +``` + +## Caching with Compact Identifiers + +Identifiers.org/n2t.net compact identifier resolver records do not change frequently. This reality is useful for caching resolver records and their URL patterns for performance reasons. Builders of systems that use compact identifier-based DRS URIs should cache prefix resolver records from identifiers.org/n2t.net and occasionally refresh the records (such as every 24 hours). This approach will reduce the burden on these community services since we anticipate many DRS URIs will be regularly resolved in workflow systems. Alternatively, system builders may decide to directly mirror the registries themselves, instructions are provided on the identifiers.org/n2t.net websites. + +## Security with Compact Identifiers + +As mentioned earlier, identifiers.org/n2t.net performs some basic verification of new prefixes and provider code mirror registrations on their sites. However, builders of systems that consume and resolve DRS URIs may have certain security compliance requirements and regulations that prohibit relying on an external site for resolving compact identifiers. In this case, systems under these security and compliance constraints may wish to whitelist certain compact identifier resolvers and/or vet records from identifiers.org/n2t.net before enabling in their systems. + +## Accession Encoding to Valid DRS IDs + +The compact identifier format used by identifiers.org/n2t.net does not percent-encode reserved URI characters but, instead, relies on the first ":" character to separate prefix from accession. Since these accessions can contain any characters, and characters like "/" will interfere with DRS API calls, you *must* percent encode the accessions extracted from DRS compact identifier-based URIs when using as DRS IDs in subsequent DRS GET requests. An easy way for a DRS client to handle this is to get the initial DRS object JSON response from whatever redirects the compact identifier resolves to, then look for the `self_uri` in the JSON, which will give you the correctly percent-encoded DRS ID for subsequent DRS API calls such as the `access` method. + +## Additional Examples + +For additional examples, see the document [More Background on Compact Identifiers](./more-background-on-compact-identifiers.html). diff --git a/openapi/tags/DrsApiPrinciples.md b/openapi/tags/DrsApiPrinciples.md new file mode 100644 index 000000000..5c540e712 --- /dev/null +++ b/openapi/tags/DrsApiPrinciples.md @@ -0,0 +1,120 @@ +## DRS IDs + +Each implementation of DRS can choose its own id scheme, as long as it follows these guidelines: + +* DRS IDs are strings made up of uppercase and lowercase letters, decimal digits, hypen, period, underscore and tilde [A-Za-z0-9.-_~]. See [RFC 3986 § 2.3](https://datatracker.ietf.org/doc/html/rfc3986#section-2.3). +* DRS IDs can contain other characters, but they MUST be encoded into valid DRS IDs whenever they are used in API calls. This is because non-encoded IDs may interfere with the interpretation of the objects/{id}/access endpoint. To overcome this limitation use percent-encoding of the ID, see [RFC 3986 § 2.4](https://datatracker.ietf.org/doc/html/rfc3986#section-2.4) +* One DRS ID MUST always return the same object data (or, in the case of a collection, the same set of objects). This constraint aids with reproducibility. +* DRS implementations MAY have more than one ID that maps to the same object. +* DRS version 1.x does NOT support semantics around multiple versions of an object. (For example, there’s no notion of “get latest version” or “list all versions”.) Individual implementations MAY choose an ID scheme that includes version hints. + +## DRS URIs + +For convenience, including when passing content references to a [WES server](https://github.com/ga4gh/workflow-execution-service-schemas), we define a [URI scheme](https://en.wikipedia.org/wiki/Uniform_Resource_Identifier#Generic_syntax) for DRS-accessible content. This section documents the syntax of DRS URIs, and the rules clients follow for translating a DRS URI into a URL that they use for making the DRS API calls described in this spec. + +There are two styles of DRS URIs, Hostname-based and Compact Identifier-based, both using the `drs://` URI scheme. DRS servers may choose either style when exposing references to their content;. DRS clients MUST support resolving both styles. + +Tip: +> See [Appendix: Background Notes on DRS URIs](#tag/Background-Notes-on-DRS-URIs) for more information on our design motivations for DRS URIs. + +### Hostname-based DRS URIs + +Hostname-based DRS URIs are simpler than compact identifier-based URIs. They contain the DRS server name and the DRS ID only and can be converted directly into a fetchable URL based on a simple rule. They take the form: + +``` +drs:/// +``` + +DRS URIs of this form mean *\"you can fetch the content with DRS id \ from the DRS server at \\"*. +For example, here are the client resolution steps if the URI is: + +``` +drs://drs.example.org/314159 +``` + +1. The client parses the string to extract the hostname of “drs.example.org” and the id of “314159”. +2. The client makes a GET request to the DRS server, using the standard DRS URL syntax: + +``` +GET https://drs.example.org/ga4gh/drs/v1/objects/314159 +``` + +The protocol is always https and the port is always the standard 443 SSL port. It is invalid to include a different port in a DRS hostname-based URI. + +Tip: +> See the [Appendix: Hostname-Based URIs](#tag/Hostname-Based-URIs) for information on how hostname-based DRS URI resolution to URLs is likely to change in the future, when the DRS v2 major release happens. + +### Compact Identifier-based DRS URIs + +Compact Identifier-based DRS URIs use resolver registry services (specifically, [identifiers.org](https://identifiers.org/) and [n2t.net (Name-To-Thing)](https://n2t.net/)) to provide a layer of indirection between the DRS URI and the DRS server name — the actual DNS name of the DRS server is not present in the URI. This approach is based on the Joint Declaration of Data Citation Principles as detailed by [Wimalaratne et al (2018)](https://www.nature.com/articles/sdata201829). + +For more information, see the document [More Background on Compact Identifiers](./more-background-on-compact-identifiers.html). + +Compact Identifiers take the form: + +``` +drs://[provider_code/]namespace:accession +``` + +Together, provider code and the namespace are referred to as the `prefix`. The provider code is optional and is used by identifiers.org/n2t.net for compact identifier resolver mirrors. Both the `provider_code` and `namespace` disallow spaces or punctuation, only lowercase alphanumerical characters, underscores and dots are allowed (e.g. [A-Za-z0-9._]). + +Tip: +> See the [Appendix: Compact Identifier-Based URIs](#tag/Compact-Identifier-Based-URIs) for more background on Compact Identifiers and resolver registry services like identifiers.org/n2t.net (aka meta-resolvers), how to register prefixes, possible caching strategies, and security considerations. + +#### For DRS Servers + +If your DRS implementation will issue DRS URIs based *on your own* compact identifiers, you MUST first register a new prefix with identifiers.org (which is automatically mirrored to n2t.net). You will also need to include a provider resolver resource in this registration which links the prefix to your DRS server, so that DRS clients can get sufficient information to make a successful DRS GET request. For clarity, we recommend you choose a namespace beginning with `drs`. + +#### For DRS Clients + +A DRS client parses the DRS URI compact identifier components to extract the prefix and the accession, and then uses meta-resolver APIs to locate the actual DRS server. For example, here are the client resolution steps if the URI is: + +``` +drs://drs.42:314159 +``` + +1. The client parses the string to extract the prefix of `drs.42` and the accession of `314159`, using the first occurrence of a colon (":") character after the initial `drs://` as a delimiter. (The colon character is not allowed in a Hostname-based DRS URI, making it easy to tell them apart.) + +2. The client makes API calls to a meta-resolver to look up the URL pattern for the namespace. (See [Calling Meta-Resolver APIs for Compact Identifier-Based DRS URIs](#section/Calling-Meta-Resolver-APIs-for-Compact-Identifier-Based-DRS-URIs) for details.) The URL pattern is a string containing a `{$id}` parameter, such as: + +``` +https://drs.myexample.org/ga4gh/drs/v1/objects/{$id} +``` + +3. The client generates a DRS URL from the URL template by replacing {$id} with the accession it extracted in step 1. It then makes a GET request to the DRS server: + +``` +GET https://drs.myexample.org/ga4gh/drs/v1/objects/314159 +``` + +4. The client follows any HTTP redirects returned in step 3, in case the resolver goes through an extra layer of redirection. + +For performance reasons, DRS clients SHOULD cache the URL pattern returned in step 2, with a suggested 24 hour cache life. + +### Choosing a URI Style + +DRS servers can choose to issue either hostname-based or compact identifier-based DRS URIs, and can be confident that compliant DRS clients will support both. DRS clients must be able to accommodate both URI types. Tradeoffs that DRS server builders, and third parties who need to cite DRS objects in datasets, workflows or elsewhere, may want to consider include: + +*Table 1: Choosing a URI Style* + +| | Hostname-based | Compact Identifier-based | +|-------------------|----------------|--------------------------| +| URI Durability | URIs are valid for as long as the server operator maintains ownership of the published DNS address. (They can of course point that address at different physical serving infrastructure as often as they would like.) | URIs are valid for as long as the server operator maintains ownership of the published compact identifier resolver namespace. (They also depend on the meta-resolvers like identifiers.org/n2t.net remaining operational, which is intended to be essentially forever.) | +| Client Efficiency | URIs require minimal client logic, and no network requests, to resolve. | URIs require small client logic, and 1-2 cacheable network requests, to resolve. | +| Security | Servers have full control over their own security practices. | Server operators, in addition to maintaining their own security practices, should confirm they are comfortable with the resolver registry security practices, including protection against denial of service and namespace-hijacking attacks. (See the [Appendix: Compact Identifier-Based URIs](#tag/Compact-Identifier-Based-URIs) for more information on resolver registry security.) | + +## DRS Datatypes + +DRS v1 supports two types of content: + +* a *blob* is like a file — it’s a single blob of bytes, represented by a `DrsObject` without a `contents` array +* a *bundle* is like a folder — it’s a collection of other DRS content (either blobs or bundles), represented by a `DrsObject` with a `contents` array + +## Read-only + +DRS v1 is a read-only API. We expect that each implementation will define its own mechanisms and interfaces (graphical and/or programmatic) for adding and updating data. + +## Standards + +The DRS API specification is written in OpenAPI and embodies a RESTful service philosophy. It uses JSON in requests and responses and standard HTTPS on port 443 for information transport. Optionally, it +supports authenitcation and authorization using the [GA4GH Passport](https://github.com/ga4gh-duri/ga4gh-duri.github.io/tree/master/researcher_ids) standard. diff --git a/openapi/tags/HostnameBasedURIs.md b/openapi/tags/HostnameBasedURIs.md new file mode 100644 index 000000000..e53d39dcd --- /dev/null +++ b/openapi/tags/HostnameBasedURIs.md @@ -0,0 +1,3 @@ +## Encoding DRS IDs + +In hostname-based DRS URIs, the ID is always percent-encoded to ensure special characters do not interfere with subsequent DRS endpoint calls. As such, ":" is not allowed in the URI and is a convenient way of differentiating from a compact identifier-based DRS URI. Also, if a given DRS service implementation uses compact identifier accessions as their DRS IDs, they must be percent encoded before using them as DRS IDs in hostname-based DRS URIs and subsequent GET requests to a DRS service endpoint. diff --git a/openapi/tags/Introduction.md b/openapi/tags/Introduction.md new file mode 100644 index 000000000..8201a0786 --- /dev/null +++ b/openapi/tags/Introduction.md @@ -0,0 +1,3 @@ +The Data Repository Service (DRS) API provides a generic interface to data repositories so data consumers, including workflow systems, can access data objects in a single, standard way regardless of where they are stored and how they are managed. The primary functionality of DRS is to map a logical ID to a means for physically retrieving the data represented by the ID. The sections below describe the characteristics of those IDs, the types of data supported, how they can be pointed to using URIs, and how clients can use these URIs to ultimately make successful DRS API requests. This document also describes the DRS API in detail and provides information on the specific endpoints, request formats, and responses. This specification is intended for developers of DRS-compatible services and of clients that will call these DRS services. + +The key words MUST, MUST NOT, REQUIRED, SHALL, SHALL NOT, SHOULD, SHOULD NOT, RECOMMENDED, MAY, and OPTIONAL in this document are to be interpreted as described in [RFC 2119](https://datatracker.ietf.org/doc/html/rfc2119). diff --git a/openapi/tags/Motivation.md b/openapi/tags/Motivation.md new file mode 100644 index 000000000..27735db9f --- /dev/null +++ b/openapi/tags/Motivation.md @@ -0,0 +1,66 @@ + + + + + +
+ Data sharing requires portable data, consistent with the FAIR data principles (findable, accessible, interoperable, reusable). Today’s researchers and clinicians are surrounded by potentially useful data, but often need bespoke tools and processes to work with each dataset. Today’s data publishers don’t have a reliable way to make their data useful to all (and only) the people they choose. And today’s data controllers are tasked with implementing standard controls of non-standard mechanisms for data access. + + + + Figure 1: there’s an ocean of data, with many different tools to drink from it, but no guarantee that any tool will work with any subset of the data + +
+ + + + + + +
+ We need a standard way for data producers to make their data available to data consumers, that supports the control needs of the former and the access needs of the latter. And we need it to be interoperable, so anyone who builds access tools and systems can be confident they’ll work with all the data out there, and anyone who publishes data can be confident it will work with all the tools out there. + + + + Figure 2: by defining a standard Data Repository API, and adapting tools to use it, every data publisher can now make their data useful to every data consumer + +
+ + + + + + +
+ We envision a world where: +
    +
  • + there are many many data consumers, working in research and in care, who can use the tools of their choice to access any and all data that they have permission to see +
    • +
  • + there are many data access tools and platforms, supporting discovery, visualization, analysis, and collaboration +
    • +
  • + there are many data repositories, each with their own policies and characteristics, which can be accessed by a variety of tools +
    • +
  • + there are many data publishing tools and platforms, supporting a variety of data lifecycles and formats +
    • +
  • + there are many many data producers, generating data of all types, who can use the tools of their choice to make their data as widely available as is appropriate +
    • +
+ 		+ + + Figure 3: a standard Data Repository API enables an ecosystem of data producers and consumers + +
+ +This spec defines a standard **Data Repository Service (DRS) API** (“the yellow box”), to enable that ecosystem of data producers and consumers. Our goal is that the only thing data consumers need to know about a data repo is *\"here’s the DRS endpoint to access it\"*, and the only thing data publishers need to know to tap into the world of consumption tools is *\"here’s how to tell it where my DRS endpoint lives\"*. + +## Federation + +The world’s biomedical data is controlled by groups with very different policies and restrictions on where their data lives and how it can be accessed. A primary purpose of DRS is to support unified access to disparate and distributed data. (As opposed to the alternative centralized model of "let’s just bring all the data into one single data repository”, which would be technically easier but is no more realistic than “let’s just bring all the websites into one single web host”.) + +In a DRS-enabled world, tool builders don’t have to worry about where the data their tools operate on lives — they can count on DRS to give them access. And tool users only need to know which DRS server is managing the data they need, and whether they have permission to access it; they don’t have to worry about how to physically get access to, or (worse) make a copy of the data. For example, if I have appropriate permissions, I can run a pooled analysis where I run a single tool across data managed by different DRS servers, potentially in different locations. diff --git a/openapi/tags/ServiceRegistry.md b/openapi/tags/ServiceRegistry.md new file mode 100644 index 000000000..34467fd3f --- /dev/null +++ b/openapi/tags/ServiceRegistry.md @@ -0,0 +1,36 @@ +The [GA4GH Service Registry API specification](https://github.com/ga4gh-discovery/ga4gh-service-registry) allows information about GA4GH-compliant web services, including DRS services, to be aggregated into registries and made available via a standard API. The following considerations should be followed when registering DRS services within a service registry. + +* The DRS service attributes returned by `/service-info` (i.e. `id`, `name`, `description`, etc.) should have the same values as the registry entry for that service. +* The value of the `type` object's `artifact` property should be `drs` (i.e. the same as it appears in `service-info`) +* Each entry in a Service Registry must have a `url`, indicating the base URL to the web service. For DRS services, the registered `url` must include everything up to +the standardized `/ga4gh/drs/v1` path. Clients should be able to assume that: + + Adding `/ga4gh/drs/v1/objects/{object_id}` to the registered `url` will hit the `DrsObject` endpoint + + Adding `/ga4gh/drs/v1/service-info` to the the registered `url` will hit the Service Info endpoint + +Example listing of a DRS API registration from a service registry's `/services` endpoint: + +``` +[ + { + "id": "com.example.drs", + "name": "Example DRS API", + "type": { + "group": "org.ga4gh", + "artifact": "drs", + "version": "1.2.0" + }, + "description": "The Data Repository Service (DRS) API ...", + "organization": { + "id": "com.example", + "name": "Example Company" + }, + "contactUrl": "mailto:support@example.com", + "documentationUrl": "https://docs.example.com/docs/drs", + "createdAt": "2021-08-09T00:00:00Z", + "updatedAt": "2021-08-09T12:30:00Z", + "environment": "production", + "version": "1.13.4", + "url": "https://drs-service.example.com" + } +] +``` diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index 693d8d25b..000000000 --- a/package-lock.json +++ /dev/null @@ -1,2489 +0,0 @@ -{ - "name": "Data-Repository-Service-openapi-spec", - "version": "1.0.0", - "lockfileVersion": 1, - "requires": true, - "dependencies": { - "@types/babel-types": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/@types/babel-types/-/babel-types-7.0.4.tgz", - "integrity": "sha512-WiZhq3SVJHFRgRYLXvpf65XnV6ipVHhnNaNvE8yCimejrGglkg38kEj0JcizqwSHxmPSjcTlig/6JouxLGEhGw==" - }, - "@types/babylon": { - "version": "6.16.3", - "resolved": "https://registry.npmjs.org/@types/babylon/-/babylon-6.16.3.tgz", - "integrity": "sha512-lyJ8sW1PbY3uwuvpOBZ9zMYKshMnQpXmeDHh8dj9j2nJm/xrW0FgB5gLSYOArj5X0IfaXnmhFoJnhS4KbqIMug==", - "requires": { - "@types/babel-types": "*" - } - }, - "acorn": { - "version": "3.3.0", - "resolved": "http://registry.npmjs.org/acorn/-/acorn-3.3.0.tgz", - "integrity": "sha1-ReN/s56No/JbruP/U2niu18iAXo=" - }, - "acorn-globals": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-3.1.0.tgz", - "integrity": "sha1-/YJw9x+7SZawBPqIDuXUZXOnMb8=", - "requires": { - "acorn": "^4.0.4" - }, - "dependencies": { - "acorn": { - "version": "4.0.13", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-4.0.13.tgz", - "integrity": "sha1-EFSVrlNh1pe9GVyCUZLhrX8lN4c=" - } - } - }, - "align-text": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/align-text/-/align-text-0.1.4.tgz", - "integrity": "sha1-DNkKVhCT810KmSVsIrcGlDP60Rc=", - "requires": { - "kind-of": "^3.0.2", - "longest": "^1.0.1", - "repeat-string": "^1.5.2" - }, - "dependencies": { - "kind-of": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", - "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", - "requires": { - "is-buffer": "^1.1.5" - } - } - } - }, - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" - }, - "argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "requires": { - "sprintf-js": "~1.0.2" - } - }, - "array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" - }, - "array-union": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz", - "integrity": "sha1-mjRBDk9OPaI96jdb5b5w8kd47Dk=", - "requires": { - "array-uniq": "^1.0.1" - } - }, - "array-uniq": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz", - "integrity": "sha1-r2rId6Jcx/dOBYiUdThY39sk/bY=" - }, - "asap": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", - "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=" - }, - "babel-runtime": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz", - "integrity": "sha1-llxwWGaOgrVde/4E/yM3vItWR/4=", - "requires": { - "core-js": "^2.4.0", - "regenerator-runtime": "^0.11.0" - } - }, - "babel-types": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-types/-/babel-types-6.26.0.tgz", - "integrity": "sha1-o7Bz+Uq0nrb6Vc1lInozQ4BjJJc=", - "requires": { - "babel-runtime": "^6.26.0", - "esutils": "^2.0.2", - "lodash": "^4.17.4", - "to-fast-properties": "^1.0.3" - } - }, - "babylon": { - "version": "6.18.0", - "resolved": "https://registry.npmjs.org/babylon/-/babylon-6.18.0.tgz", - "integrity": "sha512-q/UEjfGJ2Cm3oKV71DJz9d25TPnq5rhBVL2Q4fA5wcC3jcrdn7+SssEybFIxwAvvP+YCsCYNKughoF33GxgycQ==" - }, - "balanced-match": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" - }, - "bower": { - "version": "1.8.4", - "resolved": "https://registry.npmjs.org/bower/-/bower-1.8.4.tgz", - "integrity": "sha1-54dqB23rgTf30GUl3F6MZtuC8oo=" - }, - "brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "requires": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "center-align": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/center-align/-/center-align-0.1.3.tgz", - "integrity": "sha1-qg0yYptu6XIgBBHL1EYckHvCt60=", - "requires": { - "align-text": "^0.1.3", - "lazy-cache": "^1.0.3" - } - }, - "character-parser": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/character-parser/-/character-parser-2.2.0.tgz", - "integrity": "sha1-x84o821LzZdE5f/CxfzeHHMmH8A=", - "requires": { - "is-regex": "^1.0.3" - } - }, - "clean-css": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/clean-css/-/clean-css-4.2.1.tgz", - "integrity": "sha512-4ZxI6dy4lrY6FHzfiy1aEOXgu4LIsW2MhwG0VBKdcoGoH/XLFgaHSdLTGr4O8Be6A8r3MOphEiI8Gc1n0ecf3g==", - "requires": { - "source-map": "~0.6.0" - } - }, - "code-point-at": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", - "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" - }, - "colors": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/colors/-/colors-0.5.1.tgz", - "integrity": "sha1-fQAj6usVTo7p/Oddy5I9DtFmd3Q=" - }, - "commander": { - "version": "2.17.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.17.1.tgz", - "integrity": "sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg==" - }, - "component-emitter": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz", - "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY=" - }, - "concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" - }, - "connect": { - "version": "3.6.6", - "resolved": "https://registry.npmjs.org/connect/-/connect-3.6.6.tgz", - "integrity": "sha1-Ce/2xVr3I24TcTWnJXSFi2eG9SQ=", - "requires": { - "debug": "2.6.9", - "finalhandler": "1.1.0", - "parseurl": "~1.3.2", - "utils-merge": "1.0.1" - } - }, - "constantinople": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/constantinople/-/constantinople-3.1.2.tgz", - "integrity": "sha512-yePcBqEFhLOqSBtwYOGGS1exHo/s1xjekXiinh4itpNQGCu4KA1euPh1fg07N2wMITZXQkBz75Ntdt1ctGZouw==", - "requires": { - "@types/babel-types": "^7.0.0", - "@types/babylon": "^6.16.2", - "babel-types": "^6.26.0", - "babylon": "^6.18.0" - } - }, - "content-disposition": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", - "integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ=" - }, - "content-type": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" - }, - "cookie-signature": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" - }, - "cookiejar": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.2.tgz", - "integrity": "sha512-Mw+adcfzPxcPeI+0WlvRrr/3lGVO0bD75SxX6811cxSh1Wbxx7xZBGK1eVtDf6si8rg2lhnUjsVLMFMfbRIuwA==" - }, - "core-js": { - "version": "2.5.7", - "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.5.7.tgz", - "integrity": "sha512-RszJCAxg/PP6uzXVXL6BsxSXx/B05oJAQ2vkJRjyjrEcNVycaqOmNb5OTxZPE3xa5gwZduqza6L9JOCenh/Ecw==" - }, - "cors": { - "version": "2.8.4", - "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.4.tgz", - "integrity": "sha1-K9OB8usgECAQXNUOpZ2mMJBpRoY=", - "requires": { - "object-assign": "^4", - "vary": "^1" - } - }, - "cross-spawn": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", - "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=", - "requires": { - "lru-cache": "^4.0.1", - "shebang-command": "^1.2.0", - "which": "^1.2.9" - } - }, - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "decamelize": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" - }, - "deep-is": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", - "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=" - }, - "destroy": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", - "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" - }, - "doctypes": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/doctypes/-/doctypes-1.1.0.tgz", - "integrity": "sha1-6oCxBqh1OHdOijpKWv4pPeSJ4Kk=" - }, - "ebnf-parser": { - "version": "0.1.10", - "resolved": "https://registry.npmjs.org/ebnf-parser/-/ebnf-parser-0.1.10.tgz", - "integrity": "sha1-zR9rpHfFY4xAyX7ZtXLbW6tdgzE=" - }, - "ee-first": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" - }, - "encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" - }, - "entities": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz", - "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA=" - }, - "escape-html": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" - }, - "estraverse": { - "version": "0.0.4", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-0.0.4.tgz", - "integrity": "sha1-AaCTLf7ldGhKWYr1pnw7+bZCjbI=" - }, - "esutils": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", - "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=" - }, - "execa": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", - "integrity": "sha1-lEvs00zEHuMqY6n68nrVpl/Fl3c=", - "requires": { - "cross-spawn": "^5.0.1", - "get-stream": "^3.0.0", - "is-stream": "^1.1.0", - "npm-run-path": "^2.0.0", - "p-finally": "^1.0.0", - "signal-exit": "^3.0.0", - "strip-eof": "^1.0.0" - } - }, - "extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" - }, - "fast-levenshtein": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" - }, - "finalhandler": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.0.tgz", - "integrity": "sha1-zgtoVbRYU+eRsvzGgARtiCU91/U=", - "requires": { - "debug": "2.6.9", - "encodeurl": "~1.0.1", - "escape-html": "~1.0.3", - "on-finished": "~2.3.0", - "parseurl": "~1.3.2", - "statuses": "~1.3.1", - "unpipe": "~1.0.0" - } - }, - "foreach": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz", - "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k=" - }, - "forwarded": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" - }, - "fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" - }, - "function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" - }, - "get-caller-file": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz", - "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==" - }, - "get-stdin": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-5.0.1.tgz", - "integrity": "sha1-Ei4WFZHiH/TFJTAwVpPyDmOTo5g=" - }, - "get-stream": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", - "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" - }, - "github-markdown": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/github-markdown/-/github-markdown-3.2.0.tgz", - "integrity": "sha512-mtc8/f3DOCoxLi/lVgCpx5I71YxusLiFqgDJZWPcx+wd+fa3z6+hOTmuKhekawq+/4ftwp93EipxDIf5P7a7Vg==", - "requires": { - "get-stdin": "^5.0.1", - "globby": "^6.1.0", - "highlight.js": "^9.12.0", - "markdown-it": "^8.3.1", - "minimist": "^1.2.0", - "pify": "^3.0.0", - "primer-css": "^9.0.0", - "pug": "^2.0.0-rc.2" - }, - "dependencies": { - "pify": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", - "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=" - } - } - }, - "github-markdown-css": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/github-markdown-css/-/github-markdown-css-2.10.0.tgz", - "integrity": "sha512-RX5VUC54uX6Lvrm226M9kMzsNeOa81MnKyxb3J0G5KLjyoOySOZgwyKFkUpv6iUhooiUZdogk+OTwQPJ4WttYg==" - }, - "globby": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-6.1.0.tgz", - "integrity": "sha1-9abXDoOV4hyFj7BInWTfAkJNUGw=", - "requires": { - "array-union": "^1.0.1", - "glob": "^7.0.3", - "object-assign": "^4.0.1", - "pify": "^2.0.0", - "pinkie-promise": "^2.0.0" - }, - "dependencies": { - "glob": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", - "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "requires": { - "brace-expansion": "^1.1.7" - } - } - } - }, - "has": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", - "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "requires": { - "function-bind": "^1.1.1" - } - }, - "help": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/help/-/help-3.0.2.tgz", - "integrity": "sha1-luGQ1KCkU7icLLSwWrOOOo+f2t0=" - }, - "highlight.js": { - "version": "9.12.0", - "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-9.12.0.tgz", - "integrity": "sha1-5tnb5Xy+/mB1HwKvM2GVhwyQwB4=" - }, - "inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "requires": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" - }, - "interpret": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.1.0.tgz", - "integrity": "sha1-ftGxQQxqDg94z5XTuEQMY/eLhhQ=" - }, - "invert-kv": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz", - "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY=" - }, - "is-buffer": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" - }, - "is-expression": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-expression/-/is-expression-3.0.0.tgz", - "integrity": "sha1-Oayqa+f9HzRx3ELHQW5hwkMXrJ8=", - "requires": { - "acorn": "~4.0.2", - "object-assign": "^4.0.1" - }, - "dependencies": { - "acorn": { - "version": "4.0.13", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-4.0.13.tgz", - "integrity": "sha1-EFSVrlNh1pe9GVyCUZLhrX8lN4c=" - } - } - }, - "is-fullwidth-code-point": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", - "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", - "requires": { - "number-is-nan": "^1.0.0" - } - }, - "is-promise": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", - "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=" - }, - "is-regex": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz", - "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=", - "requires": { - "has": "^1.0.1" - } - }, - "is-stream": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", - "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" - }, - "isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" - }, - "jison-lex": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/jison-lex/-/jison-lex-0.2.1.tgz", - "integrity": "sha1-rEuBXozOUTLrErXfz+jXB7iETf4=", - "requires": { - "lex-parser": "0.1.x", - "nomnom": "1.5.2" - } - }, - "js-base64": { - "version": "2.4.9", - "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-2.4.9.tgz", - "integrity": "sha512-xcinL3AuDJk7VSzsHgb9DvvIXayBbadtMZ4HFPx8rUszbW1MuNMlwYVC4zzCZ6e1sqZpnNS5ZFYOhXqA39T7LQ==" - }, - "js-stringify": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/js-stringify/-/js-stringify-1.0.2.tgz", - "integrity": "sha1-Fzb939lyTyijaCrcYjCufk6Weds=" - }, - "json-pointer": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/json-pointer/-/json-pointer-0.6.0.tgz", - "integrity": "sha1-jlAFUKaqxUZKRzN32leqbMIoKNc=", - "requires": { - "foreach": "^2.0.4" - } - }, - "jstransformer": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/jstransformer/-/jstransformer-1.0.0.tgz", - "integrity": "sha1-7Yvwkh4vPx7U1cGkT2hwntJHIsM=", - "requires": { - "is-promise": "^2.0.0", - "promise": "^7.0.1" - } - }, - "lazy-cache": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz", - "integrity": "sha1-odePw6UEdMuAhF07O24dpJpEbo4=" - }, - "lcid": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz", - "integrity": "sha1-MIrMr6C8SDo4Z7S28rlQYlHRuDU=", - "requires": { - "invert-kv": "^1.0.0" - } - }, - "lex-parser": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/lex-parser/-/lex-parser-0.1.4.tgz", - "integrity": "sha1-ZMTwJfF/1Tv7RXY/rrFvAVp0dVA=" - }, - "linkify-it": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-2.0.3.tgz", - "integrity": "sha1-2UpGSPmxwXnWT6lykSaL22zpQ08=", - "requires": { - "uc.micro": "^1.0.1" - } - }, - "locate-path": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", - "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", - "requires": { - "p-locate": "^2.0.0", - "path-exists": "^3.0.0" - }, - "dependencies": { - "path-exists": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=" - } - } - }, - "lodash": { - "version": "4.17.11", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", - "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" - }, - "lodash.get": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", - "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=" - }, - "lodash.isequal": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", - "integrity": "sha1-QVxEePK8wwEgwizhDtMib30+GOA=" - }, - "longest": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/longest/-/longest-1.0.1.tgz", - "integrity": "sha1-MKCy2jj3N3DoKUoNIuZiXtd9AJc=" - }, - "lru-cache": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.3.tgz", - "integrity": "sha512-fFEhvcgzuIoJVUF8fYr5KR0YqxD238zgObTps31YdADwPPAp82a4M8TrckkWyx7ekNlf9aBcVn81cFwwXngrJA==", - "requires": { - "pseudomap": "^1.0.2", - "yallist": "^2.1.2" - } - }, - "markdown-it": { - "version": "8.4.2", - "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-8.4.2.tgz", - "integrity": "sha512-GcRz3AWTqSUphY3vsUqQSFMbgR38a4Lh3GWlHRh/7MRwz8mcu9n2IO7HOh+bXHrR9kOPDl5RNCaEsrneb+xhHQ==", - "requires": { - "argparse": "^1.0.7", - "entities": "~1.1.1", - "linkify-it": "^2.0.0", - "mdurl": "^1.0.1", - "uc.micro": "^1.0.5" - } - }, - "mdurl": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz", - "integrity": "sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4=" - }, - "media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" - }, - "mem": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/mem/-/mem-1.1.0.tgz", - "integrity": "sha1-Xt1StIXKHZAP5kiVUFOZoN+kX3Y=", - "requires": { - "mimic-fn": "^1.0.0" - } - }, - "merge-descriptors": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" - }, - "methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" - }, - "mime-db": { - "version": "1.36.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.36.0.tgz", - "integrity": "sha512-L+xvyD9MkoYMXb1jAmzI/lWYAxAMCPvIBSWur0PZ5nOf5euahRLVqH//FKW9mWp2lkqUgYiXPgkzfMUFi4zVDw==" - }, - "mime-types": { - "version": "2.1.20", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.20.tgz", - "integrity": "sha512-HrkrPaP9vGuWbLK1B1FfgAkbqNjIuy4eHlIYnFi7kamZyLLrGlo2mpcx0bBmNpKqBtYtAfGbodDddIgddSJC2A==", - "requires": { - "mime-db": "~1.36.0" - } - }, - "mimic-fn": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", - "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==" - }, - "minimist": { - "version": "1.2.0", - "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", - "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" - }, - "mkdirp": { - "version": "0.5.1", - "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", - "requires": { - "minimist": "0.0.8" - }, - "dependencies": { - "minimist": { - "version": "0.0.8", - "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" - } - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "nomnom": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/nomnom/-/nomnom-1.5.2.tgz", - "integrity": "sha1-9DRUSKhTz71cDSYyDyR3qwUm/i8=", - "requires": { - "colors": "0.5.x", - "underscore": "1.1.x" - }, - "dependencies": { - "underscore": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.1.7.tgz", - "integrity": "sha1-QLq4S60Z0jAJbo1u9ii/8FXYPbA=" - } - } - }, - "npm-run-path": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", - "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=", - "requires": { - "path-key": "^2.0.0" - } - }, - "number-is-nan": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", - "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" - }, - "object-assign": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" - }, - "on-finished": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", - "requires": { - "ee-first": "1.1.1" - } - }, - "once": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/once/-/once-1.3.3.tgz", - "integrity": "sha1-suJhVXzkwxTsgwTz+oJmPkKXyiA=", - "requires": { - "wrappy": "1" - } - }, - "p-finally": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" - }, - "p-limit": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", - "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", - "requires": { - "p-try": "^1.0.0" - } - }, - "p-locate": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", - "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", - "requires": { - "p-limit": "^1.1.0" - } - }, - "p-try": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", - "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=" - }, - "parseurl": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz", - "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M=" - }, - "path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" - }, - "path-key": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", - "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=" - }, - "path-parse": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==" - }, - "path-to-regexp": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" - }, - "pify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=" - }, - "pinkie": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", - "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA=" - }, - "pinkie-promise": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", - "integrity": "sha1-ITXW36ejWMBprJsXh3YogihFD/o=", - "requires": { - "pinkie": "^2.0.0" - } - }, - "portfinder": { - "version": "1.0.17", - "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.17.tgz", - "integrity": "sha512-syFcRIRzVI1BoEFOCaAiizwDolh1S1YXSodsVhncbhjzjZQulhczNRbqnUl9N31Q4dKGOXsNDqxC2BWBgSMqeQ==", - "requires": { - "async": "^1.5.2", - "debug": "^2.2.0", - "mkdirp": "0.5.x" - }, - "dependencies": { - "async": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", - "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=" - } - } - }, - "primer-alerts": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/primer-alerts/-/primer-alerts-1.5.1.tgz", - "integrity": "sha512-2dyRO6ZgZF9ZR67gg+viCtsYV9CG+z6UARW8DAf4CEDkeiR4K46R6kKSC/WveEv8LPJ0MF+L4IXjqvn5kqK7zA==", - "requires": { - "primer-support": "4.4.1" - } - }, - "primer-avatars": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/primer-avatars/-/primer-avatars-1.4.1.tgz", - "integrity": "sha512-kd+GPMZqdXZ+N07CYeCWmRHCHUCeeCFNWldbgCg07NRfU+Ne9n01CLR/DcNuG27oP5A30ubvHnoEofvkgG3ynw==", - "requires": { - "primer-support": "4.4.1" - } - }, - "primer-base": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/primer-base/-/primer-base-1.5.1.tgz", - "integrity": "sha512-7tv0/V5sSRucp65NQXEyC8E2GP9EumLSKFDrjbl0liZTwSve8HUm7TmzgDXkO5pWd0VSfc7+5G1qrrdsGNmkPQ==", - "requires": { - "primer-support": "4.4.1" - } - }, - "primer-blankslate": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/primer-blankslate/-/primer-blankslate-1.4.1.tgz", - "integrity": "sha512-osgVtGY6UikfK1vuoBWijxT1C+SsD8RQxvYS8RFMDh6bReEf45//3n0NJVksRT8GdGUS8atRnrsWzRCtDcEwAg==", - "requires": { - "primer-support": "4.4.1" - } - }, - "primer-box": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/primer-box/-/primer-box-2.5.1.tgz", - "integrity": "sha512-jfJjDLQlaM9e5pyzcJHALEb3Gml5uEoDDAoWKiKwvxEhl6da+5DB+HGbEt3/KUqt9B3e9Omy6IBvvRfgnqae7A==", - "requires": { - "primer-support": "4.4.1" - } - }, - "primer-breadcrumb": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/primer-breadcrumb/-/primer-breadcrumb-1.4.1.tgz", - "integrity": "sha512-cmDjIXSXClLQcrWDeJkZJWXkacfCluZU23mfsVM8K0oZyhiHZjbleOdsbwGXMKpFSEJ61wi4zvF9ZMkx1s8EdA==", - "requires": { - "primer-marketing-support": "1.3.1", - "primer-support": "4.4.1" - } - }, - "primer-buttons": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/primer-buttons/-/primer-buttons-2.4.1.tgz", - "integrity": "sha512-wOb0FMkRI/sWntorY9KXzY/OVxT5P/V+xlotbEC3+SPfzTLG+vJCs5rsamMu7S4TzhOx675/DTSpMo83iY4h3w==", - "requires": { - "primer-support": "4.4.1" - } - }, - "primer-cards": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/primer-cards/-/primer-cards-0.5.1.tgz", - "integrity": "sha512-iXqvCgL311UBoRbC/ioFYSUunFnS8rUGkyALtN3/qAw3LoeyE1vjcooV0f1ja/xOblH0vGa32HjEKo2sBTwQ7g==", - "requires": { - "primer-marketing-support": "1.3.1", - "primer-support": "4.4.1" - } - }, - "primer-core": { - "version": "6.4.1", - "resolved": "https://registry.npmjs.org/primer-core/-/primer-core-6.4.1.tgz", - "integrity": "sha512-IxOLJ3BGLLFhDQ329zGgBMLaflqTn/WKAbRhl3XGIVh2vyRLBr8XPhzTfWfeFKt9WQ3ljzZkwip1GsMX9qqb1Q==", - "requires": { - "primer-base": "1.5.1", - "primer-box": "2.5.1", - "primer-buttons": "2.4.1", - "primer-forms": "1.4.1", - "primer-layout": "1.4.1", - "primer-navigation": "1.4.1", - "primer-support": "4.4.1", - "primer-table-object": "1.4.1", - "primer-tooltips": "1.4.1", - "primer-truncate": "1.4.1", - "primer-utilities": "4.8.1" - } - }, - "primer-css": { - "version": "9.6.0", - "resolved": "https://registry.npmjs.org/primer-css/-/primer-css-9.6.0.tgz", - "integrity": "sha512-qzTck5gvQevHvI3sUgP0D2QzLdmAqwd9h1rucMObOIbp8xQcM8zZGFNo71FBu7TxNu4A00McNvnadSNOgmnJnA==", - "requires": { - "primer-alerts": "1.5.1", - "primer-avatars": "1.4.1", - "primer-base": "1.5.1", - "primer-blankslate": "1.4.1", - "primer-box": "2.5.1", - "primer-breadcrumb": "1.4.1", - "primer-buttons": "2.4.1", - "primer-cards": "0.5.1", - "primer-core": "6.4.1", - "primer-forms": "1.4.1", - "primer-labels": "1.5.1", - "primer-layout": "1.4.1", - "primer-markdown": "3.7.1", - "primer-marketing": "5.4.1", - "primer-marketing-support": "1.3.1", - "primer-marketing-type": "1.4.1", - "primer-marketing-utilities": "1.4.1", - "primer-navigation": "1.4.1", - "primer-page-headers": "1.4.1", - "primer-page-sections": "1.4.1", - "primer-product": "5.4.1", - "primer-support": "4.4.1", - "primer-table-object": "1.4.1", - "primer-tables": "1.4.1", - "primer-tooltips": "1.4.1", - "primer-truncate": "1.4.1", - "primer-utilities": "4.8.1" - } - }, - "primer-forms": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/primer-forms/-/primer-forms-1.4.1.tgz", - "integrity": "sha512-wr7Ieoyy9iHPtR1zEaWEwPRA3PHo1eaOj3Q4VhYOBRv3rk+H3/Z49hey7PLyocPnvbF1GdG2s5/VkZUdgBEuqg==", - "requires": { - "primer-support": "4.4.1" - } - }, - "primer-labels": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/primer-labels/-/primer-labels-1.5.1.tgz", - "integrity": "sha512-dsX98awj7UYKvhed+j47ChldBX9sV5OSO3MhhVAHon7Kj5kxPCLFkTw/YlOxOHFX2VlbP/5jMM5W46xnSERy1g==", - "requires": { - "primer-support": "4.4.1" - } - }, - "primer-layout": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/primer-layout/-/primer-layout-1.4.1.tgz", - "integrity": "sha512-x4u8twf8XGFL3mKplh0XM7jUjTL9Is/97BmlehZMQE+740G/gywPo2CpOV2GMCxWzmhQVJhib8At1+UvN+qvZQ==", - "requires": { - "primer-support": "4.4.1" - } - }, - "primer-markdown": { - "version": "3.7.1", - "resolved": "https://registry.npmjs.org/primer-markdown/-/primer-markdown-3.7.1.tgz", - "integrity": "sha512-62I7tZaCCnOgjc2yE1cuu4WTwbym/eNIpEMB0CSWvFf8ZiTVKC5dNpFIwT1ipE35IjgOVopdHdqdPzMXoxKNAg==", - "requires": { - "primer-support": "4.4.1" - } - }, - "primer-marketing": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/primer-marketing/-/primer-marketing-5.4.1.tgz", - "integrity": "sha512-ccadKuKA4kQDTaopHsj+lOujdxa8d14Ff8fq5HLvHlrpgKvdVwuKtIwNL2ryuas1FVeTddxa3lBnlAcuVTwWIQ==", - "requires": { - "primer-breadcrumb": "1.4.1", - "primer-cards": "0.5.1", - "primer-marketing-support": "1.3.1", - "primer-marketing-type": "1.4.1", - "primer-marketing-utilities": "1.4.1", - "primer-page-headers": "1.4.1", - "primer-page-sections": "1.4.1", - "primer-support": "4.4.1", - "primer-tables": "1.4.1" - } - }, - "primer-marketing-support": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/primer-marketing-support/-/primer-marketing-support-1.3.1.tgz", - "integrity": "sha512-GDwQ4TdZNS4p6UbSMxv7j7DlgegEDU43k2QKFJZ9EAtBN/rOKkf9gBa31yEiJQvgG7wZ84CvioObYtw885TL7g==" - }, - "primer-marketing-type": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/primer-marketing-type/-/primer-marketing-type-1.4.1.tgz", - "integrity": "sha512-cJGHvDkCy1bYiM2EDOwc+k7Y61DfvOIWambU5WmsGq7fUI92MRWqpyWZpYVOFEWqItRjLnxPv4myJCm0itKIRQ==", - "requires": { - "primer-marketing-support": "1.3.1", - "primer-support": "4.4.1" - } - }, - "primer-marketing-utilities": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/primer-marketing-utilities/-/primer-marketing-utilities-1.4.1.tgz", - "integrity": "sha512-dbic/+lYITBnZKujg6s3GF0Mo3jhuiq1ps0a3negkBsxKsNyU68hoYEl2bN6UI5L2BX9GyzCLd58N1jyjV4uCw==", - "requires": { - "primer-marketing-support": "1.3.1", - "primer-support": "4.4.1" - } - }, - "primer-navigation": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/primer-navigation/-/primer-navigation-1.4.1.tgz", - "integrity": "sha512-fMVrR8l/JtTXLzwf+8nHeBvoIQKysfbMLbU3VArSMaQp1/IwRS9eT4NijdpGoIeh2tmxD4nA+BM/dWHfjMucAw==", - "requires": { - "primer-support": "4.4.1" - } - }, - "primer-page-headers": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/primer-page-headers/-/primer-page-headers-1.4.1.tgz", - "integrity": "sha512-kmSi4Sys2dqt74sO1b5LcJq/EAnLe9p8t6oAs4PfkwgYXAdJPwHyTfe2+fueHYgqi07AlK3bnr1gw9rFen475Q==", - "requires": { - "primer-marketing-support": "1.3.1", - "primer-support": "4.4.1" - } - }, - "primer-page-sections": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/primer-page-sections/-/primer-page-sections-1.4.1.tgz", - "integrity": "sha512-WR5abovsjAKlbZjn4q7+eLCEA3gnwh/tuZDJnZ3l2V5O+IpHYVXI5Boi6QxbQM3mbHOL19NJhQEyfcHXBe7AQw==", - "requires": { - "primer-marketing-support": "1.3.1", - "primer-support": "4.4.1" - } - }, - "primer-product": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/primer-product/-/primer-product-5.4.1.tgz", - "integrity": "sha512-W1sVne7TPc2FF+c8GHqWnWtQQOzkr7R6mT1wwafwsY8GiCCkUxOtn0JPORrHzx67FV1GSWVMJ49F7pQPcl1Zcw==", - "requires": { - "primer-alerts": "1.5.1", - "primer-avatars": "1.4.1", - "primer-blankslate": "1.4.1", - "primer-labels": "1.5.1", - "primer-markdown": "3.7.1", - "primer-support": "4.4.1" - } - }, - "primer-support": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/primer-support/-/primer-support-4.4.1.tgz", - "integrity": "sha512-stQEoF4NfWy8JOVASUHxGY+Ot1eBguPH8rWoeLQy16zKzcS16kRccfvGbBXyv0G/aA+DdL8ZmjlXB2ubJ+wBJg==" - }, - "primer-table-object": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/primer-table-object/-/primer-table-object-1.4.1.tgz", - "integrity": "sha512-OkE3knDjLlzSot0/Q9O/b5GuKWTaxFyB/2CcZttA3WizAkxlkV4ql/Xy8mFr6WxBQORkBrrbxWUZC+Ulj88ZIQ==", - "requires": { - "primer-support": "4.4.1" - } - }, - "primer-tables": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/primer-tables/-/primer-tables-1.4.1.tgz", - "integrity": "sha512-Dk9ttoxIDigcJQ0vhh3VDkOL+/spGdEJacRFvNsysS4IbDOUDilXLZFcUZB2wCbhXFHS/CObk+/3zoW39J/6tg==", - "requires": { - "primer-marketing-support": "1.3.1", - "primer-support": "4.4.1" - } - }, - "primer-tooltips": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/primer-tooltips/-/primer-tooltips-1.4.1.tgz", - "integrity": "sha512-Id0g033elSx7Sy7+HDzha4Tuv24QxPzVtT15IHSOXXa900NjZqR1HHQIyMd1EkVfPt2sZ3Z0/k0cTvdqXs6eJQ==", - "requires": { - "primer-support": "4.4.1" - } - }, - "primer-truncate": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/primer-truncate/-/primer-truncate-1.4.1.tgz", - "integrity": "sha512-spBAPx7944txGXLCNGv+WZnPG4MiMhqn3srKlpGL2nmfcLxI+geviD+GhY/b42GyCgxRhcucrt/jxKYfsaBdww==", - "requires": { - "primer-support": "4.4.1" - } - }, - "primer-utilities": { - "version": "4.8.1", - "resolved": "https://registry.npmjs.org/primer-utilities/-/primer-utilities-4.8.1.tgz", - "integrity": "sha512-3wu8GUsJVVa0IIUdIKP+ZVSFPmX8v2NQWjBSJK00GbSPwthztOnFEoE40Ru7wAuUDNXWu9zMnx2i0lcyDu+yBg==", - "requires": { - "primer-support": "4.4.1" - } - }, - "process-nextick-args": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", - "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" - }, - "promise": { - "version": "7.3.1", - "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz", - "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==", - "requires": { - "asap": "~2.0.3" - } - }, - "pseudomap": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", - "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" - }, - "pug": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/pug/-/pug-2.0.3.tgz", - "integrity": "sha1-ccuoJTfJWl6rftBGluQiH1Oqh44=", - "requires": { - "pug-code-gen": "^2.0.1", - "pug-filters": "^3.1.0", - "pug-lexer": "^4.0.0", - "pug-linker": "^3.0.5", - "pug-load": "^2.0.11", - "pug-parser": "^5.0.0", - "pug-runtime": "^2.0.4", - "pug-strip-comments": "^1.0.3" - } - }, - "pug-attrs": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/pug-attrs/-/pug-attrs-2.0.3.tgz", - "integrity": "sha1-owlflw5kFR972tlX7vVftdeQXRU=", - "requires": { - "constantinople": "^3.0.1", - "js-stringify": "^1.0.1", - "pug-runtime": "^2.0.4" - } - }, - "pug-code-gen": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/pug-code-gen/-/pug-code-gen-2.0.1.tgz", - "integrity": "sha1-CVHsgyJddNjPxHan+Zolm199BQw=", - "requires": { - "constantinople": "^3.0.1", - "doctypes": "^1.1.0", - "js-stringify": "^1.0.1", - "pug-attrs": "^2.0.3", - "pug-error": "^1.3.2", - "pug-runtime": "^2.0.4", - "void-elements": "^2.0.1", - "with": "^5.0.0" - } - }, - "pug-error": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/pug-error/-/pug-error-1.3.2.tgz", - "integrity": "sha1-U659nSm7A89WRJOgJhCfVMR/XyY=" - }, - "pug-filters": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/pug-filters/-/pug-filters-3.1.0.tgz", - "integrity": "sha1-JxZVVbwEwjbkqisDZiRt+gIbYm4=", - "requires": { - "clean-css": "^4.1.11", - "constantinople": "^3.0.1", - "jstransformer": "1.0.0", - "pug-error": "^1.3.2", - "pug-walk": "^1.1.7", - "resolve": "^1.1.6", - "uglify-js": "^2.6.1" - } - }, - "pug-lexer": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/pug-lexer/-/pug-lexer-4.0.0.tgz", - "integrity": "sha1-IQwYRX7y4XYCQnQMXmR715TOwng=", - "requires": { - "character-parser": "^2.1.1", - "is-expression": "^3.0.0", - "pug-error": "^1.3.2" - } - }, - "pug-linker": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/pug-linker/-/pug-linker-3.0.5.tgz", - "integrity": "sha1-npp65ABWgtAn3uuWsAD4juuDoC8=", - "requires": { - "pug-error": "^1.3.2", - "pug-walk": "^1.1.7" - } - }, - "pug-load": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/pug-load/-/pug-load-2.0.11.tgz", - "integrity": "sha1-5kjlftET/iwfRdV4WOorrWvAFSc=", - "requires": { - "object-assign": "^4.1.0", - "pug-walk": "^1.1.7" - } - }, - "pug-parser": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/pug-parser/-/pug-parser-5.0.0.tgz", - "integrity": "sha1-45Stmz/KkxI5QK/4hcBuRKt+aOQ=", - "requires": { - "pug-error": "^1.3.2", - "token-stream": "0.0.1" - } - }, - "pug-runtime": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/pug-runtime/-/pug-runtime-2.0.4.tgz", - "integrity": "sha1-4XjhvaaKsujArPybztLFT9iM61g=" - }, - "pug-strip-comments": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/pug-strip-comments/-/pug-strip-comments-1.0.3.tgz", - "integrity": "sha1-8VWVkiBu3G+FMQ2s9K+0igJa9Z8=", - "requires": { - "pug-error": "^1.3.2" - } - }, - "pug-walk": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/pug-walk/-/pug-walk-1.1.7.tgz", - "integrity": "sha1-wA1cUSi6xYBr7BXSt+fNq+QlMfM=" - }, - "punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" - }, - "regenerator-runtime": { - "version": "0.11.1", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", - "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==" - }, - "repeat-string": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", - "integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc=" - }, - "require-dir": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/require-dir/-/require-dir-1.0.0.tgz", - "integrity": "sha512-PUJcQVTP4n6F8Un1GEEWhqnmBMfukVsL5gqwBxt7RF+nP+9hSOLJ/vSs5iUoXw1UWDgzqg9B/IIb15kfQKWsAQ==" - }, - "require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" - }, - "require-main-filename": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz", - "integrity": "sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE=" - }, - "resolve": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.8.1.tgz", - "integrity": "sha512-AicPrAC7Qu1JxPCZ9ZgCZlY35QgFnNqc+0LtbRNxnVw4TXvjQ72wnuL9JQcEBgXkI9JM8MsT9kaQoHcpCRJOYA==", - "requires": { - "path-parse": "^1.0.5" - } - }, - "right-align": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/right-align/-/right-align-0.1.3.tgz", - "integrity": "sha1-YTObci/mo1FWiSENJOFMlhSGE+8=", - "requires": { - "align-text": "^0.1.1" - } - }, - "safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" - }, - "set-blocking": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" - }, - "setprototypeof": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", - "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==" - }, - "shebang-command": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", - "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", - "requires": { - "shebang-regex": "^1.0.0" - } - }, - "shebang-regex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", - "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=" - }, - "shelljs": { - "version": "0.7.8", - "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.7.8.tgz", - "integrity": "sha1-3svPh0sNHl+3LhSxZKloMEjprLM=", - "requires": { - "glob": "^7.0.0", - "interpret": "^1.0.0", - "rechoir": "^0.6.2" - }, - "dependencies": { - "glob": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", - "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "rechoir": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", - "integrity": "sha1-hSBLVNuoLVdC4oyWdW70OvUOM4Q=", - "requires": { - "resolve": "^1.1.6" - } - }, - "resolve": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.8.1.tgz", - "integrity": "sha512-AicPrAC7Qu1JxPCZ9ZgCZlY35QgFnNqc+0LtbRNxnVw4TXvjQ72wnuL9JQcEBgXkI9JM8MsT9kaQoHcpCRJOYA==", - "requires": { - "path-parse": "^1.0.5" - } - } - } - }, - "showdown": { - "version": "1.8.6", - "resolved": "https://registry.npmjs.org/showdown/-/showdown-1.8.6.tgz", - "integrity": "sha1-kepO47elRIqspoIKTifmkMatdxw=", - "requires": { - "yargs": "^10.0.3" - } - }, - "signal-exit": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", - "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" - }, - "slash": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-1.0.0.tgz", - "integrity": "sha1-xB8vbDn8FtHNF61LXYlhFK5HDVU=" - }, - "source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" - }, - "sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" - }, - "statuses": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz", - "integrity": "sha1-+vUbnrdKrvOzrPStX2Gr8ky3uT4=" - }, - "string-width": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", - "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", - "requires": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" - } - }, - "strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "requires": { - "ansi-regex": "^2.0.0" - } - }, - "strip-eof": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", - "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=" - }, - "swagger-repo": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/swagger-repo/-/swagger-repo-1.5.1.tgz", - "integrity": "sha512-ZZD0clWcQNfbomM3DKHuFMLLePxSMDBumgGnrY1iGvSfkTRFX7lIU4H8H1cQsZtY9oJpJdhuzv35xpiGRvAULw==", - "requires": { - "body-parser": "^1.15.2", - "commander": "^2.9.0", - "cors": "^2.7.1", - "express": "^4.13.4", - "glob": "^7.0.0", - "js-yaml": "^3.5.3", - "json-pointer": "^0.6.0", - "jsonpath": "^1.0.0", - "lodash": "^4.5.0", - "mkdirp": "^0.5.1", - "require-dir": "^1.0.0", - "swagger-editor": "^2.10.3", - "swagger-ui": "^2.2.0", - "sway": "^1.0.0" - }, - "dependencies": { - "JSONSelect": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/JSONSelect/-/JSONSelect-0.4.0.tgz", - "integrity": "sha1-oI7cxn6z/L6Z7WMIVTRKDPKCu40=" - }, - "accepts": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz", - "integrity": "sha1-63d99gEXI6OxTopywIBcjoZ0a9I=", - "requires": { - "mime-types": "~2.1.18", - "negotiator": "0.6.1" - } - }, - "argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "requires": { - "sprintf-js": "~1.0.2" - } - }, - "asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" - }, - "body-parser": { - "version": "1.18.3", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz", - "integrity": "sha1-WykhmP/dVTs6DyDe0FkrlWlVyLQ=", - "requires": { - "bytes": "3.0.0", - "content-type": "~1.0.4", - "debug": "2.6.9", - "depd": "~1.1.2", - "http-errors": "~1.6.3", - "iconv-lite": "0.4.23", - "on-finished": "~2.3.0", - "qs": "6.5.2", - "raw-body": "2.3.3", - "type-is": "~1.6.16" - } - }, - "bytes": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", - "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" - }, - "chance": { - "version": "1.0.16", - "resolved": "https://registry.npmjs.org/chance/-/chance-1.0.16.tgz", - "integrity": "sha512-2bgDHH5bVfAXH05SPtjqrsASzZ7h90yCuYT2z4mkYpxxYvJXiIydBFzVieVHZx7wLH1Ag2Azaaej2/zA1XUrNQ==" - }, - "cjson": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/cjson/-/cjson-0.2.1.tgz", - "integrity": "sha1-c82KrWXZ4VBfmvF0TTt5wVJ2gqU=" - }, - "combined-stream": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.6.tgz", - "integrity": "sha1-cj599ugBrFYTETp+RFqbactjKBg=", - "requires": { - "delayed-stream": "~1.0.0" - } - }, - "cookie": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz", - "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s=" - }, - "core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" - }, - "deep-extend": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.4.2.tgz", - "integrity": "sha1-SLaZwn4zS/ifEIkr5DL25MfTSn8=" - }, - "delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" - }, - "depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" - }, - "deref": { - "version": "0.6.4", - "resolved": "http://registry.npmjs.org/deref/-/deref-0.6.4.tgz", - "integrity": "sha1-vVqW1F2+0wEbuBvfaN31S+jhvU4=", - "requires": { - "deep-extend": "^0.4.0" - } - }, - "drange": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/drange/-/drange-1.0.2.tgz", - "integrity": "sha512-bve7maXvfKW+vcsRpP8gzEDzkTg8O6AoCGvi/52pnllzhl/nmex8XLrHOUEQ42Z8GshcyftvG+E4s5vcd/qo0Q==" - }, - "escodegen": { - "version": "0.0.21", - "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-0.0.21.tgz", - "integrity": "sha1-U9ZSz6EDA4gnlFilJmxf/HCcY8M=", - "requires": { - "esprima": "~1.0.2", - "estraverse": "~0.0.4", - "source-map": ">= 0.1.2" - }, - "dependencies": { - "esprima": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-1.0.4.tgz", - "integrity": "sha1-n1V+CPw7TSbs6d00+Pv0drYlha0=" - } - } - }, - "esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" - }, - "esutils": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", - "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=" - }, - "etag": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" - }, - "express": { - "version": "4.16.3", - "resolved": "https://registry.npmjs.org/express/-/express-4.16.3.tgz", - "integrity": "sha1-avilAjUNsyRuzEvs9rWjTSL37VM=", - "requires": { - "accepts": "~1.3.5", - "array-flatten": "1.1.1", - "body-parser": "1.18.2", - "content-disposition": "0.5.2", - "content-type": "~1.0.4", - "cookie": "0.3.1", - "cookie-signature": "1.0.6", - "debug": "2.6.9", - "depd": "~1.1.2", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "1.1.1", - "fresh": "0.5.2", - "merge-descriptors": "1.0.1", - "methods": "~1.1.2", - "on-finished": "~2.3.0", - "parseurl": "~1.3.2", - "path-to-regexp": "0.1.7", - "proxy-addr": "~2.0.3", - "qs": "6.5.1", - "range-parser": "~1.2.0", - "safe-buffer": "5.1.1", - "send": "0.16.2", - "serve-static": "1.13.2", - "setprototypeof": "1.1.0", - "statuses": "~1.4.0", - "type-is": "~1.6.16", - "utils-merge": "1.0.1", - "vary": "~1.1.2" - }, - "dependencies": { - "body-parser": { - "version": "1.18.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz", - "integrity": "sha1-h2eKGdhLR9hZuDGZvVm84iKxBFQ=", - "requires": { - "bytes": "3.0.0", - "content-type": "~1.0.4", - "debug": "2.6.9", - "depd": "~1.1.1", - "http-errors": "~1.6.2", - "iconv-lite": "0.4.19", - "on-finished": "~2.3.0", - "qs": "6.5.1", - "raw-body": "2.3.2", - "type-is": "~1.6.15" - } - }, - "iconv-lite": { - "version": "0.4.19", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz", - "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ==" - }, - "qs": { - "version": "6.5.1", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", - "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==" - }, - "raw-body": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.2.tgz", - "integrity": "sha1-vNYMd9Prk83gBQKVw/N5OJvIj4k=", - "requires": { - "bytes": "3.0.0", - "http-errors": "1.6.2", - "iconv-lite": "0.4.19", - "unpipe": "1.0.0" - }, - "dependencies": { - "depd": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz", - "integrity": "sha1-V4O04cRZ8G+lyif5kfPQbnoxA1k=" - }, - "http-errors": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz", - "integrity": "sha1-CgAsyFcHGSp+eUbO7cERVfYOxzY=", - "requires": { - "depd": "1.1.1", - "inherits": "2.0.3", - "setprototypeof": "1.0.3", - "statuses": ">= 1.3.1 < 2" - } - }, - "setprototypeof": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz", - "integrity": "sha1-ZlZ+NwQ+608E2RvWWMDL77VbjgQ=" - } - } - }, - "statuses": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", - "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" - } - } - }, - "faker": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/faker/-/faker-3.1.0.tgz", - "integrity": "sha1-D5CPr05uwCUk5UpX5DLFwBPgjJ8=" - }, - "finalhandler": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", - "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", - "requires": { - "debug": "2.6.9", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "on-finished": "~2.3.0", - "parseurl": "~1.3.2", - "statuses": "~1.4.0", - "unpipe": "~1.0.0" - }, - "dependencies": { - "statuses": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", - "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" - } - } - }, - "form-data": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.2.tgz", - "integrity": "sha1-SXBJi+YEwgwAXU9cI67NIda0kJk=", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "1.0.6", - "mime-types": "^2.1.12" - } - }, - "formidable": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.2.1.tgz", - "integrity": "sha512-Fs9VRguL0gqGHkXS5GQiMCr1VhZBxz0JnJs4JmMp/2jL18Fmbzvv7vOFRU+U8TBkHEE/CX1qDXzJplVULgsLeg==" - }, - "fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" - }, - "glob": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", - "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "graphlib": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/graphlib/-/graphlib-2.1.5.tgz", - "integrity": "sha512-XvtbqCcw+EM5SqQrIetIKKD+uZVNQtDPD1goIg7K73RuRZtVI5rYMdcCVSHm/AS1sCBZ7vt0p5WgXouucHQaOA==", - "requires": { - "lodash": "^4.11.1" - } - }, - "http-errors": { - "version": "1.6.3", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", - "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", - "requires": { - "depd": "~1.1.2", - "inherits": "2.0.3", - "setprototypeof": "1.1.0", - "statuses": ">= 1.4.0 < 2" - } - }, - "iconv-lite": { - "version": "0.4.23", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz", - "integrity": "sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==", - "requires": { - "safer-buffer": ">= 2.1.2 < 3" - } - }, - "ipaddr.js": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.8.0.tgz", - "integrity": "sha1-6qM9bd16zo9/b+DJygRA5wZzix4=" - }, - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "jison": { - "version": "0.4.13", - "resolved": "https://registry.npmjs.org/jison/-/jison-0.4.13.tgz", - "integrity": "sha1-kEFwfWIkE2f1iDRTK58ZwsNvrHg=", - "requires": { - "JSONSelect": "0.4.0", - "cjson": "~0.2.1", - "ebnf-parser": "~0.1.9", - "escodegen": "0.0.21", - "esprima": "1.0.x", - "jison-lex": "0.2.x", - "lex-parser": "~0.1.3", - "nomnom": "1.5.2" - }, - "dependencies": { - "esprima": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-1.0.4.tgz", - "integrity": "sha1-n1V+CPw7TSbs6d00+Pv0drYlha0=" - } - } - }, - "js-yaml": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.12.0.tgz", - "integrity": "sha512-PIt2cnwmPfL4hKNwqeiuz4bKfnzHTBv6HyVgjahA6mPLwPDzjDWrplJBMjHUFxku/N3FlmrbyPclad+I+4mJ3A==", - "requires": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - } - }, - "json-refs": { - "version": "2.1.7", - "resolved": "https://registry.npmjs.org/json-refs/-/json-refs-2.1.7.tgz", - "integrity": "sha1-uesB/in16j6Sh48VrqEK04taz4k=", - "requires": { - "commander": "^2.9.0", - "graphlib": "^2.1.1", - "js-yaml": "^3.8.3", - "native-promise-only": "^0.8.1", - "path-loader": "^1.0.2", - "slash": "^1.0.0", - "uri-js": "^3.0.2" - } - }, - "json-schema-faker": { - "version": "0.2.16", - "resolved": "https://registry.npmjs.org/json-schema-faker/-/json-schema-faker-0.2.16.tgz", - "integrity": "sha1-UdPKSJVdj+c09ZHXR7ckU75aePI=", - "requires": { - "chance": "~1.0.1", - "deref": "~0.6.3", - "faker": "~3.1.0", - "randexp": "~0.4.2" - } - }, - "jsonpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/jsonpath/-/jsonpath-1.0.0.tgz", - "integrity": "sha1-Rc2dTE0NaCXZC9fkD4PxGCsT3Qc=", - "requires": { - "esprima": "1.2.2", - "jison": "0.4.13", - "static-eval": "2.0.0", - "underscore": "1.7.0" - }, - "dependencies": { - "esprima": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-1.2.2.tgz", - "integrity": "sha1-dqD9Zvz+FU/SkmZ9wmQBl1CxZXs=" - } - } - }, - "levn": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", - "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", - "requires": { - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2" - } - }, - "lodash": { - "version": "4.17.10", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.10.tgz", - "integrity": "sha512-UejweD1pDoXu+AD825lWwp4ZGtSwgnpZxb3JDViD7StjQz+Nb/6l093lx4OQ0foGWNRoc19mWy7BzL+UAK2iVg==" - }, - "mime": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz", - "integrity": "sha512-KI1+qOZu5DcW6wayYHSzR/tXKCDC5Om4s1z2QJjDULzLcmf3DvzS7oluY4HCTrc+9FiKmWUgeNLg7W3uIQvxtQ==" - }, - "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "native-promise-only": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/native-promise-only/-/native-promise-only-0.8.1.tgz", - "integrity": "sha1-IKMYwwy0X3H+et+/eyHJnBRy7xE=" - }, - "negotiator": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", - "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=" - }, - "optionator": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz", - "integrity": "sha1-NkxeQJ0/TWMB1sC0wFu6UBgK62Q=", - "requires": { - "deep-is": "~0.1.3", - "fast-levenshtein": "~2.0.4", - "levn": "~0.3.0", - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2", - "wordwrap": "~1.0.0" - } - }, - "path-loader": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/path-loader/-/path-loader-1.0.8.tgz", - "integrity": "sha512-/JQCrTcrteaPB8IHefEAQbmBQReKj51A+yTyc745TBbO4FOySw+/l3Rh0zyad0Nrd87TMROlmFANQwCRsuvN4w==", - "requires": { - "native-promise-only": "^0.8.1", - "superagent": "^3.8.3" - } - }, - "prelude-ls": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", - "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=" - }, - "proxy-addr": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.4.tgz", - "integrity": "sha512-5erio2h9jp5CHGwcybmxmVqHmnCBZeewlfJ0pex+UW7Qny7OOZXTtH56TGNyBizkgiOwhJtMKrVzDTeKcySZwA==", - "requires": { - "forwarded": "~0.1.2", - "ipaddr.js": "1.8.0" - } - }, - "qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" - }, - "randexp": { - "version": "0.4.9", - "resolved": "https://registry.npmjs.org/randexp/-/randexp-0.4.9.tgz", - "integrity": "sha512-maAX1cnBkzIZ89O4tSQUOF098xjGMC8N+9vuY/WfHwg87THw6odD2Br35donlj5e6KnB1SB0QBHhTQhhDHuTPQ==", - "requires": { - "drange": "^1.0.0", - "ret": "^0.2.0" - } - }, - "range-parser": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", - "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4=" - }, - "raw-body": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.3.tgz", - "integrity": "sha512-9esiElv1BrZoI3rCDuOuKCBRbuApGGaDPQfjSflGxdy4oyzqghxu6klEkkVIvBje+FF0BX9coEv8KqW6X/7njw==", - "requires": { - "bytes": "3.0.0", - "http-errors": "1.6.3", - "iconv-lite": "0.4.23", - "unpipe": "1.0.0" - } - }, - "readable-stream": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", - "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "ret": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/ret/-/ret-0.2.2.tgz", - "integrity": "sha512-M0b3YWQs7R3Z917WRQy1HHA7Ba7D8hvZg6UE5mLykJxQVE2ju0IXbGlaHPPlkY+WN7wFP+wUMXmBFA0aV6vYGQ==" - }, - "safe-buffer": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", - "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==" - }, - "send": { - "version": "0.16.2", - "resolved": "https://registry.npmjs.org/send/-/send-0.16.2.tgz", - "integrity": "sha512-E64YFPUssFHEFBvpbbjr44NCLtI1AohxQ8ZSiJjQLskAdKuriYEP6VyGEsRDH8ScozGpkaX1BGvhanqCwkcEZw==", - "requires": { - "debug": "2.6.9", - "depd": "~1.1.2", - "destroy": "~1.0.4", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "~1.6.2", - "mime": "1.4.1", - "ms": "2.0.0", - "on-finished": "~2.3.0", - "range-parser": "~1.2.0", - "statuses": "~1.4.0" - }, - "dependencies": { - "statuses": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", - "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" - } - } - }, - "serve-static": { - "version": "1.13.2", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.13.2.tgz", - "integrity": "sha512-p/tdJrO4U387R9oMjb1oj7qSMaMfmOyd4j9hOFoxZe2baQszgHcSWjuya/CiT5kgZZKRudHNOA0pYXOl8rQ5nw==", - "requires": { - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "parseurl": "~1.3.2", - "send": "0.16.2" - } - }, - "source-map": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", - "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==", - "optional": true - }, - "sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" - }, - "static-eval": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/static-eval/-/static-eval-2.0.0.tgz", - "integrity": "sha512-6flshd3F1Gwm+Ksxq463LtFd1liC77N/PX1FVVc3OzL3hAmo2fwHFbuArkcfi7s9rTNsLEhcRmXGFZhlgy40uw==", - "requires": { - "escodegen": "^1.8.1" - }, - "dependencies": { - "escodegen": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.11.0.tgz", - "integrity": "sha512-IeMV45ReixHS53K/OmfKAIztN/igDHzTJUhZM3k1jMhIZWjk45SMwAtBsEXiJp3vSPmTcu6CXn7mDvFHRN66fw==", - "requires": { - "esprima": "^3.1.3", - "estraverse": "^4.2.0", - "esutils": "^2.0.2", - "optionator": "^0.8.1", - "source-map": "~0.6.1" - } - }, - "esprima": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-3.1.3.tgz", - "integrity": "sha1-/cpRzuYTOJXjyI1TXOSdv/YqRjM=" - }, - "estraverse": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz", - "integrity": "sha1-De4/7TH81GlhjOc0IJn8GvoL2xM=" - }, - "source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "optional": true - } - } - }, - "statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } - }, - "superagent": { - "version": "3.8.3", - "resolved": "https://registry.npmjs.org/superagent/-/superagent-3.8.3.tgz", - "integrity": "sha512-GLQtLMCoEIK4eDv6OGtkOoSMt3D+oq0y3dsxMuYuDvaNUvuT8eFBuLmfR0iYYzHC1e8hpzC6ZsxbuP6DIalMFA==", - "requires": { - "component-emitter": "^1.2.0", - "cookiejar": "^2.1.0", - "debug": "^3.1.0", - "extend": "^3.0.0", - "form-data": "^2.3.1", - "formidable": "^1.2.0", - "methods": "^1.1.1", - "mime": "^1.4.1", - "qs": "^6.5.1", - "readable-stream": "^2.3.5" - }, - "dependencies": { - "debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", - "requires": { - "ms": "2.0.0" - } - } - } - }, - "swagger-editor": { - "version": "2.10.5", - "resolved": "https://registry.npmjs.org/swagger-editor/-/swagger-editor-2.10.5.tgz", - "integrity": "sha1-pDFsyw1Ap30w2t+R8PTbfkdflIo=" - }, - "swagger-methods": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/swagger-methods/-/swagger-methods-1.0.4.tgz", - "integrity": "sha512-xrKFLbrZ6VxRsg+M3uJozJtsEpNI/aPfZsOkoEjXw8vhAqdMIqwTYGj1f4dmUgvJvCdZhV5iArgtqXgs403ltg==" - }, - "sway": { - "version": "1.0.0", - "resolved": "http://registry.npmjs.org/sway/-/sway-1.0.0.tgz", - "integrity": "sha1-No/8Dpa9hCJu0bmzPWa+V9oE8Jo=", - "requires": { - "debug": "^2.2.0", - "js-base64": "^2.1.9", - "js-yaml": "^3.5.2", - "json-refs": "^2.1.5", - "json-schema-faker": "^0.2.8", - "lodash": "^4.2.0", - "native-promise-only": "^0.8.1", - "path-to-regexp": "^1.2.1", - "swagger-methods": "^1.0.0", - "swagger-schema-official": "2.0.0-bab6bed", - "z-schema": "^3.16.1" - }, - "dependencies": { - "isarray": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" - }, - "path-to-regexp": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.7.0.tgz", - "integrity": "sha1-Wf3g9DW62suhA6hOnTvGTpa5k30=", - "requires": { - "isarray": "0.0.1" - } - } - } - }, - "type-check": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", - "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", - "requires": { - "prelude-ls": "~1.1.2" - } - }, - "uri-js": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-3.0.2.tgz", - "integrity": "sha1-+QuFhQf4HepNz7s8TD2/orVX+qo=", - "requires": { - "punycode": "^2.1.0" - } - }, - "validator": { - "version": "10.7.1", - "resolved": "https://registry.npmjs.org/validator/-/validator-10.7.1.tgz", - "integrity": "sha512-tbB5JrTczfeHKLw3PnFRzGFlF1xUAwSgXEDb66EuX1ffCirspYpDEZo3Vc9j38gPdL4JKrDc5UPFfgYiw1IWRQ==" - }, - "z-schema": { - "version": "3.23.0", - "resolved": "https://registry.npmjs.org/z-schema/-/z-schema-3.23.0.tgz", - "integrity": "sha512-D8XV0BiHuQbWNEgu68RpjFZJ0C7jt+WYoszXKOohe54TdoTTauUvBQx+lsYCdalGIjGTFdQs5dxKvCUonUERzQ==", - "requires": { - "commander": "^2.7.1", - "lodash.get": "^4.0.0", - "lodash.isequal": "^4.0.0", - "validator": "^10.0.0" - } - } - } - }, - "swagger-schema-official": { - "version": "2.0.0-bab6bed", - "resolved": "https://registry.npmjs.org/swagger-schema-official/-/swagger-schema-official-2.0.0-bab6bed.tgz", - "integrity": "sha1-cAcEaNbSl3ylI3suUZyn0Gouo/0=" - }, - "swagger-ui": { - "version": "2.2.10", - "resolved": "https://registry.npmjs.org/swagger-ui/-/swagger-ui-2.2.10.tgz", - "integrity": "sha1-sl56IWZOXZC/OR2zDbCN5B6FLXs=" - }, - "to-fast-properties": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-1.0.3.tgz", - "integrity": "sha1-uDVx+k2MJbguIxsG46MFXeTKGkc=" - }, - "token-stream": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/token-stream/-/token-stream-0.0.1.tgz", - "integrity": "sha1-zu78cXp2xDFvEm0LnbqlXX598Bo=" - }, - "type-is": { - "version": "1.6.16", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz", - "integrity": "sha512-HRkVv/5qY2G6I8iab9cI7v1bOIdhm94dVjQCPFElW9W+3GeDOSHmy2EBYe4VTApuzolPcmgFTN3ftVJRKR2J9Q==", - "requires": { - "media-typer": "0.3.0", - "mime-types": "~2.1.18" - } - }, - "uc.micro": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-1.0.5.tgz", - "integrity": "sha512-JoLI4g5zv5qNyT09f4YAvEZIIV1oOjqnewYg5D38dkQljIzpPT296dbIGvKro3digYI1bkb7W6EP1y4uDlmzLg==" - }, - "uglify-js": { - "version": "2.8.29", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-2.8.29.tgz", - "integrity": "sha1-KcVzMUgFe7Th913zW3qcty5qWd0=", - "requires": { - "source-map": "~0.5.1", - "uglify-to-browserify": "~1.0.0", - "yargs": "~3.10.0" - }, - "dependencies": { - "camelcase": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-1.2.1.tgz", - "integrity": "sha1-m7UwTS4LVmmLLHWLCKPqqdqlijk=" - }, - "cliui": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-2.1.0.tgz", - "integrity": "sha1-S0dXYP+AJkx2LDoXGQMukcf+oNE=", - "requires": { - "center-align": "^0.1.1", - "right-align": "^0.1.1", - "wordwrap": "0.0.2" - } - }, - "source-map": { - "version": "0.5.7", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", - "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=" - }, - "window-size": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.1.0.tgz", - "integrity": "sha1-VDjNLqk7IC76Ohn+iIeu58lPnJ0=" - }, - "wordwrap": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz", - "integrity": "sha1-t5Zpu0LstAn4PVg8rVLKF+qhZD8=" - }, - "yargs": { - "version": "3.10.0", - "resolved": "http://registry.npmjs.org/yargs/-/yargs-3.10.0.tgz", - "integrity": "sha1-9+572FfdfB0tOMDnTvvWgdFDH9E=", - "requires": { - "camelcase": "^1.0.2", - "cliui": "^2.1.0", - "decamelize": "^1.0.0", - "window-size": "0.1.0" - } - } - } - }, - "uglify-to-browserify": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/uglify-to-browserify/-/uglify-to-browserify-1.0.2.tgz", - "integrity": "sha1-bgkk1r2mta/jSeOabWMoUKD4grc=", - "optional": true - }, - "underscore": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz", - "integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk=" - }, - "unpipe": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" - }, - "util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" - }, - "utils-merge": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" - }, - "vary": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" - }, - "void-elements": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/void-elements/-/void-elements-2.0.1.tgz", - "integrity": "sha1-wGavtYK7HLQSjWDqkjkulNXp2+w=" - }, - "which": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", - "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", - "requires": { - "isexe": "^2.0.0" - } - }, - "with": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/with/-/with-5.1.1.tgz", - "integrity": "sha1-+k2qktrzLE6pTtRTyB8EaGtXXf4=", - "requires": { - "acorn": "^3.1.0", - "acorn-globals": "^3.0.0" - } - }, - "wordwrap": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", - "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=" - }, - "wrap-ansi": { - "version": "2.1.0", - "resolved": "http://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", - "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", - "requires": { - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1" - } - }, - "wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" - }, - "y18n": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz", - "integrity": "sha1-bRX7qITAhnnA136I53WegR4H+kE=" - }, - "yallist": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", - "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" - }, - "yargs": { - "version": "10.1.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-10.1.2.tgz", - "integrity": "sha512-ivSoxqBGYOqQVruxD35+EyCFDYNEFL/Uo6FcOnz+9xZdZzK0Zzw4r4KhbrME1Oo2gOggwJod2MnsdamSG7H9ig==", - "requires": { - "cliui": "^4.0.0", - "decamelize": "^1.1.1", - "find-up": "^2.1.0", - "get-caller-file": "^1.0.1", - "os-locale": "^2.0.0", - "require-directory": "^2.1.1", - "require-main-filename": "^1.0.1", - "set-blocking": "^2.0.0", - "string-width": "^2.0.0", - "which-module": "^2.0.0", - "y18n": "^3.2.1", - "yargs-parser": "^8.1.0" - }, - "dependencies": { - "ansi-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", - "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=" - }, - "cliui": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-4.1.0.tgz", - "integrity": "sha512-4FG+RSG9DL7uEwRUZXZn3SS34DiDPfzP0VOiEwtUWlE+AR2EIg+hSyvrIgUUfhdgR/UkAeW2QHgeP+hWrXs7jQ==", - "requires": { - "string-width": "^2.1.1", - "strip-ansi": "^4.0.0", - "wrap-ansi": "^2.0.0" - } - }, - "find-up": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", - "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", - "requires": { - "locate-path": "^2.0.0" - } - }, - "is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" - }, - "os-locale": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-2.1.0.tgz", - "integrity": "sha512-3sslG3zJbEYcaC4YVAvDorjGxc7tv6KVATnLPZONiljsUncvihe9BQoVCEs0RZ1kmf4Hk9OBqlZfJZWI4GanKA==", - "requires": { - "execa": "^0.7.0", - "lcid": "^1.0.0", - "mem": "^1.1.0" - } - }, - "string-width": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", - "requires": { - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^4.0.0" - } - }, - "strip-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", - "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", - "requires": { - "ansi-regex": "^3.0.0" - } - }, - "which-module": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", - "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" - } - } - }, - "yargs-parser": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-8.1.0.tgz", - "integrity": "sha512-yP+6QqN8BmrgW2ggLtTbdrOyBNSI7zBa4IykmiV5R1wl1JWNxQvWhMfMdmzIYtKU7oP3OOInY/tl2ov3BDjnJQ==", - "requires": { - "camelcase": "^4.1.0" - }, - "dependencies": { - "camelcase": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz", - "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=" - } - } - } - } -} diff --git a/package.json b/package.json deleted file mode 100644 index dabfe3c46..000000000 --- a/package.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "Data-Repository-Service-openapi-spec", - "version": "1.1.0", - "dependencies": { - "bower": "^1.7.7", - "connect": "^3.4.1", - "cors": "^2.7.1", - "github-markdown": "^3.2.0", - "github-markdown-css": "^2.10.0", - "help": "^3.0.2", - "portfinder": "^1.0.3", - "shelljs": "^0.7.0", - "showdown": "^1.8.6", - "swagger-repo": "^1.5.1", - "swagger-ui": "^2.1.4" - }, - "private": true, - "scripts": { - "build": "node ./scripts/buildui.js", - "swagger": "swagger-repo" - } -} diff --git a/pages/more-background-on-compact-identifiers/openapi.yaml b/pages/more-background-on-compact-identifiers/openapi.yaml new file mode 100644 index 000000000..929b023e5 --- /dev/null +++ b/pages/more-background-on-compact-identifiers/openapi.yaml @@ -0,0 +1,29 @@ +openapi: 3.0.3 +info: + title: More Background on Compact Identifiers + version: 1.2.0 + x-logo: + url: 'https://www.ga4gh.org/wp-content/themes/ga4gh-theme/gfx/GA-logo-horizontal-tag-RGB.svg' + termsOfService: 'https://www.ga4gh.org/terms-and-conditions/' + contact: + name: GA4GH Cloud Work Stream + email: ga4gh-cloud@ga4gh.org + license: + name: Apache 2.0 + url: 'https://raw.githubusercontent.com/ga4gh/data-repository-service-schemas/master/LICENSE' +tags: + - name: About + description: + $ref: ./tags/About.md + - name: Background on Compact Identifier-Based URIs + description: + $ref: ./tags/BackgroundOnCompactIdentiferBasedURIs.md + - name: Registering a DRS Server on a Meta-Resolver + description: + $ref: ./tags/RegisteringOnMetaResolver.md + - name: Example DRS Client Compact Identifier-Based URI Resolution Process - Existing Compact Identifier Provider + description: + $ref: ./tags/ExampleExistingProvider.md + - name: Example DRS Client Compact Identifier-Based URI Resolution Process - Registering a new Compact Identifier for Your DRS Server + description: + $ref: ./tags/ExampleRegisterIdentifier.md diff --git a/pages/more-background-on-compact-identifiers/tags/About.md b/pages/more-background-on-compact-identifiers/tags/About.md new file mode 100644 index 000000000..5099f8955 --- /dev/null +++ b/pages/more-background-on-compact-identifiers/tags/About.md @@ -0,0 +1 @@ +This document contains more examples of resolving compact identifier-based DRS URIs than we could fit in the DRS specification or appendix. It’s provided here for your reference as a supplement to the specification. diff --git a/pages/more-background-on-compact-identifiers/tags/BackgroundOnCompactIdentiferBasedURIs.md b/pages/more-background-on-compact-identifiers/tags/BackgroundOnCompactIdentiferBasedURIs.md new file mode 100644 index 000000000..4dfcf1838 --- /dev/null +++ b/pages/more-background-on-compact-identifiers/tags/BackgroundOnCompactIdentiferBasedURIs.md @@ -0,0 +1,32 @@ +Compact identifiers refer to locally-unique persistent identifiers that have been namespaced to provide global uniqueness. See ["Uniform resolution of compact identifiers for biomedical data"](https://www.biorxiv.org/content/10.1101/101279v3) for an excellent introduction to this topic. By using compact identifiers in DRS URIs, along with a resolver registry (identifiers.org/n2t.net), systems can identify the current resolver when they need to translate a DRS URI into a fetchable URL. This allows a project to issue compact identifiers in DRS URIs and not be concerned if the project name or DRS hostname changes in the future, the current resolver can always be found through the identifiers.org/n2t.net registries. Together the identifiers.org/n2t.net systems support the resolver lookup for over 700 compact identifiers formats used in the research community, making it possible for a DRS server to use any of these as DRS IDs (or to register a new compact identifier type and resolver service of their own). + +We use a DRS URI scheme rather than [Compact URIs (CURIEs)](https://en.wikipedia.org/wiki/CURIE) directly since we feel that systems consuming DRS objects will be able to better differentiate a DRS URI. CURIEs are widely used in the research community and we feel the fact that they can point to a wide variety of entities (HTML documents, PDFs, identities in data models, etc) makes it more difficult for systems to unambiguously identify entities as DRS objects. + +Still, to make compact identifiers work in DRS URIs we leverage the CURIE format used by identifiers.org/n2t.net. Compact identifiers have the form: + +``` +prefix:accession +``` + +The prefix can be divided into a `provider_code` (optional) and `namespace`. The `accession` here is an Ark, DOI, Data GUID, or another issuers’s local ID for the object being pointed to: + +``` +[provider_code/]namespace:accession +``` + +Both the `provider_code` and `namespace` disallow spaces or punctuation, only lowercase alphanumerical characters, underscores and dots are allowed. + +[Examples](https://n2t.net/e/compact_ids.html) include (from n2t.net): + +``` +PDB:2gc4 +Taxon:9606 +DOI:10.5281/ZENODO.1289856 +ark:/47881/m6g15z54 +IGSN:SSH000SUA +``` + +Tip: +> DRS URIs using compact identifiers with resolvers registered in identifiers.org/n2t.net can be distinguished from the hostname-based DRS URIs below based on the required ":" which is not allowed in hostname-based URI. + +See the documentation on [n2t.net](https://n2t.net/e/compact_ids.html) and [identifiers.org](https://docs.identifiers.org/) for much more information on the compact identifiers used there and details about the resolution process. diff --git a/pages/more-background-on-compact-identifiers/tags/ExampleExistingProvider.md b/pages/more-background-on-compact-identifiers/tags/ExampleExistingProvider.md new file mode 100644 index 000000000..bf269bef0 --- /dev/null +++ b/pages/more-background-on-compact-identifiers/tags/ExampleExistingProvider.md @@ -0,0 +1,43 @@ +A DRS client identifies the a DRS URI compact identifier components using the first occurance of "/" (optional) and ":" characters. These are not allowed inside the provider_code (optional) or the namespace. The ":" character is not allowed in a Hostname-based DRS URI, providing a convenient mechanism to differentiate them. Once the provider_code (optional) and namespace are extracted from a DRS compact identifier-based URI, a client can use services on identifiers.org to identify available resolvers. + +*Let’s look at a specific example DRS compact identifier-based URI that uses DOIs, a popular compact identifier, and walk through the process that a client would use to resolve it. Keep in mind, the resolution process is the same from the client perspective if a given DRS server is using an existing compact identifier type (DOIs, ARKs, Data GUIDs) or creating their own compact identifier type for their DRS server and registering it on identifiers.org/n2t.net.* + +Starting with the DRS URI: + +``` +drs://doi:10.5072/FK2805660V +``` + +with a namespace of "doi", the following GET request will return information about the namespace: + +``` +GET https://registry.api.identifiers.org/restApi/namespaces/search/findByPrefix?prefix=doi +``` + +This information then points to resolvers for the "doi" namespace. This "doi" namespace was assigned a namespace ID of 75 by identifiers.org. This "id" has nothing to do with compact identifier accessions (which are used in the URL pattern as `{$id}` below) or DRS IDs. This namespace ID (75 below) is purely an identifiers.org internal ID for use with their APIs: + +``` +GET https://registry.api.identifiers.org/restApi/resources/search/findAllByNamespaceId?id=75 +``` + +This returns enough information to, ultimately, identify one or more resolvers and each have a URL pattern that, for DRS-supporting systems, provides a URL template for making a successful DRS GET request. For example, the DOI urlPattern is: + +``` +urlPattern: "https://doi.org/{$id}" +``` + +And the `{$id}` here refers to the accession from the compact identifier (in this example the accession is `10.5072/FK2805660V`). If applicable, a provide code can be supplied in the above requests to specify a particular mirror if there are multiple resolvers for this namespace. In the case of DOIs, you only get a single resolver. + +Given this information you now know you can make a GET on the URL: + +``` +GET https://doi.org/10.5072/FK2805660V +``` + +*The URL above is valid for a DOI object but it is not actually a DRS server! Instead, it redirects to a DRS server through a series of HTTPS redirects. This is likely to be common when working with existing compact identifiers like DOIs or ARKs. Regardless, the redirect should eventually lead to a DRS URL that percent-encodes the accession as a DRS ID in a DRS object API call. For a **hypothetical** example, here’s what a redirect to a DRS API URL might ultimately look. A client doesn’t have to do anything other than follow the HTTPS redirects. The link between the DOI resolver on doi.org and the DRS server URL below is the result of the DRS server registering their data objects with a DOI issuer.* + +``` +GET https://drs.example.org/ga4gh/drs/v1/objects/10.5072%2FFK2805660V +``` + +IDs in DRS hostname-based URIs/URLs are always percent-encoded to eliminate ambiguity even though the DRS compact identifier-based URIs and the identifier.orgs API do not percent-encode accessions. This was done in order to 1) follow the CURIE conventions of identifiers.org/n2t.net for compact identifier-based DRS URIs and 2) to aid in readability for users who understand they are working with compact identifiers. **The general rule of thumb, when using a compact identifier accession as a DRS ID in a DRS API call, make sure to percent-encode it. An easy way for a DRS client to handle this is to get the initial DRS object JSON response from whatever redirects the compact identifier resolves to, then look for the** `self_uri` **in the JSON, which will give you the correctly percent-encoded DRS ID for subsequent DRS API calls such as the** `access` **method.** diff --git a/pages/more-background-on-compact-identifiers/tags/ExampleRegisterIdentifier.md b/pages/more-background-on-compact-identifiers/tags/ExampleRegisterIdentifier.md new file mode 100644 index 000000000..f08eaa4e8 --- /dev/null +++ b/pages/more-background-on-compact-identifiers/tags/ExampleRegisterIdentifier.md @@ -0,0 +1,49 @@ +See the documentation on [n2t.net](https://n2t.net/e/compact_ids.html) and [identifiers.org](https://docs.identifiers.org/) for adding your own compact identifier type and registering your DRS server as a resolver. We document this in more detail in the [main specification document](./index.html). + +Now the question is how does a client resolve your newly registered compact identifier for your DRS server? *It turns out, whether specific to a DRS implementation or using existing compact identifiers like ARKs or DOIs, the DRS client resolution process for compact identifier-based URIs is exactly the same.* We briefly run through process below for a new compact identifier as an example but, again, a client will not need to do anything different from the resolution process documented in "DRS Client Compact Identifier-Based URI Resolution Process - Existing Compact Identifier Provider". + +Now we can issue DRS URI for our data objects like: + +``` +drs://mydrsprefix:12345 +``` + +This is a little simpler than working with DOIs or other existing compact identifier issuers out there since we can create our own IDs and not have to allocate them through a third-party service (see "Issuing Existing Compact Identifiers for Use with Your DRS Server" below). + +With a namespace of "mydrsprefix", the following GET request will return information about the namespace: + +``` +GET https://registry.api.identifiers.org/restApi/namespaces/search/findByPrefix?prefix=mydrsprefix +``` + +*Of course, this is a hypothetical example so the actual API call won’t work but you can see the GET request is identical to "DRS Client Compact Identifier-Based URI Resolution Process - Existing Compact Identifier Provider".* + +This information then points to resolvers for the "mydrsprefix" namespace. Hypothetically, this "mydrsprefix" namespace was assigned a namespace ID of 1829 by identifiers.org. This "id" has nothing to do with compact identifier accessions (which are used in the URL pattern as `{$id}` below) or DRS IDs. This namespace ID (1829 below) is purely an identifiers.org internal ID for use with their APIs: + +``` +GET https://registry.api.identifiers.org/restApi/resources/search/findAllByNamespaceId?id=1829 +``` + +*Like the previous GET request this URL won’t work but you can see the GET request is identical to "DRS Client Compact Identifier-Based URI Resolution Process - Existing Compact Identifier Provider".* + +This returns enough information to, ultimately, identify one or more resolvers and each have a URL pattern that, for DRS-supporting systems, provides a URL template for making a successful DRS GET request. For example, the "mydrsprefix" urlPattern is: + +``` +urlPattern: "https://mydrs.server.org/ga4gh/drs/v1/objects/{$id}" +``` + +And the `{$id}` here refers to the accession from the compact identifier (in this example the accession is `12345`). If applicable, a provide code can be supplied in the above requests to specify a particular mirror if there are multiple resolvers for this namespace. + +Given this information you now know you can make a GET on the URL: + +``` +GET https://mydrs.server.org/ga4gh/drs/v1/objects/12345 +``` + +So, compared to using a third party service like DOIs and ARKs, this would be a direct pointer to a DRS server. However, just as with "DRS Client Compact Identifier-Based URI Resolution Process - Existing Compact Identifier Provider", the client should always be prepared to follow HTTPS redirects. + +*To summarize, a client resolving a custom compact identifier registered for a single DRS server is actually the same as resolving using a third-party compact identifier service like ARKs or DOIs with a DRS server, just make sure to follow redirects in all cases.* + +**Note: Issuing Existing Compact Identifiers for Use with Your DRS Server** + +See the documentation on [n2t.net](https://n2t.net/e/compact_ids.html) and [identifiers.org](https://docs.identifiers.org/) for information about all the compact identifiers that are supported. You can choose to use an existing compact identifier provider for your DRS server, as we did in the example above using DOIs ("DRS Client Compact Identifier-Based URI Resolution Process - Existing Compact Identifier Provider"). Just keep in mind, each provider will have their own approach for generating compact identifiers and associating them with a DRS data object URL. Some compact identifier providers, like DOIs, provide a method whereby you can register in their network and get your own prefix, allowing you to mint your own accessions. Other services, like the University of California’s [EZID](https://ezid.cdlib.org/) service, provide accounts and a mechanism to mint accessions centrally for each of your data objects. For experimentation we recommend you take a look at the EZID website that allows you to create DOIs and ARKs and associate them with your data object URLs on your DRS server for testing purposes. diff --git a/pages/more-background-on-compact-identifiers/tags/RegisteringOnMetaResolver.md b/pages/more-background-on-compact-identifiers/tags/RegisteringOnMetaResolver.md new file mode 100644 index 000000000..9ba58b635 --- /dev/null +++ b/pages/more-background-on-compact-identifiers/tags/RegisteringOnMetaResolver.md @@ -0,0 +1,9 @@ +See the documentation on the [n2t.net](https://n2t.net/e/compact_ids.html) and [identifiers.org](https://docs.identifiers.org/) meta-resolvers for adding your own compact identifier type and registering your DRS server as a resolver. You can register new prefixes (or mirrors by adding resource provider codes) for free using a simple online form. + +Keep in mind, while anyone can register prefixes, the identifiers.org/n2t.net sites do basic hand curation to verify new prefix and resource (provider code) requests. See those sites for more details on their security practices. For more information see + +Starting with the prefix for our new compact identifier, let’s register the namespace `mydrsprefix` on identifiers.org/n2t.net and use 5-digit numeric IDs as our accessions. We will then link this to the DRS server at https://mydrs.server.org/ga4gh/drs/v1/ by filling in the provider details. Here’s what that the registration for our new namespace looks like on [identifiers.org](https://registry.identifiers.org/prefixregistrationrequest): + +![Prefix Register 1](/data-repository-service-schemas/public/img/prefix_register_1.png) + +![Prefix Register 2](/data-repository-service-schemas/public/img/prefix_register_2.png) diff --git a/docs/asciidoc/figure1.png b/public/img/figure1.png similarity index 100% rename from docs/asciidoc/figure1.png rename to public/img/figure1.png diff --git a/docs/asciidoc/figure2.png b/public/img/figure2.png similarity index 100% rename from docs/asciidoc/figure2.png rename to public/img/figure2.png diff --git a/docs/asciidoc/figure3.png b/public/img/figure3.png similarity index 100% rename from docs/asciidoc/figure3.png rename to public/img/figure3.png diff --git a/docs/asciidoc/prefix_register_1.png b/public/img/prefix_register_1.png similarity index 100% rename from docs/asciidoc/prefix_register_1.png rename to public/img/prefix_register_1.png diff --git a/docs/asciidoc/prefix_register_2.png b/public/img/prefix_register_2.png similarity index 100% rename from docs/asciidoc/prefix_register_2.png rename to public/img/prefix_register_2.png diff --git a/python/examples/compliance_testing.ipynb b/python/examples/compliance_testing.ipynb deleted file mode 100644 index 79a4c9c20..000000000 --- a/python/examples/compliance_testing.ipynb +++ /dev/null @@ -1,289 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# DOS Compliance Testing\n", - "\n", - "The ga4gh-dos-schemas packages includes a number of \"compliance tests\" - a standardized test harness that can be used to evaluate the compliance of a given service to the Data Object Service schema. The compliance tests can supplement the test suite of a DOS implementation, or can be pointed at a remote DOS endpoint. \n", - "\n", - "The compliance tests are straightforward to use and can generally be integrated into your project after implementing only one or two methods - one method to make requests to the DOS implementation under test, and another to prepare that implementation for testing.\n", - "\n", - "Let's start with a simple example. If you wanted to test a remote DOS implementation, you could write something like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from ga4gh.dos.test.compliance import AbstractComplianceTest\n", - "import json\n", - "import requests\n", - "import unittest\n", - "\n", - "basepath = 'https://dos.commons.ucsc-cgp-dev.org/ga4gh/dos/v1'\n", - "\n", - "\n", - "class RemoteTest(AbstractComplianceTest):\n", - " @classmethod\n", - " def _make_request(self, meth, path, headers=None, body=None):\n", - " # Where :param:`path` is like `/dataobjects` or `/databundles/{data_bundle_id}`\n", - " # Harcoded access token\n", - " headers = headers or {}\n", - " headers['access_token'] = 'f4ce9d3d23f4ac9dfdc3c825608dc660'\n", - " # Make the request here - :param:`body` is a JSON-formatted string. We\n", - " # convert it here so that Requests will automatically add the right headers.\n", - " r = requests.request(method=meth, url=basepath + path, headers=headers, json=json.loads(body))\n", - " return r.content, r.status_code" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To test the implementation at `https://dos.commons.ucsc-cgp-dev.org/`, all we need to do is implement the `_make_request` method that makes an (authenticated) request to the chosen endpoint given a DOS endpoint (such as `/databundles` or `/service-info`), a method, and request content.\n", - "\n", - "Before we run the tests, there's one thing to note - the service that we are testing (dos-azul-lambda dev) does not have data bundle support yet. Luckily, we can pare down what tests we run based on what features the implementation under test supports by setting the `supports` class variable like so:" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "RemoteTest.supports = ['GetDataObject', 'ListDataObjects', 'UpdateDataObject']" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now, only compliance tests that utilize the `GetDataObject`, `ListDataObjects`, and `UpdateDataObject` DOS operations will run. With that handled, we can run the tests.\n", - "\n", - "`AbstractComplianceTest` subclasses `unittest.TestCase`, so we can run it using a test runner like nose or by using the unittest command hook (e.g. `python -m unittest ...`). For the purposes of running the notebook, we can set up a test suite and run it like so:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "..s.ss..F.....\n", - "======================================================================\n", - "FAIL: test_list_data_objects_by_checksum (__main__.RemoteTest)\n", - "----------------------------------------------------------------------\n", - "Traceback (most recent call last):\n", - " File \"/home/natan/cgl/data-object-service-schemas/python/ga4gh/dos/test/compliance.py\", line 55, in wrapper\n", - " return func(self)\n", - " File \"/home/natan/cgl/data-object-service-schemas/python/ga4gh/dos/test/compliance.py\", line 317, in test_list_data_objects_by_checksum\n", - " self.assertEqual(len(r['data_objects']), 1)\n", - "AssertionError: 10 != 1\n", - "\n", - "----------------------------------------------------------------------\n", - "Ran 14 tests in 14.933s\n", - "\n", - "FAILED (failures=1, skipped=3)\n" - ] - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import sys\n", - "import logging\n", - "\n", - "# First, we need to tweak the normal test logging structure to account\n", - "# for the fact we're running in a notebook...\n", - "handler = logging.getLogger().handlers[0]\n", - "handler.setLevel(logging.WARNING)\n", - "\n", - "# Now, set up the testing harness\n", - "suite = unittest.TestLoader().loadTestsFromTestCase(RemoteTest)\n", - "# By default, TextTestRunner outputs to sys.stderr.\n", - "runner = unittest.TextTestRunner(verbosity=1, stream=sys.stdout)\n", - "runner.run(suite)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The data in the dev instance can vary, but all the tests that are run should generally pass. Looking at the output, you can see the `s` for tests that were skipped because they weren't specified in the `supports` parameter.\n", - "\n", - "This pattern is easily extended to DOS implementations built on Chalice:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "# import chalice\n", - "# from my_chalice_app import chalice_app\n", - "\n", - "# We don't have a Chalice app on hand; this code is a non-functional example\n", - "# and will fail if run. We set the `chalice` and `my_chalice_app` names so that\n", - "# the code runs and doesn't make a fuss...\n", - "chalice = {}\n", - "my_chalice_app = {}\n", - "\n", - "class ChaliceTest(AbstractComplianceTest):\n", - " @classmethod\n", - " def setUpClass(cls):\n", - " cls.lg = chalice.LocalGateway(chalice_app, chalice.Config())\n", - "\n", - " @classmethod\n", - " def _make_request(self, meth, path, headers=None, body=None):\n", - " headers = headers or {}\n", - " r = self.lg.handle_request(method=meth, path='/ga4gh/dos/v1' + path,\n", - " headers=headers, body=body)\n", - " return r['body'], r['statusCode']" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "A similar implementation to the above is used in [DataBiosphere/dos-azul-lambda](https://github.com/DataBiosphere/dos-azul-lambda).\n", - "\n", - "This pattern can also be easily extended to test implementations built on Flask. (The reference server included in `ga4gh.dos.server` is built on Connexion, which is itself built on Flask. We test it against the compliance tests in addition to the other integration tests, so it makes a good example. You can check out [Travis](https://travis-ci.org/ga4gh/data-object-service-schemas) to see the latest build results.)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "import werkzeug.datastructures\n", - "\n", - "import ga4gh.dos.server\n", - "from ga4gh.dos.test.compliance import AbstractComplianceTest\n", - "\n", - "class FlaskTest(AbstractComplianceTest):\n", - " @classmethod\n", - " def setUpClass(cls):\n", - " # :mod:`ga4gh.dos.server` is built on top of :mod:`connexion`,\n", - " # which is built on top of :mod:`flask`, which is built on top\n", - " # of :mod:`werkzeug`, which means we can do some cool nice\n", - " # things with testing.\n", - " app = ga4gh.dos.server.configure_app().app\n", - " cls.client = app.test_client()\n", - "\n", - " # Populate our new server with some test data objects and bundles\n", - " for data_obj in cls.generate_data_objects(250):\n", - " cls.dos_request('POST', '/dataobjects', body={'data_object': data_obj})\n", - " for data_bdl in cls.generate_data_bundles(250):\n", - " cls.dos_request('POST', '/databundles', body={'data_bundle': data_bdl})\n", - "\n", - " @classmethod\n", - " def _make_request(cls, meth, path, headers=None, body=None):\n", - " # For documentation on this function call, see\n", - " # :class:`werkzeug.test.EnvironBuilder` and :meth:`werkzeug.test.Client.get`.\n", - " headers = werkzeug.datastructures.Headers(headers)\n", - " r = cls.client.open(method=meth, path='/ga4gh/dos/v1' + path,\n", - " data=body, headers=headers)\n", - " return r.data, r.status_code" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And if we run it, all tests should pass:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "............" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ERROR:connexion.decorators.validation:http://localhost/ga4gh/dos/v1/dataobjects/c610f6ec-ce5d-11e8-9acb-484520e6c2c4 validation error: 'data_object' is a required property\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "..\n", - "----------------------------------------------------------------------\n", - "Ran 14 tests in 7.360s\n", - "\n", - "OK\n" - ] - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import sys\n", - "suite = unittest.TestLoader().loadTestsFromTestCase(FlaskTest)\n", - "runner.run(suite)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In conclusion, the compliance tests provide an easy way to supplement existing test suites by testing the compliance of a DOS implementation under test to the Data Object Service schema. This is part of a larger effort to automate the testing and compliance of DOS implementations that is being actively developed that we hope will streamline development and use of DOS." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/python/examples/demo.py b/python/examples/demo.py deleted file mode 100644 index 0eb9c836a..000000000 --- a/python/examples/demo.py +++ /dev/null @@ -1,275 +0,0 @@ -# With app.py running start this demo -from ga4gh.dos.client import Client - - -def main(): - local_client = Client('http://localhost:8080/ga4gh/dos/v1/') - client = local_client.client - models = local_client.models - - # CreateDataObject - print("..........Create an object............") - Checksum = models.get_model('Checksum') - URL = models.get_model('URL') - CreateDataObjectRequest = models.get_model('CreateDataObjectRequest') - DataObject = models.get_model('CreateDataObjectRequest') - create_data_object = DataObject( - name="abc", - size="12345", - checksums=[Checksum(checksum="def", type="md5")], - urls=[URL(url="a"), URL(url="b")]) - create_request = CreateDataObjectRequest(data_object=create_data_object) - create_response = client.CreateDataObject(body=create_request).result() - data_object_id = create_response['data_object_id'] - print(data_object_id) - - # GetDataObject - print("..........Get the Object we just created..............") - get_object_response = client.GetDataObject( - data_object_id=data_object_id).result() - data_object = get_object_response.data_object - print((data_object.id)) - - # UpdateDataObject - print("..........Update that object.................") - UpdateDataObjectRequest = models.get_model('UpdateDataObjectRequest') - update_data_object = DataObject( - name="abc", - size="12345", - checksums=[Checksum(checksum="def", type="md5")], - urls=[URL(url="a"), URL(url="b"), URL(url="c")]) - update_request = UpdateDataObjectRequest(data_object=update_data_object) - update_response = client.UpdateDataObject( - data_object_id=data_object_id, body=update_request).result() - updated_object = client.GetDataObject( - data_object_id=update_response['data_object_id']).result().data_object - print((updated_object.version)) - - # Get the old DataObject - print("..........Get the old Data Object.................") - old_data_object = client.GetDataObject( - data_object_id=update_response['data_object_id'], - version=data_object.version).result().data_object - print((old_data_object.version)) - - # ListDataObjects - print("..........List Data Objects...............") - ListDataObjectsRequest = models.get_model('ListDataObjectsRequest') - list_request = ListDataObjectsRequest() - list_response = client.ListDataObjects(body=list_request).result() - print((len(list_response.data_objects))) - - # Get all versions of a DataObject - print("..........Get all Versions...............") - versions_response = client.GetDataObjectVersions( - data_object_id=old_data_object.id).result() - print((len(versions_response.data_objects))) - - # DeleteDataObject - print("..........Delete the Object...............") - delete_response = client.DeleteDataObject( - data_object_id=data_object_id).result() - print((delete_response.data_object_id)) - try: - client.GetDataObject( - data_object_id=update_response['data_object_id']).result() - except Exception as e: - print(('The object no longer exists, 404 not found. {}'.format(e))) - - # Create a Data Object specifying your own version - print(".......Create a Data Object with our own version..........") - my_data_object = DataObject( - name="abc", - size="12345", - checksums=[Checksum(checksum="def", type="md5")], - urls=[URL(url="a"), URL(url="b")], - version="great-version") - create_request = CreateDataObjectRequest(data_object=my_data_object) - create_response = client.CreateDataObject(body=create_request).result() - data_object_id = create_response['data_object_id'] - data_object = client.GetDataObject( - data_object_id=data_object_id).result().data_object - print((data_object.version)) - - # Create a Data Object specifying your own ID - print("..........Create a Data Object with our own ID..............") - my_data_object = DataObject( - id="myid", - file_name="abc", - checksums=[Checksum(checksum="def", type="md5")], - urls=[URL(url="a"), URL(url="b")]) - create_request = CreateDataObjectRequest(data_object=my_data_object) - create_response = client.CreateDataObject(body=create_request).result() - data_object_id = create_response['data_object_id'] - print(data_object_id) - - # Page through a listing of data objects - print("..........Page through a listing of Objects..............") - for i in range(100): - my_data_object = DataObject( - name="OBJ{}".format(i), - aliases=["OBJ{}".format(i)], - size=str(10 * i), - checksums=[Checksum(checksum="def{}".format(i), type="md5")], - urls=[URL(url="http://{}".format(i))]) - create_request = CreateDataObjectRequest(data_object=my_data_object) - client.CreateDataObject(body=create_request).result() - list_request = ListDataObjectsRequest(page_size=10) - list_response = client.ListDataObjects(body=list_request).result() - ids = [x.id for x in list_response.data_objects] - print((list_response.next_page_token)) - print(ids) - - list_request = ListDataObjectsRequest( - page_size=10, page_token=list_response.next_page_token) - list_response = client.ListDataObjects(body=list_request).result() - ids = [x.id for x in list_response.data_objects] - print(ids) - - # Find a DataObject by alias - print("..........List Objects by alias..............") - object_list_request = ListDataObjectsRequest(alias="OBJ1") - object_list_response = client.ListDataObjects( - body=object_list_request).result() - print((object_list_response.data_objects[0].aliases)) - - # Find a DataObject by checksum - print("..........List Objects by checksum..............") - object_list_request = ListDataObjectsRequest( - checksum=Checksum(checksum="def1")) - object_list_response = client.ListDataObjects( - body=object_list_request).result() - print((object_list_response.data_objects[0].checksums)) - - # Find a DataObject by URL - print("..........List Objects by url..............") - object_list_request = ListDataObjectsRequest(url="http://1") - object_list_response = client.ListDataObjects( - body=object_list_request).result() - print((object_list_response.data_objects[0].urls)) - - # CreateDataBundle - print("..........Create a Data Bundle............") - Checksum = models.get_model('Checksum') - URL = models.get_model('URL') - CreateDataBundleRequest = models.get_model('CreateDataBundleRequest') - DataBundle = models.get_model('DataBundle') - create_data_bundle = DataBundle( - name="abc", - size="12345", - checksums=[Checksum(checksum="def", type="md5")], - data_object_ids=[x.id for x in list_response.data_objects]) - create_request = CreateDataBundleRequest(data_bundle=create_data_bundle) - create_response = client.CreateDataBundle(body=create_request).result() - data_bundle_id = create_response['data_bundle_id'] - print(data_bundle_id) - - # GetDataBundle - print("..........Get the Bundle we just created..............") - get_bundle_response = client.GetDataBundle( - data_bundle_id=data_bundle_id).result() - data_bundle = get_bundle_response.data_bundle - print(data_bundle) - print((data_bundle.id)) - - # UpdateDataBundle - print("..........Update that Bundle.................") - UpdateDataBundleRequest = models.get_model('UpdateDataBundleRequest') - update_data_bundle = DataBundle( - name="abc", - size="12345", - data_object_ids=[x.id for x in list_response.data_objects], - checksums=[Checksum(checksum="def", type="md5")], - aliases=["ghi"]) - update_request = UpdateDataBundleRequest( - data_bundle_id=data_bundle.id, - data_bundle=update_data_bundle) - update_response = client.UpdateDataBundle( - data_bundle_id=data_bundle_id, - body=update_request).result() - updated_bundle = client.GetDataBundle( - data_bundle_id=update_response['data_bundle_id']).result().data_bundle - print(updated_bundle) - print(data_bundle) - print((updated_bundle.version)) - print((updated_bundle.aliases)) - assert updated_bundle.aliases[0] == 'ghi' - - # ListDataBundles - print("..........List Data Bundles...............") - ListDataBundlesRequest = models.get_model('ListDataBundlesRequest') - list_request = ListDataBundlesRequest() - list_response = client.ListDataBundles(body=list_request).result() - print((len(list_response.data_bundles))) - - # Get all versions of a DataBundle - print("..........Get all Versions of a Bundle...............") - versions_response = client.GetDataBundleVersions( - data_bundle_id=data_bundle.id).result() - print((len(versions_response.data_bundles))) - - # Get a DataObject from a bundle - print("..........Get an Object in a Bundle..............") - get_bundle_response = client.GetDataBundle( - data_bundle_id=data_bundle_id).result() - data_bundle = get_bundle_response.data_bundle - data_object = client.GetDataObject( - data_object_id=data_bundle.data_object_ids[0]).result().data_object - print((data_object.urls)) - - # Get all DataObjects from a bundle - print("..........Get all Objects in a Bundle..............") - get_bundle_response = client.GetDataBundle( - data_bundle_id=data_bundle_id).result() - data_bundle = get_bundle_response.data_bundle - bundle_objects = [] - for data_object_id in data_bundle.data_object_ids: - bundle_objects.append(client.GetDataObject( - data_object_id=data_object_id).result().data_object) - print([x.name for x in bundle_objects]) - - # DeleteDataBundle - print("..........Delete the Bundle...............") - delete_response = client.DeleteDataBundle( - data_bundle_id=data_bundle_id).result() - print((delete_response.data_bundle_id)) - try: - client.GetDataBundle( - data_bundle_id=update_response['data_bundle_id']).result() - except Exception as e: - print(('The object no longer exists, 404 not found. {}'.format(e))) - - # Page through a listing of Data Bundles - print("..........Page through a listing of Data Bundles..............") - for i in range(100): - my_data_bundle = DataBundle( - name="BDL{}".format(i), - aliases=["BDL{}".format(i)], - size=str(10 * i), - data_object_ids=data_bundle.data_object_ids, - checksums=[Checksum(checksum="def", type="md5")],) - create_request = CreateDataBundleRequest(data_bundle=my_data_bundle) - client.CreateDataBundle(body=create_request).result() - list_request = ListDataBundlesRequest(page_size=10) - list_response = client.ListDataBundles(body=list_request).result() - ids = [x['id'] for x in list_response.data_bundles] - print((list_response.next_page_token)) - print(ids) - - list_request = ListDataBundlesRequest( - page_size=10, page_token=list_response.next_page_token) - list_response = client.ListDataBundles(body=list_request).result() - ids = [x['id'] for x in list_response.data_bundles] - print(ids) - - # Find a DataBundle by alias - print("..........List Data Bundles by alias..............") - list_request = ListDataBundlesRequest( - alias=list_response.data_bundles[0].aliases[0]) - alias_list_response = client.ListDataBundles(body=list_request).result() - print((list_response.data_bundles[0].aliases[0])) - print((alias_list_response.data_bundles[0].aliases[0])) - - -if __name__ == '__main__': - main() diff --git a/python/examples/gdc_dos.py b/python/examples/gdc_dos.py deleted file mode 100644 index b726dc2e0..000000000 --- a/python/examples/gdc_dos.py +++ /dev/null @@ -1,119 +0,0 @@ -# With app.py running start this demo, it will load data from GDC public API -# into the service. -import requests -import pytz -from datetime import datetime -import dateutil - -from ga4gh.dos.client import Client - -config = { - 'validate_requests': False, - 'validate_responses': False, - 'use_models': False -} - -local_client = Client('http://localhost:8080/ga4gh/dos/v1/', config=config) -client = local_client.client -models = local_client.models - - -GDC_URL = 'https://api.gdc.cancer.gov' - -""" -{'data': {'hits': [ -{u'data_type': u'Annotated Somatic Mutation', -u'updated_datetime': u'2017-06-17T22:26:30.596775-05:00', -u'created_datetime': u'2017-06-17T19:12:16.993774-05:00', -u'file_name': u'a6c070d8-0619-4c55-b679-0420ace91903.vep.vcf.gz', -u'md5sum': u'd26f933e8b38c5dfba4aa57e47bb4c4c', -u'data_format': u'VCF', -u'submitter_id': u'TCGA-AK-3447-01A-01W-0886-08_TCGA-AK-...', -u'access': u'controlled', -u'state': u'live', -u'file_id': u'ba6c070d8-0619-4c55-b679-0420ace91903', -u'data_category': u'Simple Nucleotide Variation', -u'file_size': 202612, -u'acl': [u'phs000178'], -u'type': u'annotated_somatic_mutation', -u'id': u'a6c070d8-0619-4c55-b679-0420ace91903', -u'file_state': u'submitted', -u'experimental_strategy': u'WXS'}], -u'pagination': - {u'count': 10, - u'sort': u'', - u'from': 350, - u'pages': 31086, - u'total': 310858, - u'page': 36, - u'size': 10}}, -u'warnings': {}}} -""" - - -def gdc_to_ga4gh(gdc): - """ - Accepts a gdc dictionary and returns a CreateDataObjectRequest - :return: - """ - DataObject = models.get_model('DataObject') - CreateDataObjectRequest = models.get_model('CreateDataObjectRequest') - URL = models.get_model('URL') - Checksum = models.get_model('Checksum') - print((str(gdc.get('file_size')))) - create_data_object = DataObject( - id=gdc.get('file_id'), - name=gdc.get('file_name'), - size=str(gdc.get('file_size')), - created=dateutil.parser.parse(gdc['created_datetime']), - updated=dateutil.parser.parse(gdc['updated_datetime']), - version=gdc.get('version'), - mime_type=gdc.get('file_mime_type'), - checksums=[Checksum(checksum=gdc.get('md5sum'), type='md5')], - urls=[ - URL( - url="{}/data/{}".format(GDC_URL, gdc.get('file_id')), - system_metadata=gdc)], - description=gdc.get('file_description'), - aliases=[gdc['file_id'], gdc['file_name']]) - create_request = CreateDataObjectRequest(data_object=create_data_object) - return create_request - - -def post_dos(gdc): - """ - Takes a GDC hit and indexes it into GA4GH. - :param gdc: - :return: - """ - create_request = gdc_to_ga4gh(gdc) - create_response = client.CreateDataObject(body=create_request).result() - return create_response - - -def load_gdc(): - """ - Gets data from GDC and loads it to DOS. - :return: - """ - response = requests.post( - '{}/files?size=100&related_files=true'.format( - GDC_URL), json={}).json() - hits = response['data']['hits'] - # Initialize to kick off paging - pagination = {} - pagination['pages'] = 1 - pagination['page'] = 0 - page_length = 10000 - while int(pagination.get('page')) < int(pagination.get('pages')): - list(map(post_dos, hits)) - next_record = pagination.get('page') * page_length - response = requests.post( - '{}/files?size=100&related_files=true&from={}'.format( - GDC_URL, next_record), json={}).json() - hits = response['data']['hits'] - pagination = response['data']['pagination'] - - -if __name__ == '__main__': - load_gdc() diff --git a/python/examples/gdc_notebook.ipynb b/python/examples/gdc_notebook.ipynb deleted file mode 100644 index 24b3c1859..000000000 --- a/python/examples/gdc_notebook.ipynb +++ /dev/null @@ -1,441 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# GA4GH Data Access Example" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For this example, metadata have been loaded into a test data registry so they can be accessed using GA4GH methods. `python gdc_dos.py`" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Import the client and models" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This will import a Python client and models for accessing data as defined in the schemas." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "from ga4gh.dos.client import Client\n", - "local_client = Client('http://localhost:8080/ga4gh/dos/v1')\n", - "client = local_client.client\n", - "models = local_client.models" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Listing Data Objects" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To list the existing Data Objects, we send a ListDataObjectsRequest to the `ListDataObjects` method!" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of Data Objects: 2882 \n" - ] - } - ], - "source": [ - "ListDataObjectsRequest = models.get_model('ListDataObjectsRequest')\n", - "list_request = client.ListDataObjects(page_size=10000000)\n", - "list_response = list_request.result()\n", - "print(\"Number of Data Objects: {} \".format(len(list_response.data_objects)))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "These Data Object messages are for testing purposes only but should contain enough to retrieve their contents from GDC servers." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "url: https://api.gdc.cancer.gov/data/72fa5f40-1ade-4088-8390-b8bc970d12f1, file_size (B): 516230\n" - ] - } - ], - "source": [ - "data_objects = list_response.data_objects\n", - "data_object = data_objects[11]\n", - "print('url: {}, file_size (B): {}'.format(data_object.urls[0].url, data_object.size))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Filter Public Data" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We want to use this service to eventually download data, but first we must find data we have access to." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of public Data Objects: 1341\n" - ] - } - ], - "source": [ - "public_data_objects = filter(\n", - " lambda x: x['urls'][0]['system_metadata']['access'] == 'open', \n", - " data_objects)\n", - "print('Number of public Data Objects: {}'.format(len(public_data_objects)))\n", - "\n", - "public_data_object = public_data_objects[0]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Download a file" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can then download this file and name it." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "\n", - "# https://stackoverflow.com/questions/16694907/how-to-download-large-file-in-python-with-requests-py\n", - "def download_file(url, filename):\n", - " # NOTE the stream=True parameter\n", - " r = requests.get(url, stream=True)\n", - " with open(filename, 'wb') as f:\n", - " for chunk in r.iter_content(chunk_size=1024): \n", - " if chunk: # filter out keep-alive new chunks\n", - " f.write(chunk)\n", - " #f.flush() commented by recommendation from J.F.Sebastian\n", - " return filename" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "u'944ee313-bcf5-480c-9ce8-05821746fb34.FPKM-UQ.txt.gz'" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "download_file(public_data_object.urls[0].url, data_object.name)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Verify the checksum" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Data Object messages contain checksums of the underlying files. We can validate it here." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[Checksum(checksum=u'40d83610bbae6dcf8973590c226d4d04', type=u'md5')]\n" - ] - } - ], - "source": [ - "print(public_data_object.checksums)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "40d83610bbae6dcf8973590c226d4d04\n", - "40d83610bbae6dcf8973590c226d4d04\n", - "True\n" - ] - } - ], - "source": [ - "given_checksum = public_data_object.checksums[0].checksum\n", - "\n", - "# https://stackoverflow.com/questions/3431825/generating-an-md5-checksum-of-a-file\n", - "import hashlib\n", - "def md5(fname):\n", - " hash_md5 = hashlib.md5()\n", - " with open(fname, \"rb\") as f:\n", - " for chunk in iter(lambda: f.read(4096), b\"\"):\n", - " hash_md5.update(chunk)\n", - " return hash_md5.hexdigest()\n", - "\n", - "print(md5(data_object.name))\n", - "print(given_checksum)\n", - "print(given_checksum == md5(data_object.name))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Visualizing the contents of the registry" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here, we look at the file sizes of the contents of the registry. This is a histogram where each bin is a count of the number of files with a size in that range. We plot with a log axis because of the number of very small files dominates a linear scale." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEVCAYAAAD+TqKGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGxFJREFUeJzt3Xv4XVV95/H3l4SAcguQKDYhBhHRqAPOw0VBKaVeAozC\neCWixRZFrLR2vII6BaooOvNYi2KVKkWrQtGi4oBFR0Ech1EuooKIIEZIBIPcgxcKfOePtUJ2Dues\n3/nldzm/hPfrec6T395rX9bee+392bdzEpmJJEmDbDLqCkiSZjaDQpLUZFBIkpoMCklSk0EhSWoy\nKCRJTQaFNAIRcUJEfHbU9ZCGMaODIiKOiIjLI+LuiFgRER+MiNmd8sURcX5E3BERt0TER3vKD4iI\nK+r4N0TEUZ2ygyPi/0TEnXXcT0bEVuOoW0bEvRGxun4+2Sn7k4i4MCLuiojlfcZdHhG/64z79U7Z\nZhHx9xHxq7pcH4uITRv1aC3jn0TEj+sy3hYRX4qIBZ3yD0bETXXcX0bEOztl8yLiu3W8OyPikojY\nt1N+WERcW5dxVUR8OiK27pSv7vk8EBEfqWVLIuKyunx3RMT/joglnXHn1umtqp8TxtgWcyLib2t9\n7o2IlRHxtYh4/oB1fkdEnBcROzameVFEvLY134kMvzGIiMdFxLm1rWZELG4Mu6hPm8iIeEstj4h4\nV0TcWNvjWT3tabOIOL2W3RIRb+6UNdtTHeY/R8TFdb6/jog3dcoG7o890/hmrXP3GLN7RHyn7gcr\nIuK/94zz8oi4JiLuiYifRMShnbLmPtQZbpeI+H2McWJRhzsrIm6t6+m6iPhIRCys5ftHxIOd5VwZ\nESe2pvmQzJyxH+ANwHOAOcAC4HLg2E75+cAZwObADsCPgb+uZZsCdwGvBwLYE1gN7FbLXwksBR4N\nbAt8Dfj4OOqWwBMHlO0FvBo4Cljep3w58NwB4x4PfAfYDpgP/D/gxAHDjrWMjwX+qP69GfBB4NzO\n+LsCW9S/FwBXAy+u3ZvX8k3qtA8Fbgdm1/IdgXn17y2BzwGnDKjnlrVe+9XuucDiOt1ZwF8DP+oM\n/8/AF+q2WQz8HPjzxrY4t7aNvWtbmVO37T/0W+d12U4HvtyY5kXAa8fRHsY7/AnAZ0e9j03kU9vX\nXwLPqvvD4nGMuxPwwJpxgCOAn9Z2tSXwFeDTneHfX/eLbYGnALcAS4dsT/OAVcDhdT/YCnjKMPtj\nZ5jDgYvrcs7u9P8JcFKd787AzcCLOvvUfcCBtW4HA78FHjOefQj4el32ge0FeCJl//wQsLD2ewzw\nN8BhtXt/YEXPNlgBHDrm9ppAI1kOvBX4EeVg9a/A5lPcMN8MfLXTfQ1wUKf7fwCf6DTiBB7dKb8U\nWDZg2i8GfjyOugwMis4wz2X8QXEZ8LJO9yuBmwYMO/Qy1h3k/cBPBkxrASVo396nbBPghXVej+lT\nviXwGeD8AdM+ArgBiD5ls4E3Ar/t9PsNsGen+53Adxrr+Hdrdo4x2utzO90HAT8bMOxJlIPY7ykB\n99Haf5+6fu+q/+4zxvD/ANwE3E0Jsud05nECQwZFHfYLwGeBe+p2ehJwHOUAeBPw/M7w2wCfohy0\nVgLvBWbVsp2BbwG31fX8OWBuz3oa135dt+F4g+J44MJO9xeBt3W696nr89G1+1c9y/ge4Kwh29P7\ngH8Ztm30Kd8G+BnwTB4eFL8FlnS6vwAcV//eG1jVM61bgWcNuw8BhwFnj9Veatv46qDyOsz+dIKi\n9jsbeOeY22vYDTtg5X4f+CPK2e81wNEDhn02cGfj8+wh5/ll4ORO9+vryn005UB3FfBfO+Wfr41m\nFuWsZxWw44Bpf7hfw2vUJWvjvQU4p99OQjsofl0bzdepVwC17DLg5Z3uw+u8thlQj+YyAovqOn4Q\n+A/gNT3jH0s5uCXlYL6wp/xHlLOiBP6pz3a9q5bdS2dH7hnuW8AJffrfCdxf6/buTv/fAHt1ut8F\n3DFg2icDFw3ZXtdcUTwa+DTwmcbwF9G5Qqht/A7KleJsYFnt3r7f8LXfq4Dt6/BvqW1l81p2AuML\nit8DL6jT+gzwi7peNgVeB/yiM/yXgE8AW1DOKr8PvL6WPRF4HuXEYT7lLPnD67Nfd8YZV1BQzq5/\n3m2LlKB4e6d73zrN3ShXEQk8tlP+UnpO7Brt6VuU0P6/lP3jq8CiYfbHWn4q8N8oVy29QfG+2gY3\npVyBr6Ce5FD2yW8DL6p/H1rLtxhmHwK2pgTUwrHaS21brxlUXofZn3WvKHahnEgcMOY2G2bDNna8\nV3W6P8g4bt2sx/z+oq7keZ1+T6Gcqd1fV/QZdM5aKWfBv67l9wOvGzDt51F2+ieNoz77UW5xzAU+\nSgmp2T3DDAqKfYFHUQ5Yx9WNPLeWvRf4LmUn3gH4Xl22xw2ox7DLuB3wDuCZA3bcZwAnAlv1Kd+c\ncmA8YsC0F9SG/LD1Bzyecra904Bxt6Dcvji40++zlPDdinJg+znwhwHjf5JOwNflvLPufL/vaa+r\na9l/UEL+6Y3texHrBsWrge/3DHPJmp2zd/gB07yDtbcFT2B8QfGNnm2+mrVXCVvVNjKXcpX5B+BR\nneGX0Tl775n2ocAPetbTuPZrxh8Uz6n137LT77WUg+Jiyhn8uXWaz6Lcokk6VzaUfbbfvtWvPf2s\nbvc9a1s+BfjukPvjHsCVdRkX8/Cg2Ae4nrXHoBN76nNkXdb7KVcfBw9YJw/bhyjh9o5h2kud/tJO\n9zF1mVdTT/AoQfFg7X93re85wJwxt9kwG3ZAxZaz7qX80A1/PeZ1KOVg+PROv02AX1LOqjajnLl9\nBfhgLX8yJaFfUIfdFbiud0NRLidvBf50AvWbVef19J7+fYOiz/g/BV5Y/34UJXhWUs7wj6Oc0W/S\nZ7yhlrEz/A51Pc4eUH4s8KFGPa+h52yrZz1e0af/u4Fvj7H8m1Buhay5d7sd5ZbILZTnJu8Ffj5g\n3A/0mz4lYLJfe63b68WUe7o7DJjuRawbFO8AvtAzzFnAu/oNX/u9ta6zu1h7Vfen491feoftbVes\nPVAvpDwfW3MwWPO5G7i6DvvYWu+Vtf9qOrc2WY/9mvEHxSfpPH/otIET6/xXUM7gkxISa64oHtMZ\n/iUMuFXcpz39EPjnTvn2tK/Sf0oJ400oV1d/XPsvphMUtZ3eDfxZXQcLKc8U/7KznW6jhM0mlKC6\nGdh9rH0I2J3S9ucMsx0o+/XDnuNR9p0z6t/7s+4VxTaU/ezMsbbZtLz1FBHP6fPGQ/fznMa4S4F/\nohxIf9wp2o5yW+WjmfmHzLyN8hD0oFr+NMo96Asy88HMvBY4j/Jgac20n0E5c/mLzPzmBBczKWfm\nExo3M3+Xmcdk5oLMfAKloV2emQ/2GW/MZewxm3Ir4mFvVnTKd27Uc1PgCeMc988ot3laNmHt7UMy\n8/bMPDwzd8jMp7J2h+3nm8Cea97sGEZmPpCZ51CudJ49aLCe7l9Rro66FlEOuA8bvrbptwMvB7bN\nzLmUwFjfNjKsmyhXFPMyc279bF3XI5RbJUk5qdmacntsquv0kIh4FPAyetpEbb/HZ+bizFxIOUiu\nBFZm5h2UA+xunVF2q8P0s057otw+7W6f3m3ba83+uDXlIP+vEXEL5bkUwIq6fZ8APJCZn8nM+zNz\nBSWE1xyDdgcuzszL6vJdSrlD8NwB8+3uQ/tTgunGOu+3Ai+JiCsGjPtNysnP0DLzLsqt6xcOM/B6\nfZiGKwrgAMqBcr8B5TdQzoJnUy67vwR8vpbtTDlbOoCy0XemXCIeVcufRknhVwyY9gkMuPcNPJXS\nCGZRHkJ9GLgW2DTXntFsTjlg/7L+vebMYBHlUndO7f82yhXNmnvdCyj3h4NyhrHOg8qeeoy1jC9m\n7ZtL8ykPrq7o1PH1lLO1oJyJ3szat8aeSTmIzqFc5byD8iB1zVtUh1Pv81IOoN8Gzump3z6UK56t\nevo/j3KraxZlZzyFciDevLNc29fyAynPLJ7aaCfnUZ7trHnraVPKATD7tde6vIdQLtf7Tpeyw7+v\n07095ez8lZT29oraPW/A8AfVZdqh1ulvKcG0pg4nsO5VwnIG3GPuM+zAK4ra/RXKbYut63bembVn\nxWdTTrxmUdrad1n3LPOh9TTMfk1pw1vU+e/K2A++X1nnET39t6v1DGAJ5VbuUZ3yk2sb25ZyJX0z\na996Gqs9HUC57bd7bRt/T305gsb+WOuyQ+ezZ13OBXX4rTttYpM6zCVr2gHwx5S2u3vtfgblePb8\nsfYhStB15/0/Kc9x5g9Yr0+qdfkQsKD2mwf8G4OvKLYE/gX43pjH4mEO2AMqNq4GtZ7zuJCyM6/u\nfL7WKd+dcsl/R90gZ7PuA6+X1wZ3D+Vy9gPUWziUq48He6Z9dWfcTwEnDajXAZRguJfycOzLwC6d\n8v1rg+p+LqplT6Wc4dxbG803gT064+5X1+1v6zwO75n31+i8pTDGMv4V5aHnvZTbOGcBj69lmwD/\nTrn9sppyH/ed1B2Y0sh/WKd7e23E+3Xme1Kd373139OoYdcZ5hP0eduEckb50zrfWykH+v/Us0y/\nquvgSuAFY7STObX9XVfHWVHXU/fB4HLK21Gr6zJd1btue6b5rLpO7qC+skgJzsspVwaX03kJo3d4\nykHrdMqtiZspVxfL6RMUtf73AE8eUJeHhq3dYwXFNsA/1vVwF/AD1r4i+dRa99V13b6FiQVFbzvP\nTtnH6Xm+AVwAvKfPdJ5Eae+/pZxcvbmnfLPO+vx1t3ys9lSHeQPlCuUOysPsHYfZH3umsZiHP6M4\ngLVvwt1CCeHuW4jHUE7e7qGc2L5lPPvQsNuhDrMr5Rj4mzq/a4GPdJZ1f9Y95t1W11Xz7c3MfOig\noB4RcSXlfvJto66LNm4R8WzgjZm5bNR1kfoxKCRJTTP6JzwkSaNnUEiSmgwKSVLT7LEHmXrz5s3L\nxYsXj7oakrRBufzyy3+TmfOnej4zIigWL17MZZddNupqSNIGJSJ+OR3z8daTJKnJoJAkNRkUkqQm\ng0KS1GRQSJKaDApJUpNBIUlqMigkSU0GhSSpaUZ8M3siFh973kN/Lz/54BHWRJI2Tl5RSJKaDApJ\nUpNBIUlqMigkSU0GhSSpyaCQJDUZFJKkJoNCktRkUEiSmgwKSVKTQSFJajIoJElNBoUkqcmgkCQ1\nGRSSpKZJD4qIeEpEfDwivhgRb5js6UuSptdQQRERp0fEqoi4qqf/0oi4NiKuj4hjATLzmsw8Gng5\nsO/kV1mSNJ2GvaI4A1ja7RERs4BTgQOBJcCyiFhSy14EnAecP2k1lSSNxFBBkZkXA7f39N4LuD4z\nb8jM+4CzgEPq8Odm5oHA4YOmGRFHRcR1EXHrjTfeuH61lyRNuYk8o1gA3NTpXgEsiIj9I+KUiPgE\njSuKzDwtM3fJzPmLFi2aQDUkSVNp9mRPMDMvAi6a7OlKkkZjIlcUK4EdO90Laz9J0kZkIkFxKbBL\nROwUEXOAw4BzJ6dakqSZYtjXY88ELgF2jYgVEXFkZt4PHANcAFwDnJ2ZV09dVSVJozDUM4rMXDag\n//n4CqwkbdT8CQ9JUpNBIUlqMigkSU0GhSSpyaCQJDUZFJKkJoNCktRkUEiSmgwKSVKTQSFJajIo\nJElNBoUkqcmgkCQ1GRSSpCaDQpLUZFBIkpoMCklSk0EhSWoyKCRJTQaFJKnJoJAkNRkUkqQmg0KS\n1GRQSJKaDApJUpNBIUlqMigkSU0GhSSpyaCQJDUZFJKkJoNCktRkUEiSmgwKSVKTQSFJajIoJElN\nBoUkqcmgkCQ1GRSSpCaDQpLUNHsqJhoRhwIHA1sDn8rMr0/FfCRJU2/oK4qIOD0iVkXEVT39l0bE\ntRFxfUQcC5CZX87M1wFHA6+Y3CpLkqbTeG49nQEs7faIiFnAqcCBwBJgWUQs6Qzy7louSdpADR0U\nmXkxcHtP772A6zPzhsy8DzgLOCSKDwBfy8wr+k0vIo6KiOsi4tYbb7xxfesvSZpiE32YvQC4qdO9\novb7K+C5wEsj4uh+I2bmaZm5S2bOX7Ro0QSrIUmaKlPyMDszTwFOmYppS5Km10SvKFYCO3a6F9Z+\nkqSNxESD4lJgl4jYKSLmAIcB5068WpKkmWI8r8eeCVwC7BoRKyLiyMy8HzgGuAC4Bjg7M6+emqpK\nkkZh6GcUmblsQP/zgfMnrUaSpBnFn/CQJDUZFJKkJoNCktRkUEiSmgwKSVKTQSFJajIoJElNBoUk\nqcmgkCQ1GRSSpCaDQpLUZFBIkpoMCklSk0EhSWoyKCRJTQaFJKnJoJAkNRkUkqQmg0KS1GRQSJKa\nDApJUpNBIUlqMigkSU0GhSSpyaCQJDUZFJKkJoNCktRkUEiSmgwKSVLT7FFXQDPH4mPPe+jv5Scf\nPMKaSJpJDIoNTPdgDuse0CfzQG9oSFrDoNjA9QZHv/4e6CVNhM8oJElNBoUkqclbTzPIKG4XDbp1\nJUlreEUhSWoyKCRJTQaFJKnJoJAkNU36w+yIeALwLmCbzHzpZE//kcLvQUiaKYa6ooiI0yNiVURc\n1dN/aURcGxHXR8SxAJl5Q2YeORWVlSRNv2GvKM4APgp8Zk2PiJgFnAo8D1gBXBoR52bmTya7ksNq\n/bzFhsxXWCWN0lBXFJl5MXB7T++9gOvrFcR9wFnAIZNcP0nSiE3kGcUC4KZO9wpg74jYHjgJeEZE\nHJeZ7+83ckQcBbwNmDt//vwJVEPTyWcn0iPPpD/MzszbgKOHGO404DSAPfbYIye7HpKkyTGRoFgJ\n7NjpXlj7aQzTfVbuMw5JEzGR71FcCuwSETtFxBzgMODcyamWJGmmGOqKIiLOBPYH5kXECuD4zPxU\nRBwDXADMAk7PzKunrKYbqQ3hbH/YOg4abtB/rjRoGEkzy1BBkZnLBvQ/Hzh/UmskSZpR/AkPSVKT\n/x/FNNgQbi+N2lR+WdJXeqWJ8YpCktRkUEiSmgwKSVKTzyimyCPhucSol3E6nj1srD80KY2HVxSS\npCaDQpLUZFBIkpp8RjFBvqM/M7gdpKnjFYUkqcmgkCQ1GRSSpKZH5DMK72dPr/X5vsVEttFUfr9j\nqtqObVIzmVcUkqQmg0KS1GRQSJKaDApJUpNBIUlqekS+9TTIRN88GfWvqW6sNrQ3giazHW0Iy6uN\nn1cUkqQmg0KS1GRQSJKaDApJUpNBIUlqMigkSU0GhSSpyaCQJDX5hTttUCbzS43T8QXJ1jyG+TLd\nxvqz5oPm37u+/MLhzOAVhSSpyaCQJDUZFJKkJoNCktRkUEiSmgwKSVKTQSFJajIoJElNBoUkqcmg\nkCQ1GRSSpCaDQpLUZFBIkpoMCklSk0EhSWoyKCRJTQaFJKnJoJAkNRkUkqQmg0KS1GRQSJKaDApJ\nUpNBIUlqmj3ZE4yILYCPAfcBF2Xm5yZ7HpKk6TPUFUVEnB4RqyLiqp7+SyPi2oi4PiKOrb1fDHwx\nM18HvGiS6ytJmmbD3no6A1ja7RERs4BTgQOBJcCyiFgCLARuqoM9MDnVlCSNylC3njLz4ohY3NN7\nL+D6zLwBICLOAg4BVlDC4koaQRQRRwFvA+bOnz9/3BUfxuJjz1vvYXr7Lz/54Empk0ZrmDYximlN\n1fy67XbYtj7e6Q5rfebTHWfQsgzTf33HmQob4rFlIg+zF7D2ygFKQCwAzgFeEhH/CHx10MiZeVpm\n7pKZ8xctWjSBakiSptKkP8zOzHuBP5/s6UqSRmMiVxQrgR073QtrP0nSRmQiQXEpsEtE7BQRc4DD\ngHMnp1qSpJli2NdjzwQuAXaNiBURcWRm3g8cA1wAXAOcnZlXT11VJUmjMOxbT8sG9D8fOH9SayRJ\nmlH8CQ9JUpNBIUlqMigkSU0GhSSpyaCQJDUZFJKkJoNCktRkUEiSmgwKSVKTQSFJajIoJElNBoUk\nqcmgkCQ1GRSSpCaDQpLUZFBIkpoMCklSk0EhSWoyKCRJTZMeFBGxNCKujYjrI+LYyZ6+JGl6TWpQ\nRMQs4FTgQGAJsCwilkzmPCRJ0ysyc/ImFvEs4ITMfEHtPg4gM9/fZ9ijgLcBc4EtgavXc7bzgN+s\n57gbO9dNf66XwVw3/c3U9fL4zJw/1TOZPcnTWwDc1OleAezdb8DMPA04baIzjIjLMnOPiU5nY+S6\n6c/1Mpjrpr9H+nrxYbYkqWmyg2IlsGOne2HtJ0naQE12UFwK7BIRO0XEHOAw4NxJnkevCd++2oi5\nbvpzvQzmuunvEb1eJvVhNkBEHAR8GJgFnJ6ZJ03qDCRJ02rSg0KStHHxYbYkqWmDDgq/Bd5fRJwe\nEasi4qpR12UmiYgdI+LCiPhJRFwdEW8adZ1mgojYPCK+HxE/rOvlxFHXaSaJiFkR8YOI+F+jrsuo\nbLBB4bfAm84Alo66EjPQ/cBbMnMJ8EzgjbYZAP4AHJCZuwG7A0sj4pkjrtNM8ibgmlFXYpQ22KAA\n9gKuz8wbMvM+4CzgkBHXaUbIzIuB20ddj5kmM2/OzCvq3/dQdv4Fo63V6GWxunZuWj8+vAQiYiFw\nMPDJUddllDbkoOj3LfBH/E6v4UTEYuAZwPdGW5OZod5euRJYBXwjM10vxYeBtwMPjroio7QhB4W0\nXiJiS+DfgL/JzLtHXZ+ZIDMfyMzdKV+S3SsinjbqOo1aRPwXYFVmXj7quozahhwUfgtc4xYRm1JC\n4nOZec6o6zPTZOadwIX4jAtgX+BFEbGccmv7gIj47GirNBobclCM4lvg2oBFRACfAq7JzA+Nuj4z\nRUTMj4i59e9HAc8DfjraWo1eZh6XmQszczHl+PKtzHzViKs1EhtsUGTm/cAxwAWUh5JnZ+b6/lT5\nRiUizgQuAXaNiBURceSo6zRD7Au8mnJmeGX9HDTqSs0AjwMujIgfUU7AvpGZj9hXQfVwfjNbktS0\nwV5RSJKmh0EhSWoyKCRJTQaFJKnJoJCkKTKeH+iMiP0i4oqIuD8iXtpTdkREXFc/R0xdjfszKCRp\n6pzB8F9evBF4DfD5bs+I2A44Htib8ht3x0fEtpNXxbEZFJI0Rfr9QGdE7BwR/x4Rl0fEdyLiyXXY\n5Zn5Ix7+u1IvoHy35fbMvAP4BtP8zfnZ0zkzSRKnAUdn5nURsTfwMeCAxvAj/wFUg0KSpkn9Qcp9\ngC+UX5QBYLPR1Wg4BoUkTZ9NgDvrL/UOayWwf6d7IXDRJNZpTD6jkKRpUn/W/hcR8TIoP1QZEbuN\nMdoFwPMjYtv6EPv5td+0MSgkaYoM+IHOw4EjI+KHwNXU/5kzIvaMiBXAy4BPRMTVAJl5O/Aeyg82\nXgr8Xe03fcvhjwJKklq8opAkNRkUkqQmg0KS1GRQSJKaDApJUpNBIUlqMigkSU3/H0m2WX7CkmeU\nAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "file_sizes = [float(x.size) for x in data_objects]\n", - "plt.hist(file_sizes, bins=96)\n", - "plt.title(\"n = {}, {} GB total, mean {} GB\".format(len(file_sizes), str(sum(file_sizes) / 1000000000.0), (sum(file_sizes) / len(file_sizes)) / 1000000000.0 ))\n", - "plt.yscale('symlog')\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Observe that most of the non-tiny files are around 2GB and a few files are very large." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": true - }, - "source": [ - "## Make a Data Bundle of some Data Objects" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can now organize some of the Data Objects into a bundle so we can share them together. \n", - "\n", - "For example, a few publicly available items. First, we have to gather the list of Data Objects and compute their concatenated hash." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Calculate the hash for our Objects" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "238dc9ee-c1de-4803-8e95-bc21f3a7e806\n", - "40d83610bbae6dcf8973590c226d4d04\n", - "abf6bc6f18bee84a17c9ece681de5f93\n" - ] - } - ], - "source": [ - "public_data_object_ids = [x.id for x in public_data_objects]\n", - "print(public_data_object_ids[0])\n", - "hashes = [x.checksums[0].checksum for x in public_data_objects]\n", - "print(hashes[0])\n", - "bundle_md5 = hashlib.md5()\n", - "bundle_md5.update(''.join(hashes[0:10]))\n", - "bundle_digest = bundle_md5.hexdigest()\n", - "print(bundle_digest)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Create a new Data Bundle" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "c4dbd853-a057-4910-b2b1-d16211fd4031\n" - ] - } - ], - "source": [ - "CreateDataBundleRequest = models.get_model('CreateDataBundleRequest')\n", - "DataBundle = models.get_model('DataBundle')\n", - "Checksum = models.get_model('Checksum')\n", - "my_bundle = DataBundle(\n", - " name=\"My Bundle\",\n", - " checksums=[Checksum(checksum=bundle_digest, type='md5')],\n", - " data_object_ids=public_data_object_ids[0:10],\n", - " aliases=[\"bundle-alias\", \"access:public\"])\n", - "create_request = CreateDataBundleRequest(data_bundle=my_bundle)\n", - "create_response = client.CreateDataBundle(body=create_request).result()\n", - "print(create_response.data_bundle_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's now verify the Data Bundle appears as expected:" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[u'238dc9ee-c1de-4803-8e95-bc21f3a7e806', u'10e4ee41-ed07-488f-b024-dc47103ebcdc', u'9035cbd8-439c-49d5-9c28-f6620a0268f9', u'0e9e4a38-d6c5-4c6e-8044-93d9c2bdc686', u'c3c5adb2-d5ae-4d89-9c38-fe9178ce5665', u'2e9b7043-e5c4-4168-b4e5-13a929f48846', u'4d67906b-6f98-4aa3-b095-15a53b7a5a5f', u'b0002ca4-d762-40c1-b4e0-d40e2988f6f0', u'dc0dece5-71a1-4920-a779-cd256d2f47b1', u'88b51e89-0888-40b3-bef8-4725d1857de2']\n" - ] - } - ], - "source": [ - "get_bundle_response = client.GetDataBundle(data_bundle_id=create_response.data_bundle_id).result()\n", - "print(get_bundle_response.data_bundle.data_object_ids)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 2", - "language": "python", - "name": "python2" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.12+" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/python/examples/object-type-examples.ipynb b/python/examples/object-type-examples.ipynb deleted file mode 100644 index 0e2864847..000000000 --- a/python/examples/object-type-examples.ipynb +++ /dev/null @@ -1,719 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Data Object Service Demo\n", - "\n", - "This notebook demonstrates how to use the demonstration server and client to make a simple Data Object service that makes available data from a few different sources.\n", - "\n", - "## Installing the Python package\n", - "\n", - "First, we'll install the Data Object Service Schemas package from PyPi, it includes a Python client and demonstration server.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip install ga4gh-dos-schemas" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Running the server\n", - "\n", - "Once you've installed the PyPi package, you can run the demonstration server using `ga4gh_dos_server`. Open this in a separate terminal.\n", - "\n", - "You should see something like:\n", - "\n", - "```\n", - "$ ga4gh_dos_server\n", - " * Serving Flask app \"ga4gh.dos.server\" (lazy loading)\n", - " * Environment: production\n", - " WARNING: Do not use the development server in a production environment.\n", - " Use a production WSGI server instead.\n", - " * Debug mode: on\n", - " * Running on http://0.0.0.0:8080/ (Press CTRL+C to quit)\n", - " * Restarting with stat\n", - " * Debugger is active!\n", - " * Debugger PIN: 192-487-366\n", - "```\n", - "\n", - "Your DOS is now ready to accept requests to Create, Get, and List Data Objects!" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using the Client to Access the Demo Server" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can now use the Python client to create a simple Data Object. The same could be done using cURL or wget." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "from ga4gh.dos.client import Client\n", - "client = Client(\"http://localhost:8080/ga4gh/dos/v1\")\n", - "c = client.client\n", - "models = client.models" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "At first, the service will not present any Data Objects." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "ListDataObjectsResponse(data_objects=[], next_page_token=None)" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "c.ListDataObjects().result()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can now create an simple Data Object representing a file." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "976feb684cfdb4b2337530699e1d0fbd dos.txt\r\n" - ] - } - ], - "source": [ - "!echo \"Hello DOS\" > dos.txt\n", - "!md5sum dos.txt" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "DataObject = models.get_model('DataObject')\n", - "Checksum = models.get_model('Checksum')\n", - "URL = models.get_model('URL')\n", - "hello_object = DataObject()" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "# Set the Data Object metadata\n", - "hello_object.id = 'test'\n", - "hello_object.checksums = [Checksum(checksum=\"976feb684cfdb4b2337530699e1d0fbd\", type=\"md5\")]\n", - "hello_object.urls = [URL(url=\"file://dos.txt\")]\n", - "hello_object.name = 'dos.txt'" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "CreateDataObjectResponse(data_object_id=u'test')" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Post the Data Object to the service\n", - "c.CreateDataObject(body={'data_object': hello_object}).result()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "GetDataObjectResponse(data_object=DataObject(aliases=None, checksums=[Checksum(checksum=u'976feb684cfdb4b2337530699e1d0fbd', type=u'md5')], created=datetime.datetime(2018, 5, 31, 9, 47, 9, 729521, tzinfo=tzutc()), description=None, id=u'test', mime_type=None, name=u'dos.txt', size=None, updated=datetime.datetime(2018, 5, 31, 9, 47, 9, 729536, tzinfo=tzutc()), urls=[URL(system_metadata=None, url=u'file://dos.txt', user_metadata=None)], version=u'2018-05-31T09:47:09.729541Z'))" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Get the resulting created object\n", - "c.GetDataObject(data_object_id='test').result()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using DOS With Reference FASTAs\n", - "\n", - "A useful Data Object Service might present a list of available reference FASTAs for performing downstream alignment and analysis.\n", - "\n", - "We'll index the UCSC human reference FASTAs into DOS as an example." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--2018-05-31 09:50:36-- http://hgdownload.cse.ucsc.edu/goldenPath/hg38/chromosomes/chr22.fa.gz\n", - "Resolving hgdownload.cse.ucsc.edu (hgdownload.cse.ucsc.edu)... 128.114.119.163\n", - "Connecting to hgdownload.cse.ucsc.edu (hgdownload.cse.ucsc.edu)|128.114.119.163|:80... connected.\n", - "HTTP request sent, awaiting response... 200 OK\n", - "Length: 12255678 (12M) [application/x-gzip]\n", - "Saving to: ‘chr22.fa.gz’\n", - "\n", - "chr22.fa.gz 100%[===================>] 11.69M 2.08MB/s in 5.8s \n", - "\n", - "2018-05-31 09:50:42 (2.03 MB/s) - ‘chr22.fa.gz’ saved [12255678/12255678]\n", - "\n", - "41b47ce1cc21b558409c19b892e1c0d1 chr22.fa.gz\n" - ] - } - ], - "source": [ - "!wget http://hgdownload.cse.ucsc.edu/goldenPath/hg38/chromosomes/chr22.fa.gz\n", - "!md5sum chr22.fa.gz" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "# Adding a second URL because FTP is preferred\n", - "chr22 = DataObject()\n", - "chr22.id = 'hg38-chr22'\n", - "chr22.name = 'chr22.fa.gz'\n", - "chr22.urls = [\n", - " URL(url='http://hgdownload.cse.ucsc.edu/goldenPath/hg38/chromosomes/chr22.fa.gz'),\n", - " URL(url='ftp://hgdownload.cse.ucsc.edu/goldenPath/hg38/chromosomes/chr22.fa.gz')]\n", - "chr22.checksums = [Checksum(checksum='41b47ce1cc21b558409c19b892e1c0d1', type='md5')]\n", - "chr22.aliases = ['NC_000022', 'CM000684']\n", - "chr22.size = '12255678'" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "CreateDataObjectResponse(data_object_id=u'hg38-chr22')" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Add the chr22 Data Object to the service\n", - "c.CreateDataObject(body={'data_object': chr22}).result()" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "GetDataObjectResponse(data_object=DataObject(aliases=[u'NC_000022', u'CM000684'], checksums=[Checksum(checksum=u'41b47ce1cc21b558409c19b892e1c0d1', type=u'md5')], created=datetime.datetime(2018, 5, 31, 9, 54, 56, 385181, tzinfo=tzutc()), description=None, id=u'hg38-chr22', mime_type=None, name=u'chr22.fa.gz', size=12255678L, updated=datetime.datetime(2018, 5, 31, 9, 54, 56, 385193, tzinfo=tzutc()), urls=[URL(system_metadata=None, url=u'http://hgdownload.cse.ucsc.edu/goldenPath/hg38/chromosomes/chr22.fa.gz', user_metadata=None), URL(system_metadata=None, url=u'ftp://hgdownload.cse.ucsc.edu/goldenPath/hg38/chromosomes/chr22.fa.gz', user_metadata=None)], version=u'2018-05-31T09:54:56.385197Z'))" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "c.GetDataObject(data_object_id='hg38-chr22').result()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using DOS with htsget\n", - "\n", - "Data Objects are meant to represent versioned artifacts and can represent an API resource. For example, we could use DOS as a way of exposing htsget resources.\n", - "\n", - "In the [htsget Quickstart documentation](https://htsget.readthedocs.io/en/stable/quickstart.html) a link is made to the following snippet, which will stream the BAM results to the client." - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [], - "source": [ - "!htsget http://htsnexus.rnd.dnanex.us/v1/reads/BroadHiSeqX_b37/NA12878 \\\n", - " --reference-name=2 --start=1000 --end=20000 -O NA12878_2.bam" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "eaf80af5e9e54db5936578bed06ffcdc NA12878_2.bam\n", - "-rw-r--r-- 1 david david 555749 May 31 10:00 NA12878_2.bam\n" - ] - } - ], - "source": [ - "!md5sum NA12878_2.bam\n", - "!ls -al NA12878_2.bam" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [], - "source": [ - "na12878_2 = DataObject()\n", - "na12878_2.id = 'na12878_2'\n", - "na12878_2.name = 'NA12878_2.bam'\n", - "na12878_2.checksums = [Checksum(checksum='eaf80af5e9e54db5936578bed06ffcdc', type='md5')]\n", - "na12878_2.urls = [\n", - " URL(\n", - " url=\"http://htsnexus.rnd.dnanex.us/v1/reads/BroadHiSeqX_b37/NA12878\", \n", - " system_metadata={'reference_name': 2, 'start': 1000, 'end': 20000})]\n", - "na12878_2.aliases = ['NA12878 chr 2 subset']\n", - "na12878_2.size = '555749'" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "CreateDataObjectResponse(data_object_id=u'na12878_2')" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "c.CreateDataObject(body={'data_object': na12878_2}).result()" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "GetDataObjectResponse(data_object=DataObject(aliases=[u'NA12878 chr 2 subset'], checksums=[Checksum(checksum=u'eaf80af5e9e54db5936578bed06ffcdc', type=u'md5')], created=datetime.datetime(2018, 5, 31, 10, 5, 7, 748572, tzinfo=tzutc()), description=None, id=u'na12878_2', mime_type=None, name=u'NA12878_2.bam', size=555749L, updated=datetime.datetime(2018, 5, 31, 10, 5, 7, 748583, tzinfo=tzutc()), urls=[URL(system_metadata=SystemMetadata(end=20000, reference_name=2, start=1000), url=u'http://htsnexus.rnd.dnanex.us/v1/reads/BroadHiSeqX_b37/NA12878', user_metadata=None)], version=u'2018-05-31T10:05:07.748588Z'))" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "c.GetDataObject(data_object_id='na12878_2').result()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using DOS with S3\n", - "\n", - "One of the original intentions of DOS is to create an interoperability layer over the various object stores. We can create Data Objects that point to items in S3 so that subsequent downloaders can find them.\n", - "\n", - "Using [dos_connect](https://github.com/ohsu-comp-bio/dos_connect), a DOS hosting the 1kgenomes s3 data is available." - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [], - "source": [ - "client_1kg = Client('http://ec2-52-26-45-130.us-west-2.compute.amazonaws.com:8080/ga4gh/dos/v1/')\n", - "c1kg = client_1kg.client" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [], - "source": [ - "do_1kg = c1kg.ListDataObjects().result().data_objects[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "s3://1000genomes/phase3/data/HG02885/alignment/HG02885.mapped.ILLUMINA.bwa.GWD.low_coverage.20121211.bam.cram.crai\n", - "Checksum(checksum=u'ddc4d0aea91b82a1c202a0cd1219e520', type=u'md5')\n", - "b3549308-9dd0-4fdb-92b2-5a2697521354\n" - ] - } - ], - "source": [ - "print(do_1kg.urls[0].url)\n", - "print(do_1kg.checksums[0])\n", - "print(do_1kg.id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can now use an S3 downloader to retrieve the file and confirm the checksum." - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "usage: dos-downloader [-h] [--aws_access_key AWS_ACCESS_KEY]\r\n", - " [--aws_secret_key AWS_SECRET_KEY] [--path PATH]\r\n", - " url id\r\n", - "dos-downloader: error: argument --aws_secret_key: expected one argument\r\n" - ] - } - ], - "source": [ - "!dos-downloader http://ec2-52-26-45-130.us-west-2.compute.amazonaws.com:8080/ga4gh/dos/v1/ b3549308-9dd0-4fdb-92b2-5a2697521354 --aws_secret_key $aws_secret_access_key --aws_access_key $aws_access_key_id" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DOS GDC Data\n", - "\n", - "Another demonstration in this repository asks you to create a DOS of the NCI GDC data. This process has been automated as part of a lambda: dos-gdc-lambda." - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [], - "source": [ - "cgdc = Client(\"https://dos-gdc.ucsc-cgp-dev.org/\")" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": {}, - "outputs": [], - "source": [ - "gdc_do = cgdc.client.ListDataObjects().result().data_objects[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4803fc06-e2de-44aa-b76e-f8fe9308c18d.bam\n", - "19098711404\n", - "https://api.gdc.cancer.gov/data/4803fc06-e2de-44aa-b76e-f8fe9308c18d\n" - ] - } - ], - "source": [ - "print(gdc_do.name)\n", - "print(gdc_do.size)\n", - "print(gdc_do.urls[0].url)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## DAS DOS\n", - "\n", - "UCSC Genome Browser makes available a service for getting sequence by region from named FASTA. Working with DOS is pretty easy.\n", - "\n", - "Both of these APIs allow one to make further range queries against the result.\n", - "\n", - "https://genome.ucsc.edu/FAQ/FAQdownloads.html#download23" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [], - "source": [ - "chr22.urls.append(URL(url='http://genome.ucsc.edu/cgi-bin/das/hg19/dna?segment=chr22:15000,16000'))\n", - "chr22.urls.append(URL(url='http://togows.org/api/ucsc/hg38/chr22:15000-16000.fasta'))" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "UpdateDataObjectResponse(data_object_id=u'hg38-chr22')" - ] - }, - "execution_count": 57, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "c.UpdateDataObject(body={'data_object': chr22}, data_object_id=chr22.id).result()" - ] - }, - { - "cell_type": "code", - "execution_count": 84, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(u'2018-05-31T11:33:04.156318Z', None)\n" - ] - } - ], - "source": [ - "response_chr22 = c.GetDataObject(data_object_id=chr22.id).result().data_object\n", - "# Note the change in version, in DOS versions are just arbitrary strings\n", - "print(response_chr22.version, chr22.version)\n", - "url_1 = response_chr22.urls[2].url\n", - "url_2 = response_chr22.urls[3].url" - ] - }, - { - "cell_type": "code", - "execution_count": 89, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--2018-05-31 11:40:17-- http://genome.ucsc.edu/cgi-bin/das/hg19/dna?segment=chr22:15000,16000\n", - "Resolving genome.ucsc.edu (genome.ucsc.edu)... 128.114.119.131, 128.114.119.132, 128.114.119.133, ...\n", - "Connecting to genome.ucsc.edu (genome.ucsc.edu)|128.114.119.131|:80... connected.\n", - "HTTP request sent, awaiting response... 200 OK\n", - "Length: unspecified [text/xml]\n", - "Saving to: ‘dna?segment=chr22:15000,16000.2’\n", - "\n", - "dna?segment=chr22:1 [ <=> ] 1.22K --.-KB/s in 0s \n", - "\n", - "2018-05-31 11:40:17 (62.9 MB/s) - ‘dna?segment=chr22:15000,16000.2’ saved [1246]\n", - "\n" - ] - } - ], - "source": [ - "!wget $url_1" - ] - }, - { - "cell_type": "code", - "execution_count": 90, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r\n", - "\r\n", - "\r\n", - "\r\n", - "\r\n", - "nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn\r\n", - "nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn\r\n", - "nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn\r\n", - "nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn\r\n", - "nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn\r\n" - ] - } - ], - "source": [ - "!head dna?segment=chr22:15000,16000" - ] - }, - { - "cell_type": "code", - "execution_count": 91, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--2018-05-31 11:40:20-- http://togows.org/api/ucsc/hg38/chr22:15000-16000.fasta\n", - "Resolving togows.org (togows.org)... 133.39.78.80\n", - "Connecting to togows.org (togows.org)|133.39.78.80|:80... connected.\n", - "HTTP request sent, awaiting response... 200 OK\n", - "Length: unspecified [text/plain]\n", - "Saving to: ‘chr22:15000-16000.fasta.1’\n", - "\n", - "chr22:15000-16000.f [ <=> ] 1.02K --.-KB/s in 0s \n", - "\n", - "2018-05-31 11:40:21 (49.6 MB/s) - ‘chr22:15000-16000.fasta.1’ saved [1042]\n", - "\n" - ] - } - ], - "source": [ - "!wget $url_2" - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - ">hg38:chr22:15000-16000\r\n", - "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\r\n", - "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\r\n", - "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\r\n", - "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\r\n", - "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\r\n", - "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\r\n", - "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\r\n", - "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\r\n", - "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\r\n" - ] - } - ], - "source": [ - "!head chr22:15000-16000.fasta" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 2", - "language": "python", - "name": "python2" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.14" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/python/ga4gh/__init__.py b/python/ga4gh/__init__.py deleted file mode 100644 index 3627ee094..000000000 --- a/python/ga4gh/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# -*- coding: utf-8 -*- -""" -This top-level package simply declares the ga4gh namespace so it can -be used across modules. In this project it contains the single -:mod:`ga4gh.drs` module. -""" -__import__('pkg_resources').declare_namespace(__name__) diff --git a/python/ga4gh/drs/__init__.py b/python/ga4gh/drs/__init__.py deleted file mode 100644 index ae2d18393..000000000 --- a/python/ga4gh/drs/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# -*- coding: utf-8 -*- -__version__ = "0.6.0" diff --git a/python/ga4gh/drs/client.py b/python/ga4gh/drs/client.py deleted file mode 100644 index fda157350..000000000 --- a/python/ga4gh/drs/client.py +++ /dev/null @@ -1,169 +0,0 @@ -# -*- coding: utf-8 -*- -""" -This module exposes a single class :class:`ga4gh.drs.client.Client`, which -exposes the HTTP methods of the Data Object Service as named Python functions. - -This makes it easy to access resources that are described following these -schemas, and uses bravado to dynamically generate the client functions -following the OpenAPI schema. - -It currently assumes that the service also hosts the swagger.json, in a style -similar to the demonstration server, :mod:`ga4gh.drs.server`. -""" -try: # for python3 compat - import urlparse -except ImportError: - import urllib.parse as urlparse - -from bravado.client import SwaggerClient -from bravado.swagger_model import Loader -from bravado.requests_client import RequestsClient -from bravado_core.exception import SwaggerValidationError -from bravado_core.formatter import SwaggerFormat - -import ga4gh.drs.schema - -DEFAULT_CONFIG = { - 'validate_requests': True, - 'validate_responses': True -} - - -def validate_int64(test): - """ - Accepts an int64 and checks for numerality. Throws a Swagger Validation - exception when failing the test. - - :param test: - :return: - :raises SwaggerValidationError: - """ - if str(test) != test: - raise SwaggerValidationError('int64 are serialized as strings') - - -# This is to support serializing int64 as strings on the wire. JavaScript -# only supports up to 2^53. -int64_format = SwaggerFormat( - format='int64', - to_wire=lambda i: str(i), - to_python=lambda i: int(i), - validate=validate_int64, # jsonschema validates integer - description="Converts [wire]str:int64 <=> python long" -) - - -class Client: - """ - This class is the instantiated to create a new connection to a DOS. It - connects to the service to download the swagger.json and returns a client - in the DataObjectService namespace:: - - from ga4gh.drs.client import Client - client = Client(url='http://localhost:8000/ga4gh/drs/v1') - - models = client.models - c = client.client - - # Will return a Data Object by identifier - c.GetDataObject(data_object_id="abc").result() - - # To access models in the Data Object Service namespace: - ListDataObjectRequest = models.get_model('ListDataObjectsRequest') - - # And then instantiate a request with our own query: - my_request = ListDataObjectsRequest(alias="doi:10.0.1.1/1234") - - # Finally, send the request to the service and evaluate the response. - c.ListDataObjects(body=my_request).result() - - If you want to use the client against a DOS implementation that does - not present a ``swagger.json``, then you can use the local schema:: - - client = Client(url='http://example.com/drs-base-path/', local=True) - - Note that since this uses the local schema, some operations that are - not implemented by the implementation under test may fail. - - The class accepts a configuration dictionary that maps directly to the - bravado configuration. - - For more information on configuring the client, see - `bravado documentation - `_. - """ - def __init__(self, url=None, config=DEFAULT_CONFIG, http_client=None, request_headers=None, local=False): - """ - Instantiates :class:`~bravado.client.SwaggerClient`. - - For further documentation, refer to the documentation - for :meth:`bravado.client.SwaggerClient.from_url` and - :meth:`bravado.client.SwaggerClient.from_spec`. - - :param str url: the URL of a Swagger specification. If ``local`` - is True, this should be the base URL of a DOS - implementation (see ``local``). - :param dict config: see :meth:`bravado.client.SwaggerClient` - :param http_client: see :meth:`bravado.client.SwaggerClient` - :param request_headers: see :meth:`beravado.client.SwaggerClient` - :param bool local: set this to True to load the local schema. - If this is True, the ``url`` parameter should - point to the host and base path of the - implementation under test:: - - Client(url='https://example.com/ga4gh/drs/v1/', local=True) - - If False, the ``url`` parameter should point to a - Swagger specification (``swagger.json``). - """ - self._config = config - config['formats'] = [int64_format] - if local: - # :meth:`bravado.client.SwaggerClient.from_spec` takes a schema - # as a Python dictionary, which we can conveniently expose - # via :func:`ga4gh.drs.schema.present_schema`. - schema = ga4gh.drs.schema.present_schema() - - # Set schema['host'] and schema['basePath'] to the provided - # values if specified, otherwise leave them as they are - url = urlparse.urlparse(url) - schema['host'] = url.netloc or schema['host'] - schema['basePath'] = url.path or schema['basePath'] - - self.models = SwaggerClient.from_spec(spec_dict=schema, - config=config, - http_client=http_client) - else: - # If ``local=False``, ``url`` is expected to be a ``swagger.json`` - swagger_path = '{}/swagger.json'.format(url.rstrip("/")) - self.models = SwaggerClient.from_url(swagger_path, - config=config, - http_client=http_client, - request_headers=request_headers) - self.client = self.models.DataRepositoryService - - @classmethod - def config(cls, url, http_client=None, request_headers=None): - """ - Accepts an optionally configured requests client with authentication - details set. - - :param url: The URL of the service to connect to - :param http_client: The http_client to use, \ - defaults to :func:`RequestsClient` - :param request_headers: The headers to set on each request. - :return: - """ - swagger_path = '{}/swagger.json'.format(url.rstrip('/')) - http_client = http_client or RequestsClient() - loader = Loader(http_client, request_headers=request_headers) - spec_dict = loader.load_spec(swagger_path) - return spec_dict - - -def main(): - print('client') - - -if __name__ == '__main__': - main() diff --git a/python/ga4gh/drs/controllers.py b/python/ga4gh/drs/controllers.py deleted file mode 100644 index a8c20b4a9..000000000 --- a/python/ga4gh/drs/controllers.py +++ /dev/null @@ -1,433 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Data Repository Service Controller Functions - -These controller functions for the demo server implement an opinionated version -of DRS by providing uuid's to newly create objects, and using timestamp -versions. - -Initializes an in-memory dictionary for storing Objects. -""" -import uuid -import datetime -from dateutil.parser import parse - - -DEFAULT_PAGE_SIZE = 100 - -# Our in memory registry -objects = {} -bundles = {} - -# Application logic - - -def now(): - """ - Returns the current time in string format. - :return: Current ISO time. - """ - return str(datetime.datetime.now().isoformat("T") + "Z") - - -def get_most_recent(key): - """ - Gets the most recent Object for a key. - :param key: - :return: - """ - max = {'updated': '01-01-1965 00:00:00Z'} - if key not in objects: - raise KeyError("object not found!") - for version in objects[key].keys(): - object = objects[key][version] - if parse(object['updated']) > parse(max['updated']): - max = object - return max - - -# TODO refactor to higher order function -def get_most_recent_bundle(key): - """ - Returns the most recent bundle for the given key. - - :param key: - :return: - """ - max = {'updated': '01-01-1965 00:00:00Z'} - for version in bundles[key].keys(): - bundle = bundles[key][version] - if parse(bundle['updated']) > parse(max['updated']): - max = bundle - return max - - -def filter_objects(predicate): - """ - Filters data objects according to a function that acts on each item - returning either True or False per item. - """ - return [get_most_recent(x[0]) for x in filter(predicate, objects.items())] - - -def filter_bundles(predicate): - """ - Filters data bundles according to a function that acts on each item - returning either True or False per item. - :param predicate: A function used to test items - :return: List of Data Bundles - """ - return [ - get_most_recent_bundle(x[0]) for x in filter( - predicate, bundles.items())] - - -def add_created_timestamps(doc): - """ - Adds created and updated timestamps to the document. - :param doc: A document to be timestamped - :return doc: The timestamped document - """ - doc['created'] = now() - doc['updated'] = now() - return doc - - -def add_updated_timestamps(doc): - """ - Adds created and updated timestamps to the document. - """ - doc['updated'] = now() - return doc - - -stores = { - 'objects': objects, - 'bundles': bundles -} - - -def create(body, key): - """ - Creates a new document at the given key by adding necessary metadata - and storing in the in-memory store. - :param body: - :param key: - :return: - """ - store = stores[key] - doc = add_created_timestamps(body) - version = doc.get('version', None) - if not version: - doc['version'] = now() - if doc.get('id', None): - temp_id = str(uuid.uuid4()) - if store.get(doc['id'], None): - # issue an identifier if a valid one hasn't been provided - doc['id'] = temp_id - else: - temp_id = str(uuid.uuid4()) - doc['id'] = temp_id - store[doc['id']] = {} - store[doc['id']][doc['version']] = doc - return doc - -# Data Object Controllers - - -def CreateObject(**kwargs): - """ - Creates a new Data Object by issuing an identifier if it is not - provided. - - :param kwargs: - :return: - """ - # TODO Safely create - body = kwargs['body']['object'] - doc = create(body, 'objects') - return({"object_id": doc['id']}, 200) - - -def GetObject(**kwargs): - """ - Get a Data Object by object_id. - :param kwargs: - :return: - """ - object_id = kwargs['object_id'] - version = kwargs.get('version', None) - # Implementation detail, this server uses integer version numbers. - # Get the Data Object from our dictionary - object_key = objects.get(object_id, None) - if object_key and not version: - object = get_most_recent(object_id) - return({"object": object}, 200) - elif object_key and objects[object_id].get(version, None): - object = objects[object_id][version] - return ({"object": object}, 200) - else: - return({'msg': "The requested Data " - "Object wasn't found", 'status_code': 404}, 404) - - -def GetObjectVersions(**kwargs): - """ - Returns all versions of a Data Object. - :param kwargs: - :return: - """ - object_id = kwargs['object_id'] - # Implementation detail, this server uses integer version numbers. - # Get the Data Object from our dictionary - object_versions_dict = objects.get(object_id, None) - object_versions = [x[1] for x in object_versions_dict.items()] - if object_versions: - return({"objects": object_versions}, 200) - else: - return({'msg': "The requested Data " - "Object wasn't found", 'status_code': 404}, 404) - - -def UpdateObject(**kwargs): - """ - Update a Data Object by creating a new version. - - :param kwargs: - :return: - """ - object_id = kwargs['object_id'] - body = kwargs['body']['object'] - # Check to make sure we are updating an existing document. - try: - old_object = get_most_recent(object_id) - except KeyError: - return "Data object not found", 404 - # Upsert the new body in place of the old document - doc = add_updated_timestamps(body) - doc['created'] = old_object['created'] - # We need to safely set the version if they provided one that - # collides we'll pad it. If they provided a good one, we will - # accept it. If they don't provide one, we'll give one. - new_version = doc.get('version', None) - if not new_version or new_version in objects[object_id].keys(): - doc['version'] = now() - doc['id'] = old_object['id'] - objects[object_id][doc['version']] = doc - return({"object_id": object_id}, 200) - - -def DeleteObject(**kwargs): - """ - Delete a Data Object by object_id. - - :param kwargs: - :return: - """ - object_id = kwargs['object_id'] - del objects[object_id] - return({"object_id": object_id}, 200) - - -def ListObjects(**kwargs): - """ - Returns a list of Data Objects matching a ListObjectsRequest. - - :param kwargs: alias, url, checksum, checksum_type, page_size, page_token - :return: - """ - def filterer(item): - """ - This filter is defined as a closure to set gather the kwargs from - the request. It returns true or false depending on whether to - include the item in the filter. - :param item: - :return: bool - """ - selected = get_most_recent(item[0]) # dict.items() gives us a tuple - sel_checksum = selected.get('checksums', []) - - if kwargs.get('checksum', None): - if kwargs['checksum'] not in [i['checksum'] for i in sel_checksum]: - return False - if kwargs.get('checksum_type', None): - if kwargs['checksum_type'] not in [i['type'] for i in sel_checksum]: - return False - if kwargs.get('url', None): - if kwargs['url'] not in [i['url'] for i in selected.get('urls', [])]: - return False - if kwargs.get('alias', None): - if kwargs['alias'] not in selected.get('aliases', []): - return False - return True - - # Lazy since we're in memory - filtered = filter_objects(filterer) - page_size = int(kwargs.get('page_size', DEFAULT_PAGE_SIZE)) - # We'll page if there's a provided token or if we have too many - # objects. - if len(filtered) > page_size or kwargs.get('page_token', None): - start_index = int(kwargs.get('page_token', 0)) * page_size - end_index = start_index + page_size - # First fill a page - page = filtered[start_index:min(len(filtered), end_index)] - if len(filtered[start_index:]) - len(page) > 0: - # If there is more than one page left of results - next_page_token = int(kwargs.get('page_token', 0)) + 1 - return ( - {"objects": page, - "next_page_token": str(next_page_token)}, 200) - else: - return ({"objects": page}, 200) - else: - page = filtered - return({"objects": page}, 200) - - -# Data Bundle Controllers - - -def CreateBundle(**kwargs): - """ - Create a Data Bundle, issuing a new identifier if one is not provided. - - :param kwargs: - :return: - """ - body = kwargs['body']['bundle'] - doc = create(body, 'bundles') - return({"bundle_id": doc['id']}, 200) - - -def GetBundle(**kwargs): - """ - Get a Data Bundle by identifier. - - :param kwargs: - :return: - """ - bundle_id = kwargs['bundle_id'] - version = kwargs.get('version', None) - # Implementation detail, this server uses integer version numbers. - # Get the Data Object from our dictionary - bundle_key = bundles.get(bundle_id, None) - if bundle_key and not version: - bundle = get_most_recent_bundle(bundle_id) - return({"bundle": bundle}, 200) - elif bundle_key and objects[bundle_id].get(version, None): - bundle = bundles[bundle_id][version] - return ({"bundle": bundle}, 200) - else: - return({'msg': "The requested Data " - "Bundle wasn't found", 'status_code': 404}, 404) - - -def UpdateBundle(**kwargs): - """ - Updates a Data Bundle to include new metadata by upserting the new - bundle. - - :param kwargs: - :return: - """ - bundle_id = kwargs['bundle_id'] - body = kwargs['body']['bundle'] - # Check to make sure we are updating an existing document. - old_bundle = get_most_recent_bundle(bundle_id) - # Upsert the new body in place of the old document - doc = add_updated_timestamps(body) - doc['created'] = old_bundle['created'] - # We need to safely set the version if they provided one that - # collides we'll pad it. If they provided a good one, we will - # accept it. If they don't provide one, we'll give one. - new_version = doc.get('version', None) - if not new_version or new_version in bundles[bundle_id].keys(): - doc['version'] = now() - doc['id'] = old_bundle['id'] - bundles[bundle_id][doc['version']] = doc - return({"bundle_id": bundle_id}, 200) - - -def GetBundleVersions(**kwargs): - """ - Get all versions of a Data Bundle. - - :param kwargs: - :return: - """ - bundle_id = kwargs['bundle_id'] - bundle_versions_dict = bundles.get(bundle_id, None) - bundle_versions = [x[1] for x in bundle_versions_dict.items()] - if bundle_versions: - return({"bundles": bundle_versions}, 200) - else: - return({'msg': "The requested Data " - "Bundle wasn't found", 'status_code': 404}, 404) - - -def DeleteBundle(**kwargs): - """ - Deletes a Data Bundle by ID. - - :param kwargs: - :return: - """ - bundle_id = kwargs['bundle_id'] - del bundles[bundle_id] - return(kwargs, 200) - - -def ListBundles(**kwargs): - """ - Takes a ListBundles request and returns the bundles that match - that request. Possible kwargs: alias, url, checksum, checksum_type, page_size, page_token - - :param kwargs: ListBundles request. - :return: - """ - def filterer(item): - """ - This filter is defined as a closure to set gather the kwargs from - the request. It returns true or false depending on whether to - include the item in the filter. - :param item: - :rtype: bool - """ - selected = get_most_recent_bundle(item[0]) - sel_checksum = selected.get('checksums', []) - if kwargs.get('checksum', None): - if kwargs['checksum'] not in [i['checksum'] for i in sel_checksum]: - return False - if kwargs.get('checksum_type', None): - if kwargs['checksum_type'] not in [i['type'] for i in sel_checksum]: - return False - if kwargs.get('alias', None): - if kwargs['alias'] not in selected.get('aliases', []): - return False - return True - # Lazy since we're in memory - filtered = filter_bundles(filterer) - page_size = int(kwargs.get('page_size', DEFAULT_PAGE_SIZE)) - # We'll page if there's a provided token or if we have too many - # objects. - if len(filtered) > page_size: - start_index = int(kwargs.get('page_token', 0)) * page_size - end_index = start_index + page_size - # First fill a page - page = filtered[start_index:min(len(filtered), end_index)] - if len(filtered[start_index:]) - len(page) > 0: - # If there is more than one page left of results - next_page_token = int(kwargs.get('page_token', 0)) + 1 - return ( - {"bundles": page, - "next_page_token": str(next_page_token)}, 200) - else: - return ({"bundles": page}, 200) - else: - page = filtered - return({"bundles": page}, 200) - - -def GetServiceInfo(**kwargs): - import ga4gh.drs.schema - return ga4gh.drs.schema.present_schema()['info'], 200 diff --git a/python/ga4gh/drs/data_repository_service.swagger.yaml b/python/ga4gh/drs/data_repository_service.swagger.yaml deleted file mode 120000 index 9d18830c6..000000000 --- a/python/ga4gh/drs/data_repository_service.swagger.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../openapi/data_repository_service.swagger.yaml \ No newline at end of file diff --git a/python/ga4gh/drs/schema.py b/python/ga4gh/drs/schema.py deleted file mode 100644 index fbde3b28c..000000000 --- a/python/ga4gh/drs/schema.py +++ /dev/null @@ -1,56 +0,0 @@ -# -*- coding: utf-8 -*- -import os.path - -import swagger_spec_validator.common - -cd = os.path.dirname(os.path.realpath(__file__)) -SWAGGER_PATH = os.path.join(cd, 'data_repository_service.swagger.yaml') - - -def present_schema(): - """ - Presents the OpenAPI 2.0 schema as a dictionary. - :rvtype: dict - """ - return swagger_spec_validator.common.read_file(SWAGGER_PATH) - - -def from_chalice_routes(routes, base_path=''): - """ - Given a :obj:`chalice.Chalice.routes` objects, computes the proper - subset of the Data Object Service schema and presents it as an - OpenAPI 2.0 JSON schema. - :rvtype: dict - :param chalice.Chalice.routes routes: - :param str base_path: the base path of the endpoints listed in `routes`. - This is only necessary if a base path is manually - prepended to each endpoint your service exposes, - e.g. ``@app.route('/ga4gh/drs/v1/dataobjects')``. - This string will be stripped from the beginning of - each path in the `routes` object if it is present. - The schema will be updated with this value. - """ - schema = present_schema() - - # Sanitize the routes that are provided so we can compare them easily. - schema['basePath'] = base_path.rstrip('/').lower() or schema['basePath'].lower() - sanitized = {} - for chalice_path, chalice_methods in routes.items(): - path = chalice_path.lower() - if path.startswith(schema['basePath']): - path = path.replace(schema['basePath'], '', 1) - sanitized[path] = [i.lower() for i in chalice_methods] - routes = sanitized - - # Next, remove from the generated schema paths that are not defined - # in :obj:`routes`. - schema['paths'] = {k: v for k, v in schema['paths'].items() - if k in routes.keys()} - - # Loop over the remaining paths in the generated schema and remove - # methods that are not listed in :obj:`routes`. - for path in schema['paths'].keys(): - schema['paths'][path] = {k: v for k, v in schema['paths'][path].items() - if k in [r.lower() for r in routes[path]]} - - return schema diff --git a/python/ga4gh/drs/server.py b/python/ga4gh/drs/server.py deleted file mode 100644 index d04173625..000000000 --- a/python/ga4gh/drs/server.py +++ /dev/null @@ -1,39 +0,0 @@ -# -*- coding: utf-8 -*- -""" -DRS Demonstration Server - -Running this server will start an ephemeral Data Repository Service (its registry -contents won't be saved after exiting). It uses the connexion module -to translate the OpenAPI schema into named controller functions. - -These functions are described in :mod:`ga4gh.drs.controllers` and -are meant to provide a simple implementation of DOS. -""" -import connexion -from flask_cors import CORS - -# These are imported by name by connexion so we assert it here. -from ga4gh.drs.controllers import * # noqa -from ga4gh.drs.schema import SWAGGER_PATH - - -def configure_app(): - # The model name has to match what is in - # tools/prepare_swagger.sh controller. - app = connexion.App( - "ga4gh.drs.server", - swagger_ui=True, - swagger_json=True) - app.add_api(SWAGGER_PATH) - - CORS(app.app) - return app - - -def main(): - app = configure_app() - app.run(port=8080, debug=True) - - -if __name__ == '__main__': - main() diff --git a/python/ga4gh/drs/test/__init__.py b/python/ga4gh/drs/test/__init__.py deleted file mode 100644 index bee266244..000000000 --- a/python/ga4gh/drs/test/__init__.py +++ /dev/null @@ -1,117 +0,0 @@ -# -*- coding: utf-8 -*- -import datetime -import functools -import hashlib -import random -import unittest -import uuid - - -def test_requires(*operations): - """ - This is a decorator that identifies what DOS operations a given test - case uses (where each DOS operation is named by its `operationId` in - the schema, e.g. ListBundles, UpdateObject, GetServiceInfo, - etc.) and skips them if the operation is not supported by the - implementation under test. - - For example, given this test setup:: - - class Test(AbstractComplianceTest): - supports = ['UpdateBundles'] - - @test_requires('UpdateBundles') - def test_update_data_bundles(self): - self.drs_request('PUT', '/databundles/1234') - - @test_requires('ListBundles', 'UpdateBundles') - def test_list_and_update_data_bundles(self): - self.drs_request('GET', '/databundles') - self.drs_request('PUT', '/databundles/1234') - - ``test_update_data_bundles`` would run and ``test_list_and_update_data_bundles`` - would be skipped. - - :param str \*operations: the operations supported by the decorated - test case - """ - def decorator(func): - @functools.wraps(func) - def wrapper(self): - unsupported = [op for op in operations if op not in self.supports] - if unsupported: - raise unittest.SkipTest("not supported: " + ", ".join(unsupported)) - return func(self) - return wrapper - return decorator - - -class DataRepositoryServiceTest(unittest.TestCase): - @staticmethod - def generate_objects(amount): - """ - Yields a specified number of data objects with random attributes. - - :param int amount: the amount of data objects to generate - """ - # Defines sane default random values for each field type. The - # field types are defined in the schema and manually assigned here. - # They are specified as lambdas so that we can generate a new - # value each time, instead of having a single value for each - # invocation of :meth:`generate_data_objects`. - types = { - # uuid4() produces UUIDs that are easier to quickly - # differentiate visually, as opposed to uuid1() which - # produces UUIDs based on the hostname and current time. - # (It doesn't matter too much but it's a nice convenience - # thing to have.) - 'string': lambda: str(uuid.uuid4()), - # `long` - 'str-int64': lambda: str(random.randint(-(2**63) + 1, 2**63 - 1)), - # Swagger expects a RFC 3339 compliant datetime object. - # See https://stackoverflow.com/a/8556555 - 'str-date-time': lambda: datetime.datetime.utcnow().isoformat('T') + 'Z' - } - for _ in range(amount): - yield { - 'id': types['string'](), - 'name': types['string'](), - # `size` can't be negative, but there's a possibility that - # not calling :func:`abs` on the 'size' key could result in - # a really entertaining bug down the line so I'm going to - # leave it like that - 'size': types['str-int64'](), - 'created': types['str-date-time'](), - 'updated': types['str-date-time'](), - 'version': types['string'](), - 'mime_type': types['string'](), - 'checksums': [{ - # Encode for Python 3 compat - 'checksum': hashlib.md5(types['string']().encode('utf-8')).hexdigest(), - # I believe that this field will soon become an `enum` in the schema. - # Ideally, the available choices should be pulled from the schema... - 'type': random.choice(['md5', 'multipart-md5', 'sha256', 'sha512']) - }], - 'urls': [ - {'url': types['string']()}, - {'url': types['string']()} - ], - 'description': types['string'](), - 'aliases': [types['string']()], - } - - @staticmethod - def generate_bundles(amount): - """ - Yields a specified number of data bundles with random attributes. - - :param int amount: the amount of data bundles to generate - """ - for bdl in DataRepositoryServiceTest.generate_objects(amount): - del bdl['name'] - del bdl['size'] - del bdl['mime_type'] - del bdl['urls'] - # See :var:`generate_data_objects.types` above - bdl['object_ids'] = [str(uuid.uuid4()), str(uuid.uuid4())] - yield bdl diff --git a/python/ga4gh/drs/test/compliance.py b/python/ga4gh/drs/test/compliance.py deleted file mode 100644 index 5a2975ad3..000000000 --- a/python/ga4gh/drs/test/compliance.py +++ /dev/null @@ -1,481 +0,0 @@ -# -*- coding: utf-8 -*- -import json -import logging -import random -import time -try: - import urllib.parse as urllib # For Python 3 compat -except ImportError: - import urllib -import uuid - -import ga4gh.drs.schema -from ga4gh.drs.test import DataRepositoryServiceTest, test_requires - -logging.basicConfig(level=logging.DEBUG) -logger = logging.getLogger(__name__) - - -class AbstractComplianceTest(DataRepositoryServiceTest): - """ - This class implements a number of compliance tests for Object Service - implementations. It is meant to provide a single, standardized test - harness to verify that a given DOS implementation acts in a manner - consistent with the schema. - - Using the test harness is pretty straightforward, and only requires - implementing a method that can make requests to the service under test - (:meth:`~AbstractComplianceTest._make_request`). As this class subclasses - :class:`unittest.TestCase`, all the functions exposed to a subclass - of :class:`unittest.TestCase` (e.g. :meth:`~unittest.TestCase.setUpClass`) - are available for use. - - This test suite does not perform any authentication testing. Requests made - during testing are made with the assumption that they will be properly - authenticated in :meth:`_make_request` or similar. - - For a service built using Chalice, you would likely be able to write - something similar to this:: - - from ga4gh.drs.test.compliance import AbstractComplianceTest - from chalice import LocalGateway, Config - from my_chalice_app import chalice_app - - class TestApp(AbstractComplianceTest): - @classmethod - def setUpClass(cls): - cls.lg = LocalGateway(chalice_app, Config()) - - @classmethod - def _make_request(self, meth, path, headers=None, body=None) - headers = headers or {} - r = self.lg.handle_request(method=meth, path='/ga4gh/drs/v1' + path, - headers=headers, body=body) - return r['body'], r['statusCode'] - - You would then be able to run the compliance test suite however you - normally run your tests (e.g. ``nosetests`` or ``python -m unittest discover``). - - :var supports: a list of supported DOS operations. By default, this is - the list of all DOS operations, named by the `operationId` - key in the schema:: - - supports = ['GetServiceInfo', 'GetBundleVersions', - 'CreateBundle', 'ListBundles', - 'UpdateObject', 'GetObject', ...] - - Adding / removing operations from this list will adjust - which tests are run. So, doing something like:: - - class Test(AbstractComplianceTest): - self.supports = ['ListObjects'] - - would skip all tests calling UpdateBundle, GetBundle, - and any other endpoint that is not ListObjects. - """ - # Populate :var:`supports` with the `operationId` of each DOS endpoint - # specified in the schema. - supports = [] - for path in ga4gh.drs.schema.present_schema()['paths'].values(): - for method in path.values(): - supports.append(method['operationId']) - - @classmethod - def _make_request(cls, meth, path, headers=None, body=None): - """ - Method that makes requests to a DOS implementation under test - given a method, path, request headers, and a request body. - - The provided path is the path provided in the Object Service - schema - this means that in your implementation of this method, - you might need to prepend the provided path with your ``basePath``, - e.g. ``/ga4gh/drs/v1``. - - This method should return a tuple of the raw request content as a - string and the return code of the request as an int. - - :param str meth: the HTTP method to use in the request (i.e. GET, - PUT, etc.) - :param str path: path to make a request to, sans hostname (e.g. - `/bundles`) - :param dict headers: headers to include with the request - :param dict body: data to be included in the request body (serialized - as JSON) - :rtype: tuple - :returns: a tuple of the response body as a JSON-formatted string and the - response code as an int - """ - raise NotImplementedError - - @classmethod - def drs_request(cls, meth, path, headers=None, body=None, expected_status=200): - """ - Wrapper function around :meth:`AbstractComplianceTest._make_request`. - Logs the request being made, makes the request with - :meth:`._make_request`, checks for errors, and performs transparent - JSON de/serialization. - - It is assumed that any request made through this function is a - request made to the underlying DOS implementation - e.g., - ``self.drs_request('https://example.com/')`` should be expected - to fail. - - :param str meth: the HTTP method to use in the request (i.e. GET, - PUT, etc.) - :param str path: path to make a request to, sans hostname (e.g. - `/bundles`) - :param dict headers: headers to include with the request - :param dict body: data to be included in the request body - (**not** serialized as JSON) - :param int expected_status: expected HTTP status code. If the status - code is not expected, an error will be - raised. - :rtype: dict - :returns: the response body - """ - # Log the request being made, make the request itself, then log the response. - logger.debug("%s %s", meth, path) - # DOS only really speaks JSON, so we can assume that if data is being - # sent with a request, that data will be JSON - headers = headers or {} - if body and 'Content-Type' not in headers: - headers['Content-Type'] = 'application/json' - request, status = cls._make_request(meth=meth, path=path, headers=headers, - body=json.dumps(body)) - logger.info("{meth} {path} [{status}]".format(**locals())) - - # Check to make sure the return code is what we expect - msg = "{meth} {path} returned {status}, expected {expected_status}: {request}" - # We could use :meth:`assertEqual` here, but if we do, - # :meth:`drs_request` must be an instance method. Since the only - # advantage we really lose is a prettier error message, we can - # be a little verbose this one time. - # It's preferable that :meth:`drs_request` be defined as a class method - # to allow one-time server setup to be performed in meth:`setUpClass`, - # which must necessarily be a class method. - if not status == expected_status: - raise AssertionError(msg.format(**locals())) - - # Return the deserialized request body - return json.loads(request) - - @staticmethod - def get_query_url(path, **kwargs): - """ - Returns the given path with the provided kwargs concatenated as - query parameters, e.g.:: - - >>> self.get_query_url('/objects', alias=123) - '/objects?alias=123' - - :param str path: URL path without query parameters - :param kwargs: query parameters - :rtype: str - """ - return path + '?' + urllib.urlencode(kwargs) - - def get_random_object(self): - """ - Retrieves a 'random' data object by performing a ListObjects - request with a large page size then randomly selecting a data - object from the response. - - As this test utilizes the ListObjects operation, be sure to - specify that as a test requirement with :func:`test_requires` - when using this context manager in a test case. - - Usage:: - - obj, url = self.get_random_object() - - :returns: a random data object as a dict and its relative URL - (e.g. '/objects/abcdefg-12345') as a string - :rtype: tuple - """ - r = self.drs_request('GET', self.get_query_url('/objects', page_size=100)) - obj = random.choice(r['objects']) - url = '/objects/' + obj['id'] - return obj, url - - def get_random_bundle(self): - """ - Retrieves a 'random' data bundle. Similar to :meth:`get_random_object` - but retrieves a data bundle instead. - """ - r = self.drs_request('GET', self.get_query_url('/bundles', page_size=100)) - bdl = random.choice(r['bundles']) - url = '/bundles/' + bdl['id'] - return bdl, url - - # # ListObject tests - @test_requires('ListObjects') - def test_list_objects_simple(self): - """ - Smoke test to verify that `GET /objects` returns a response. - """ - r = self.drs_request('GET', '/objects') - self.assertTrue(r) - - @test_requires('ListObjects') - def test_list_objects_by_checksum(self): - """ - Test that filtering by checksum in ListObjects works nicely. - Since we can assume that checksums are unique between data - objects, we can test this functionality by selecting a random - data object then using ListObjects with a checksum parameter - and asserting that only one result is returned and that the - result returned is the same as the one queried. - """ - obj, _ = self.get_random_object() - for cs in obj['checksums']: - url = self.get_query_url('/objects', checksum=cs['checksum'], checksum_type=cs['type']) - r = self.drs_request('GET', url) - self.assertEqual(len(r['objects']), 1) - self.assertEqual(r['objects'][0]['id'], obj['id']) - - @test_requires('ListObjects') - def test_list_objects_by_alias(self): - """ - Tests that filtering by alias in ListObjects works. We do - this by selecting a random data object with ListObjects - then performing another ListObjects query but filtering - by the alias, then checking that every returned object contains - the proper aliases. - """ - reference_obj, _ = self.get_random_object() - url = self.get_query_url('/objects', alias=reference_obj['aliases'][0]) - queried_objs = self.drs_request('GET', url)['objects'] - for queried_obj in queried_objs: - self.assertIn(reference_obj['aliases'][0], queried_obj['aliases']) - - @test_requires('ListObjects') - def test_list_objects_with_nonexist_alias(self): - """ - Test to ensure that looking up a nonexistent alias returns an - empty list. - """ - alias = str(uuid.uuid1()) # An alias that is unlikely to exist - body = self.drs_request('GET', self.get_query_url('/objects', alias=alias)) - self.assertEqual(len(body['objects']), 0) - - @test_requires('ListObjects') - def test_list_objects_paging(self): - """ - Demonstrates basic paging features. - """ - # Test the page_size parameter - r = self.drs_request('GET', self.get_query_url('/objects', page_size=3)) - self.assertEqual(len(r['objects']), 3) - r = self.drs_request('GET', self.get_query_url('/objects', page_size=7)) - self.assertEqual(len(r['objects']), 7) - - # Next, given that the adjusting page_size works, we can test that paging - # works by making a ListObjects request with page_size=2, then making - # two requests with page_size=1, and comparing that the results are the same. - both = self.drs_request('GET', self.get_query_url('/objects', page_size=2)) - self.assertEqual(len(both['objects']), 2) - first = self.drs_request('GET', self.get_query_url('/objects', page_size=1)) - self.assertEqual(len(first['objects']), 1) - second = self.drs_request('GET', self.get_query_url('/objects', page_size=1, - page_token=first['next_page_token'])) - self.assertEqual(len(second['objects']), 1) - self.assertEqual(first['objects'][0], both['objects'][0]) - self.assertEqual(second['objects'][0], both['objects'][1]) - - @test_requires('ListObjects') - def test_list_object_querying(self): - """ - Tests if ListObject handles multiple query parameters correctly. - """ - # ListObjects supports querying by checksum, URL, and alias. - # To test this, let us take a data object with a unique checksum, - # URL, and alias: - obj, _ = self.get_random_object() - - def query(expected_results, expected_object=None, **kwargs): - """ - Makes a ListObject query with parameters specifying - the checksum, URL, and alias of the ``obj`` data object above. - - :param int expected_results: the amount of results to expect - from the ListObjects request - :param dict expected_object: if expected_results is 1, then - if only one object is returned - from the query, assert that the - returned object is this object - :param kwargs: query parameters for the ListObjects request - """ - args = { - 'url': obj['urls'][0]['url'], - 'alias': obj['aliases'][0], - 'checksum': obj['checksums'][0]['checksum'], - 'checksum_type': obj['checksums'][0]['type'] - } - args.update(kwargs) - url = self.get_query_url('/objects', **args) - r = self.drs_request('GET', url) - self.assertEqual(len(r['objects']), expected_results) - if expected_object and expected_results == 1: - self.assertEqual(expected_object, r['objects'][0]) - - rand = str(uuid.uuid1()) - - # If the data object we selected has a unique checksum, alias, and URL, - # then when we make a ListObjects requesting all three of those - # parameters, we should receive exactly one data object back - the one - # we chose above. - query(expected_results=1, expected_object=obj) - - # That said, if we query for the above checksum and alias but also - # query for a URL that is unlikely to exist, then we should receive - # no results, as the search criteria should be logically ANDed together. - # If `expected_results != 0`, then it is likely that the criteria are - # being ORed. - query(expected_results=0, url=rand) - - # And to finish up the test, we repeat the test directly aforementioned - # on the other two attributes we expect to be unique. - query(expected_results=0, alias=rand) - query(expected_results=0, checksum=rand) - - # # GetObject tests - @test_requires('ListObjects', 'GetObject') - def test_get_object(self): - """ - Lists Objects and then gets one by ID. - """ - obj_1, url = self.get_random_object() - obj_2 = self.drs_request('GET', url)['object'] - # Test that the data object randomly chosen via `/objects` - # can be retrieved via `/objects/{object_id}` - self.assertEqual(obj_1, obj_2) - - @test_requires('ListBundles', 'GetBundle') - def test_get_bundle(self): - """ - Lists data bundles and then gets one by ID. - """ - bdl_1, url = self.get_random_bundle() - bdl_2 = self.drs_request('GET', url)['bundle'] - # Test that the data object randomly chosen via `/bundles` - # can be retrieved via `/bundles/{bundle_id}` - self.assertEqual(bdl_1, bdl_2) - - @test_requires('ListBundles') - def test_list_bundles_with_nonexist_alias(self): - """ - Test to ensure that searching for data bundles with a nonexistent - alias returns an empty list. - """ - alias = str(uuid.uuid1()) # An alias that is unlikely to exist - body = self.drs_request('GET', self.get_query_url('/bundles', alias=alias)) - self.assertEqual(len(body['bundles']), 0) - - @test_requires('GetBundle') - def test_get_nonexistent_bundle(self): - """ - Verifies that requesting a data bundle that doesn't exist results in HTTP 404 - """ - bdl, url = self.get_random_bundle() - self.drs_request('GET', '/bundles/NonexistentBundle', - body={'bundle': bdl}, expected_status=404) - - @test_requires('UpdateObject') - def test_update_nonexistent_object(self): - """ - Verifies that trying to update a data object that doesn't exist - returns HTTP 404 - """ - obj, url = self.get_random_object() - self.drs_request('PUT', '/objects/NonexistentObjID', expected_status=404, - body={'object': obj, 'object_id': obj['id']}) - - @test_requires('GetObject', 'ListObjects') - def test_update_object_with_bad_request(self): - """ - Verifies that attempting to update a data object with a malformed - request returns HTTP 400 - """ - _, url = self.get_random_object() - self.drs_request('PUT', url, expected_status=400, body={'abc': ''}) - - @test_requires('ListObjects', 'UpdateObject', 'GetObject') - def test_alias_update(self): - """ - Demonstrates updating a data object with a given alias. - """ - alias = 'daltest:' + str(uuid.uuid1()) - # First, select a "random" object that we can test - object, url = self.get_random_object() - - # Try and update with no changes. - self.drs_request('PUT', url, body={'object': object}) - # We specify the Content-Type since Chalice looks for it when - # deserializing the request body server-side - - # Test adding an alias (acceptably unique to try - # retrieving the object by the alias) - object['aliases'].append(alias) - - # Try and update, this time with a change. - update_response = self.drs_request('PUT', url, - body={'object': object}) - self.assertEqual(object['id'], update_response['object_id']) - - time.sleep(2) - - # Test and see if the update took place by retrieving the object - # and checking its aliases - get_response = self.drs_request('GET', url) - self.assertEqual(update_response['object_id'], get_response['object']['id']) - self.assertIn(alias, get_response['object']['aliases']) - - # Testing the update again by using a DOS ListObjectsRequest - # to locate the object by its new alias. - list_request = { - 'alias': alias, - # We know the alias is unique, so even though page_size > 1 - # we expect only one result. - 'page_size': 10 - } - list_url = self.get_query_url('/objects', **list_request) - list_response = self.drs_request('GET', list_url) - self.assertEqual(1, len(list_response['objects'])) - self.assertIn(alias, list_response['objects'][0]['aliases']) - - # # Tear down and remove the test alias - # params['body']['object']['aliases'].remove(alias) - # self.drs_request('PUT', url, **params) - - @test_requires('ListObjects', 'UpdateObject') - def test_full_object_update(self): - """ - Demonstrates updating multiple fields of a data object at once. - This incidentally also tests object conversion. - """ - # First, select a "random" object that we can test - object, url = self.get_random_object() - - # Make a new data object that is different from the data object we retrieved - attributes = { - # 'name' and 'description' are optional fields and might not be present - 'name': object.get('name', '') + 'test-suffix', - # See Biosphere/drs-azul-lambda#87 - # 'description': object.get('description', '') + 'Change This', - 'urls': [ - {'url': 'https://cgl.genomics.ucsc.edu/'}, - {'url': 'https://github.com/Biosphere'} - ] - } - object.update(attributes) - - # Now update the old data object with the new attributes we added - self.drs_request('PUT', url, body={'object': object}) - time.sleep(2) # Give the server some time to catch up - - # Test and see if the update took place - get_response = self.drs_request('GET', url)['object'] - # We only compare the change attributes as DOS implementations - # can update timestamps server-side - self.assertEqual(get_response['name'], object['name']) - self.assertEqual(get_response['urls'], object['urls']) diff --git a/python/test/__init__.py b/python/test/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/python/test/test_compliance.py b/python/test/test_compliance.py deleted file mode 100644 index 3aad4c052..000000000 --- a/python/test/test_compliance.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- -import werkzeug.datastructures - -import ga4gh.drs.server -from ga4gh.drs.test.compliance import AbstractComplianceTest - -# We set this so that `nose` doesn't try and run the abstract tests. -# (If that happens, all of the tests fail since :meth:`_make_request` -# raises a :exc:`NotImplementedError` for each of the test cases. -AbstractComplianceTest.__test__ = False - - -class TestCompliance(AbstractComplianceTest): - """ - Runs the :class:`~ga4gh.drs.test.compliance.AbstractComplianceTest` - against :mod:`ga4gh.drs.server`. - """ - # See above - if we don't explicitly set :var:`__test__` here, - # this test suite won't run as we adjust the value of the variable - # in the superclass above. - __test__ = True - - @classmethod - def setUpClass(cls): - # :mod:`ga4gh.drs.server` is built on top of :mod:`connexion`, - # which is built on top of :mod:`flask`, which is built on top - # of :mod:`werkzeug`, which means we can do some cool nice - # things with testing. - app = ga4gh.drs.server.configure_app().app - cls.client = app.test_client() - - # Populate our new server with some test data objects and bundles - for data_obj in cls.generate_objects(250): - cls.drs_request('POST', '/objects', body={'object': data_obj}) - for data_bdl in cls.generate_bundles(250): - cls.drs_request('POST', '/bundles', body={'bundle': data_bdl}) - - @classmethod - def _make_request(cls, meth, path, headers=None, body=None): - # For documentation on this function call, see - # :class:`werkzeug.test.EnvironBuilder` and :meth:`werkzeug.test.Client.get`. - headers = werkzeug.datastructures.Headers(headers) - r = cls.client.open(method=meth, path='/ga4gh/drs/v1' + path, - data=body, headers=headers) - return r.data, r.status_code diff --git a/python/test/test_package.py b/python/test/test_package.py deleted file mode 100644 index a038f5384..000000000 --- a/python/test/test_package.py +++ /dev/null @@ -1,106 +0,0 @@ -# -*- coding: utf-8 -*- -import os.path -import unittest - -import openapi_spec_validator -import requests -import swagger_spec_validator -import yaml - - -class TestPackage(unittest.TestCase): - @classmethod - def setUpClass(cls): - cwd = os.path.dirname(os.path.realpath(__file__)) - spec_dir = os.path.join(cwd, '../../openapi') - cls.swagger_path = os.path.join(spec_dir, 'data_repository_service.swagger.yaml') - cls.smartapi_path = os.path.join(spec_dir, 'data_repository_service.smartapi.yaml') - cls.openapi_path = os.path.join(spec_dir, 'data_repository_service.openapi.yaml') - - # The :func:`unittest.skipUnless` calls depend on class variables, - # which means that we can't decorate the test cases conventionally - # and have to do so after the class variables we need are instantiated. - openapi_dec = unittest.skipUnless(os.path.exists(cls.openapi_path), "Generated schema not found.") - cls.test_openapi_schema_validity = openapi_dec(cls.test_openapi_schema_validity) - smartapi_dec = unittest.skipUnless(os.path.exists(cls.smartapi_path), "Generated schema not found.") - cls.test_smartapi_schema_validity = smartapi_dec(cls.test_smartapi_schema_validity) - - def test_version_consensus(self): - from ga4gh.drs import __version__ - with open(self.swagger_path, 'r') as f: - spec_version = yaml.safe_load(f)['info']['version'] - assert __version__ == spec_version - - def test_swagger_schema_validity(self): - """Validate the Swagger schema using swagger_spec_validator.""" - # We always expect the Swagger schema to exist since it's the - # reference implementation from which the OpenAPI (3.0) and SmartAPI - # schemas are generated, so we won't include a silent skip condition - # like we would for :meth:`test_openapi_schema_validity` or - # :meth:`test_smartapi_schema_validity`. - path = os.path.abspath(self.swagger_path) - swagger_spec_validator.validate_spec_url('file://' + path) - - def test_openapi_schema_validity(self): - """ - Validate the generated OpenAPI schema. Will be skipped if the - generated schema file is not present. - """ - # p1c2u/openapi-spec-validator supports validation of both Swagger - # and OpenAPI schemas, but since Yelp/swagger_spec_validator is - # installed with the other dependencies anyway, it's easier to just - # leave it in place. - path = os.path.abspath(self.openapi_path) - openapi_spec_validator.validate_v3_spec_url('file://' + path) - - def test_smartapi_schema_validity(self): - """ - Validates the generated SmartAPI schema by temporarily uploading - it to a third-party pastebin then using the SmartAPI API to - validate the schema. Will be skipped if the generated schema - file is not present. - """ - schema = {'file': open(self.smartapi_path, 'rb')} - post = requests.post('https://file.io/', files=schema) - self.assertTrue(post.json()['success'], post.json()) - validate = requests.get('https://smart-api.info/api/validate?url=' + post.json()['link']) - # A couple notes on the /validate endpoint, since it appears to be - # mostly undocumented: - # * The endpoint will always return HTTP 200 regardless of whether - # or not the schema is valid - # * If the schema is invalid, the endpoint will return something - # like {'success': false} - # * If the schema is valid, the endpoint will return something - # like {'valid': true} - self.assertTrue(validate.json()['valid'], validate.json()) - - def test_chalice_schema_generation(self): - """ - Validate that the schema generated by :func:`ga4gh.drs.schema.from_chalice_routes` - is valid. - """ - from ga4gh.drs.schema import from_chalice_routes - routes = { - # Test a path that does not exist in the schema - '/PathThatDoesNotExist': {'GET': None}, - # Test a valid path with a nonexistent method - '/ga4gh/drs/v1/bundles': {'MethodThatDoesNotExist': None}, - # Test a path with a different case than what is defined in the schema - '/GA4GH/DRS/V1/BUNDLES/{bundle_id}': {'GET': None}, - # Test multiple methods - '/ga4gh/drs/v1/objects/{object_id}': {'GET': None, - 'PUT': None} - } - schema = from_chalice_routes(routes, base_path='/ga4gh/drs/v1') - paths = schema['paths'] - - self.assertNotIn('/PathThatDoesNotExist', paths.keys()) - # Test that base path is correctly stripped - self.assertNotIn('/ga4gh/drs/v1/bundles', paths.keys()) - self.assertIn('/bundles/{bundle_id}', paths.keys()) - self.assertNotIn('MethodThatDoesNotExist', paths['/bundles'].keys()) - self.assertIn('get', paths['/objects/{object_id}'].keys()) - self.assertIn('put', paths['/objects/{object_id}'].keys()) - self.assertNotIn('/objects', paths.keys()) - # Make sure that the schema is intact - self.assertIn('200', paths['/objects/{object_id}']['get']['responses'].keys()) diff --git a/python/test/test_server.py b/python/test/test_server.py deleted file mode 100644 index b550b193b..000000000 --- a/python/test/test_server.py +++ /dev/null @@ -1,426 +0,0 @@ -# -*- coding: utf-8 -*- -import logging -import subprocess -import time - -import bravado.exception -import jsonschema.exceptions - -import ga4gh.drs -import ga4gh.drs.test -import ga4gh.drs.client - -SERVER_URL = 'http://localhost:8080/ga4gh/drs/v1' -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) -logging.captureWarnings(True) -# Make scrolling through test logs more useful -logging.getLogger('swagger_spec_validator.ref_validators').setLevel(logging.INFO) -logging.getLogger('bravado_core.model').setLevel(logging.INFO) -logging.getLogger('swagger_spec_validator.validator20').setLevel(logging.INFO) - - -class TestServer(ga4gh.drs.test.DataRepositoryServiceTest): - @classmethod - def setUpClass(cls): - cls._server_process = subprocess.Popen(['ga4gh_drs_server'], stdout=subprocess.PIPE, - stderr=subprocess.PIPE, shell=False) - time.sleep(2) - local_client = ga4gh.drs.client.Client(SERVER_URL) - cls._models = local_client.models - cls._client = local_client.client - - @classmethod - def tearDownClass(cls): - logger.info('Tearing down server process (PID %d)', cls._server_process.pid) - cls._server_process.kill() - cls._server_process.wait() - - def generate_bundle(self, **kwargs): - """ - Generates a Bundle with bravado. - Same arguments as :meth:`generate_object`. - """ - bdl_model = self._models.get_model('Bundle') - bdl = next(self.generate_bundles(1)) - bdl.update(kwargs) - return bdl_model.unmarshal(bdl) - - def generate_object(self, **kwargs): - """ - Generates a Object with bravado. - :param kwargs: fields to set in the generated data object - """ - obj_model = self._models.get_model('Object') - obj = next(self.generate_objects(1)) - obj.update(kwargs) - return obj_model.unmarshal(obj) - - def request(self, operation_id, query={}, **params): - """ - Make a request to the DOS server with :class:`ga4gh.drs.client.Client`. - :param str operation_id: the name of the operation ID to call (e.g. - ListBundles, DeleteObject, etc.) - :param dict query: parameters to include in the query / path - :param \*\*params: parameters to include in the request body - (that would normally be provided to the - Request model) - :returns: response body of the request as a schema model (e.g. - ListBundlesResponse) - """ - request_name = operation_id + 'Request' - # These two in particular are special cases as they are the only - # models that utilize query parameters - if request_name in ['ListBundlesRequest', 'ListObjectsRequest']: - params = self._models.get_model(request_name)(**params).marshal() - elif request_name in self._models.swagger_spec.definitions: - params = {'body': self._models.get_model(request_name)(**params)} - params.update(query) - return getattr(self._client, operation_id)(**params).result() - - def assertSameObject(self, obj_1, obj_2, check_version=True): - """ - Verifies that the two provided data objects are the same by - comparing them key-by-key. - - :param bool check_version: set to True to check if the version - key is the same, False otherwise. - This option is provided as some DOS - implementations will touch the version - key on their own, and some will not - :raises AssertionError: if the provided objects are not the same - :rtype: bool - :returns: True if the objects are the same - """ - # ctime and mtime can be touched server-side - ignored = ['created', 'updated'] - if not check_version: - ignored.append('version') - for k in obj_1.__dict__['_Model__dict'].keys(): - if k in ignored: - continue - error = "Mismatch on '%s': %s != %s" % (k, obj_1[k], obj_2[k]) - self.assertEqual(obj_1[k], obj_2[k], error) - return True - - def assertSameBundle(self, *args, **kwargs): - """ - Wrapper around :meth:`assertSameObject`. Has the exact same - arguments and functionality, as the method by which data objects - and data bundles are compared are similar. - - This method is provided so that the test code can be semantically - correct. - """ - return self.assertSameObject(*args, **kwargs) - - def test_create_object(self): - """Smoke test to verify functionality of the CreateObject endpoint.""" - # First, create a data object. - obj = self.generate_object() - response = self.request('CreateObject', object=obj) - # Then, verify that the data object id returned by the server is the - # same id that we sent to it. - self.assertEqual(response['object_id'], obj.id, - "Mismatch between data object ID in request and response") - # Now that we know that things look fine at the surface level, - # verify that we can retrieve the data object by its ID. - response = self.request('GetObject', object_id=obj.id) - # Finally, ensure that the returned data object is the same as the - # one we sent. - self.assertSameObject(obj, response.object) - - def test_duplicate_checksums(self): - """ validate expected behavior of multiple creates of same checksum """ - # Create a data object (:var:`obj_1`) and save its checksum - # for later. - obj_1 = self.generate_object() - # There's some bug that causes a RecursionError if :var:`obj_1_checksum` - # is passed to :meth:`self._client.ListObjects` without first being - # casted to a string... - obj_1_checksum = str(obj_1.checksums[0].checksum) - obj_1_checksum_type = str(obj_1.checksums[0].type) - self.request('CreateObject', object=obj_1) - # Create another data object (:var:`obj_2`) but with the - # same checksum as :var:`obj_1`. - obj_2 = self.generate_object() - obj_2.checksums[0].checksum = obj_1_checksum - obj_2.checksums[0].type = obj_1_checksum_type - self.request('CreateObject', object=obj_2) - # There are now two data objects with the same checksum on the - # server. We can retrieve them using a ListObjects request. - # Even though we're only expecting two data objects to be - # returned by this query, we specify a high page_size - that way, - # if we receive more than two data objects in the response, we - # know something is up. - response = self.request('ListObjects', page_size=100, - checksum=obj_1_checksum, - checksum_type=obj_1_checksum_type) - self.assertEqual(len(response.objects), 2) - # Finally, confirm that the server returned both data objects - # that we created, and that they're all intact. - try: - self.assertSameObject(obj_1, response.objects[0]) - except AssertionError: - self.assertSameObject(obj_2, response.objects[0]) - try: - self.assertSameObject(obj_2, response.objects[1]) - except AssertionError: - self.assertSameObject(obj_1, response.objects[1]) - - def test_update_object(self): - # Create a data object using CreateObject, then retrieve it - # using GetObject to make sure it exists. - old_obj = self.generate_object() - self.request('CreateObject', object=old_obj) - response = self.request('GetObject', object_id=old_obj.id) - server_obj = response.object - self.assertSameObject(old_obj, server_obj) - # Now that we have a shiny new data object, let's update all of - # its attributes - we can do this quickly by generating a new - # data object and updating all of the attributes of the old object - # with that of the new one. (All the attributes except the id: we - # need to be careful that the id of the data object we send in the - # request body is the same as the original data object, or the data - # object's id will be changed, rendering this exercise moot.) - new_obj = self.generate_object(id=old_obj.id) - self.request('UpdateObject', object=new_obj, - query={'object_id': old_obj.id}) - response = self.request('GetObject', object_id=old_obj.id) - server_obj = response.object - # The data object should now be updated. If we use the GetObject - # endpoint to retrieve the updated data object from the server, - # it should be the same as the one we have in memory. - response = self.request('GetObject', object_id=old_obj.id) - server_obj = response.object - self.assertSameObject(server_obj, new_obj, check_version=False) - - # TODO: DOS server currently does not support updating a data object id but - # it should. - # def test_update_object_id(self): - # """ - # Test that updating a data object's id works correctly - # """ - # # Create a data object - # obj_1 = self.generate_object() - # self.request('CreateObject', object=obj_1) - # # Confirm that the data object we just created exists server-side - # response = self.request('GetObject', object_id=obj_1.id) - # self.assertSameObject(obj_1, response.object) - # # Update the id of the data object we created to something different - # obj_2 = response.object - # obj_2.id = 'new-data-object-id' - # self.request('UpdateObject', object=obj_2, - # query={'object_id': obj_1.id}) - # # Try to retrieve the data object by its new id and its old id - # # The former should succeed: - # response = self.request('GetObject', object_id=obj_2.id) - # self.assertSameObject(response.object, obj_2) - # # And the latter should fail: - # with self.assertRaises(bravado.exception.HTTPNotFound) as ctx: - # self.request('GetObject', object_id=obj_1.id) - # self.assertEqual(ctx.exception.status_code, 404) - - def test_object_long_serialization(self): - # Specify `size` as an int gte 2^31 - 1 (int32 / Javascript's - # maximum int size) but lte 2^63 - 1 (int64 / maximum int size - # in schema) to test json serialization/casting (see #63) - obj = self.generate_object(size=2**63 - 1) - self.request('CreateObject', object=obj) - # Now check to make sure that nothing was lost in transit - retrieved_obj = self.request('GetObject', object_id=obj.id).object - self.assertEqual(obj.size, retrieved_obj.size) - - def test_delete_object(self): - # Create a data object - obj = self.generate_object() - self.request('CreateObject', object=obj) - # Make sure it exists! - response = self.request('GetObject', object_id=obj.id) - self.assertSameObject(obj, response.object) - # Begone foul data object - self.request('DeleteObject', object_id=obj.id) - # Make sure it's gone - with self.assertRaises(bravado.exception.HTTPNotFound) as ctx: - self.request('GetObject', object_id=obj.id) - self.assertEqual(ctx.exception.status_code, 404) - - def test_list_object_querying(self): - obj = self.generate_object() - self.request('CreateObject', object=obj) - # We should be able to retrieve the data object by a unique alias, ... - results = self.request('ListObjects', query={'alias': obj.aliases[0]}) - self.assertEqual(len(results['objects']), 1) - results = self.request('ListObjects', # by a unique checksum... - query={'checksum': obj.checksums[0].checksum, - 'checksum_type': obj.checksums[0].type}) - self.assertEqual(len(results['objects']), 1) - results = self.request('ListObjects', # and by a unique url.. - query={'url': obj.urls[0].url}) - self.assertEqual(len(results['objects']), 1) - # The more advanced ListObjects testing is left to :meth:`ComplianceTest.test_list_object_querying`. - - def test_object_versions(self): - obj = self.generate_object() - self.request('CreateObject', object=obj) - # Make a GetObjectVersions request to see retrieve all the - # stored versions of this data object. As we've just created it, - # there should onlty be one version. - r = self.request('GetObjectVersions', object_id=obj.id) - self.assertEqual(len(r['objects']), 1) - obj.version = 'great-version' # Now make a new version and upload it - obj.name = 'greatest-change' # technically unnecessary, but just in case - self.request('UpdateObject', object=obj, - query={'object_id': obj.id}) - # Now that we've added another version, a GetObjectVersions - # query should confirm that there are now two versions - r = self.request('GetObjectVersions', object_id=obj.id) - self.assertEqual(len(r['objects']), 2) - - def test_bundles(self): - ids = [] # Create data objects to populate the data bundle with - names = [] - aliases = [] - for i in range(10): - obj = self.generate_object() - ids.append(obj.id) - names.append(obj.name) - aliases.append(obj.aliases[0]) - self.request('CreateObject', object=obj) - # Make sure that the data objects we just created exist - for id_ in ids: - self.request('GetObject', object_id=id_) - - # Mint a data bundle with the data objects we just created then - # check to verify its existence - bundle = self.generate_bundle(object_ids=ids) - self.request('CreateBundle', bundle=bundle) - server_bdl = self.request('GetBundle', bundle_id=bundle.id).bundle - self.assertSameBundle(server_bdl, bundle) - - logger.info("..........Update that Bundle.................") - server_bdl.aliases = ['ghi'] - update_bundle = server_bdl - update_response = self.request('UpdateBundle', bundle=update_bundle, - query={'bundle_id': bundle.id}) - logger.info("..........Get that Bundle.................") - updated_bundle = self.request('GetBundle', bundle_id=update_response['bundle_id']).bundle - logger.info('updated_bundle.aliases: %r', updated_bundle.aliases) - logger.info('updated_bundle.updated: %r', updated_bundle.updated) - logger.info('bundle.aliases: %r', bundle.aliases) - logger.info('bundle.updated: %r', bundle.updated) - self.assertEqual(updated_bundle.aliases[0], 'ghi') - - logger.info("..........List Bundles...............") - list_response = self.request('ListBundles') - logger.info(len(list_response.bundles)) - - logger.info("..........Get all Versions of a Bundle...............") - versions_response = self.request('GetBundleVersions', bundle_id=bundle.id) - logger.info(len(versions_response.bundles)) - - logger.info("..........Get an Object in a Bundle..............") - bundle = self.request('GetBundle', bundle_id=bundle.id).bundle - object = self.request('GetObject', object_id=bundle.object_ids[0]).object - logger.info(object.urls) - - logger.info("..........Get all Objects in a Bundle..............") - bundle = self.request('GetBundle', bundle_id=bundle.id).bundle - bundle_objects = [] - for object_id in bundle.object_ids: - bundle_objects.append(self._client.GetObject( - object_id=object_id).result().object) - logger.info([x.name for x in bundle_objects]) - - logger.info("..........Delete the Bundle...............") - delete_response = self.request('DeleteBundle', bundle_id=bundle.id) - logger.info(delete_response.bundle_id) - with self.assertRaises(bravado.exception.HTTPNotFound): - self.request('GetBundle', bundle_id=update_response['bundle_id']) - - logger.info("..........Page through a listing of Bundles......") - for i in range(100): - num = "BDL{}".format(i) - my_bundle = self.generate_bundle(name=num, aliases=[num], object_ids=bundle.object_ids) - self.request('CreateBundle', bundle=my_bundle) - list_response = self.request('ListBundles', page_size=10) - ids = [x['id'] for x in list_response.bundles] - logger.info(list_response.next_page_token) - logger.info(ids) - - list_response = self.request('ListBundles', page_size=10, page_token=list_response.next_page_token) - ids = [x['id'] for x in list_response.bundles] - logger.info(ids) - - logger.info("..........List Bundles by alias..............") - alias_list_response = self.request('ListBundles', alias=list_response.bundles[0].aliases[0]) - logger.info(list_response.bundles[0].aliases[0]) - logger.info(alias_list_response.bundles[0].aliases[0]) - - def test_list_bundle_querying(self): - ids = [] # Create data objects to populate the data bundle with - names = [] - aliases = [] - for i in range(10): - obj = self.generate_object() - ids.append(obj.id) - names.append(obj.name) - aliases.append(obj.aliases[0]) - self.request('CreateObject', object=obj) - # Make sure that the data objects we just created exist - for id_ in ids: - self.request('GetObject', object_id=id_) - - # Mint a data bundle with the data objects we just created then - # check to verify its existence - bundle = self.generate_bundle(object_ids=ids) - self.request('CreateBundle', bundle=bundle) - results = self.request('ListBundles', query={'alias': bundle.aliases[0]}) - self.assertEqual(len(results['bundles']), 1) - results = self.request('ListBundles', # by a unique checksum... - query={'checksum': bundle.checksums[0].checksum, - 'checksum_type': bundle.checksums[0].type}) - self.assertEqual(len(results['bundles']), 1) - - def test_get_nonexistent_bundle(self): - """Test querying GetBundle with a nonexistent data bundle.""" - with self.assertRaises(bravado.exception.HTTPNotFound) as ctx: - self._client.GetBundle(bundle_id='nonexistent-key').result() - self.assertEqual(ctx.exception.status_code, 404) - - def test_schema_required(self): - """ - Tests that the server properly rejects a request - missing a parameter that is marked as required. - """ - CreateObjectRequest = self._models.get_model('CreateObjectRequest') - Object = self._models.get_model('CreateObjectRequest') - object = Object(name='random-name', size='1') # Missing the `id` parameter - create_request = CreateObjectRequest(object=object) - - with self.assertRaises(jsonschema.exceptions.ValidationError) as ctx: - self._client.CreateObject(body=create_request) - self.assertIn('required property', ctx.exception.message) - - def test_service_info(self): - r = self._client.GetServiceInfo().result() - self.assertEqual(ga4gh.drs.__version__, r.version) - - -class TestServerWithLocalClient(TestServer): - """ - Runs all of the test cases in the :class:`TestServer` test suite but - using :class:`ga4gh.drs.client.Client` when loaded locally. (In fact, - this suite is exactly the same as :class:`TestServer` except with - :meth:`setUpClass` modified to load the client locally.) Running all - the same tests is a little overkill but they're fast enough that it - really doesn't make a difference at all. - """ - @classmethod - def setUpClass(cls): - cls._server_process = subprocess.Popen(['ga4gh_drs_server'], stdout=subprocess.PIPE, - stderr=subprocess.PIPE, shell=False) - time.sleep(2) - local_client = ga4gh.drs.client.Client(SERVER_URL, local=True) - cls._models = local_client.models - cls._client = local_client.client diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index df5c92c79..000000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -flake8==3.5.0 -nose==1.3.7 -Sphinx==1.7.4 -openapi-spec-validator==0.2.4 diff --git a/scripts/buildui.js b/scripts/buildui.js deleted file mode 100755 index ce0588f0a..000000000 --- a/scripts/buildui.js +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env node -'use strict'; -var path = require('path'); - -require('shelljs/global'); -set('-e'); -set('-v'); - -mkdir('-p', 'spec') -mkdir('-p', 'web_deploy') - -cp('openapi/data_repository_service.swagger.yaml', 'spec/swagger.yaml'); - -exec('npm run swagger bundle -- -o web_deploy/swagger.json'); -exec('npm run swagger bundle -- --yaml -o web_deploy/swagger.yaml'); - -var SWAGGER_UI_DIST = path.dirname(require.resolve('swagger-ui')); -//rm('-rf', 'web_deploy/swagger-ui/') -cp('-R', SWAGGER_UI_DIST, 'web_deploy/swagger-ui/') -ls('web_deploy/swagger-ui') -sed('-i', 'http://petstore.swagger.io/v2/swagger.json', '../swagger.json', 'web_deploy/swagger-ui/index.html') diff --git a/scripts/fetchpages.sh b/scripts/fetchpages.sh deleted file mode 100644 index 2053efbdd..000000000 --- a/scripts/fetchpages.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -v - -REPO_URL="https://github.com/$TRAVIS_REPO_SLUG" -rm -rf .ghpages-tmp -mkdir -p .ghpages-tmp -cd .ghpages-tmp -git clone --depth=1 --branch=gh-pages $REPO_URL . -cp -Rn . ../ -cd .. -rm -rf .ghpages-tmp diff --git a/scripts/stagepages.sh b/scripts/stagepages.sh deleted file mode 100644 index ce103e6bc..000000000 --- a/scripts/stagepages.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -set -ev - -if [ "$TRAVIS_BRANCH" != "gh-pages" ]; then - if [ "$TRAVIS_BRANCH" == "master" ]; then - branchpath="." - else - branch=$(echo "$TRAVIS_BRANCH" | awk '{print tolower($0)}') - branchpath="preview/$branch" - fi - echo $branchpath - mkdir -p "$branchpath/docs" - cp docs/html5/index.html "$branchpath/docs/" - cp docs/html5/more_background_on_compact_identifiers.html "$branchpath/docs/" - cp docs/pdf/index.pdf "$branchpath/docs/" - cp docs/pdf/more_background_on_compact_identifiers.pdf "$branchpath/docs/" - cp docs/asciidoc/*.png "$branchpath/docs/" - cp openapi/data_repository_service.swagger.yaml "$branchpath/swagger.yaml" - cp -R web_deploy/* "$branchpath/" -fi - -# do some cleanup, these cause the gh-pages deploy to break -# rm -rf node_modules -# rm -rf web_deploy -# rm -rf spec diff --git a/settings.gradle b/settings.gradle deleted file mode 100644 index a65a13387..000000000 --- a/settings.gradle +++ /dev/null @@ -1 +0,0 @@ -rootProject.name = 'data-repository-service-schemas' diff --git a/setup.py b/setup.py deleted file mode 100644 index a1048f152..000000000 --- a/setup.py +++ /dev/null @@ -1,75 +0,0 @@ -# -*- coding: utf-8 -*- -import sys - -# Get version -sys.path.insert(0, 'python/') -from ga4gh.drs import __version__ # noqa - -# First, we try to use setuptools. If it's not available locally, -# we fall back on ez_setup. -try: - from setuptools import setup -except ImportError: - from ez_setup import use_setuptools - use_setuptools() - from setuptools import setup - -with open("README.md") as readmeFile: - long_description = readmeFile.read() - - -setup( - name="ga4gh_drs_schemas", - description="GA4GH Data Repository Service Schemas", - packages=[ - "ga4gh", - "ga4gh.drs", - 'ga4gh.drs.test' - ], - namespace_packages=["ga4gh"], - url="https://github.com/ga4gh/data-repository-service-schemas", - entry_points={ - 'console_scripts': [ - 'ga4gh_drs_server=ga4gh.drs.server:main', - 'ga4gh_drs_client=ga4gh.drs.client:main', - ] - }, - package_dir={'': 'python'}, - long_description=long_description, - long_description_content_type='text/markdown', - install_requires=[ - 'connexion==1.4.2', - 'Flask-Cors==3.0.4', - 'bravado-core==4.13.4', - 'bravado==9.2.2', - 'jsonschema>=2.6.0,<3', - # These dependencies listed below are dependencies of jsonschema[format]. - # We specify them here manually because of pypa/pip#4957. In summary, - # between the dependencies listed above, both jsonschema and - # jsonschema[format] are identified as sub-dependencies. Due to a bug in - # pip, only the former is installed, and not the latter, causing - # installation to fail silently on some setups. (Related to #137.) - 'jsonpointer>1.33', - 'rfc3987', - 'strict-rfc3339', - 'webcolors' - ], - license='Apache License 2.0', - package_data={ - 'ga4gh.drs': ['data_repository_service.swagger.yaml'], - '': ['openapi/data_repository_service.swagger.yaml'] - }, - zip_safe=False, - author="Global Alliance for Genomics and Health", - author_email="theglobalalliance@genomicsandhealth.org", - classifiers=[ - 'Development Status :: 3 - Alpha', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.6', - 'Topic :: Scientific/Engineering :: Bio-Informatics', - ], - version=__version__, - keywords=['genomics'], -)