From b0e7f8d714c1f4d053cda0c415939e8c736ed144 Mon Sep 17 00:00:00 2001 From: Pat Nadolny Date: Fri, 7 Jul 2023 17:01:39 -0400 Subject: [PATCH] refresh contributor instructions (#1417) --- .github/ISSUE_TEMPLATE/new_plugin.yml | 86 ++++++++++++ .github/ISSUE_TEMPLATE/new_tap.yml | 122 ------------------ .github/PULL_REQUEST_TEMPLATE.md | 46 ------- CONTRIBUTING.md | 67 ++++++---- README.md | 2 +- static/markdown/add-a-tap.md | 100 +------------- .../plugin_definitions/yaml_lint_fix.py | 91 ------------- 7 files changed, 130 insertions(+), 384 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/new_plugin.yml delete mode 100644 .github/ISSUE_TEMPLATE/new_tap.yml delete mode 100644 .github/PULL_REQUEST_TEMPLATE.md delete mode 100644 utility_scripts/plugin_definitions/yaml_lint_fix.py diff --git a/.github/ISSUE_TEMPLATE/new_plugin.yml b/.github/ISSUE_TEMPLATE/new_plugin.yml new file mode 100644 index 000000000..f2ede5c6f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/new_plugin.yml @@ -0,0 +1,86 @@ +name: 🚰 New Plugin +description: Request a new Plugin to Meltano Hub +title: "Add Plugin: " +labels: [] +assignees: [taylormurphy, pnadolny13] + +body: +- type: markdown + attributes: + value: | + ## New Plugin Request + + Please fill out the following information to request a new plugin to be added to the Meltano Hub. + + If you'd like you can follow the steps in the [CONTRIBUTING.md](https://github.com/meltano/hub/blob/main/CONTRIBUTING.md) to add it yourself, + otherwise someone on the Meltano team will get it added. + +- type: dropdown + id: plugin_type + attributes: + label: What is the plugin type? + multiple: true + options: + - extractor + - loader + - utility + - mapper + - files + +- type: input + id: name + attributes: + label: Repository URL + description: The URL for the repository that contains the plugin code + placeholder: ex. https://github.com/MeltanoLabs/tap-github + validations: + required: true + +- type: input + id: name + attributes: + label: Extension Repository URL + description: The URL for the plugin extension repository. Only for utility plugins. + placeholder: ex. https://github.com/meltano/airflow-ext + validations: + required: false + +- type: input + id: description + attributes: + label: Description + description: General description of what the company behind the API or database does + placeholder: ex. Online Marketing Platform + validations: + required: true + +- type: input + id: url + attributes: + label: Domain URL + description: URL of the developer documentation or website + placeholder: ex. https://developers.hubspot.com/docs/api/overview + validations: + required: true + +- type: textarea + attributes: + label: Logo Image + description: | + Upload a logo image for the new plugin. The ideal image is an SVG or PNG, has no text over the logo, and has a transparent background. + + Sometimes and image already exists, for example if an extractor is listed on the MeltanoHub and you want to add a loader. + In that case, you can leave a note of what existing plugin to use. + + Tip: You can attach images by clicking this area to highlight it and then dragging files in. + validations: + required: false + +- type: textarea + attributes: + label: Any other notes? + description: | + Links? References? Anything that will give us more context. + + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/new_tap.yml b/.github/ISSUE_TEMPLATE/new_tap.yml deleted file mode 100644 index 8a7a8ff55..000000000 --- a/.github/ISSUE_TEMPLATE/new_tap.yml +++ /dev/null @@ -1,122 +0,0 @@ -name: 🚰 New Singer Tap or Target -description: Submit a new Singer tap/target to Meltano Hub -title: "[Singer Tap/Target] " -labels: [] -assignees: [taylormurphy, pnadolny13] - -body: -- type: markdown - attributes: - value: | - All Singer definitions are stored in `/_data/meltano/extractors/` or `/_data/meltano/loaders/` and logo images are in `/assets/logos/<extractors or loaders>`. The minimal steps for adding a tap or target are the following: - Reference the plugin definition [syntax docs](https://docs.meltano.com/reference/plugin-definition-syntax), for more details. - -- type: input - id: description - attributes: - label: Description - description: General description of what the company behind the API does - validations: - required: true - -- type: input - id: url - attributes: - label: Entity URL - description: URL of the developer documentation or website - validations: - required: true - -- type: input - id: name - attributes: - label: Name - description: The unique name of the connector - -- type: input - id: singer_name - attributes: - label: Singer Name - description: The Singer specific name of the connector. Typically tap-<name> - -- type: markdown - attributes: - value: | - ## Detailed Steps (GitHub web-based Editor) - - - Open the web-based editor in [MeltanoHub repo](https://github.dev/meltano/hub) - - - Add your yaml config file in `/_data/meltano/<extractors or loaders>` by clicking "New File" in the left sidebar Explorer view. - - ![Explorer View](https://github.com/meltano/hub/blob/main/.github/ISSUE_TEMPLATE/images/add-new-file.png) - - You can use the sample below to get started. - - - Upload your png image in `/assets/logos/<extractors or loaders>` by dragging a file from your computer to the Explorer view. - - - Commit your changes in the Source Control view - - ![Source Control View](https://github.com/meltano/hub/blob/main/.github/ISSUE_TEMPLATE/images/commit.png) - - - You'll be prompted to create a fork since you don't have access to edit directly. This automatically create a fork under your organization or username. - -- type: markdown - attributes: - value: | - ```yaml - description: General description of what the company behind the API does - label: Properly formatted label of the connector e.g. GitLab - name: The unique name of the connector - logo_url: /assets/logos/extractors/gitlab.png - namespace: The namespace e.g. tap_gitlab - variant: Name of the GitHub/GitLab namespace - pip_url: git+<git_url>.git or pip install name - repo: repo URL - capabilities: - # Chose the capabilities that the connector supports - # Checkout for a list and details - https://hub.meltano.com/singer/docs#singer-connector-capabilities - # Tap Capability Options - - about - - activate-version - - batch - - stream-maps - - schema-flattening - - catalog - - properties - - discover - - state - - test - - log-based - # Target Capability Options - - about - - activate-version - - batch - - stream-maps - - schema-flattening - - soft-delete - - hard-delete - - datatype-failsafe - - record-flattening - settings_group_validation: - # The set of required settings. - - - api_key - - start_date - settings: - - name: api_key - label: API Key - kind: password - description: The API key for this source. - - name: start_date - label: Start date - description: Start date of when to start retrieve data from. - - name: my_other_setting - label: My Other Setting - description: Some other setting - domain_url: URL of the developer documentation or website - maintenance_status: "Options: active, beta, development, inactive, unknown" - keywords: - # Attributes about the plugin for easier searching. Include `meltano_sdk` here if built using the SDK. - # Other examples could be source type: `api`, `file`, `database`, etc. or cloud name `aws`, `gcp`, etc. - - api - - meltano_sdk - ``` diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 63bdf6eb0..000000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,46 +0,0 @@ -All Singer definitions are stored in `/_data/taps/` or `/_data/targets`. The minimal requirement for adding a tap or target will match the following format: - -```yaml -description: General description of what the company behind the API does -entity_type: Review the JSON Schema for latest options. Currently api, file, database. -entity_url: URL of the developer documentation or website -label: Properly formatted label of the connector -name: The unique name of the connector -singer_name: The Singer specific name of the connector. Typically tap-<name> or target-<name> -type: tap or target -variants: # an array of variant types -- default: true or false - maintenance_status: "Options: active, beta, development, inactive, unknown" - meltano_sdk: true or false - name: Name of the GitHub/GitLab namespace - pip_url: git+<git_url>.git or pip instal name - repo: repo URL - capabilities: - - catalog - - discover - settings: - # Can be an empty array if unknown: [] - # Describe the list of supported settings, for example: - - name: project_ids - kind: array - label: Project IDs - description: Array of project IDs. - - name: username - label: Username - description: Credentials used when for connecting to the source. - - name: password - kind: password - label: Password - description: Credentials used when for connecting to the source. -``` - -## Checklist - -- [ ] Add/update the file in the appropriate folder (`/taps` or `/targets`). The name of the file should match the name of the tap. If there is already one, add a descriptor to the name such as `-search`. -- [ ] Add/update the PNG logo image in `/assets/logos/<taps or targets>`. The image name must match the YAML file name. -- [ ] Tag `@tayloramurphy` or `@pnadolny13` to flag it for review. Or post to the [#hub](https://meltano.slack.com/archives/C01UGBSJNG5) channel on Meltano slack. - -## Reviewer Checklist - -- [ ] Validate file against JSON Schema. https://www.jsonschema.net/home is an option. -- [ ] Build website locally and validate everything works. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 76b6e5959..fee639dc5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,12 +6,52 @@ Let's build together! Please see our [Contributor Guide](https://docs.meltano.co for more information on contributing to Meltano. We believe that everyone can contribute and we welcome all contributions. -If you're not sure what to work on, here are some [ideas to get you started](https://github.com/meltano/meltano/issues?q=is%3Aissue+is%3Aopen+label%3A%22accepting+merge+requests%22+). +If you're not sure what to work on, here are some [ideas to get you started](https://github.com/meltano/hub/issues?q=is%3Aissue+is%3Aopen+label%3A%22Accepting+Pull+Requests%22) Chat with us in [#contributing](https://meltano.slack.com/archives/C013Z450LCD) on [Slack](https://meltano.com/slack). Contributors are expected to follow our [Code of Conduct](https://docs.meltano.com/contribute/#code-of-conduct). +## hub-utils CLI + +MeltanoHub has some conventions and patterns that need to be followed in order for the plugin pages to render properly and CI tests to pass. +This can sometimes make it difficult for our team and community members to contribute changes. +To help with this we created a [hub-utils CLI](https://github.com/meltano/hub-utils) that offers an interactive CLI interface for common operations. +For example it will help you add or update plugins, along the way it will prompt for information it needs to fulfil the plugin definition and if the plugin is SDK based it will scrape most of the information for you so you don't need to provide it. +It does it's best to default to the correct answers and fill common descriptions (i.e. start_date) and labels for you. + +This assumes you already have pipx installed, see the meltano [install-pipx docs](https://docs.meltano.com/guide/installation-guide#install-pipx) for details. + +```bash +pipx install git+https://github.com/meltano/hub-utils.git +``` + +The CLI assumes your terminal is in the root of the hub repository, if you need to run it outside the hub repository root you can set the path using the `HUB_ROOT_PATH` environment variable. + +## Add or Updating Plugins + +To add a new plugin or variant of an existing one, run the following command and provide any input that it prompts for. + +```bash +hub-utils add +``` + +To update an existing variant, run the following command and provide any input that it prompts for. + +```bash +hub-utils update-definition +``` + +If you chose to make any manual changes to the yaml files, make sure you run the yaml linters to fix any linting violations before creating a PR. + +```bash +# Automatically attempt to fix any lint violations +hub-utils yamllint fix _data/meltano/extractors/tap-3plcentral/bytecodeio.yml + +# Check for lint violations +hub-utils yamllint lint _data/meltano/extractors/tap-3plcentral/bytecodeio.yml +``` + ## Automated plugin testing Repo maintainers with `write` access are able to perform automated plugin testing using slash commands in any issue or pull request. @@ -35,31 +75,6 @@ yarn yarn add --global @gridsome/cli ``` -### Linters - -You can use `pre-commit` to run the linters before committing. - -```console -pipx install pre-commit -pre-commit install -``` - -This will run the linters on all files that are staged for commit. Included linters: - -- [yamlllint](https://yamllint.readthedocs.io/en/stable/) - -Theres a utility script that makes a best effort to help reformat yaml files to conform to the -yamllint rules. -To use the script run: - -```bash -# Run on a single file -poetry run python utility_scripts/plugin_definitions/yaml_lint_fix.py _data/meltano/extractors/tap-3plcentral/bytecodeio.yml - -# Run on all .yml files in the `_data/` directory including subdirectories -poetry run python utility_scripts/plugin_definitions/yaml_lint_fix.py -``` - ## Build and serve the project Build for development and get live updates as files are changed: diff --git a/README.md b/README.md index c0969cb3f..09f3a1026 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ _Not familiar with Meltano?_ - **Starts simple**: Meltano is pip-installable and comes in a prepackaged docker container, you can have your first ELT pipeline running within minutes. - **Has DataOps out-of-the-box**: Meltano provides tools that make DataOps best practices easy to use in every project. -- **Integrates with everything**: 350+ natively supported data sources & targets, as well as additional plugins like great expectations or dbt are natively available. +- **Integrates with everything**: 600+ natively supported data sources & targets, as well as additional plugins like great expectations or dbt are natively available. - **Is Easily customizable**: Meltano isn't just extensible, it's built to be extended! The SDK for Singer Connectors & EDK for Meltano Components are easy to use. Meltano Hub helps you find all of the connectors and components created across the data community. - **Is a Mature system**: Developed since [2018](https://handbook.meltano.com/timeline), runs in production at large companies like GitLab, and currently powers over a million pipeline runs monthly. - **Has first class ELT tooling built-in**: Extract data from any data source, load into any target, use inline maps to transform on data on the fly, and test the incoming data, all in one package. diff --git a/static/markdown/add-a-tap.md b/static/markdown/add-a-tap.md index fb0732eaf..a2f7a3fdf 100644 --- a/static/markdown/add-a-tap.md +++ b/static/markdown/add-a-tap.md @@ -5,102 +5,6 @@ description: "Find out how to get your plugins on the MeltanoHub" # Add a Plugin to the MeltanoHub -1. Clone the MeltanoHub Repo +1. Create an [issue to add a new plugin](https://github.com/meltano/hub/issues/new?assignees=taylormurphy%2Cpnadolny13&labels=&projects=&template=new_plugin.yml&title=Add+Plugin%3A+%3Cinsert+plugin+name%3E) -``` -git clone git@github.com:meltano/hub.git -``` - -2. Add or Update Plugin Definition - -All Singer definitions are stored in `/_data/meltano/extractors/` or `/_data/meltano/loaders/` and logo images are in /assets/logos/<extractors or loaders>. Each variant of a plugin is stored under a directory named after the plugin (e.g.`/_data/meltano/extractors/tap-github/meltanolabs.yml`). - -Use the following template to create your YAML file. - -```yaml -capabilities: - # Chose the capabilities that the connector supports - # Checkout for a list and details - https://hub.meltano.com/singer/docs#singer-connector-capabilities - # Tap Capability Options - - about - - activate-version - - batch - - stream-maps - - schema-flattening - - catalog - - properties - - discover - - state - - test - - log-based - # Target Capability Options - - about - - activate-version - - batch - - stream-maps - - schema-flattening - - soft-delete - - hard-delete - - datatype-failsafe - - record-flattening -description: Code hosting platform # General description of what the company behind the API does -domain_url: URL of the developer documentation or website -keywords: - # Attributes about the plugin for easier searching. Include `meltano_sdk` here if built using the SDK. - # Other examples could be source type: `api`, `file`, `database`, etc. or cloud name `aws`, `gcp`, etc. - - api - - meltano_sdk -label: GitHub -logo_url: /assets/logos/extractors/github.png -maintenance_status: active # Options: active, beta, development, inactive, unknown -name: tap-github # The unique name of the connector -namespace: tap_github # The namespace e.g. tap_github -pip_url: git+<git_url>.git or pip install name -repo: https://github.com/MeltanoLabs/tap-github -settings: - - name: api_key - label: API Key - kind: password - description: The API key for this source. - - name: start_date - label: Start date - description: Start date of when to start retrieve data from. - - name: my_other_setting - label: My Other Setting - description: Some other setting -settings_group_validation: - # The set of required settings. - - api_key - - start_date -variant: meltanolabs # the github namespace -``` - -3. Add a Logo - -Add or update the logo image in `/assets/logos/<extractors or loaders>`. - -4. Add or Update default_variants.yml - -If the plugin is new or if the default variant is changing then update the `/_data/default_variants.yml` file with the plugin name and its associated default variant name. - -5. Add or Update maintainers.yml - -The plugin variant name is usually the github namespaces that the plugin lives in. -If its not already listed in `/_data/maintainers.yml` then they should be added along with a link and label. - -6. Lint Your Changes - -Optionally run the yaml_lint_fix.py script over the files you updated to make them conform with the yamlint settings for this repo. - -``` -poetry install -poetry run python utility_scripts/plugin_definitions/yaml_lint_fix.py _data/meltano/extractors/tap-github/meltanolabs.yml -poetry run python utility_scripts/plugin_definitions/yaml_lint_fix.py _data/default_variants.yml -poetry run python utility_scripts/plugin_definitions/yaml_lint_fix.py _data/maintainers.yml -``` - -See the [CONTRIBUTING.md](https://github.com/meltano/hub/blob/main/CONTRIBUTING.md#linters) for more details. - -7. Open a PR and Get Your PR Reviewed - -[Open a pull request](https://github.com/meltano/hub/pulls) on the MeltanoHub repo and tag `@tayloramurphy` or `@pnadolny13` to flag it for review. You can optionally post to the [#hub](https://meltano.slack.com/archives/C01UGBSJNG5) channel on Meltano's [Slack](https://meltano.com/slack) workspace. +2. See [CONTRIBUTING.md](https://github.com/meltano/hub/blob/main/CONTRIBUTING.md) if you would like to attempt to add it yourself using the [`hub-utils` CLI](https://github.com/meltano/hub-utils). Otherwise a member of the Meltano team will add it for you. diff --git a/utility_scripts/plugin_definitions/yaml_lint_fix.py b/utility_scripts/plugin_definitions/yaml_lint_fix.py deleted file mode 100644 index 6c3e3f716..000000000 --- a/utility_scripts/plugin_definitions/yaml_lint_fix.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -import subprocess -import sys -from collections import OrderedDict - -from ruamel.yaml import YAML - -yaml = YAML() -yaml.preserve_quotes = True -yaml.default_flow_style = False - -def insert_newlines(string, every=160): - # TODO: this is not working because editing strings causes them - # to get wrapped in double quotes which looks ugly. - return string - # lines = [] - # for i in range(0, len(string), every): - # lines.append(string[i:i+every]) - # return '\n'.join(lines) - - -def process(v): - output = None - if isinstance(v, dict): - output = fix_yaml_dict_format(v) - elif isinstance(v, list): - new_l = [] - for i in v: - if isinstance(i, str): - new_l.append(i) - else: - new_l.append(process(i)) - output = new_l - elif isinstance(v, str): - if len(v) > 160: - v = insert_newlines(v, every=160) - output = v - else: - output = v - return output - - -def fix_yaml_dict_format(od): - res = OrderedDict() - for k, v in sorted(od.items()): - res[k] = process(v) - return dict(res) - - -def fix_yaml(yml_path): - """ - Reads in the yaml file and attempts to fix it before - overwriting the existing contents. - """ - print(f"Fixing: {yml_path}") - with open(yml_path, "r") as plugin_file: - plugin_data = yaml.load(plugin_file) - with open(yml_path, "w") as plugin_file: - updated_dict = fix_yaml_dict_format(plugin_data) - yaml.dump(updated_dict, plugin_file) - - -def run_yamllint(path): - print(f"Linting: {path}") - subprocess.run(f"yamllint {path} -c .yamllint.yaml".split(" ")) - - -def find_all_yamls(f_path="_data/"): - for root, subdirs, files in os.walk(f_path): - for file in files: - if file.endswith(".yml"): - yield os.path.join(root, file) - for subdir in subdirs: - find_all_yamls(os.path.join(root, f_path, subdir)) - - -if __name__ == "__main__": - """ - This script iterates all the plugin definition files and fixes - their formatting, then runs yamllint to validate the changes. - You can optionally provide a file path to target only a single file, - otherwise the default is to iterate all yml files in the `_data/` directory. - """ - if len(sys.argv) > 1: - file_path = sys.argv[1] - fix_yaml(file_path) - run_yamllint(file_path) - else: - for yaml_file in find_all_yamls(): - fix_yaml(yaml_file) - run_yamllint(yaml_file)