diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/Dockerfile b/airbyte-integrations/connectors/source-dolead-exchange-rates/Dockerfile new file mode 100644 index 0000000000000..a18a503974185 --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.9-slim + +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +WORKDIR /airbyte/integration_code +COPY source_dolead_exchange_rates ./source_dolead_exchange_rates +COPY main.py ./ +COPY setup.py ./ +RUN pip install . + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.name=airbyte/source-dolead-exchange-rates diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/README.md b/airbyte-integrations/connectors/source-dolead-exchange-rates/README.md new file mode 100644 index 0000000000000..c3bd8c0eaf34d --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/README.md @@ -0,0 +1,166 @@ +# Dolead Exchange Rates Source + +This is the repository for the Dolead Exchange Rates source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/dolead-exchange-rates). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/dolead-exchange-rates) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_dolead_exchange_rates/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source dolead-exchange-rates test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name source-dolead-exchange-rates build +``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-dolead-exchange-rates:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") +``` + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-dolead-exchange-rates:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +``` +Please use this as an example. This is not optimized. + +2. Build your image: +```bash +docker build -t airbyte/source-dolead-exchange-rates:dev . +# Running the spec command against your patched connector +docker run airbyte/source-dolead-exchange-rates:dev spec +```` + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-dolead-exchange-rates:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-dolead-exchange-rates:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-dolead-exchange-rates:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-dolead-exchange-rates:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` + +### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +Please run acceptance tests via [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command): +```bash +airbyte-ci connectors --name source-dolead-exchange-rates test +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-dolead-exchange-rates test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/dolead-exchange-rates.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/acceptance-test-config.yml b/airbyte-integrations/connectors/source-dolead-exchange-rates/acceptance-test-config.yml new file mode 100644 index 0000000000000..86f55d1b1ca8d --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/acceptance-test-config.yml @@ -0,0 +1,39 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-dolead-exchange-rates:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_dolead_exchange_rates/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.jsonl" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" +# TODO uncomment this block this block if your connector implements incremental sync: +# tests: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# future_state: +# future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/__init__.py b/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/__init__.py new file mode 100644 index 0000000000000..c941b30457953 --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/abnormal_state.json new file mode 100644 index 0000000000000..52b0f2c2118f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/acceptance.py new file mode 100644 index 0000000000000..9e6409236281f --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..5a04ebdeb1be9 --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "eur", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "usd", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..f3732995784f2 --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "todo-wrong-field": "this should be an incomplete config file, used in standard tests" +} diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/sample_config.json new file mode 100644 index 0000000000000..ecc4913b84c74 --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/sample_state.json new file mode 100644 index 0000000000000..3587e579822d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/main.py b/airbyte-integrations/connectors/source-dolead-exchange-rates/main.py new file mode 100644 index 0000000000000..b7e377ee71c0a --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_dolead_exchange_rates import SourceDoleadExchangeRates + +if __name__ == "__main__": + source = SourceDoleadExchangeRates() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/metadata.yaml b/airbyte-integrations/connectors/source-dolead-exchange-rates/metadata.yaml new file mode 100644 index 0000000000000..62c37502743b6 --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/metadata.yaml @@ -0,0 +1,30 @@ +data: + allowedHosts: + hosts: + - TODO # Please change to the hostname of the source. + registries: + oss: + enabled: false + cloud: + enabled: false + connectorBuildOptions: + # Please update to the latest version of the connector base image. + # https://hub.docker.com/r/airbyte/python-connector-base + # Please use the full address with sha256 hash to guarantee build reproducibility. + baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27 + connectorSubtype: api + connectorType: source + definitionId: 88de4e93-eb2b-4e1f-9d5b-9094a83ef02f + dockerImageTag: 0.1.0 + dockerRepository: airbyte/source-dolead-exchange-rates + githubIssueLabel: source-dolead-exchange-rates + icon: dolead-exchange-rates.svg + license: MIT + name: Dolead Exchange Rates + releaseDate: TODO + supportLevel: community + releaseStage: alpha + documentationUrl: https://docs.airbyte.com/integrations/sources/dolead-exchange-rates + tags: + - language:python +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/requirements.txt b/airbyte-integrations/connectors/source-dolead-exchange-rates/requirements.txt new file mode 100644 index 0000000000000..d6e1198b1ab1f --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/sample_files/config.json b/airbyte-integrations/connectors/source-dolead-exchange-rates/sample_files/config.json new file mode 100644 index 0000000000000..f47a7b5dc67a6 --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/sample_files/config.json @@ -0,0 +1 @@ +{"input_currency": "eur"} diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-dolead-exchange-rates/sample_files/configured_catalog.json new file mode 100644 index 0000000000000..ed7aa3a81defc --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/sample_files/configured_catalog.json @@ -0,0 +1,38 @@ +{ + "streams": [ + { + "stream": { + "name": "usd", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "input_currency": { + "type": "string" + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "eur", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "input_currency": { + "type": "string" + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/setup.py b/airbyte-integrations/connectors/source-dolead-exchange-rates/setup.py new file mode 100644 index 0000000000000..a4bcb34fc6f2b --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/setup.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.2", +] + +TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest~=6.2", + "pytest-mock~=3.6.1", + "connector-acceptance-test", +] + +setup( + name="source_dolead_exchange_rates", + description="Source implementation for Dolead Exchange Rates.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/source_dolead_exchange_rates/__init__.py b/airbyte-integrations/connectors/source-dolead-exchange-rates/source_dolead_exchange_rates/__init__.py new file mode 100644 index 0000000000000..ddadfbabed757 --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/source_dolead_exchange_rates/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceDoleadExchangeRates + +__all__ = ["SourceDoleadExchangeRates"] diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/source_dolead_exchange_rates/schemas/TODO.md b/airbyte-integrations/connectors/source-dolead-exchange-rates/source_dolead_exchange_rates/schemas/TODO.md new file mode 100644 index 0000000000000..cf1efadb3c9c9 --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/source_dolead_exchange_rates/schemas/TODO.md @@ -0,0 +1,25 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. + +The schema of a stream is the return value of `Stream.get_json_schema`. + +## Static schemas +By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. + +Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. + +## Dynamic schemas +If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). + +## Dynamically modifying static schemas +Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: +``` +def get_json_schema(self): + schema = super().get_json_schema() + schema['dynamically_determined_property'] = "property" + return schema +``` + +Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/source_dolead_exchange_rates/schemas/eur.json b/airbyte-integrations/connectors/source-dolead-exchange-rates/source_dolead_exchange_rates/schemas/eur.json new file mode 100644 index 0000000000000..61a35f856cc7e --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/source_dolead_exchange_rates/schemas/eur.json @@ -0,0 +1,7 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/source_dolead_exchange_rates/schemas/usd.json b/airbyte-integrations/connectors/source-dolead-exchange-rates/source_dolead_exchange_rates/schemas/usd.json new file mode 100644 index 0000000000000..d38261e556417 --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/source_dolead_exchange_rates/schemas/usd.json @@ -0,0 +1,6 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/source_dolead_exchange_rates/source.py b/airbyte-integrations/connectors/source-dolead-exchange-rates/source_dolead_exchange_rates/source.py new file mode 100644 index 0000000000000..2cbdc68a3a07a --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/source_dolead_exchange_rates/source.py @@ -0,0 +1,106 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import json +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + +""" +TODO: Most comments in this class are instructive and should be deleted after the source is implemented. + +This file provides a stubbed example of how to use the Airbyte CDK to develop both a source connector which supports full refresh or and an +incremental syncs from an HTTP API. + +The various TODOs are both implementation hints and steps - fulfilling all the TODOs should be sufficient to implement one basic and one incremental +stream from a source. This pattern is the same one used by Airbyte internally to implement connectors. + +The approach here is not authoritative, and devs are free to use their own judgement. + +There are additional required TODOs in the files within the integration_tests folder and the spec.yaml file. +""" + + +# Basic full refresh stream +class DoleadExchangeRatesStream(HttpStream, ABC): + url_base = "https://cdn.jsdelivr.net/npm/@fawazahmed0/currency-api@latest/v1/currencies/" + primary_key = None + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def get_json_schema(self): + schema = super().get_json_schema() + schema['properties']["date"] = {"type": "string"} + schema['properties']["input_currency"] = {"type": "string"} + schema['properties']["output_currency"] = {"type": "string"} + schema['properties']["rate"] = {"type": "number"} + return schema + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def path(self, **kwargs) -> str: + # This defines the path to the endpoint that we want to hit. + return f"{self.output_currency}.min.json" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, + next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {"input_currency": self.output_currency} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json() + + +class Usd(DoleadExchangeRatesStream): + output_currency = "usd" + input_currencies = ["eur", "gbp", "cad", "aud"] + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response = super().parse_response(response) + record = [] + for input_currency in self.input_currencies: + line = {} + rate = response[self.output_currency][input_currency] + rate = 1 / rate + line["date"] = response["date"] + line["input_currency"] = input_currency + line["output_currency"] = self.output_currency + line["rate"] = rate + record.append(line) + return record + + +class Eur(DoleadExchangeRatesStream): + output_currency = "eur" + input_currencies = ["usd", "gbp", "cad", "aud"] + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response = super().parse_response(response) + record = [] + for input_currency in self.input_currencies: + line = {} + rate = response[self.output_currency][input_currency] + rate = 1 / rate + line["date"] = response["date"] + line["input_currency"] = input_currency + line["output_currency"] = self.output_currency + line["rate"] = rate + record.append(line) + return record + + +# Source +class SourceDoleadExchangeRates(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + return True, None + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + return [Eur(), Usd()] diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/source_dolead_exchange_rates/spec.yaml b/airbyte-integrations/connectors/source-dolead-exchange-rates/source_dolead_exchange_rates/spec.yaml new file mode 100644 index 0000000000000..78ebae838c9b9 --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/source_dolead_exchange_rates/spec.yaml @@ -0,0 +1,7 @@ +documentationUrl: https://docsurl.com +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Dolead Exchange Rates Spec + type: object + required: + [] \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/unit_tests/__init__.py b/airbyte-integrations/connectors/source-dolead-exchange-rates/unit_tests/__init__.py new file mode 100644 index 0000000000000..c941b30457953 --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-dolead-exchange-rates/unit_tests/test_incremental_streams.py new file mode 100644 index 0000000000000..65f94dcb64c1d --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/unit_tests/test_incremental_streams.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from airbyte_cdk.models import SyncMode +from pytest import fixture +from source_dolead_exchange_rates.source import IncrementalDoleadExchangeRatesStream + + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(IncrementalDoleadExchangeRatesStream, "path", "v0/example_endpoint") + mocker.patch.object(IncrementalDoleadExchangeRatesStream, "primary_key", "test_primary_key") + mocker.patch.object(IncrementalDoleadExchangeRatesStream, "__abstractmethods__", set()) + + +def test_cursor_field(patch_incremental_base_class): + stream = IncrementalDoleadExchangeRatesStream() + # TODO: replace this with your expected cursor field + expected_cursor_field = [] + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_base_class): + stream = IncrementalDoleadExchangeRatesStream() + # TODO: replace this with your input parameters + inputs = {"current_stream_state": None, "latest_record": None} + # TODO: replace this with your expected updated stream state + expected_state = {} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_stream_slices(patch_incremental_base_class): + stream = IncrementalDoleadExchangeRatesStream() + # TODO: replace this with your input parameters + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + # TODO: replace this with your expected stream slices list + expected_stream_slice = [None] + assert stream.stream_slices(**inputs) == expected_stream_slice + + +def test_supports_incremental(patch_incremental_base_class, mocker): + mocker.patch.object(IncrementalDoleadExchangeRatesStream, "cursor_field", "dummy_field") + stream = IncrementalDoleadExchangeRatesStream() + assert stream.supports_incremental + + +def test_source_defined_cursor(patch_incremental_base_class): + stream = IncrementalDoleadExchangeRatesStream() + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(patch_incremental_base_class): + stream = IncrementalDoleadExchangeRatesStream() + # TODO: replace this with your expected checkpoint interval + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/unit_tests/test_source.py b/airbyte-integrations/connectors/source-dolead-exchange-rates/unit_tests/test_source.py new file mode 100644 index 0000000000000..775616a307cee --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/unit_tests/test_source.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_dolead_exchange_rates.source import SourceDoleadExchangeRates + + +def test_check_connection(mocker): + source = SourceDoleadExchangeRates() + logger_mock, config_mock = MagicMock(), MagicMock() + assert source.check_connection(logger_mock, config_mock) == (True, None) + + +def test_streams(mocker): + source = SourceDoleadExchangeRates() + config_mock = MagicMock() + streams = source.streams(config_mock) + # TODO: replace this with your streams number + expected_streams_number = 2 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-dolead-exchange-rates/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-dolead-exchange-rates/unit_tests/test_streams.py new file mode 100644 index 0000000000000..276b87767908a --- /dev/null +++ b/airbyte-integrations/connectors/source-dolead-exchange-rates/unit_tests/test_streams.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_dolead_exchange_rates.source import DoleadExchangeRatesStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(DoleadExchangeRatesStream, "path", "v0/example_endpoint") + mocker.patch.object(DoleadExchangeRatesStream, "primary_key", "test_primary_key") + mocker.patch.object(DoleadExchangeRatesStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = DoleadExchangeRatesStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request parameters + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = DoleadExchangeRatesStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected next page token + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = DoleadExchangeRatesStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected parced object + expected_parsed_object = {} + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = DoleadExchangeRatesStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request headers + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = DoleadExchangeRatesStream() + # TODO: replace this with your expected http request method + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = DoleadExchangeRatesStream() + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = DoleadExchangeRatesStream() + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time