diff --git a/airbyte-config-oss/init-oss/src/main/resources/icons/kubecost.svg b/airbyte-config-oss/init-oss/src/main/resources/icons/kubecost.svg new file mode 100644 index 000000000000..51b2f7eb1dd2 --- /dev/null +++ b/airbyte-config-oss/init-oss/src/main/resources/icons/kubecost.svg @@ -0,0 +1,1596 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-integrations/connectors/source-kubecost/.dockerignore b/airbyte-integrations/connectors/source-kubecost/.dockerignore new file mode 100644 index 000000000000..a0d11ec5b5b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_kubecost +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-kubecost/Dockerfile b/airbyte-integrations/connectors/source-kubecost/Dockerfile new file mode 100644 index 000000000000..24a8a8d69512 --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_kubecost ./source_kubecost + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-kubecost diff --git a/airbyte-integrations/connectors/source-kubecost/README.md b/airbyte-integrations/connectors/source-kubecost/README.md new file mode 100644 index 000000000000..c11715ed41e9 --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/README.md @@ -0,0 +1,82 @@ +# Kubecost Source + +This is the repository for the Kubecost configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/kubecost). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-kubecost:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/kubecost) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_kubecost/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source kubecost test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-kubecost:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-kubecost:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-kubecost:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-kubecost:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-kubecost:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-kubecost:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with Docker, run: +``` +./acceptance-test-docker.sh +``` + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-kubecost:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-kubecost:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-kubecost/__init__.py b/airbyte-integrations/connectors/source-kubecost/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-kubecost/acceptance-test-config.yml b/airbyte-integrations/connectors/source-kubecost/acceptance-test-config.yml new file mode 100644 index 000000000000..13df3ac323d6 --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/acceptance-test-config.yml @@ -0,0 +1,39 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-kubecost:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_kubecost/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.jsonl" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" +# TODO uncomment this block this block if your connector implements incremental sync: +# tests: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# future_state: +# future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-kubecost/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-kubecost/acceptance-test-docker.sh new file mode 100755 index 000000000000..b6d65deeccb4 --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/acceptance-test-docker.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-kubecost/build.gradle b/airbyte-integrations/connectors/source-kubecost/build.gradle new file mode 100644 index 000000000000..46661d88eddf --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-connector-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_kubecost' +} diff --git a/airbyte-integrations/connectors/source-kubecost/integration_tests/__init__.py b/airbyte-integrations/connectors/source-kubecost/integration_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-kubecost/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-kubecost/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-kubecost/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-kubecost/integration_tests/acceptance.py new file mode 100644 index 000000000000..9e6409236281 --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-kubecost/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-kubecost/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..36f0468db0d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/integration_tests/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "customers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "employees", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-kubecost/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-kubecost/integration_tests/invalid_config.json new file mode 100644 index 000000000000..f3732995784f --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "todo-wrong-field": "this should be an incomplete config file, used in standard tests" +} diff --git a/airbyte-integrations/connectors/source-kubecost/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-kubecost/integration_tests/sample_config.json new file mode 100644 index 000000000000..ecc4913b84c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-kubecost/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-kubecost/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-kubecost/main.py b/airbyte-integrations/connectors/source-kubecost/main.py new file mode 100644 index 000000000000..4cfb84128b49 --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_kubecost import SourceKubecost + +if __name__ == "__main__": + source = SourceKubecost() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-kubecost/metadata.yaml b/airbyte-integrations/connectors/source-kubecost/metadata.yaml new file mode 100644 index 000000000000..0d939733ffc8 --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/metadata.yaml @@ -0,0 +1,17 @@ +data: + registries: + oss: + enabled: false + connectorSubtype: api + connectorType: source + definitionId: 9dd432d2-5389-4068-a7b4-e31217e65a58 + dockerImageTag: 0.1.0 + dockerRepository: airbyte/source-kubecost + githubIssueLabel: source-kubecost + icon: kubecost.svg + license: MIT + name: Kubecost + releaseDate: TODO + releaseStage: alpha + supportUrl: https://docs.airbyte.com/integrations/sources/kubecost +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-kubecost/requirements.txt b/airbyte-integrations/connectors/source-kubecost/requirements.txt new file mode 100644 index 000000000000..cc57334ef619 --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/connector-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-kubecost/setup.py b/airbyte-integrations/connectors/source-kubecost/setup.py new file mode 100644 index 000000000000..6ea33b76c6f2 --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.2", + "pytest-mock~=3.6.1", + "connector-acceptance-test", +] + +setup( + name="source_kubecost", + description="Source implementation for Kubecost.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-kubecost/source_kubecost/__init__.py b/airbyte-integrations/connectors/source-kubecost/source_kubecost/__init__.py new file mode 100644 index 000000000000..534f2e02decb --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/source_kubecost/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceKubecost + +__all__ = ["SourceKubecost"] diff --git a/airbyte-integrations/connectors/source-kubecost/source_kubecost/manifest.yaml b/airbyte-integrations/connectors/source-kubecost/source_kubecost/manifest.yaml new file mode 100644 index 000000000000..32242f9f5e31 --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/source_kubecost/manifest.yaml @@ -0,0 +1,81 @@ +version: 0.36.1 +type: DeclarativeSource +check: + type: CheckStream + stream_names: + - allocation +streams: + - type: DeclarativeStream + name: allocation + primary_key: [] + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + properties: + data: + type: array + required: + - data + type: array + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: '{{ config[''endpoint''] }}' + path: /model/allocation + http_method: GET + request_parameters: + window: '{{ stream_interval.start_time }},{{ stream_interval.end_time }}' + reconcile: 'true' + shareIdle: 'true' + accumulate: 'true' + shareNamespaces: cert-manager,istio-system,kube-system,kubecost + request_headers: {} + request_body_json: {} + authenticator: + type: NoAuth + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: NoPagination + incremental_sync: + type: DatetimeBasedCursor + step: P1D + cursor_field: windws.start + datetime_format: '%Y-%m-%dT%H:%M:%SZ' + lookback_window: '' + cursor_granularity: PT1S + start_datetime: + type: MinMaxDatetime + datetime: '{{ config[''start_date''] }}' + datetime_format: '%Y-%m-%dT%H:%M:%SZ' + end_datetime: + type: MinMaxDatetime + datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' + datetime_format: '%Y-%m-%dT%H:%M:%SZ' +spec: + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + type: object + required: + - endpoint + - start_date + properties: + endpoint: + type: string + title: Endpoint + default: http://kubecost-staging.prima.it + start_date: + type: string + title: Start date + format: date-time + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + additionalProperties: true + documentation_url: https://example.org + type: Spec diff --git a/airbyte-integrations/connectors/source-kubecost/source_kubecost/source.py b/airbyte-integrations/connectors/source-kubecost/source_kubecost/source.py new file mode 100644 index 000000000000..4b072ae4fba4 --- /dev/null +++ b/airbyte-integrations/connectors/source-kubecost/source_kubecost/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceKubecost(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"})