From f2125be0fac7933c4c46b12634d2a7d2b1e9b945 Mon Sep 17 00:00:00 2001 From: Alex Boyd <859286+aaboyd@users.noreply.github.com> Date: Thu, 27 Jun 2024 16:22:55 -0400 Subject: [PATCH] Update GItlab from upstream, update pipelines and jobs schemas to incorporate all properties (#16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add streams for group and project variables (#64) * Add support for Group Variables and Project Variables * Update README * Add config for fetching group/project variables - defaults both to False * Update README * Fixup trailing commas Co-authored-by: Warren Ersly Co-authored-by: Warren Ersly * Update README.md * docs: add hyperlinks to main and legacy branches * feat: Add `ci_config_path` to projects schema (#97) * chore(deps): Bump requests from 2.20.0 to 2.31.0 (#99) Bumps [requests](https://github.com/psf/requests) from 2.20.0 to 2.31.0.
Release notes

Sourced from requests's releases.

v2.31.0

2.31.0 (2023-05-22)

Security

  • Versions of Requests between v2.3.0 and v2.30.0 are vulnerable to potential forwarding of Proxy-Authorization headers to destination servers when following HTTPS redirects.

    When proxies are defined with user info (https://user:pass@proxy:8080), Requests will construct a Proxy-Authorization header that is attached to the request to authenticate with the proxy.

    In cases where Requests receives a redirect response, it previously reattached the Proxy-Authorization header incorrectly, resulting in the value being sent through the tunneled connection to the destination server. Users who rely on defining their proxy credentials in the URL are strongly encouraged to upgrade to Requests 2.31.0+ to prevent unintentional leakage and rotate their proxy credentials once the change has been fully deployed.

    Users who do not use a proxy or do not supply their proxy credentials through the user information portion of their proxy URL are not subject to this vulnerability.

    Full details can be read in our Github Security Advisory and CVE-2023-32681.

v2.30.0

2.30.0 (2023-05-03)

Dependencies

v2.29.0

2.29.0 (2023-04-26)

Improvements

  • Requests now defers chunked requests to the urllib3 implementation to improve standardization. (#6226)
  • Requests relaxes header component requirements to support bytes/str subclasses. (#6356)

... (truncated)

Changelog

Sourced from requests's changelog.

2.31.0 (2023-05-22)

Security

  • Versions of Requests between v2.3.0 and v2.30.0 are vulnerable to potential forwarding of Proxy-Authorization headers to destination servers when following HTTPS redirects.

    When proxies are defined with user info (https://user:pass@proxy:8080), Requests will construct a Proxy-Authorization header that is attached to the request to authenticate with the proxy.

    In cases where Requests receives a redirect response, it previously reattached the Proxy-Authorization header incorrectly, resulting in the value being sent through the tunneled connection to the destination server. Users who rely on defining their proxy credentials in the URL are strongly encouraged to upgrade to Requests 2.31.0+ to prevent unintentional leakage and rotate their proxy credentials once the change has been fully deployed.

    Users who do not use a proxy or do not supply their proxy credentials through the user information portion of their proxy URL are not subject to this vulnerability.

    Full details can be read in our Github Security Advisory and CVE-2023-32681.

2.30.0 (2023-05-03)

Dependencies

2.29.0 (2023-04-26)

Improvements

  • Requests now defers chunked requests to the urllib3 implementation to improve standardization. (#6226)
  • Requests relaxes header component requirements to support bytes/str subclasses. (#6356)

2.28.2 (2023-01-12)

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=requests&package-manager=pip&previous-version=2.20.0&new-version=2.31.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/MeltanoLabs/tap-gitlab/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Managed by Pulumi * Managed by Pulumi * chore(deps): Bump requests from 2.31.0 to 2.32.0 (#101) Bumps [requests](https://github.com/psf/requests) from 2.31.0 to 2.32.0.
Release notes

Sourced from requests's releases.

v2.32.0

2.32.0 (2024-05-20)

🐍 PYCON US 2024 EDITION 🐍

Security

Improvements

  • verify=True now reuses a global SSLContext which should improve request time variance between first and subsequent requests. It should also minimize certificate load time on Windows systems when using a Python version built with OpenSSL 3.x. (#6667)
  • Requests now supports optional use of character detection (chardet or charset_normalizer) when repackaged or vendored. This enables pip and other projects to minimize their vendoring surface area. The Response.text() and apparent_encoding APIs will default to utf-8 if neither library is present. (#6702)

Bugfixes

  • Fixed bug in length detection where emoji length was incorrectly calculated in the request content-length. (#6589)
  • Fixed deserialization bug in JSONDecodeError. (#6629)
  • Fixed bug where an extra leading / (path separator) could lead urllib3 to unnecessarily reparse the request URI. (#6644)

Deprecations

  • Requests has officially added support for CPython 3.12 (#6503)
  • Requests has officially added support for PyPy 3.9 and 3.10 (#6641)
  • Requests has officially dropped support for CPython 3.7 (#6642)
  • Requests has officially dropped support for PyPy 3.7 and 3.8 (#6641)

Documentation

  • Various typo fixes and doc improvements.

Packaging

  • Requests has started adopting some modern packaging practices. The source files for the projects (formerly requests) is now located in src/requests in the Requests sdist. (#6506)
  • Starting in Requests 2.33.0, Requests will migrate to a PEP 517 build system using hatchling. This should not impact the average user, but extremely old versions of packaging utilities may have issues with the new packaging format.

New Contributors

... (truncated)

Changelog

Sourced from requests's changelog.

2.32.0 (2024-05-20)

Security

Improvements

  • verify=True now reuses a global SSLContext which should improve request time variance between first and subsequent requests. It should also minimize certificate load time on Windows systems when using a Python version built with OpenSSL 3.x. (#6667)
  • Requests now supports optional use of character detection (chardet or charset_normalizer) when repackaged or vendored. This enables pip and other projects to minimize their vendoring surface area. The Response.text() and apparent_encoding APIs will default to utf-8 if neither library is present. (#6702)

Bugfixes

  • Fixed bug in length detection where emoji length was incorrectly calculated in the request content-length. (#6589)
  • Fixed deserialization bug in JSONDecodeError. (#6629)
  • Fixed bug where an extra leading / (path separator) could lead urllib3 to unnecessarily reparse the request URI. (#6644)

Deprecations

  • Requests has officially added support for CPython 3.12 (#6503)
  • Requests has officially added support for PyPy 3.9 and 3.10 (#6641)
  • Requests has officially dropped support for CPython 3.7 (#6642)
  • Requests has officially dropped support for PyPy 3.7 and 3.8 (#6641)

Documentation

  • Various typo fixes and doc improvements.

Packaging

  • Requests has started adopting some modern packaging practices. The source files for the projects (formerly requests) is now located in src/requests in the Requests sdist. (#6506)
  • Starting in Requests 2.33.0, Requests will migrate to a PEP 517 build system using hatchling. This should not impact the average user, but extremely old versions of packaging utilities may have issues with the new packaging format.
Commits
  • d6ebc4a v2.32.0
  • 9a40d12 Avoid reloading root certificates to improve concurrent performance (#6667)
  • 0c030f7 Merge pull request #6702 from nateprewitt/no_char_detection
  • 555b870 Allow character detection dependencies to be optional in post-packaging steps
  • d6dded3 Merge pull request #6700 from franekmagiera/update-redirect-to-invalid-uri-test
  • bf24b7d Use an invalid URI that will not cause httpbin to throw 500
  • 2d5f547 Pin 3.8 and 3.9 runners back to macos-13 (#6688)
  • f1bb07d Merge pull request #6687 from psf/dependabot/github_actions/github/codeql-act...
  • 60047ad Bump github/codeql-action from 3.24.0 to 3.25.0
  • 31ebb81 Merge pull request #6682 from frenzymadness/pytest8
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=requests&package-manager=pip&previous-version=2.31.0&new-version=2.32.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/MeltanoLabs/tap-gitlab/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feat: add on_predicate decorator to handle 429 (#102) ## Description This MR aims to cover the case HTTP `429` "Too Many Requests" ([RFC685](https://www.rfc-editor.org/rfc/rfc6585#section-4))i n a more specific way rather than the current way to handle any exception between the HTTP status code 400 and 500. This is motivated by the issue opened by @kgpayne [here](https://github.com/MeltanoLabs/tap-gitlab/issues/94). ## Implementation The first decorator to be executed and added on top will be only evaluated when the HTTP code is 429 and then backoff whatever time is in the `Retry-After` header that gitlab returns (see [here](https://docs.gitlab.com/ee/administration/settings/user_and_ip_rate_limits.html#response-headers)). The example is literally an implementation of the example supplied in the README.md from the backoff library [here](https://github.com/litl/backoff?tab=readme-ov-file#backoffruntime) + adding the same amount of max_times. --------- Co-authored-by: Alejandro Martinez Co-authored-by: Edgar Ramírez Mondragón <16805946+edgarrmondragon@users.noreply.github.com> * fix: Bump `backoff` to 2.2.1 (#103) Follows from #102 * fix: Bump singer-python to 6.0.1 (#104) Related: * #102 * #103 * Fix things after merge * Update schemas for pipelines and jobs --------- Signed-off-by: dependabot[bot] Co-authored-by: wersly Co-authored-by: Warren Ersly Co-authored-by: Warren Ersly Co-authored-by: Aaron ("AJ") Steers Co-authored-by: Francis Potter Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: MeltyBot <105875157+MeltyBot@users.noreply.github.com> Co-authored-by: Alejandro Martínez Otal Co-authored-by: Alejandro Martinez Co-authored-by: Edgar Ramírez Mondragón <16805946+edgarrmondragon@users.noreply.github.com> --- README.md | 18 +- setup.py | 8 +- tap_gitlab/__init__.py | 115 +++++-- tap_gitlab/schemas/group_variables.json | 26 ++ tap_gitlab/schemas/jobs.json | 371 +++++++++++++++++---- tap_gitlab/schemas/pipelines.json | 40 ++- tap_gitlab/schemas/pipelines_extended.json | 119 ++++++- tap_gitlab/schemas/project_variables.json | 26 ++ tap_gitlab/schemas/projects.json | 3 + 9 files changed, 602 insertions(+), 124 deletions(-) create mode 100644 tap_gitlab/schemas/group_variables.json create mode 100644 tap_gitlab/schemas/project_variables.json diff --git a/README.md b/README.md index 7ee6833..f7255e3 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,14 @@ This is a [Singer](https://singer.io) tap that produces JSON-formatted data following the [Singer spec](https://github.com/singer-io/getting-started/blob/master/SPEC.md). +## Important notice + +- This repository's default branch [`legacy-stable`](https://github.com/MeltanoLabs/tap-gitlab/tree/legacy-stable) is kept for compatibility reasons but is no longer under active development. +- New development is being performed against the [`main`](https://github.com/MeltanoLabs/tap-gitlab/tree/main) branch, which is based on a port to the Meltano SDK in Pull Request #65. +- For a stable experience, users of this tap should begin pinning their installations to a specific [release](https://github.com/MeltanoLabs/tap-gitlab/releases) instead of branch references. More instructions are provided within the `README.md` of the `main` branch. + +## About this tap + It is based on v0.5.1 of , but contains [many additional improvements](./CHANGELOG.md). This tap: @@ -26,6 +34,8 @@ This tap: - [Epics](https://docs.gitlab.com/ee/api/epics.html) (only available for GitLab Ultimate and GitLab.com Gold accounts) - [Epic Issues](https://docs.gitlab.com/ee/api/epic_issues.html) (only available for GitLab Ultimate and GitLab.com Gold accounts) - [Vulnerabilities](https://docs.gitlab.com/ee/api/project_vulnerabilities.html) + - [Group Variables](https://docs.gitlab.com/ee/api/group_level_variables.html) + - [Project Variables](https://docs.gitlab.com/ee/api/project_level_variables.html) - Outputs the schema for each resource - Incrementally pulls data based on the input state @@ -68,7 +78,9 @@ pip install git+https://gitlab.com/meltano/tap-gitlab.git "start_date": "2018-01-01T00:00:00Z", "ultimate_license": true, "fetch_merge_request_commits": false, - "fetch_pipelines_extended": false + "fetch_pipelines_extended": false, + "fetch_group_variables": false, + "fetch_project_variables": false } ``` @@ -80,6 +92,10 @@ pip install git+https://gitlab.com/meltano/tap-gitlab.git If `fetch_pipelines_extended` is true (defaults to false), then for every Pipeline fetched with `sync_pipelines` (which returns N pages containing all pipelines per project), also fetch extended details of each of these pipelines with `sync_pipelines_extended`. Similar concerns as those related to `fetch_merge_request_commits` apply here - every pipeline fetched with `sync_pipelines_extended` requires a separate API call. + If `fetch_group_variables` is true (defaults to false), then Group-level CI/CD variables will be retrieved for each available / specified group. This feature is treated as an opt-in to prevent users from accidentally extracting any potential secrets stored as Group-level CI/CD variables. + + If `fetch_project_variables` is true (defaults to false), then Project-level CI/CD variables will be retrieved for each available / specified project. This feature is treated as an opt-in to prevent users from accidentally extracting any potential secrets stored as Project-level CI/CD variables. + 4. [Optional] Create the initial state file You can provide JSON file that contains a date for the API endpoints diff --git a/setup.py b/setup.py index 491421c..c360d84 100644 --- a/setup.py +++ b/setup.py @@ -11,10 +11,10 @@ classifiers=['Programming Language :: Python :: 3 :: Only'], py_modules=['tap_gitlab'], install_requires=[ - 'singer-python==5.9.1', - 'requests==2.20.0', + 'singer-python==6.0.1', + 'requests==2.32.0', 'strict-rfc3339==0.7', - 'backoff==1.8.0', + 'backoff==2.2.1', 'psutil==5.8.0', 'gitlocal@git+https://{}@github.com/minwareco/gitlocal.git'.format(os.environ.get("GITHUB_TOKEN", "")) ], @@ -40,6 +40,8 @@ "tags.json", "releases.json", "vulnerabilities.json", + "project_variables.json", + "group_variables.json" ], }, include_package_data=True, diff --git a/tap_gitlab/__init__.py b/tap_gitlab/__init__.py index a109139..23499cc 100644 --- a/tap_gitlab/__init__.py +++ b/tap_gitlab/__init__.py @@ -29,7 +29,12 @@ 'private_token': None, 'start_date': None, 'groups': '', - 'ultimate_license': False + 'ultimate_license': False, + 'fetch_merge_request_commits': False, + 'fetch_merge_request_notes': False, + 'fetch_pipelines_extended': False, + 'fetch_group_variables': False, + 'fetch_project_variables': False, } STATE = {} CATALOG = None @@ -132,6 +137,7 @@ def load_schema(entity): 'url': '/groups/{id}/subgroups', 'schema': load_schema('groups'), 'key_properties': ['id'], + 'replication_method': 'FULL_TABLE', }, 'group_milestones': { 'url': '/groups/{id}/milestones', @@ -225,16 +231,34 @@ def load_schema(entity): 'key_properties': ['id'], 'replication_method': 'FULL_TABLE', }, - 'vulnerabilities': { - 'url': '/projects/{id}/vulnerabilities', - 'schema': load_schema('vulnerabilities'), - 'key_properties': ['id'], - 'replication_method': 'FULL_TABLE', - }, + 'vulnerabilities': { + 'url': '/projects/{id}/vulnerabilities', + 'schema': load_schema('vulnerabilities'), + 'key_properties': ['id'], + 'replication_method': 'FULL_TABLE', + }, + 'project_variables': { + 'url': '/projects/{id}/variables', + 'schema': load_schema('project_variables'), + 'key_properties': ['project_id', 'key'], + 'replication_method': 'FULL_TABLE', + }, + 'group_variables': { + 'url': '/groups/{id}/variables', + 'schema': load_schema('group_variables'), + 'key_properties': ['group_id', 'key'], + 'replication_method': 'FULL_TABLE', + } } ULTIMATE_RESOURCES = ("epics", "epic_issues") -STREAM_CONFIG_SWITCHES = ('merge_request_commits', 'merge_request_notes', 'pipelines_extended') +STREAM_CONFIG_SWITCHES = ( + 'merge_request_commits', + 'merge_request_notes', + 'pipelines_extended', + 'group_variables', + 'project_variables', +) LOGGER = singer.get_logger() SESSION = requests.Session() @@ -276,17 +300,15 @@ def get_start(entity): STATE[entity] = max(dates_to_compare).isoformat() return STATE[entity] - -# TODO : when singer-python updates the backoff module -# we should update this to pull the exact time to wait from the header -@backoff.on_predicate(backoff.expo, - lambda x: x.status_code == 429, - max_tries=10, - jitter=backoff.random_jitter) +@backoff.on_predicate(backoff.runtime, + predicate=lambda r: r.status_code == 429, + max_tries=5, + value=lambda r: int(r.headers.get("Retry-After")), + jitter=None) @backoff.on_exception(backoff.expo, (requests.exceptions.RequestException), max_tries=5, - giveup=lambda e: e.response is not None and 400 <= e.response.status_code < 500, # pylint: disable=line-too-long + giveup=lambda e: e.response is not None and e.response.status_code != 429 and 400 <= e.response.status_code < 500, # pylint: disable=line-too-long factor=2) def request(url, params=None): params = params or {} @@ -307,12 +329,6 @@ def request(url, params=None): LOGGER.info("Skipping request to {}".format(url)) LOGGER.info("Reason: {} - {}".format(resp.status_code, resp.content)) raise ResourceInaccessible - # if we are being rate limited, let the backoff logic run - elif resp.status_code != 429 and resp.status_code >= 400: - LOGGER.critical( - "Error making request to GitLab API: GET {} [{} - {}]".format( - url, resp.status_code, resp.content)) - sys.exit(1) return resp @@ -1004,6 +1020,8 @@ def sync_group(gid, pids, gitLocal): sync_labels(data, "group") + sync_variables(data, "group") + if CONFIG['ultimate_license']: sync_epics(data) @@ -1017,6 +1035,8 @@ def sync_group(gid, pids, gitLocal): def sync_pipelines(project): entity = "pipelines" stream = CATALOG.get_stream(entity) + + LOGGER.info('Stream Pipelines: {}'.format(stream.is_selected())) if stream is None or not stream.is_selected(): return @@ -1030,7 +1050,12 @@ def sync_pipelines(project): with Transformer(pre_hook=format_timestamp) as transformer: for row in gen_request(url): - transformed_row = transformer.transform(row, RESOURCES[entity]["schema"], mdata) + pipeline_record = { + **row, + 'project_id': project['id'], + '_sdc_repository': project['path_with_namespace'] + } + transformed_row = transformer.transform(pipeline_record, RESOURCES[entity]["schema"], mdata) # Write the Pipeline record singer.write_record(entity, transformed_row, time_extracted=utils.now()) @@ -1058,8 +1083,12 @@ def sync_pipelines_extended(project, pipeline): with Transformer(pre_hook=format_timestamp) as transformer: for row in gen_request(url): - row['project_id'] = project['id'] - transformed_row = transformer.transform(row, RESOURCES[entity]["schema"], mdata) + pipeline_extended_record = { + **row, + 'project_id': project['id'], + '_sdc_repository': project['path_with_namespace'] + } + transformed_row = transformer.transform(pipeline_extended_record, RESOURCES[entity]["schema"], mdata) singer.write_record(entity, transformed_row, time_extracted=utils.now()) @@ -1088,13 +1117,13 @@ def sync_jobs(project, pipeline): url = get_url(entity=entity, id=project['id'], secondary_id=pipeline['id']) with Transformer(pre_hook=format_timestamp) as transformer: for row in gen_request(url): - row['project_id'] = project['id'] - flatten_id(row, 'user') - flatten_id(row, 'commit') - flatten_id(row, 'pipeline') - flatten_id(row, 'runner') - - transformed_row = transformer.transform(row, RESOURCES[entity]['schema'], mdata) + job_record = { + **row, + 'project_id': project['id'], + 'pipeline_id': pipeline['id'], + '_sdc_repository': project['path_with_namespace'] + } + transformed_row = transformer.transform(job_record, RESOURCES[entity]['schema'], mdata) singer.write_record(entity, transformed_row, time_extracted=utils.now()) def write_repository(raw_repo): @@ -1123,6 +1152,21 @@ def write_repository(raw_repo): rec = transformer.transform(repo, Schema.to_dict(stream.schema), metadata=metadata.to_map(stream.metadata)) singer.write_record('repositories', rec, time_extracted=extraction_time) +def sync_variables(entity, element="project"): + stream_name = "{}_variables".format(element) + stream = CATALOG.get_stream(stream_name) + if stream is None or not stream.is_selected(): + return + mdata = metadata.to_map(stream.metadata) + + url = get_url(entity=element + "_variables", id=entity['id']) + + with Transformer(pre_hook=format_timestamp) as transformer: + for row in gen_request(url): + row[element + '_id'] = entity['id'] + transformed_row = transformer.transform(row, RESOURCES[element + "_variables"]["schema"], mdata) + singer.write_record(element + "_variables", transformed_row, time_extracted=utils.now()) + def sync_project(pid, gitLocal): url = get_url(entity="projects", id=pid) @@ -1190,6 +1234,8 @@ def sync_project(pid, gitLocal): sync_tags(data) sync_pipelines(data) sync_vulnerabilities(data) + sync_variables(data) + def do_sync(): LOGGER.info("Starting sync") @@ -1285,6 +1331,11 @@ def main_impl(): CONFIG.update(args.config) CONFIG['ultimate_license'] = truthy(CONFIG['ultimate_license']) + CONFIG['fetch_merge_request_commits'] = truthy(CONFIG['fetch_merge_request_commits']) + CONFIG['fetch_merge_request_notes'] = truthy(CONFIG['fetch_merge_request_notes']) + CONFIG['fetch_pipelines_extended'] = truthy(CONFIG['fetch_pipelines_extended']) + CONFIG['fetch_group_variables'] = truthy(CONFIG['fetch_group_variables']) + CONFIG['fetch_project_variables'] = truthy(CONFIG['fetch_project_variables']) if '/api/' not in CONFIG['api_url']: CONFIG['api_url'] += '/api/v4' diff --git a/tap_gitlab/schemas/group_variables.json b/tap_gitlab/schemas/group_variables.json new file mode 100644 index 0000000..2352ed0 --- /dev/null +++ b/tap_gitlab/schemas/group_variables.json @@ -0,0 +1,26 @@ +{ + "type": "object", + "properties": { + "group_id": { + "type": ["null", "integer"] + }, + "variable_type": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "protected": { + "type": ["null", "boolean"] + }, + "masked": { + "type": ["null", "boolean"] + }, + "environment_scope": { + "type": ["null", "string"] + } + } +} diff --git a/tap_gitlab/schemas/jobs.json b/tap_gitlab/schemas/jobs.json index cf2f370..b17beb5 100644 --- a/tap_gitlab/schemas/jobs.json +++ b/tap_gitlab/schemas/jobs.json @@ -1,35 +1,204 @@ { "type": "object", "properties": { - "id": { - "type": ["null", "integer"] + "_sdc_repository": { + "type": "string" }, "project_id": { - "type": ["null", "integer"] + "type": "integer" + }, + "pipeline_id": { + "type": "integer" + }, + "id": { + "type": "integer" }, "user_id": { - "type": ["null", "integer"] + "type": [ + "null", + "integer" + ] }, - "commit_id": { - "type": ["null", "string"] + "commit": { + "type": "object", + "properties": { + "author_email": { + "type": "string" + }, + "author_name": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "id": { + "type": "string" + }, + "message": { + "type": "string" + }, + "short_id": { + "type": "string" + }, + "title": { + "type": "string" + } + } }, - "pipeline_id": { - "type": ["null", "integer"] + "pipeline": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "project_id": { + "type": "integer" + }, + "ref": { + "type": "string" + }, + "sha": { + "type": "string" + }, + "status": { + "type": "string" + } + } + }, + "runner": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": "integer" + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "ip_address": { + "type": [ + "null", + "string" + ] + }, + "active": { + "type": "boolean" + }, + "paused": { + "type": "boolean" + }, + "is_shared": { + "type": "boolean" + }, + "runner_type": { + "type": "string" + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "online": { + "type": "boolean" + }, + "status": { + "type": "string" + } + } }, - "runner_id": { - "type": ["null", "integer"] + "runner_manager": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": "integer" + }, + "system_id": { + "type": "string" + }, + "version": { + "type": [ + "null", + "string" + ] + }, + "revision": { + "type": [ + "null", + "string" + ] + }, + "platform": { + "type": [ + "null", + "string" + ] + }, + "architecture": { + "type": [ + "null", + "string" + ] + }, + "is_shared": { + "type": "boolean" + }, + "ip_address": { + "type": [ + "null", + "string" + ] + }, + "status": { + "type": "string" + }, + "created_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "contacted_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + } + } }, "name": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "stage": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "ref": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "web_url": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "status": { "anyOf": [ @@ -52,74 +221,142 @@ } ] }, - "duration": { - "anyOf": [ - { - "type": "number", - "format": "float" - }, - { - "type": "null" - } + "failure_reason": { + "type": [ + "null", + "string" ] }, + "tag": { + "type": "boolean" + }, + "duration": { + "type": [ + "null", + "number" + ], + "format": "float" + }, + "queued_duration": { + "type": [ + "null", + "number" + ], + "format": "float" + }, "coverage": { - "anyOf": [ - { - "type": "number", - "format": "float" - }, - { - "type": "null" - } - ] + "type": [ + "null", + "number" + ], + "format": "float" + }, + "archived": { + "type": "boolean" }, "allow_failure": { - "type": ["null", "boolean"] + "type": [ + "null", + "boolean" + ] }, "created_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] + "type": [ + "null", + "string" + ], + "format": "date-time" }, "started_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] + "type": [ + "null", + "string" + ], + "format": "date-time" }, "finished_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "erased_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "artifacts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "file_type": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "filename": { + "type": "string" + }, + "file_format": { + "type": [ + "null", + "string" + ] + } } - ] + } }, "artifacts_expire_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "tag_list": { + "items": { + "type": "string" + }, + "type": "array" + }, + "project": { + "type": "object", + "properties": { + "ci_job_token_scope_enabled": { + "type": "boolean" + } + } + }, + "user": { + "type": "object", + "properties": { + "name": { + "type": [ + "null", + "string" + ] }, - { - "type": "null" + "username": { + "type": "string" + }, + "id": { + "type": "integer" + }, + "state": { + "type": "string" + }, + "avatar_url": { + "type": "string" + }, + "web_url": { + "type": "string" } - ] + } } } -} \ No newline at end of file +} diff --git a/tap_gitlab/schemas/pipelines.json b/tap_gitlab/schemas/pipelines.json index 2ded96b..175f934 100644 --- a/tap_gitlab/schemas/pipelines.json +++ b/tap_gitlab/schemas/pipelines.json @@ -1,20 +1,50 @@ { "type": "object", "properties": { + "_sdc_repository": { + "type": "string" + }, "id": { - "type": ["null", "integer"] + "type": "integer" + }, + "iid": { + "type": "integer" + }, + "project_id": { + "type": "integer" }, "status": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] + }, + "source": { + "type": "string" }, "ref": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "sha": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] }, "web_url": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "created_at": { "anyOf": [ diff --git a/tap_gitlab/schemas/pipelines_extended.json b/tap_gitlab/schemas/pipelines_extended.json index 1864702..8809aea 100644 --- a/tap_gitlab/schemas/pipelines_extended.json +++ b/tap_gitlab/schemas/pipelines_extended.json @@ -1,44 +1,86 @@ { "type": "object", "properties": { - "project_id": { - "type": ["null", "integer"] + "_sdc_repository": { + "type": "string" }, "id": { - "type": ["null", "integer"] + "type": "integer" + }, + "iid": { + "type": "integer" + }, + "project_id": { + "type": "integer" + }, + "name": { + "type": [ + "null", + "string" + ] }, "status": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] + }, + "source": { + "type": "string" }, "ref": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "sha": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "before_sha": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "tag": { - "type": ["null", "boolean"] + "type": [ + "null", + "boolean" + ] }, "yaml_errors": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "user": { "type": "object", "properties": { "name": { - "type": "string" + "type": [ + "null", + "string" + ] }, "username": { - "type": "string" + "type": "string" }, "id": { - "type": "integer" + "type": "integer" }, "state": { - "type": "string" + "type": "string" + }, + "avatar_url": { + "type": "string" + }, + "web_url": { + "type": "string" } } }, @@ -107,18 +149,63 @@ } ] }, - "coverage": { + "queued_duration": { "anyOf": [ { - "type": "number" + "type": "integer" }, { "type": "null" } ] }, + "coverage": { + "type": [ + "null", + "number" + ], + "format": "float" + }, "web_url": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] + }, + "detailed_status": { + "type": "object", + "properties": { + "icon": { + "type": "string" + }, + "text": { + "type": "string" + }, + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "tooltip": { + "type": "string" + }, + "has_details": { + "type": "boolean" + }, + "details_path": { + "type": "string" + }, + "illustration": { + "type": [ + "null", + "string" + ] + }, + "favicon": { + "type": "string" + } + } } } } diff --git a/tap_gitlab/schemas/project_variables.json b/tap_gitlab/schemas/project_variables.json new file mode 100644 index 0000000..62f0b49 --- /dev/null +++ b/tap_gitlab/schemas/project_variables.json @@ -0,0 +1,26 @@ +{ + "type": "object", + "properties": { + "project_id": { + "type": ["null", "integer"] + }, + "variable_type": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "protected": { + "type": ["null", "boolean"] + }, + "masked": { + "type": ["null", "boolean"] + }, + "environment_scope": { + "type": ["null", "string"] + } + } +} diff --git a/tap_gitlab/schemas/projects.json b/tap_gitlab/schemas/projects.json index 159197b..dc177e0 100644 --- a/tap_gitlab/schemas/projects.json +++ b/tap_gitlab/schemas/projects.json @@ -13,6 +13,9 @@ "builds_enabled": { "type": ["null", "boolean"] }, + "ci_config_path": { + "type": ["null", "string"] + }, "container_registry_enabled": { "type": ["null", "boolean"] },