diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index e4b8633316..d57b44ed2f 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -28,8 +28,7 @@ RUN pip3 install --upgrade pip setuptools wheel && \ {{ " " }}-r ./{{ item.name }}/ci_requirements.txt {%- endif -%} {%- endfor %} -{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt \ - pipdeptree && \ +{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \ rm -rf /root/.cache/pip {% if pulp_env is defined and pulp_env %} diff --git a/.ci/assets/ci_constraints.txt b/.ci/assets/ci_constraints.txt index 14b515968b..2617a40896 100644 --- a/.ci/assets/ci_constraints.txt +++ b/.ci/assets/ci_constraints.txt @@ -5,8 +5,3 @@ pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.* tablib!=3.6.0 # 3.6.0: This release introduced a regression removing the "html" optional dependency. - - - -# Newer version seem to have a conflict around packaging, that pip fails to resolve in time. Remove this when this starts to impose an issue. -pipdeptree<=3.23.1 diff --git a/.github/template_gitref b/.github/template_gitref index 4fb36e9605..73f0df798b 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-387-ge627e91 +2021.08.26-389-g444ab6c diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 3146d2a94c..81edc4ad54 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -120,7 +120,7 @@ if [ "$TEST" = "azure" ]; then command: "azurite-blob --blobHost 0.0.0.0"' vars/main.yaml sed -i -e '$a azure_test: true\ pulp_scenario_settings: {"domain_enabled": true}\ -pulp_scenario_env: {"otel_bsp_max_export_batch_size": 1, "otel_bsp_max_queue_size": 1, "otel_exporter_otlp_endpoint": "http://localhost:4318", "otel_exporter_otlp_protocol": "http/protobuf", "pulp_otel_enabled": "true"}\ +pulp_scenario_env: {}\ ' vars/main.yaml fi @@ -166,5 +166,5 @@ if [[ "$TEST" = "azure" ]]; then fi echo ::group::PIP_LIST -cmd_prefix bash -c "pip3 list && pipdeptree" +cmd_prefix bash -c "pip3 list" echo ::endgroup:: diff --git a/.github/workflows/scripts/pre_before_script.sh b/.github/workflows/scripts/pre_before_script.sh deleted file mode 100644 index 143f743961..0000000000 --- a/.github/workflows/scripts/pre_before_script.sh +++ /dev/null @@ -1,8 +0,0 @@ -set -euv - - - -if [ "$TEST" = "azure" ]; then - cmd_stdin_prefix bash -c "cat > /var/lib/pulp/scripts/otel_server.py" < pulpcore/tests/functional/assets/otel_server.py - cmd_user_prefix nohup python3 /var/lib/pulp/scripts/otel_server.py & -fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a63a412cfb..f2e2c9d822 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -155,5 +155,5 @@ jobs: docker logs pulp || true docker exec pulp ls -latr /etc/yum.repos.d/ || true docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + docker exec pulp bash -c "pip3 list" || true ... diff --git a/.gitignore b/.gitignore index 2d068ce3f3..01492e3be0 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ tags # Vagrant .vagrant Vagrantfile +__pycache__/ # Built documentation docs/_build diff --git a/pulp_file/tests/functional/api/test_telemetry_collection.py b/pulp_file/tests/functional/api/test_telemetry_collection.py deleted file mode 100644 index 2e6345a8c3..0000000000 --- a/pulp_file/tests/functional/api/test_telemetry_collection.py +++ /dev/null @@ -1,92 +0,0 @@ -import requests -import uuid - -from urllib.parse import urljoin, urlparse -from django.conf import settings - -from pulpcore.client.pulp_file import FileFileDistribution, RepositoryAddRemoveContent - - -def test_get_requests( - file_distribution_api_client, - file_bindings, - file_repo_with_auto_publish, - file_content_unit_with_name_factory, - gen_object_with_cleanup, - monitor_task, - received_otel_span, - test_path, -): - """Test if content-app correctly returns mime-types based on filenames.""" - content_units = [ - file_content_unit_with_name_factory("otel_test_file1.tar.gz"), - file_content_unit_with_name_factory("otel_test_file2.xml.gz"), - file_content_unit_with_name_factory("otel_test_file3.txt"), - ] - units_to_add = list(map(lambda f: f.pulp_href, content_units)) - data = RepositoryAddRemoveContent(add_content_units=units_to_add) - monitor_task( - file_bindings.RepositoriesFileApi.modify(file_repo_with_auto_publish.pulp_href, data).task - ) - - data = FileFileDistribution( - name=str(uuid.uuid4()), - base_path=str(uuid.uuid4()), - repository=file_repo_with_auto_publish.pulp_href, - ) - distribution = gen_object_with_cleanup(file_distribution_api_client, data) - - for content_unit in content_units: - url = urljoin(distribution.base_url, content_unit.relative_path) - content_path = urlparse(url).path - - s = requests.Session() - s.headers = {"User-Agent": test_path} - - if ( - settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem" - and settings.REDIRECT_TO_OBJECT_STORAGE - ): - status_code = 302 - else: - status_code = 200 - - s.get(url, allow_redirects=False) - assert received_otel_span( - { - "http.method": "GET", - "http.target": content_path, - "http.status_code": status_code, - "http.user_agent": test_path, - } - ) - - s.get(url + "fail") - assert received_otel_span( - { - "http.method": "GET", - "http.target": content_path + "fail", - "http.status_code": 404, - "http.user_agent": test_path, - } - ) - - s.post(url, data={}) - assert received_otel_span( - { - "http.method": "POST", - "http.target": content_path, - "http.status_code": 405, - "http.user_agent": test_path, - } - ) - - s.head(url, allow_redirects=False) - assert received_otel_span( - { - "http.method": "HEAD", - "http.target": content_path, - "http.status_code": status_code, - "http.user_agent": test_path, - } - ) diff --git a/pulpcore/tests/functional/api/test_status.py b/pulpcore/tests/functional/api/test_status.py index 624ae4c806..10b372b569 100644 --- a/pulpcore/tests/functional/api/test_status.py +++ b/pulpcore/tests/functional/api/test_status.py @@ -59,40 +59,24 @@ @pytest.mark.parallel -def test_get_authenticated(test_path, status_api_client, received_otel_span): +def test_get_authenticated(test_path, pulpcore_bindings): """GET the status path with valid credentials. Verify the response with :meth:`verify_get_response`. """ - response = status_api_client.status_read() + response = pulpcore_bindings.StatusApi.status_read() verify_get_response(response.to_dict(), STATUS) - assert received_otel_span( - { - "http.method": "GET", - "http.target": "/pulp/api/v3/status/", - "http.status_code": 200, - "http.user_agent": test_path, - } - ) @pytest.mark.parallel -def test_get_unauthenticated(test_path, status_api_client, anonymous_user, received_otel_span): +def test_get_unauthenticated(test_path, pulpcore_bindings, anonymous_user): """GET the status path with no credentials. Verify the response with :meth:`verify_get_response`. """ with anonymous_user: - response = status_api_client.status_read() + response = pulpcore_bindings.StatusApi.status_read() verify_get_response(response.to_dict(), STATUS) - assert received_otel_span( - { - "http.method": "GET", - "http.target": "/pulp/api/v3/status/", - "http.status_code": 200, - "http.user_agent": test_path, - } - ) @pytest.mark.parallel @@ -102,7 +86,6 @@ def test_post_authenticated( status_api_client, pulpcore_bindings, pulp_api_v3_url, - received_otel_span, ): """POST the status path with valid credentials. @@ -118,14 +101,6 @@ def test_post_authenticated( pulpcore_bindings.client.request("POST", status_url, headers={"User-Agent": test_path}) assert e.value.status == 405 - assert received_otel_span( - { - "http.method": "POST", - "http.target": f"{pulp_api_v3_path}status/", - "http.status_code": 405, - "http.user_agent": test_path, - } - ) @pytest.mark.parallel diff --git a/pulpcore/tests/functional/assets/otel_server.py b/pulpcore/tests/functional/assets/otel_server.py deleted file mode 100644 index cd19ca3555..0000000000 --- a/pulpcore/tests/functional/assets/otel_server.py +++ /dev/null @@ -1,104 +0,0 @@ -import asyncio -import json -import logging -import os -import sys -import threading - -from aiohttp import web - -from opentelemetry.proto.trace.v1.trace_pb2 import TracesData - - -class ThreadedAiohttpServer(threading.Thread): - def __init__(self, app, host, port, ssl_ctx): - super().__init__() - self.app = app - self.host = host - self.port = port - self.ssl_ctx = ssl_ctx - self.loop = asyncio.new_event_loop() - - async def arun(self): - runner = web.AppRunner(self.app) - await runner.setup() - site = web.TCPSite(runner, host=self.host, port=self.port, ssl_context=self.ssl_ctx) - await site.start() - async with self.shutdown_condition: - await self.shutdown_condition.wait() - await runner.cleanup() - - def run(self): - asyncio.set_event_loop(self.loop) - self.shutdown_condition = asyncio.Condition() - self.loop.run_until_complete(self.arun()) - - async def astop(self): - async with self.shutdown_condition: - self.shutdown_condition.notify_all() - - def stop(self): - asyncio.run_coroutine_threadsafe(self.astop(), self.loop) - - -def _otel_collector(): - if ( - os.environ.get("PULP_OTEL_ENABLED") != "true" - or os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT") != "http://localhost:4318" - or os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL") != "http/protobuf" - ): - logging.info("Telemetry was not configured. Exiting...") - sys.exit(0) - else: - logging.info("Booting up the otel collector server...") - - spans = [] - - async def _null_handler(request): - raise web.HTTPOk() - - async def _traces_handler(request): - traces_data = TracesData() - traces_data.ParseFromString(await request.read()) - for resource_span in traces_data.resource_spans: - for scope_span in resource_span.scope_spans: - for span in scope_span.spans: - attrs = { - item.key: getattr(item.value, item.value.WhichOneof("value")) - for item in span.attributes - } - spans.append(attrs) - raise web.HTTPOk() - - async def _test_handler(request): - try: - attrs = await request.json() - except json.decoder.JSONDecodeError: - raise web.HTTPNotFound() - - matched_span = next( - (span for span in spans if all((span.get(k) == v for k, v in attrs.items()))), - None, - ) - if matched_span: - raise web.HTTPOk() - else: - raise web.HTTPNotFound() - - app = web.Application() - app.add_routes( - [ - web.post("/v1/metrics", _null_handler), - web.post("/v1/traces", _traces_handler), - web.post("/test", _test_handler), - ] - ) - - host = "127.0.0.1" - port = 4318 - collector_server = ThreadedAiohttpServer(app, host, port, ssl_ctx=None) - collector_server.start() - - -if __name__ == "__main__": - _otel_collector() diff --git a/releasing.md b/releasing.md index b1d93e889d..03a69b84b3 100644 --- a/releasing.md +++ b/releasing.md @@ -9,19 +9,21 @@ This document outlines the steps to perform a release. ### Determine if a Release is Required +- Make sure to have GitPython python package installed - Run the release checker script: ``` python3 .ci/scripts/check_release.py ``` -### Create a New Y-branch (e.g., 3.23) -- If a new minor version (Y) is needed, trigger a [Create New Release Branch](https://github.com/pulp/pulpcore/actions/workflows/create-branch.yml) job via the GitHub Actions. +### Release a New Y-version (e.g., 3.23.0) +- If a new minor version (Y) is needed, trigger a [Create New Release Branch](https://github.com/pulp/pulpcore/actions/workflows/create-branch.yml) job from the main branch via the GitHub Actions. - Look for the "Bump minor version" pull request and merge it. +- Trigger a [Release Pipeline](https://github.com/pulp/pulpcore/actions/workflows/release.yml) job by specifying the new release branch (X.**Y**) via the GitHub Actions. ### Release a New Z-version (Patch Release) (e.g., 3.23.1, 3.22.12) - Trigger a [Release Pipeline](https://github.com/pulp/pulpcore/actions/workflows/release.yml) job by specifying the release branch (X.Y) via the GitHub Actions. -### Final Steps (Optional but Recommended) -- Ensure the new version appears on PyPI. -- Verify that the changelog has been updated by looking for the "Update Changelog" pull request. -- Post a brief announcement about the new release on the [Pulp Discourse](https://discourse.pulpproject.org/). +## Final Steps +- Ensure the new version appears on PyPI (it should appear after [Publish Release](https://github.com/pulp/pulpcore/actions/workflows/publish.yml) workflow succeeds). +- Verify that the changelog has been updated by looking for the "Update Changelog" pull request (A new PR should be available on the next day). +- [optional] Post a brief announcement about the new release on the [Pulp Discourse](https://discourse.pulpproject.org/). diff --git a/template_config.yml b/template_config.yml index cebb19b3e8..68e7b7a37e 100644 --- a/template_config.yml +++ b/template_config.yml @@ -43,12 +43,7 @@ post_job_template: null pre_job_template: null pulp_env: PULP_CA_BUNDLE: /etc/pulp/certs/pulp_webserver.crt -pulp_env_azure: - otel_bsp_max_export_batch_size: 1 - otel_bsp_max_queue_size: 1 - otel_exporter_otlp_endpoint: http://localhost:4318 - otel_exporter_otlp_protocol: http/protobuf - pulp_otel_enabled: 'true' +pulp_env_azure: {} pulp_env_gcp: {} pulp_env_s3: {} pulp_scheme: https