From ef24f36e9bea1eb98eaf7d81f956701bd3d56217 Mon Sep 17 00:00:00 2001 From: Ross Whitfield Date: Mon, 8 Jul 2024 14:40:52 +1000 Subject: [PATCH 1/3] Switch to using Authorization header --- docker-compose.yml | 2 +- src/webmon_app/reporting/report/view_util.py | 43 +++++++------------ .../reporting/templates/report/detail.html | 3 ++ tests/test_livedata.py | 10 +++-- 4 files changed, 27 insertions(+), 31 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index faede670..ac3b37cb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,7 +42,7 @@ services: - CATALOG_ID=${CATALOG_ID} - CATALOG_SECRET=${CATALOG_SECRET} - GUNICORN_CMD_ARGS=--reload --workers=8 - - HTTPLIB2_CA_CERTS=/nginx.crt + - REQUESTS_CA_BUNDLE=/nginx.crt healthcheck: test: wget --no-verbose --tries=1 --spider http://localhost:8000/ht || exit 1 interval: 60s diff --git a/src/webmon_app/reporting/report/view_util.py b/src/webmon_app/reporting/report/view_util.py index 93c70896..d8dd0a2d 100644 --- a/src/webmon_app/reporting/report/view_util.py +++ b/src/webmon_app/reporting/report/view_util.py @@ -9,9 +9,9 @@ import json import datetime import string -import httplib2 import re import hashlib +import requests from reporting.report.models import ( DataRun, RunStatus, @@ -51,22 +51,6 @@ def generate_key(instrument: str, run_id: int): return hashlib.sha1(f"{instrument.upper()}{secret_key}{run_id}".encode("utf-8")).hexdigest() -def append_key(input_url, instrument, run_id): - """ - Append a live data secret key to a url - - :param input_url: url to modify - :param instrument: instrument name - :param run_id: run number - """ - client_key = generate_key(instrument, run_id) - if client_key is None: - return input_url - # Determine whether this is the first query string argument of the url - delimiter = "&" if "/?" in input_url else "?" - return "%s%skey=%s" % (input_url, delimiter, client_key) - - def fill_template_values(request, **template_args): """ Fill the template argument items needed to populate @@ -510,11 +494,12 @@ def get_plot_template_dict(run_object=None, instrument=None, run_id=None): "plot_label_x": "", "plot_label_y": "", "update_url": None, + "key": generate_key(instrument, run_id), } url_template = string.Template(settings.LIVE_DATA_SERVER) live_data_url = url_template.substitute(instrument=instrument, run_number=run_id) - live_data_url = "https://%s:%s%s" % ( + live_data_url = "https://{}:{}{}".format( settings.LIVE_DATA_SERVER_DOMAIN, settings.LIVE_DATA_SERVER_PORT, live_data_url, @@ -524,7 +509,7 @@ def get_plot_template_dict(run_object=None, instrument=None, run_id=None): html_data = get_plot_data_from_server(instrument, run_id, "html") if html_data is not None: plot_dict["html_data"] = html_data - plot_dict["update_url"] = append_key("%s/html/" % live_data_url, instrument, run_id) + plot_dict["update_url"] = live_data_url + "/html/" if extract_ascii_from_div(html_data) is not None: plot_dict["data_url"] = reverse("report:download_reduced_data", args=[instrument, run_id]) return plot_dict @@ -534,7 +519,7 @@ def get_plot_template_dict(run_object=None, instrument=None, run_id=None): # Third, local json data for the d3 plots if json_data: - plot_dict["update_url"] = append_key("%s/json/" % live_data_url, instrument, run_id) + plot_dict["update_url"] = live_data_url + "/json/" plot_data, x_label, y_label = extract_d3_data_from_json(json_data) if plot_data is not None: @@ -558,13 +543,17 @@ def get_plot_data_from_server(instrument, run_id, data_type="json"): try: url_template = string.Template(settings.LIVE_DATA_SERVER) live_data_url = url_template.substitute(instrument=instrument, run_number=run_id) - live_data_url += "/%s/" % data_type - live_data_url = append_key(live_data_url, instrument, run_id) - conn = httplib2.HTTPSConnectionWithTimeout(settings.LIVE_DATA_SERVER_DOMAIN, timeout=1.5) - conn.request("GET", live_data_url) - data_request = conn.getresponse() - if data_request.status == 200: - json_data = data_request.read() + live_data_url = "https://{}:{}{}/{}/".format( + settings.LIVE_DATA_SERVER_DOMAIN, + settings.LIVE_DATA_SERVER_PORT, + live_data_url, + data_type, + ) + key = generate_key(instrument, run_id) + headers = {} if key is None else {"Authorization": key} + data_request = requests.get(live_data_url, timeout=1.5, headers=headers) + if data_request.status_code == 200: + json_data = data_request.text except: # noqa: E722 logging.exception("Could not pull data from live data server:") return json_data diff --git a/src/webmon_app/reporting/templates/report/detail.html b/src/webmon_app/reporting/templates/report/detail.html index af64c494..70a0ffdf 100644 --- a/src/webmon_app/reporting/templates/report/detail.html +++ b/src/webmon_app/reporting/templates/report/detail.html @@ -46,6 +46,9 @@ function get_plot_update() { $.ajax({type: "GET", url: "{{update_url}}", + headers: { + "Authorization": "{{key}}" + }, success: function(data) { if (div_data.localeCompare(data) != 0) { div_data = data; diff --git a/tests/test_livedata.py b/tests/test_livedata.py index 894b3a4e..41538475 100644 --- a/tests/test_livedata.py +++ b/tests/test_livedata.py @@ -64,10 +64,12 @@ def test_reduction_request_livedata(self): key = generate_key(self.instrument, self.run_number) ssl_crt_filename = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../nginx/nginx.crt") + key = generate_key(self.instrument, self.run_number) # first check that the there isn't an existing plot, should 404 response = requests.get( - f"{LIVEDATA_TEST_URL}/plots/{self.instrument}/{self.run_number}/update/html/?key={key}", + f"{LIVEDATA_TEST_URL}/plots/{self.instrument}/{self.run_number}/update/html/", verify=ssl_crt_filename, + headers={"Authorization": key}, ) assert response.status_code == 404 @@ -76,8 +78,9 @@ def test_reduction_request_livedata(self): # the data should now be on livedata response = requests.get( - f"{LIVEDATA_TEST_URL}/plots/{self.instrument}/{self.run_number}/update/html/?key={key}", + f"{LIVEDATA_TEST_URL}/plots/{self.instrument}/{self.run_number}/update/html/", verify=ssl_crt_filename, + headers={"Authorization": key}, ) assert response.status_code == 200 assert "Example Plot Data" in response.text @@ -87,7 +90,8 @@ def test_reduction_request_livedata(self): # now verify that the run report page is templated correctly client = self.get_session() page = client.get(f"{WEBMON_TEST_URL}/report/{self.instrument}/{self.run_number}/") - assert f"https://172.16.238.222:443/plots/arcs/214583/update/html/?key={key}" in page.text + assert 'url: "https://172.16.238.222:443/plots/arcs/214583/update/html/"' in page.text + assert f'"Authorization": "{key}"' in page.text def generate_key(instrument, run_id): From 7e329d611ee6fc028f3a778d167bcc65a22e8673 Mon Sep 17 00:00:00 2001 From: Ross Whitfield Date: Mon, 8 Jul 2024 15:07:24 +1000 Subject: [PATCH 2/3] Update tests --- .../tests/test_report/test_view_util.py | 31 +++++-------------- 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/src/webmon_app/reporting/tests/test_report/test_view_util.py b/src/webmon_app/reporting/tests/test_report/test_view_util.py index 741e9cde..08f5e88b 100644 --- a/src/webmon_app/reporting/tests/test_report/test_view_util.py +++ b/src/webmon_app/reporting/tests/test_report/test_view_util.py @@ -13,9 +13,9 @@ import os import json from reporting import dasmon, report, reporting_app -import httplib2 +import requests -_ = [dasmon, report, reporting_app, os, httplib2] +_ = [dasmon, report, reporting_app, os, requests] class ViewUtilTest(TestCase): @@ -82,21 +82,6 @@ def test_generate_key(self): refval = "d556af22f61bf04e6eb79d88f5c9031230b29b33" self.assertEqual(rst, refval) - def test_append_key(self): - from reporting.report.view_util import append_key - - # case: client_key is None - input_url = "www.test.xyz" - inst = "test_instrument" - run_id = 1 - rst = append_key(input_url, inst, run_id) - self.assertEqual(rst, input_url) - # case: client_key is not None - with self.settings(LIVE_PLOT_SECRET_KEY="test_key"): - rst = append_key(input_url, inst, run_id) - refval = f"{input_url}?key=d556af22f61bf04e6eb79d88f5c9031230b29b33" - self.assertEqual(rst, refval) - @mock.patch(("reporting.dasmon.view_util.fill_template_values"), return_value="passed") def test_fill_template_values(self, mockDasmonTemplateFiller): from reporting.report.view_util import fill_template_values @@ -304,17 +289,15 @@ def test_get_plot_template_dict(self, mockGetDataFromServer): rst = get_plot_template_dict(run, inst, run_id) self.assertTrue("html_data" in rst.keys()) - @mock.patch("httplib2.HTTPSConnectionWithTimeout") - def test_get_plot_data_from_server(self, mockHTTPSCon): + @mock.patch("requests.get") + def test_get_plot_data_from_server(self, mockRequestsGet): from reporting.report.view_util import get_plot_data_from_server # mock external dependencies getresponse_return = mock.MagicMock() - getresponse_return.status = 200 - getresponse_return.read = mock.MagicMock(return_value="test") - con_return = mock.MagicMock() - con_return.getresponse = mock.MagicMock(return_value=getresponse_return) - mockHTTPSCon.return_value = con_return + getresponse_return.status_code = 200 + getresponse_return.text = "test" + mockRequestsGet.return_value = getresponse_return # test inst = Instrument.objects.get(name="test_instrument") run = DataRun.objects.get(run_number=1, instrument_id=inst) From 3e5b72f9cfc8b80084ba41a72f575ded3820f544 Mon Sep 17 00:00:00 2001 From: Ross Whitfield Date: Tue, 9 Jul 2024 08:10:08 +1000 Subject: [PATCH 3/3] Remove unused imports --- src/webmon_app/reporting/tests/test_report/test_view_util.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/webmon_app/reporting/tests/test_report/test_view_util.py b/src/webmon_app/reporting/tests/test_report/test_view_util.py index 08f5e88b..4713862a 100644 --- a/src/webmon_app/reporting/tests/test_report/test_view_util.py +++ b/src/webmon_app/reporting/tests/test_report/test_view_util.py @@ -10,12 +10,7 @@ from reporting.report.models import Task from reporting.report.models import WorkflowSummary -import os import json -from reporting import dasmon, report, reporting_app -import requests - -_ = [dasmon, report, reporting_app, os, requests] class ViewUtilTest(TestCase):