From a3a3438815d9b0ecb5abf9c4817173c1f97af5c2 Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 2 Aug 2023 19:51:23 +0100 Subject: [PATCH 01/50] Initial gh action implementation --- src/action_handler.py | 175 ++++++++++++++++++++++++++++++++++++++++++ src/utils.py | 25 ++++++ 2 files changed, 200 insertions(+) create mode 100644 src/action_handler.py diff --git a/src/action_handler.py b/src/action_handler.py new file mode 100644 index 0000000..5fdda58 --- /dev/null +++ b/src/action_handler.py @@ -0,0 +1,175 @@ +# src/get_num_square.py +import os +import datetime +from dataclasses import dataclass, asdict +import requests +import time +import json + +from openapi.validation import parse_resolve_and_validate_openapi_spec +from static_analysis import LANGUAGE_ANALYSERS +from utils import upload_api_spec_to_firetail, logger + + +@dataclass +class GitHubContext: + sha: str + repositoryName: str + repositoryId: str + repositoryOwner: str + ref: str + headCommitUsername: str + actor: str + actorId: str + workflowRef: str + eventName: str + private: bool + runId: str + timeTriggered: int + timeTriggeredUTCString: str + file_urls: list[str] + + +@dataclass +class FireTailRequestBody: + collection_uuid: str + spec_data: dict + spec_type: str + context: GitHubContext | None = None + + +def get_spec_type(spec_data: dict) -> str: + if spec_data.get("openapi", "").startswith("3.1"): + return "OAS3.1" + if spec_data.get("swagger"): + return "SWAGGER2" + return "OAS3.0" + + +def load_openapi_spec(api_spec_location: str) -> dict: + try: + openapi_spec = parse_resolve_and_validate_openapi_spec( + api_spec_location, lambda: open(api_spec_location, "r").read() + ) + except FileNotFoundError: + raise Exception(f"Could not find OpenAPI spec at {api_spec_location}") + if openapi_spec is None: + # TODO: a much more helpful error message here + raise Exception(f"File at {api_spec_location} is not a valid OpenAPI spec") + return openapi_spec + + +def handler(): + FIRETAIL_API_TOKEN = os.environ.get("FIRETAIL_API_TOKEN") + if FIRETAIL_API_TOKEN is None: + raise Exception("Missing environment variable 'FIRETAIL_API_TOKEN") + FIRETAIL_API_URL = os.environ.get("FIRETAIL_API_URL", "https://api.saas.eu-west-1.prod.firetail.app") + + CONTEXT = os.environ.get("CONTEXT") + if CONTEXT is not None: + CONTEXT = json.loads(CONTEXT) + CONTEXT = GitHubContext( + sha=CONTEXT.get("sha", ""), + repositoryId=CONTEXT.get("repository_id", ""), + repositoryName=CONTEXT.get("event", {}).get("repository", {}).get("name", ""), + repositoryOwner=CONTEXT.get("repository_owner", ""), + ref=CONTEXT.get("ref", ""), + headCommitUsername=CONTEXT.get("event", {}).get("head_commit", {}).get("author", {}).get("username", ""), + actor=CONTEXT.get("actor", ""), + actorId=CONTEXT.get("actor_id", ""), + workflowRef=CONTEXT.get("workflow_ref", ""), + eventName=CONTEXT.get("event_name", ""), + private=CONTEXT.get("event", {}).get("repository", {}).get("private"), + runId=CONTEXT.get("run_id"), + timeTriggered=int(time.time() * 1000 * 1000), + timeTriggeredUTCString=datetime.datetime.now(datetime.timezone.utc).isoformat(), + file_urls=[], + ) + + # If API_SPEC_LOCATION is set then we upload the OpenAPI spec at that location + COLLECTION_UUID = os.environ.get("COLLECTION_UUID") + API_SPEC_LOCATION = os.environ.get("API_SPEC_LOCATION") + if API_SPEC_LOCATION is None: + logger.info("API_SPEC_LOCATION is not set, skipping direct upload step.") + elif COLLECTION_UUID is None: + logger.info("COLLECTION_UUID is not set, skipping direct upload step.") + else: + # If we have a CONTEXT then we can add the API_SPEC_LOCATION to the file_urls + if CONTEXT is not None: + CONTEXT.file_urls.append(API_SPEC_LOCATION) + + OPENAPI_SPEC = load_openapi_spec(API_SPEC_LOCATION) + + FIRETAIL_API_RESPONSE = requests.post( + url=f"{FIRETAIL_API_URL}/code_repository/spec", + json=asdict( + FireTailRequestBody( + collection_uuid=COLLECTION_UUID, + spec_data=OPENAPI_SPEC, + spec_type=get_spec_type(OPENAPI_SPEC), + context=CONTEXT, + ) + ), + headers={"x-ft-api-key": FIRETAIL_API_TOKEN}, + ) + if FIRETAIL_API_RESPONSE.status_code not in {201, 409}: + raise Exception(f"Failed to send FireTail API Spec. {FIRETAIL_API_RESPONSE.text}") + + logger.info(f"Successfully uploaded OpenAPI spec to Firetail: {API_SPEC_LOCATION}") + + API_UUID = os.environ.get("API_UUID") + if API_UUID is None: + logger.info("API_UUID is not set, skipping static analysis step.") + return + + STATIC_ANALYSIS_ROOT_DIR = os.environ.get("STATIC_ANALYSIS_ROOT_DIR", "/") + STATIC_ANALYSIS_LANGUAGES = map( + lambda v: v.strip(), os.environ.get("STATIC_ANALYSIS_LANGUAGES", "Python,Golang,Javascript").split(",") + ) + + logger.info(f"Statically analysing files under {STATIC_ANALYSIS_ROOT_DIR}...") + + for path, _, filenames in os.walk(STATIC_ANALYSIS_ROOT_DIR): + for filename in filenames: + FULL_PATH = f"{path}/{filename}" + logger.info(f"Statically analysing {FULL_PATH}...") + + try: + FILE_CONTENTS = open(FULL_PATH, "r").read() + except Exception as e: # noqa: E722 + logger.critical(f"{FULL_PATH}: Could not read, exception: {e}") + continue + + # Check if the file is an openapi spec first. If it is, there's no point doing expensive static analysis. + OPENAPI_SPEC = parse_resolve_and_validate_openapi_spec(FULL_PATH, lambda: FILE_CONTENTS) + if OPENAPI_SPEC is not None: + logger.info(f"{FULL_PATH}: Detected OpenAPI spec, uploading to Firetail...") + upload_api_spec_to_firetail( + source=FULL_PATH, + openapi_spec=json.dumps(OPENAPI_SPEC, indent=2), + api_uuid=API_UUID, + firetail_api_url=FIRETAIL_API_URL, + firetail_api_token=FIRETAIL_API_TOKEN, + ) + continue + + for language, language_analysers in LANGUAGE_ANALYSERS.items(): + if language not in STATIC_ANALYSIS_LANGUAGES: + continue + + for language_analyser in language_analysers: + _, openapi_specs_from_analysis = language_analyser(FULL_PATH, FILE_CONTENTS) + + for openapi_spec_source, OPENAPI_SPEC in openapi_specs_from_analysis.items(): + logger.info(f"{FULL_PATH}: Created OpenAPI spec via {language} static analysis...") + upload_api_spec_to_firetail( + source=openapi_spec_source, + openapi_spec=json.dumps(OPENAPI_SPEC, indent=2), + api_uuid=API_UUID, + firetail_api_url=FIRETAIL_API_URL, + firetail_api_token=FIRETAIL_API_TOKEN, + ) + + +if __name__ == "__main__": + handler() diff --git a/src/utils.py b/src/utils.py index 458e24f..a3df48a 100644 --- a/src/utils.py +++ b/src/utils.py @@ -5,6 +5,7 @@ import github from github import Github as GithubClient +import requests from env import LOGGING_LEVEL @@ -33,3 +34,27 @@ def respect_rate_limit(func: Callable[[], FuncReturnType], github_client: Github f"waiting {sleep_duration} second(s)..." ) time.sleep(sleep_duration) + + +def upload_api_spec_to_firetail( + source: str, openapi_spec: str, api_uuid: str, firetail_api_url: str, firetail_api_token: str +): + upload_api_spec_response = requests.post( + f"{firetail_api_url}/discovery/api-repository/{api_uuid}/appspec", + headers={ + "x-ft-api-key": firetail_api_token, + "Content-Type": "application/json", + }, + json={ + "source": source, + "appspec": openapi_spec, + }, + ) + + if upload_api_spec_response.status_code not in [201, 304]: + raise Exception(f"Failed to send API Spec to FireTail. {upload_api_spec_response.text}") + + logger.info( + f"Successfully created/updated {source} API spec in Firetail SaaS, response:" + f" {upload_api_spec_response.text}" + ) From d52793e0f77cb1d2f1c380cc09996e45c6cd26e5 Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 2 Aug 2023 19:51:34 +0100 Subject: [PATCH 02/50] Initial action.yml --- action.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 action.yml diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..b413e50 --- /dev/null +++ b/action.yml @@ -0,0 +1,34 @@ +# action.yaml +name: "FireTail Github Action" +description: "A Github Action that will upload an API Spec file to the FireTail Platform and perform static analysis on your repository" +inputs: + FIRETAIL_API_TOKEN: + description: "Your FireTail API token" + required: true + FIRETAIL_API_URL: + description: "Your FireTail API token" + required: false + default: "https://api.saas.eu-west-1.prod.firetail.app" + CONTEXT: + required: false + description: "provides the github context that gets passed with the api call. this allows for determining where the change came from and by which user" + COLLECTION_UUID: + description: "UUID of the FireTail API Collection to directly upload the API spec at API_SPEC_LOCATION to" + required: false + API_SPEC_LOCATION: + description: "Path to your OpenAPI/Swagger spec file" + required: false + API_UUID: + description: "UUID of the FireTail API under which to create collections from static analysis results" + required: false + STATIC_ANALYSIS_ROOT_DIR: + description: "The root directory in your repository to perform static analysis from" + required: false + default: "/" + STATIC_ANALYSIS_LANGUAGES: + description: "A comma separated list of languages to statically analyse (currently supported are Python, Golang and Javascript)" + required: false + default: "Python,Golang,Javascript" +runs: + using: "docker" + image: "build_setup/Dockerfile.githubaction" From 6635241333ecefb963534299a6e46c301dc37593 Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 2 Aug 2023 19:51:45 +0100 Subject: [PATCH 03/50] Initial Dockerfile for gh action runtime --- build_setup/Dockerfile.githubaction | 38 +++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 build_setup/Dockerfile.githubaction diff --git a/build_setup/Dockerfile.githubaction b/build_setup/Dockerfile.githubaction new file mode 100644 index 0000000..39b8d9d --- /dev/null +++ b/build_setup/Dockerfile.githubaction @@ -0,0 +1,38 @@ +FROM golang:1.20.5-bullseye as build-golang +WORKDIR /src +COPY analysers/golang /src +RUN go build -buildmode=c-shared -o /dist/main.so . + +FROM build-golang as test-golang +RUN go test -coverprofile=coverage.out ./... +RUN go tool cover -html coverage.out -o coverage.html + +FROM python:3.11-bullseye as build-tree-sitter +WORKDIR /src +RUN apt-get update -y && apt-get upgrade -y +RUN git clone https://github.com/tree-sitter/tree-sitter-javascript +RUN python3 -m pip install tree_sitter +COPY analysers/tree-sitter/build.py build.py +RUN python3 build.py + +FROM python:3.11-bullseye as build-python +WORKDIR /github-api-discovery/src +RUN apt-get update -y && apt-get upgrade -y +COPY --from=build-golang /dist/main.so /analysers/golang/main.so +COPY --from=build-tree-sitter /dist/languages.so /analysers/tree-sitter/languages.so +COPY build_setup/requirements.txt /build_setup/requirements.txt +RUN python3 -m pip install -r /build_setup/requirements.txt +COPY src/ /github-api-discovery/src +RUN rm -rf /build_setup + +FROM build-python as test +WORKDIR /github-api-discovery +COPY setup.cfg /github-api-discovery/setup.cfg +RUN python3 -m pip install pytest pytest-cov +COPY tests/ /github-api-discovery/tests/ +RUN pytest --cov /github-api-discovery --cov-report=xml:coverage.xml -vv -x + +FROM build-python as runtime +RUN chmod +x /github-api-discovery/src/action_handler.py +CMD ["/github-api-discovery/src/action_handler.py"] +ENTRYPOINT ["python"] \ No newline at end of file From d87209dc0392b4dd38622c36187f9a5c5a9df51b Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 2 Aug 2023 20:19:41 +0100 Subject: [PATCH 04/50] Fix paths for gh action dockerfile --- build_setup/Dockerfile.githubaction | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build_setup/Dockerfile.githubaction b/build_setup/Dockerfile.githubaction index 39b8d9d..7cf718c 100644 --- a/build_setup/Dockerfile.githubaction +++ b/build_setup/Dockerfile.githubaction @@ -1,6 +1,6 @@ FROM golang:1.20.5-bullseye as build-golang WORKDIR /src -COPY analysers/golang /src +COPY ./analysers/golang /src RUN go build -buildmode=c-shared -o /dist/main.so . FROM build-golang as test-golang @@ -12,7 +12,7 @@ WORKDIR /src RUN apt-get update -y && apt-get upgrade -y RUN git clone https://github.com/tree-sitter/tree-sitter-javascript RUN python3 -m pip install tree_sitter -COPY analysers/tree-sitter/build.py build.py +COPY ./analysers/tree-sitter/build.py build.py RUN python3 build.py FROM python:3.11-bullseye as build-python @@ -20,16 +20,16 @@ WORKDIR /github-api-discovery/src RUN apt-get update -y && apt-get upgrade -y COPY --from=build-golang /dist/main.so /analysers/golang/main.so COPY --from=build-tree-sitter /dist/languages.so /analysers/tree-sitter/languages.so -COPY build_setup/requirements.txt /build_setup/requirements.txt +COPY ./build_setup/requirements.txt /build_setup/requirements.txt RUN python3 -m pip install -r /build_setup/requirements.txt -COPY src/ /github-api-discovery/src +COPY ./src/ /github-api-discovery/src RUN rm -rf /build_setup FROM build-python as test WORKDIR /github-api-discovery -COPY setup.cfg /github-api-discovery/setup.cfg +COPY ./setup.cfg /github-api-discovery/setup.cfg RUN python3 -m pip install pytest pytest-cov -COPY tests/ /github-api-discovery/tests/ +COPY ./tests/ /github-api-discovery/tests/ RUN pytest --cov /github-api-discovery --cov-report=xml:coverage.xml -vv -x FROM build-python as runtime From 09bf65bba20b3b43172b07b737576225b448b47b Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 2 Aug 2023 20:23:17 +0100 Subject: [PATCH 05/50] Remove WORKDIRs --- build_setup/Dockerfile.githubaction | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/build_setup/Dockerfile.githubaction b/build_setup/Dockerfile.githubaction index 7cf718c..2fab472 100644 --- a/build_setup/Dockerfile.githubaction +++ b/build_setup/Dockerfile.githubaction @@ -1,5 +1,4 @@ FROM golang:1.20.5-bullseye as build-golang -WORKDIR /src COPY ./analysers/golang /src RUN go build -buildmode=c-shared -o /dist/main.so . @@ -8,15 +7,13 @@ RUN go test -coverprofile=coverage.out ./... RUN go tool cover -html coverage.out -o coverage.html FROM python:3.11-bullseye as build-tree-sitter -WORKDIR /src RUN apt-get update -y && apt-get upgrade -y RUN git clone https://github.com/tree-sitter/tree-sitter-javascript RUN python3 -m pip install tree_sitter -COPY ./analysers/tree-sitter/build.py build.py +COPY ./analysers/tree-sitter/build.py /src/build.py RUN python3 build.py FROM python:3.11-bullseye as build-python -WORKDIR /github-api-discovery/src RUN apt-get update -y && apt-get upgrade -y COPY --from=build-golang /dist/main.so /analysers/golang/main.so COPY --from=build-tree-sitter /dist/languages.so /analysers/tree-sitter/languages.so @@ -26,7 +23,6 @@ COPY ./src/ /github-api-discovery/src RUN rm -rf /build_setup FROM build-python as test -WORKDIR /github-api-discovery COPY ./setup.cfg /github-api-discovery/setup.cfg RUN python3 -m pip install pytest pytest-cov COPY ./tests/ /github-api-discovery/tests/ From 8eeb47a035f338a8208d63782f3874ccf432f9f1 Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 2 Aug 2023 20:25:39 +0100 Subject: [PATCH 06/50] Paths relative to Dockerfile??? --- build_setup/Dockerfile.githubaction | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/build_setup/Dockerfile.githubaction b/build_setup/Dockerfile.githubaction index 2fab472..6c87cad 100644 --- a/build_setup/Dockerfile.githubaction +++ b/build_setup/Dockerfile.githubaction @@ -1,31 +1,31 @@ FROM golang:1.20.5-bullseye as build-golang -COPY ./analysers/golang /src +COPY ../analysers/golang /src RUN go build -buildmode=c-shared -o /dist/main.so . FROM build-golang as test-golang -RUN go test -coverprofile=coverage.out ./... +RUN go test -coverprofile=coverage.out ../... RUN go tool cover -html coverage.out -o coverage.html FROM python:3.11-bullseye as build-tree-sitter RUN apt-get update -y && apt-get upgrade -y RUN git clone https://github.com/tree-sitter/tree-sitter-javascript RUN python3 -m pip install tree_sitter -COPY ./analysers/tree-sitter/build.py /src/build.py +COPY ../analysers/tree-sitter/build.py /src/build.py RUN python3 build.py FROM python:3.11-bullseye as build-python RUN apt-get update -y && apt-get upgrade -y COPY --from=build-golang /dist/main.so /analysers/golang/main.so COPY --from=build-tree-sitter /dist/languages.so /analysers/tree-sitter/languages.so -COPY ./build_setup/requirements.txt /build_setup/requirements.txt +COPY ../build_setup/requirements.txt /build_setup/requirements.txt RUN python3 -m pip install -r /build_setup/requirements.txt -COPY ./src/ /github-api-discovery/src +COPY ../src/ /github-api-discovery/src RUN rm -rf /build_setup FROM build-python as test -COPY ./setup.cfg /github-api-discovery/setup.cfg +COPY ../setup.cfg /github-api-discovery/setup.cfg RUN python3 -m pip install pytest pytest-cov -COPY ./tests/ /github-api-discovery/tests/ +COPY ../tests/ /github-api-discovery/tests/ RUN pytest --cov /github-api-discovery --cov-report=xml:coverage.xml -vv -x FROM build-python as runtime From 042b78b41f48929706ea5808edb225c679a12cf2 Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 2 Aug 2023 20:27:04 +0100 Subject: [PATCH 07/50] Move gh action dockerfile to root --- ...kerfile.githubaction => Dockerfile.githubaction | 14 +++++++------- action.yml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) rename build_setup/Dockerfile.githubaction => Dockerfile.githubaction (75%) diff --git a/build_setup/Dockerfile.githubaction b/Dockerfile.githubaction similarity index 75% rename from build_setup/Dockerfile.githubaction rename to Dockerfile.githubaction index 6c87cad..2fab472 100644 --- a/build_setup/Dockerfile.githubaction +++ b/Dockerfile.githubaction @@ -1,31 +1,31 @@ FROM golang:1.20.5-bullseye as build-golang -COPY ../analysers/golang /src +COPY ./analysers/golang /src RUN go build -buildmode=c-shared -o /dist/main.so . FROM build-golang as test-golang -RUN go test -coverprofile=coverage.out ../... +RUN go test -coverprofile=coverage.out ./... RUN go tool cover -html coverage.out -o coverage.html FROM python:3.11-bullseye as build-tree-sitter RUN apt-get update -y && apt-get upgrade -y RUN git clone https://github.com/tree-sitter/tree-sitter-javascript RUN python3 -m pip install tree_sitter -COPY ../analysers/tree-sitter/build.py /src/build.py +COPY ./analysers/tree-sitter/build.py /src/build.py RUN python3 build.py FROM python:3.11-bullseye as build-python RUN apt-get update -y && apt-get upgrade -y COPY --from=build-golang /dist/main.so /analysers/golang/main.so COPY --from=build-tree-sitter /dist/languages.so /analysers/tree-sitter/languages.so -COPY ../build_setup/requirements.txt /build_setup/requirements.txt +COPY ./build_setup/requirements.txt /build_setup/requirements.txt RUN python3 -m pip install -r /build_setup/requirements.txt -COPY ../src/ /github-api-discovery/src +COPY ./src/ /github-api-discovery/src RUN rm -rf /build_setup FROM build-python as test -COPY ../setup.cfg /github-api-discovery/setup.cfg +COPY ./setup.cfg /github-api-discovery/setup.cfg RUN python3 -m pip install pytest pytest-cov -COPY ../tests/ /github-api-discovery/tests/ +COPY ./tests/ /github-api-discovery/tests/ RUN pytest --cov /github-api-discovery --cov-report=xml:coverage.xml -vv -x FROM build-python as runtime diff --git a/action.yml b/action.yml index b413e50..8c2d42a 100644 --- a/action.yml +++ b/action.yml @@ -31,4 +31,4 @@ inputs: default: "Python,Golang,Javascript" runs: using: "docker" - image: "build_setup/Dockerfile.githubaction" + image: "Dockerfile.githubaction" From 3437d99438d7a57f0c62a5511fa77edc722f66b4 Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 2 Aug 2023 20:30:06 +0100 Subject: [PATCH 08/50] Less relative paths --- Dockerfile.githubaction | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile.githubaction b/Dockerfile.githubaction index 2fab472..25d6b3b 100644 --- a/Dockerfile.githubaction +++ b/Dockerfile.githubaction @@ -1,9 +1,9 @@ FROM golang:1.20.5-bullseye as build-golang COPY ./analysers/golang /src -RUN go build -buildmode=c-shared -o /dist/main.so . +RUN go build -buildmode=c-shared -o /dist/main.so /src FROM build-golang as test-golang -RUN go test -coverprofile=coverage.out ./... +RUN go test -coverprofile=coverage.out /src/... RUN go tool cover -html coverage.out -o coverage.html FROM python:3.11-bullseye as build-tree-sitter @@ -11,7 +11,7 @@ RUN apt-get update -y && apt-get upgrade -y RUN git clone https://github.com/tree-sitter/tree-sitter-javascript RUN python3 -m pip install tree_sitter COPY ./analysers/tree-sitter/build.py /src/build.py -RUN python3 build.py +RUN python3 /src/build.py FROM python:3.11-bullseye as build-python RUN apt-get update -y && apt-get upgrade -y From 03e095fb1cac6f95c3005bb96fe4df97a5adf1a2 Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 2 Aug 2023 20:32:08 +0100 Subject: [PATCH 09/50] cd instead so go.mod is in current dir when go runs --- Dockerfile.githubaction | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile.githubaction b/Dockerfile.githubaction index 25d6b3b..189fc53 100644 --- a/Dockerfile.githubaction +++ b/Dockerfile.githubaction @@ -1,10 +1,10 @@ FROM golang:1.20.5-bullseye as build-golang COPY ./analysers/golang /src -RUN go build -buildmode=c-shared -o /dist/main.so /src +RUN cd /src && go build -buildmode=c-shared -o /dist/main.so FROM build-golang as test-golang -RUN go test -coverprofile=coverage.out /src/... -RUN go tool cover -html coverage.out -o coverage.html +RUN cd /src && go test -coverprofile=coverage.out ./... +RUN cd /src && go tool cover -html coverage.out -o coverage.html FROM python:3.11-bullseye as build-tree-sitter RUN apt-get update -y && apt-get upgrade -y From b5b660763589ee0c37ec395615bc39b410591825 Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 2 Aug 2023 20:34:09 +0100 Subject: [PATCH 10/50] cd for python3 build.py instead --- Dockerfile.githubaction | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.githubaction b/Dockerfile.githubaction index 189fc53..86acdf6 100644 --- a/Dockerfile.githubaction +++ b/Dockerfile.githubaction @@ -11,7 +11,7 @@ RUN apt-get update -y && apt-get upgrade -y RUN git clone https://github.com/tree-sitter/tree-sitter-javascript RUN python3 -m pip install tree_sitter COPY ./analysers/tree-sitter/build.py /src/build.py -RUN python3 /src/build.py +RUN cd /src && python3 build.py FROM python:3.11-bullseye as build-python RUN apt-get update -y && apt-get upgrade -y From 66d890d88fd029cd32b40035d4887b795c7bbc6e Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 2 Aug 2023 20:36:43 +0100 Subject: [PATCH 11/50] Clone into /src --- Dockerfile.githubaction | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.githubaction b/Dockerfile.githubaction index 86acdf6..da5a196 100644 --- a/Dockerfile.githubaction +++ b/Dockerfile.githubaction @@ -8,7 +8,7 @@ RUN cd /src && go tool cover -html coverage.out -o coverage.html FROM python:3.11-bullseye as build-tree-sitter RUN apt-get update -y && apt-get upgrade -y -RUN git clone https://github.com/tree-sitter/tree-sitter-javascript +RUN mkdir /src && cd /src && git clone https://github.com/tree-sitter/tree-sitter-javascript RUN python3 -m pip install tree_sitter COPY ./analysers/tree-sitter/build.py /src/build.py RUN cd /src && python3 build.py From 2245f992a5123f02d09474c8d445c2ab863708a0 Mon Sep 17 00:00:00 2001 From: theteacat Date: Wed, 2 Aug 2023 20:43:00 +0100 Subject: [PATCH 12/50] Rename handlers to mains --- Dockerfile.githubaction | 4 ++-- build_setup/Dockerfile | 6 +++--- src/{local_handler.py => main.py} | 0 src/{lambda_handler.py => main_awslambda.py} | 0 src/{action_handler.py => main_githubaction.py} | 0 5 files changed, 5 insertions(+), 5 deletions(-) rename src/{local_handler.py => main.py} (100%) rename src/{lambda_handler.py => main_awslambda.py} (100%) rename src/{action_handler.py => main_githubaction.py} (100%) diff --git a/Dockerfile.githubaction b/Dockerfile.githubaction index da5a196..d411c61 100644 --- a/Dockerfile.githubaction +++ b/Dockerfile.githubaction @@ -29,6 +29,6 @@ COPY ./tests/ /github-api-discovery/tests/ RUN pytest --cov /github-api-discovery --cov-report=xml:coverage.xml -vv -x FROM build-python as runtime -RUN chmod +x /github-api-discovery/src/action_handler.py -CMD ["/github-api-discovery/src/action_handler.py"] +RUN chmod +x /github-api-discovery/src/main_githubaction.py +CMD ["/github-api-discovery/src/main_githubaction.py"] ENTRYPOINT ["python"] \ No newline at end of file diff --git a/build_setup/Dockerfile b/build_setup/Dockerfile index d26716f..84682d5 100644 --- a/build_setup/Dockerfile +++ b/build_setup/Dockerfile @@ -35,8 +35,8 @@ COPY tests/ /github-api-discovery/tests/ RUN pytest --cov /github-api-discovery --cov-report=xml:coverage.xml -vv -x FROM build-python as runtime -RUN chmod +x /github-api-discovery/src/local_handler.py -CMD [ "python", "/github-api-discovery/src/local_handler.py" ] +RUN chmod +x /github-api-discovery/src/main.py +CMD [ "python", "/github-api-discovery/src/main.py" ] FROM public.ecr.aws/lambda/python:3.10 as build-python-lambda RUN yum install gcc -y @@ -56,4 +56,4 @@ RUN PYTHONPATH=${LAMBDA_TASK_ROOT} pytest --cov ${LAMBDA_TASK_ROOT} --cov-report FROM build-python-lambda as runtime-lambda RUN ls -la ${LAMBDA_TASK_ROOT} -CMD [ "lambda_handler.handler" ] \ No newline at end of file +CMD [ "main_awslambda.handler" ] \ No newline at end of file diff --git a/src/local_handler.py b/src/main.py similarity index 100% rename from src/local_handler.py rename to src/main.py diff --git a/src/lambda_handler.py b/src/main_awslambda.py similarity index 100% rename from src/lambda_handler.py rename to src/main_awslambda.py diff --git a/src/action_handler.py b/src/main_githubaction.py similarity index 100% rename from src/action_handler.py rename to src/main_githubaction.py From f887220c3d447d92933bdadd6f7edbefc73830dc Mon Sep 17 00:00:00 2001 From: theteacat Date: Thu, 3 Aug 2023 12:04:33 +0100 Subject: [PATCH 13/50] cd before pytest --- Dockerfile.githubaction | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.githubaction b/Dockerfile.githubaction index d411c61..48659d9 100644 --- a/Dockerfile.githubaction +++ b/Dockerfile.githubaction @@ -26,7 +26,7 @@ FROM build-python as test COPY ./setup.cfg /github-api-discovery/setup.cfg RUN python3 -m pip install pytest pytest-cov COPY ./tests/ /github-api-discovery/tests/ -RUN pytest --cov /github-api-discovery --cov-report=xml:coverage.xml -vv -x +RUN cd /github-api-discovery && pytest --cov . --cov-report=xml:coverage.xml -vv -x FROM build-python as runtime RUN chmod +x /github-api-discovery/src/main_githubaction.py From b34698e53ba67291a60d7acb0a07953decc4f1a2 Mon Sep 17 00:00:00 2001 From: theteacat Date: Thu, 3 Aug 2023 16:12:20 +0100 Subject: [PATCH 14/50] Move lots of stuff to utils.py --- src/main_githubaction.py | 81 ++++++++-------------------------------- src/utils.py | 75 ++++++++++++++++++++++++++++++++++++- 2 files changed, 89 insertions(+), 67 deletions(-) diff --git a/src/main_githubaction.py b/src/main_githubaction.py index 5fdda58..2a20fbb 100644 --- a/src/main_githubaction.py +++ b/src/main_githubaction.py @@ -1,62 +1,18 @@ # src/get_num_square.py import os import datetime -from dataclasses import dataclass, asdict -import requests import time import json from openapi.validation import parse_resolve_and_validate_openapi_spec from static_analysis import LANGUAGE_ANALYSERS -from utils import upload_api_spec_to_firetail, logger - - -@dataclass -class GitHubContext: - sha: str - repositoryName: str - repositoryId: str - repositoryOwner: str - ref: str - headCommitUsername: str - actor: str - actorId: str - workflowRef: str - eventName: str - private: bool - runId: str - timeTriggered: int - timeTriggeredUTCString: str - file_urls: list[str] - - -@dataclass -class FireTailRequestBody: - collection_uuid: str - spec_data: dict - spec_type: str - context: GitHubContext | None = None - - -def get_spec_type(spec_data: dict) -> str: - if spec_data.get("openapi", "").startswith("3.1"): - return "OAS3.1" - if spec_data.get("swagger"): - return "SWAGGER2" - return "OAS3.0" - - -def load_openapi_spec(api_spec_location: str) -> dict: - try: - openapi_spec = parse_resolve_and_validate_openapi_spec( - api_spec_location, lambda: open(api_spec_location, "r").read() - ) - except FileNotFoundError: - raise Exception(f"Could not find OpenAPI spec at {api_spec_location}") - if openapi_spec is None: - # TODO: a much more helpful error message here - raise Exception(f"File at {api_spec_location} is not a valid OpenAPI spec") - return openapi_spec +from utils import ( + GitHubContext, + load_openapi_spec, + upload_api_spec_to_firetail_collection, + upload_discovered_api_spec_to_firetail, + logger, +) def handler(): @@ -100,20 +56,13 @@ def handler(): OPENAPI_SPEC = load_openapi_spec(API_SPEC_LOCATION) - FIRETAIL_API_RESPONSE = requests.post( - url=f"{FIRETAIL_API_URL}/code_repository/spec", - json=asdict( - FireTailRequestBody( - collection_uuid=COLLECTION_UUID, - spec_data=OPENAPI_SPEC, - spec_type=get_spec_type(OPENAPI_SPEC), - context=CONTEXT, - ) - ), - headers={"x-ft-api-key": FIRETAIL_API_TOKEN}, + upload_api_spec_to_firetail_collection( + openapi_spec=OPENAPI_SPEC, + context=CONTEXT, + collection_uuid=COLLECTION_UUID, + firetail_api_url=FIRETAIL_API_URL, + firetail_api_token=FIRETAIL_API_TOKEN, ) - if FIRETAIL_API_RESPONSE.status_code not in {201, 409}: - raise Exception(f"Failed to send FireTail API Spec. {FIRETAIL_API_RESPONSE.text}") logger.info(f"Successfully uploaded OpenAPI spec to Firetail: {API_SPEC_LOCATION}") @@ -144,7 +93,7 @@ def handler(): OPENAPI_SPEC = parse_resolve_and_validate_openapi_spec(FULL_PATH, lambda: FILE_CONTENTS) if OPENAPI_SPEC is not None: logger.info(f"{FULL_PATH}: Detected OpenAPI spec, uploading to Firetail...") - upload_api_spec_to_firetail( + upload_discovered_api_spec_to_firetail( source=FULL_PATH, openapi_spec=json.dumps(OPENAPI_SPEC, indent=2), api_uuid=API_UUID, @@ -162,7 +111,7 @@ def handler(): for openapi_spec_source, OPENAPI_SPEC in openapi_specs_from_analysis.items(): logger.info(f"{FULL_PATH}: Created OpenAPI spec via {language} static analysis...") - upload_api_spec_to_firetail( + upload_discovered_api_spec_to_firetail( source=openapi_spec_source, openapi_spec=json.dumps(OPENAPI_SPEC, indent=2), api_uuid=API_UUID, diff --git a/src/utils.py b/src/utils.py index a3df48a..856ee67 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,3 +1,4 @@ +from dataclasses import asdict, dataclass import datetime import logging import time @@ -8,6 +9,7 @@ import requests from env import LOGGING_LEVEL +from openapi.validation import parse_resolve_and_validate_openapi_spec logger = logging.Logger(name="Firetail GitHub Scanner", level=LOGGING_LEVEL) logger_handler = logging.StreamHandler() @@ -17,6 +19,54 @@ FuncReturnType = TypeVar("FuncReturnType") +@dataclass +class GitHubContext: + sha: str + repositoryName: str + repositoryId: str + repositoryOwner: str + ref: str + headCommitUsername: str + actor: str + actorId: str + workflowRef: str + eventName: str + private: bool + runId: str + timeTriggered: int + timeTriggeredUTCString: str + file_urls: list[str] + + +@dataclass +class FireTailRequestBody: + collection_uuid: str + spec_data: dict + spec_type: str + context: GitHubContext | None = None + + +def get_spec_type(spec_data: dict) -> str: + if spec_data.get("openapi", "").startswith("3.1"): + return "OAS3.1" + if spec_data.get("swagger"): + return "SWAGGER2" + return "OAS3.0" + + +def load_openapi_spec(api_spec_location: str) -> dict: + try: + openapi_spec = parse_resolve_and_validate_openapi_spec( + api_spec_location, lambda: open(api_spec_location, "r").read() + ) + except FileNotFoundError: + raise Exception(f"Could not find OpenAPI spec at {api_spec_location}") + if openapi_spec is None: + # TODO: a much more helpful error message here + raise Exception(f"File at {api_spec_location} is not a valid OpenAPI spec") + return openapi_spec + + def get_datestamp() -> str: return datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") @@ -36,7 +86,30 @@ def respect_rate_limit(func: Callable[[], FuncReturnType], github_client: Github time.sleep(sleep_duration) -def upload_api_spec_to_firetail( +def upload_api_spec_to_firetail_collection( + openapi_spec: dict, + context: GitHubContext | None, + collection_uuid: str, + firetail_api_url: str, + firetail_api_token: str, +): + FIRETAIL_API_RESPONSE = requests.post( + url=f"{firetail_api_url}/code_repository/spec", + json=asdict( + FireTailRequestBody( + collection_uuid=collection_uuid, + spec_data=openapi_spec, + spec_type=get_spec_type(openapi_spec), + context=context, + ) + ), + headers={"x-ft-api-key": firetail_api_token}, + ) + if FIRETAIL_API_RESPONSE.status_code not in {201, 409}: + raise Exception(f"Failed to send FireTail API Spec. {FIRETAIL_API_RESPONSE.text}") + + +def upload_discovered_api_spec_to_firetail( source: str, openapi_spec: str, api_uuid: str, firetail_api_url: str, firetail_api_token: str ): upload_api_spec_response = requests.post( From 9433b4be4470c7dc067db8eed5c4198b04e714ca Mon Sep 17 00:00:00 2001 From: theteacat Date: Thu, 3 Aug 2023 16:12:34 +0100 Subject: [PATCH 15/50] Add util to get api uuid from api token --- src/utils.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/utils.py b/src/utils.py index 856ee67..ce1ea50 100644 --- a/src/utils.py +++ b/src/utils.py @@ -3,6 +3,7 @@ import logging import time from typing import Callable, TypeVar +import uuid import github from github import Github as GithubClient @@ -46,6 +47,14 @@ class FireTailRequestBody: context: GitHubContext | None = None +def get_api_uuid_from_api_token(api_token: str) -> str: + # Using the uuid lib here to make sure it's a valid UUID + try: + return str(uuid.UUID("-".join(api_token.split("-")[2:7]))) + except: # noqa: E722 + raise Exception("Failed to extract API UUID from API token") + + def get_spec_type(spec_data: dict) -> str: if spec_data.get("openapi", "").startswith("3.1"): return "OAS3.1" From 4c4e0b06874cebd26b17795b385737bc6bf40bea Mon Sep 17 00:00:00 2001 From: theteacat Date: Thu, 3 Aug 2023 16:13:12 +0100 Subject: [PATCH 16/50] Remove API_UUID env var --- action.yml | 3 --- src/main_githubaction.py | 10 +++------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/action.yml b/action.yml index 8c2d42a..4a1b358 100644 --- a/action.yml +++ b/action.yml @@ -18,9 +18,6 @@ inputs: API_SPEC_LOCATION: description: "Path to your OpenAPI/Swagger spec file" required: false - API_UUID: - description: "UUID of the FireTail API under which to create collections from static analysis results" - required: false STATIC_ANALYSIS_ROOT_DIR: description: "The root directory in your repository to perform static analysis from" required: false diff --git a/src/main_githubaction.py b/src/main_githubaction.py index 2a20fbb..9c64ca5 100644 --- a/src/main_githubaction.py +++ b/src/main_githubaction.py @@ -8,6 +8,7 @@ from static_analysis import LANGUAGE_ANALYSERS from utils import ( GitHubContext, + get_api_uuid_from_api_token, load_openapi_spec, upload_api_spec_to_firetail_collection, upload_discovered_api_spec_to_firetail, @@ -66,11 +67,6 @@ def handler(): logger.info(f"Successfully uploaded OpenAPI spec to Firetail: {API_SPEC_LOCATION}") - API_UUID = os.environ.get("API_UUID") - if API_UUID is None: - logger.info("API_UUID is not set, skipping static analysis step.") - return - STATIC_ANALYSIS_ROOT_DIR = os.environ.get("STATIC_ANALYSIS_ROOT_DIR", "/") STATIC_ANALYSIS_LANGUAGES = map( lambda v: v.strip(), os.environ.get("STATIC_ANALYSIS_LANGUAGES", "Python,Golang,Javascript").split(",") @@ -96,7 +92,7 @@ def handler(): upload_discovered_api_spec_to_firetail( source=FULL_PATH, openapi_spec=json.dumps(OPENAPI_SPEC, indent=2), - api_uuid=API_UUID, + api_uuid=get_api_uuid_from_api_token(FIRETAIL_API_TOKEN), firetail_api_url=FIRETAIL_API_URL, firetail_api_token=FIRETAIL_API_TOKEN, ) @@ -114,7 +110,7 @@ def handler(): upload_discovered_api_spec_to_firetail( source=openapi_spec_source, openapi_spec=json.dumps(OPENAPI_SPEC, indent=2), - api_uuid=API_UUID, + api_uuid=get_api_uuid_from_api_token(FIRETAIL_API_TOKEN), firetail_api_url=FIRETAIL_API_URL, firetail_api_token=FIRETAIL_API_TOKEN, ) From 51a8ad4e8aa3b92541b554a1d317a61230a87fb2 Mon Sep 17 00:00:00 2001 From: theteacat Date: Fri, 4 Aug 2023 13:21:09 +0100 Subject: [PATCH 17/50] appspec shouldn't be stringified --- src/main_githubaction.py | 8 ++++---- src/utils.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main_githubaction.py b/src/main_githubaction.py index 9c64ca5..43e3ae5 100644 --- a/src/main_githubaction.py +++ b/src/main_githubaction.py @@ -1,8 +1,8 @@ # src/get_num_square.py -import os import datetime -import time import json +import os +import time from openapi.validation import parse_resolve_and_validate_openapi_spec from static_analysis import LANGUAGE_ANALYSERS @@ -10,9 +10,9 @@ GitHubContext, get_api_uuid_from_api_token, load_openapi_spec, + logger, upload_api_spec_to_firetail_collection, upload_discovered_api_spec_to_firetail, - logger, ) @@ -109,7 +109,7 @@ def handler(): logger.info(f"{FULL_PATH}: Created OpenAPI spec via {language} static analysis...") upload_discovered_api_spec_to_firetail( source=openapi_spec_source, - openapi_spec=json.dumps(OPENAPI_SPEC, indent=2), + openapi_spec=OPENAPI_SPEC, api_uuid=get_api_uuid_from_api_token(FIRETAIL_API_TOKEN), firetail_api_url=FIRETAIL_API_URL, firetail_api_token=FIRETAIL_API_TOKEN, diff --git a/src/utils.py b/src/utils.py index ce1ea50..61bc3ad 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,13 +1,13 @@ -from dataclasses import asdict, dataclass import datetime import logging import time -from typing import Callable, TypeVar import uuid +from dataclasses import asdict, dataclass +from typing import Callable, TypeVar import github -from github import Github as GithubClient import requests +from github import Github as GithubClient from env import LOGGING_LEVEL from openapi.validation import parse_resolve_and_validate_openapi_spec @@ -119,7 +119,7 @@ def upload_api_spec_to_firetail_collection( def upload_discovered_api_spec_to_firetail( - source: str, openapi_spec: str, api_uuid: str, firetail_api_url: str, firetail_api_token: str + source: str, openapi_spec: dict, api_uuid: str, firetail_api_url: str, firetail_api_token: str ): upload_api_spec_response = requests.post( f"{firetail_api_url}/discovery/api-repository/{api_uuid}/appspec", From f3e73f0cc0fa8edde8e3716e8a4a962d5ea79738 Mon Sep 17 00:00:00 2001 From: theteacat Date: Fri, 4 Aug 2023 13:51:46 +0100 Subject: [PATCH 18/50] openapi spec shouldn't be str --- src/main_githubaction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main_githubaction.py b/src/main_githubaction.py index 43e3ae5..89813e9 100644 --- a/src/main_githubaction.py +++ b/src/main_githubaction.py @@ -91,7 +91,7 @@ def handler(): logger.info(f"{FULL_PATH}: Detected OpenAPI spec, uploading to Firetail...") upload_discovered_api_spec_to_firetail( source=FULL_PATH, - openapi_spec=json.dumps(OPENAPI_SPEC, indent=2), + openapi_spec=OPENAPI_SPEC, api_uuid=get_api_uuid_from_api_token(FIRETAIL_API_TOKEN), firetail_api_url=FIRETAIL_API_URL, firetail_api_token=FIRETAIL_API_TOKEN, From 3c889e1732b9c55bc967909b7bffb3a89cb2abc7 Mon Sep 17 00:00:00 2001 From: ciaran Date: Fri, 19 Jan 2024 10:58:09 +0000 Subject: [PATCH 19/50] black and isort after rebase --- src/main.py | 1 + src/main_awslambda.py | 1 + .../javascript/analyse_express.py | 83 ++++++++++--------- tests/javascript/test_analyse_express.py | 1 + tests/javascript/test_analyse_javascript.py | 6 +- 5 files changed, 48 insertions(+), 44 deletions(-) diff --git a/src/main.py b/src/main.py index 91d0ff0..cbeb783 100644 --- a/src/main.py +++ b/src/main.py @@ -1,4 +1,5 @@ import time + from scanning import scan from utils import logger diff --git a/src/main_awslambda.py b/src/main_awslambda.py index f33c4de..2f3ebb3 100644 --- a/src/main_awslambda.py +++ b/src/main_awslambda.py @@ -1,4 +1,5 @@ import time + from scanning import scan from utils import logger diff --git a/src/static_analysis/javascript/analyse_express.py b/src/static_analysis/javascript/analyse_express.py index bd008e0..6b94a88 100644 --- a/src/static_analysis/javascript/analyse_express.py +++ b/src/static_analysis/javascript/analyse_express.py @@ -1,13 +1,16 @@ from tree_sitter import Tree -from utils import get_datestamp from static_analysis.javascript.utils import ( - get_children_of_type, get_default_identifiers_from_import_statement, + get_children_of_type, + get_default_identifiers_from_import_statement, get_identifiers_from_variable_declarator_or_assignment_expression, - get_module_name_from_import_statement, get_module_name_from_require_args, + get_module_name_from_import_statement, + get_module_name_from_require_args, is_variable_declarator_or_assignment_expression_calling_func, is_variable_declarator_or_assignment_expression_calling_func_member, - traverse_tree_depth_first) + traverse_tree_depth_first, +) +from utils import get_datestamp def get_express_identifiers(tree: Tree) -> set[str]: @@ -25,8 +28,10 @@ def get_express_identifiers(tree: Tree) -> set[str]: case "variable_declarator" | "assignment_expression": # Pick out all the identifiers from nested assignment_expressions # E.g. 'foo = bar = baz = require("express");' - identifiers_assigned_to, last_assignment_expression = \ - get_identifiers_from_variable_declarator_or_assignment_expression(node) + ( + identifiers_assigned_to, + last_assignment_expression, + ) = get_identifiers_from_variable_declarator_or_assignment_expression(node) # If we didn't manage to extract any identifiers then we don't care if require() is involved, because we # don't have any identifiers to add to the set of express_identifiers anyway @@ -58,8 +63,10 @@ def get_app_identifiers(tree: Tree, express_identifiers: set[str]) -> set[str]: case "variable_declarator" | "assignment_expression": # Pick out all the identifiers from nested assignment_expressions # E.g. 'foo = bar = baz = express();' - identifiers_assigned_to, last_assignment_expression = \ - get_identifiers_from_variable_declarator_or_assignment_expression(node) + ( + identifiers_assigned_to, + last_assignment_expression, + ) = get_identifiers_from_variable_declarator_or_assignment_expression(node) # If we didn't manage to extract any identifiers then we don't care if express() is involved, because we # don't have any identifiers to add to the set of app_identifiers anyway @@ -69,13 +76,15 @@ def get_app_identifiers(tree: Tree, express_identifiers: set[str]) -> set[str]: # get_identifiers_from_variable_declarator returns the last assignment expression it traversed, which we # can now check to see if it actually calls express. E.g. if the variable declarator was # 'foo = bar = baz = express();', last_assignment_expression would be 'baz = express();' - is_calling_express = any([ - # we don't care about the args to express() so just [0] - is_variable_declarator_or_assignment_expression_calling_func( - last_assignment_expression, express_identifier - )[0] - for express_identifier in express_identifiers - ]) + is_calling_express = any( + [ + # we don't care about the args to express() so just [0] + is_variable_declarator_or_assignment_expression_calling_func( + last_assignment_expression, express_identifier + )[0] + for express_identifier in express_identifiers + ] + ) if not is_calling_express: continue @@ -92,8 +101,10 @@ def get_router_identifiers(tree: Tree, express_identifiers: set[str]) -> set[str case "variable_declarator" | "assignment_expression": # Pick out all the identifiers from nested assignment_expressions # E.g. 'foo = bar = baz = express();' - identifiers_assigned_to, last_assignment_expression = \ - get_identifiers_from_variable_declarator_or_assignment_expression(node) + ( + identifiers_assigned_to, + last_assignment_expression, + ) = get_identifiers_from_variable_declarator_or_assignment_expression(node) # If we didn't manage to extract any identifiers then we don't care if express() is involved, because we # don't have any identifiers to add to the set of app_identifiers anyway @@ -103,13 +114,15 @@ def get_router_identifiers(tree: Tree, express_identifiers: set[str]) -> set[str # get_identifiers_from_variable_declarator returns the last assignment expression it traversed, which we # can now check to see if it actually calls express.Router(). E.g. if the variable declarator was # 'foo = bar = baz = express.Router();', last_assignment_expression would be 'baz = express.Router();' - is_calling_express_router = any([ - # we don't care about the args to express() so just [0] - is_variable_declarator_or_assignment_expression_calling_func_member( - last_assignment_expression, express_identifier, "Router" - )[0] - for express_identifier in express_identifiers - ]) + is_calling_express_router = any( + [ + # we don't care about the args to express() so just [0] + is_variable_declarator_or_assignment_expression_calling_func_member( + last_assignment_expression, express_identifier, "Router" + )[0] + for express_identifier in express_identifiers + ] + ) if not is_calling_express_router: continue @@ -124,9 +137,7 @@ def get_paths_and_methods(tree: Tree, app_and_router_identifiers: set[str]) -> d # NOTE: This is a subset of all the methods that you can use in Express; specifically, an intersection with all the # methods supported by the OpenAPI 3 specification with the addition of "all" and "use" which in Express accept all # HTTP methods - SUPPORTED_EXPRESS_PROPERTIES = { - "all", "delete", "get", "head", "options", "patch", "post", "put", "trace", "use" - } + SUPPORTED_EXPRESS_PROPERTIES = {"all", "delete", "get", "head", "options", "patch", "post", "put", "trace", "use"} for node in traverse_tree_depth_first(tree): match node.type: @@ -174,10 +185,7 @@ def get_paths_and_methods(tree: Tree, app_and_router_identifiers: set[str]) -> d if len(string_arguments) == 1: # There should be a single string fragment within the string whose text is the path string_fragments = get_children_of_type(string_arguments[0], "string_fragment") - if ( - len(string_fragments) != 1 - or type(string_fragments[0].text) != bytes - ): + if len(string_fragments) != 1 or type(string_fragments[0].text) != bytes: continue path = string_fragments[0].text.decode("utf-8") @@ -205,15 +213,12 @@ def analyse_express(tree: Tree) -> dict | None: # definition under each of the methods, but it's good enough for now. return { "openapi": "3.0.0", - "info": { - "title": "Static Analysis - Express", - "version": get_datestamp() - }, + "info": {"title": "Static Analysis - Express", "version": get_datestamp()}, "paths": { path: { - method: { - "responses": {"default": {"description": "Discovered via static analysis"}} - } for method in methods - } for path, methods in paths.items() + method: {"responses": {"default": {"description": "Discovered via static analysis"}}} + for method in methods + } + for path, methods in paths.items() }, } diff --git a/tests/javascript/test_analyse_express.py b/tests/javascript/test_analyse_express.py index bbc6548..853d934 100644 --- a/tests/javascript/test_analyse_express.py +++ b/tests/javascript/test_analyse_express.py @@ -1,4 +1,5 @@ import datetime + import pytest import yaml diff --git a/tests/javascript/test_analyse_javascript.py b/tests/javascript/test_analyse_javascript.py index bccca59..4ef26d7 100644 --- a/tests/javascript/test_analyse_javascript.py +++ b/tests/javascript/test_analyse_javascript.py @@ -3,11 +3,7 @@ import pytest import yaml -from static_analysis.javascript.analyse_javascript import ( - JS_PARSER, - analyse_javascript, - get_imports, -) +from static_analysis.javascript.analyse_javascript import JS_PARSER, analyse_javascript, get_imports @pytest.fixture(autouse=True) From 3be4ef95fa464934bfcc5ce7901758567cb49110 Mon Sep 17 00:00:00 2001 From: ciaran Date: Fri, 19 Jan 2024 16:47:38 +0000 Subject: [PATCH 20/50] bones of git hub action with new uuid --- src/main_githubaction.py | 144 +++++++++++++++++++++++---------------- src/utils.py | 7 +- 2 files changed, 87 insertions(+), 64 deletions(-) diff --git a/src/main_githubaction.py b/src/main_githubaction.py index 89813e9..a9444dc 100644 --- a/src/main_githubaction.py +++ b/src/main_githubaction.py @@ -3,6 +3,7 @@ import json import os import time +import uuid from openapi.validation import parse_resolve_and_validate_openapi_spec from static_analysis import LANGUAGE_ANALYSERS @@ -17,103 +18,128 @@ def handler(): - FIRETAIL_API_TOKEN = os.environ.get("FIRETAIL_API_TOKEN") - if FIRETAIL_API_TOKEN is None: + firetail_api_token = os.environ.get("FIRETAIL_API_TOKEN") + if firetail_api_token is None: raise Exception("Missing environment variable 'FIRETAIL_API_TOKEN") - FIRETAIL_API_URL = os.environ.get("FIRETAIL_API_URL", "https://api.saas.eu-west-1.prod.firetail.app") - - CONTEXT = os.environ.get("CONTEXT") - if CONTEXT is not None: - CONTEXT = json.loads(CONTEXT) - CONTEXT = GitHubContext( - sha=CONTEXT.get("sha", ""), - repositoryId=CONTEXT.get("repository_id", ""), - repositoryName=CONTEXT.get("event", {}).get("repository", {}).get("name", ""), - repositoryOwner=CONTEXT.get("repository_owner", ""), - ref=CONTEXT.get("ref", ""), - headCommitUsername=CONTEXT.get("event", {}).get("head_commit", {}).get("author", {}).get("username", ""), - actor=CONTEXT.get("actor", ""), - actorId=CONTEXT.get("actor_id", ""), - workflowRef=CONTEXT.get("workflow_ref", ""), - eventName=CONTEXT.get("event_name", ""), - private=CONTEXT.get("event", {}).get("repository", {}).get("private"), - runId=CONTEXT.get("run_id"), + firetail_api_url = os.environ.get("FIRETAIL_API_URL", "https://api.saas.eu-west-1.prod.firetail.app") + + context = os.environ.get("CONTEXT") + if context: + context = json.loads(context) + context = GitHubContext( + sha=context.get("sha", ""), + repositoryId=context.get("repository_id", ""), + repositoryName=context.get("event", {}).get("repository", {}).get("name", ""), + repositoryOwner=context.get("repository_owner", ""), + ref=context.get("ref", ""), + headCommitUsername=context.get("event", {}).get("head_commit", {}).get("author", {}).get("username", ""), + actor=context.get("actor", ""), + actorId=context.get("actor_id", ""), + workflowRef=context.get("workflow_ref", ""), + eventName=context.get("event_name", ""), + private=context.get("event", {}).get("repository", {}).get("private"), + runId=context.get("run_id"), timeTriggered=int(time.time() * 1000 * 1000), timeTriggeredUTCString=datetime.datetime.now(datetime.timezone.utc).isoformat(), file_urls=[], ) # If API_SPEC_LOCATION is set then we upload the OpenAPI spec at that location - COLLECTION_UUID = os.environ.get("COLLECTION_UUID") - API_SPEC_LOCATION = os.environ.get("API_SPEC_LOCATION") - if API_SPEC_LOCATION is None: + collection_uuid = os.environ.get("COLLECTION_UUID") + api_spec_location = os.environ.get("API_SPEC_LOCATION") + if api_spec_location is None: logger.info("API_SPEC_LOCATION is not set, skipping direct upload step.") - elif COLLECTION_UUID is None: + elif collection_uuid is None: logger.info("COLLECTION_UUID is not set, skipping direct upload step.") else: # If we have a CONTEXT then we can add the API_SPEC_LOCATION to the file_urls - if CONTEXT is not None: - CONTEXT.file_urls.append(API_SPEC_LOCATION) + if context is not None: + context.file_urls.append(api_spec_location) - OPENAPI_SPEC = load_openapi_spec(API_SPEC_LOCATION) + openapi_spec = load_openapi_spec(api_spec_location) upload_api_spec_to_firetail_collection( - openapi_spec=OPENAPI_SPEC, - context=CONTEXT, - collection_uuid=COLLECTION_UUID, - firetail_api_url=FIRETAIL_API_URL, - firetail_api_token=FIRETAIL_API_TOKEN, + openapi_spec=openapi_spec, + context=context, + collection_uuid=collection_uuid, + firetail_api_url=firetail_api_url, + firetail_api_token=firetail_api_token, ) - logger.info(f"Successfully uploaded OpenAPI spec to Firetail: {API_SPEC_LOCATION}") + logger.info(f"Successfully uploaded OpenAPI spec to Firetail: {api_spec_location}") - STATIC_ANALYSIS_ROOT_DIR = os.environ.get("STATIC_ANALYSIS_ROOT_DIR", "/") - STATIC_ANALYSIS_LANGUAGES = map( + static_analysis_root_dir = os.environ.get("STATIC_ANALYSIS_ROOT_DIR", "/") + static_analysis_languages = map( lambda v: v.strip(), os.environ.get("STATIC_ANALYSIS_LANGUAGES", "Python,Golang,Javascript").split(",") ) - logger.info(f"Statically analysing files under {STATIC_ANALYSIS_ROOT_DIR}...") - - for path, _, filenames in os.walk(STATIC_ANALYSIS_ROOT_DIR): + logger.info(f"Statically analysing files under {static_analysis_root_dir}...") + external_uuids = [] + last_time = time.time() + for path, _, filenames in os.walk(static_analysis_root_dir): for filename in filenames: - FULL_PATH = f"{path}/{filename}" - logger.info(f"Statically analysing {FULL_PATH}...") + full_path = f"{path}/{filename}" + logger.info(f"Statically analysing {full_path}...") try: - FILE_CONTENTS = open(FULL_PATH, "r").read() + file_contents = open(full_path, "r").read() except Exception as e: # noqa: E722 - logger.critical(f"{FULL_PATH}: Could not read, exception: {e}") + logger.critical(f"{full_path}: Could not read, exception: {e}") continue - + last_time = time.time() # Check if the file is an openapi spec first. If it is, there's no point doing expensive static analysis. - OPENAPI_SPEC = parse_resolve_and_validate_openapi_spec(FULL_PATH, lambda: FILE_CONTENTS) - if OPENAPI_SPEC is not None: - logger.info(f"{FULL_PATH}: Detected OpenAPI spec, uploading to Firetail...") + openapi_spec = parse_resolve_and_validate_openapi_spec(full_path, lambda: file_contents) + if openapi_spec is not None: + logger.info(f"{full_path}: Detected OpenAPI spec, uploading to Firetail...") + external_uuid = str(uuid.uuid4()) upload_discovered_api_spec_to_firetail( - source=FULL_PATH, - openapi_spec=OPENAPI_SPEC, - api_uuid=get_api_uuid_from_api_token(FIRETAIL_API_TOKEN), - firetail_api_url=FIRETAIL_API_URL, - firetail_api_token=FIRETAIL_API_TOKEN, + source=full_path, + openapi_spec=openapi_spec, + api_uuid=get_api_uuid_from_api_token(firetail_api_token), + firetail_api_url=firetail_api_url, + firetail_api_token=firetail_api_token, + external_uuid=external_uuid, ) + external_uuids.append(external_uuid) + last_time = time.time() continue for language, language_analysers in LANGUAGE_ANALYSERS.items(): - if language not in STATIC_ANALYSIS_LANGUAGES: + if language not in static_analysis_languages: continue for language_analyser in language_analysers: - _, openapi_specs_from_analysis = language_analyser(FULL_PATH, FILE_CONTENTS) + _, openapi_specs_from_analysis = language_analyser(full_path, file_contents) - for openapi_spec_source, OPENAPI_SPEC in openapi_specs_from_analysis.items(): - logger.info(f"{FULL_PATH}: Created OpenAPI spec via {language} static analysis...") + for openapi_spec_source, openapi_spec in openapi_specs_from_analysis.items(): + logger.info(f"{full_path}: Created OpenAPI spec via {language} static analysis...") + external_uuid = str(uuid.uuid4()) upload_discovered_api_spec_to_firetail( source=openapi_spec_source, - openapi_spec=OPENAPI_SPEC, - api_uuid=get_api_uuid_from_api_token(FIRETAIL_API_TOKEN), - firetail_api_url=FIRETAIL_API_URL, - firetail_api_token=FIRETAIL_API_TOKEN, + openapi_spec=openapi_spec, + api_uuid=get_api_uuid_from_api_token(firetail_api_token), + firetail_api_url=firetail_api_url, + firetail_api_token=firetail_api_token, + external_uuid=external_uuid, ) + external_uuids.append(external_uuid) + last_time = time.time() + + if external_uuids == []: + return + # We have external IDs now check for finding counts + wait_time = 60 + while True: + # we loop until we have elapsed the timeout + if (time.time() - last_time) > wait_time: + break + for ex_id in external_uuids: + if has_findings_over_x(ex_id): + raise "Error - This action found errors with your spec" + + +def has_findings_over_x(ex_ID): + pass if __name__ == "__main__": diff --git a/src/utils.py b/src/utils.py index 61bc3ad..5a2df4e 100644 --- a/src/utils.py +++ b/src/utils.py @@ -119,7 +119,7 @@ def upload_api_spec_to_firetail_collection( def upload_discovered_api_spec_to_firetail( - source: str, openapi_spec: dict, api_uuid: str, firetail_api_url: str, firetail_api_token: str + source: str, openapi_spec: dict, api_uuid: str, firetail_api_url: str, firetail_api_token: str, external_id: str ): upload_api_spec_response = requests.post( f"{firetail_api_url}/discovery/api-repository/{api_uuid}/appspec", @@ -127,10 +127,7 @@ def upload_discovered_api_spec_to_firetail( "x-ft-api-key": firetail_api_token, "Content-Type": "application/json", }, - json={ - "source": source, - "appspec": openapi_spec, - }, + json={"source": source, "appspec": openapi_spec, "external_id": external_id}, ) if upload_api_spec_response.status_code not in [201, 304]: From 74603ded8ff87f69a0b371f4761cf239cfaf4f03 Mon Sep 17 00:00:00 2001 From: ciaran Date: Fri, 19 Jan 2024 16:55:42 +0000 Subject: [PATCH 21/50] cover all incidents --- src/main_githubaction.py | 70 +++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/src/main_githubaction.py b/src/main_githubaction.py index a9444dc..6d1399b 100644 --- a/src/main_githubaction.py +++ b/src/main_githubaction.py @@ -1,4 +1,3 @@ -# src/get_num_square.py import datetime import json import os @@ -22,27 +21,11 @@ def handler(): if firetail_api_token is None: raise Exception("Missing environment variable 'FIRETAIL_API_TOKEN") firetail_api_url = os.environ.get("FIRETAIL_API_URL", "https://api.saas.eu-west-1.prod.firetail.app") - + external_uuids = [] + last_time = time.time() context = os.environ.get("CONTEXT") if context: - context = json.loads(context) - context = GitHubContext( - sha=context.get("sha", ""), - repositoryId=context.get("repository_id", ""), - repositoryName=context.get("event", {}).get("repository", {}).get("name", ""), - repositoryOwner=context.get("repository_owner", ""), - ref=context.get("ref", ""), - headCommitUsername=context.get("event", {}).get("head_commit", {}).get("author", {}).get("username", ""), - actor=context.get("actor", ""), - actorId=context.get("actor_id", ""), - workflowRef=context.get("workflow_ref", ""), - eventName=context.get("event_name", ""), - private=context.get("event", {}).get("repository", {}).get("private"), - runId=context.get("run_id"), - timeTriggered=int(time.time() * 1000 * 1000), - timeTriggeredUTCString=datetime.datetime.now(datetime.timezone.utc).isoformat(), - file_urls=[], - ) + context = get_context(context) # If API_SPEC_LOCATION is set then we upload the OpenAPI spec at that location collection_uuid = os.environ.get("COLLECTION_UUID") @@ -57,36 +40,33 @@ def handler(): context.file_urls.append(api_spec_location) openapi_spec = load_openapi_spec(api_spec_location) - + external_id = str(uuid.uuid4()) upload_api_spec_to_firetail_collection( openapi_spec=openapi_spec, context=context, collection_uuid=collection_uuid, firetail_api_url=firetail_api_url, firetail_api_token=firetail_api_token, + external_id=external_id, ) - + last_time = time.time() + external_uuids.append(external_id) logger.info(f"Successfully uploaded OpenAPI spec to Firetail: {api_spec_location}") static_analysis_root_dir = os.environ.get("STATIC_ANALYSIS_ROOT_DIR", "/") static_analysis_languages = map( lambda v: v.strip(), os.environ.get("STATIC_ANALYSIS_LANGUAGES", "Python,Golang,Javascript").split(",") ) - logger.info(f"Statically analysing files under {static_analysis_root_dir}...") - external_uuids = [] - last_time = time.time() for path, _, filenames in os.walk(static_analysis_root_dir): for filename in filenames: full_path = f"{path}/{filename}" logger.info(f"Statically analysing {full_path}...") - try: file_contents = open(full_path, "r").read() except Exception as e: # noqa: E722 logger.critical(f"{full_path}: Could not read, exception: {e}") continue - last_time = time.time() # Check if the file is an openapi spec first. If it is, there's no point doing expensive static analysis. openapi_spec = parse_resolve_and_validate_openapi_spec(full_path, lambda: file_contents) if openapi_spec is not None: @@ -98,7 +78,7 @@ def handler(): api_uuid=get_api_uuid_from_api_token(firetail_api_token), firetail_api_url=firetail_api_url, firetail_api_token=firetail_api_token, - external_uuid=external_uuid, + external_id=external_uuid, ) external_uuids.append(external_uuid) last_time = time.time() @@ -110,7 +90,6 @@ def handler(): for language_analyser in language_analysers: _, openapi_specs_from_analysis = language_analyser(full_path, file_contents) - for openapi_spec_source, openapi_spec in openapi_specs_from_analysis.items(): logger.info(f"{full_path}: Created OpenAPI spec via {language} static analysis...") external_uuid = str(uuid.uuid4()) @@ -120,25 +99,48 @@ def handler(): api_uuid=get_api_uuid_from_api_token(firetail_api_token), firetail_api_url=firetail_api_url, firetail_api_token=firetail_api_token, - external_uuid=external_uuid, + external_id=external_uuid, ) external_uuids.append(external_uuid) last_time = time.time() - if external_uuids == []: + if not external_uuids: + # We don't have anything else to check, just return. return # We have external IDs now check for finding counts - wait_time = 60 + wait_time = 60 # TODO: make this configurable while True: # we loop until we have elapsed the timeout if (time.time() - last_time) > wait_time: break + for ex_id in external_uuids: - if has_findings_over_x(ex_id): + if has_findings_over_x(ex_id, firetail_api_token): raise "Error - This action found errors with your spec" -def has_findings_over_x(ex_ID): +def get_context(context): + context = json.loads(context) + return GitHubContext( + sha=context.get("sha", ""), + repositoryId=context.get("repository_id", ""), + repositoryName=context.get("event", {}).get("repository", {}).get("name", ""), + repositoryOwner=context.get("repository_owner", ""), + ref=context.get("ref", ""), + headCommitUsername=context.get("event", {}).get("head_commit", {}).get("author", {}).get("username", ""), + actor=context.get("actor", ""), + actorId=context.get("actor_id", ""), + workflowRef=context.get("workflow_ref", ""), + eventName=context.get("event_name", ""), + private=context.get("event", {}).get("repository", {}).get("private"), + runId=context.get("run_id"), + timeTriggered=int(time.time() * 1000 * 1000), + timeTriggeredUTCString=datetime.datetime.now(datetime.timezone.utc).isoformat(), + file_urls=[], + ) + + +def has_findings_over_x(ex_id: str): pass From 21826c3fc3d137a569071785b386e8d40f05f28a Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 10:06:13 +0000 Subject: [PATCH 22/50] process response from ft api --- build_setup/requirements.txt | 2 ++ src/main_githubaction.py | 18 +++++++++++++++--- tests/requirements.txt | 3 ++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/build_setup/requirements.txt b/build_setup/requirements.txt index 5cd6bf7..ff1bdcc 100644 --- a/build_setup/requirements.txt +++ b/build_setup/requirements.txt @@ -5,3 +5,5 @@ dacite==1.8.1 tree_sitter==0.20.2 PyGithub==1.59.1 jsonschema==4.19.0 +requests +git+ssh://git@github.com/FireTail-io/python-commons-library.git@v1.0.118 diff --git a/src/main_githubaction.py b/src/main_githubaction.py index 6d1399b..ba88954 100644 --- a/src/main_githubaction.py +++ b/src/main_githubaction.py @@ -4,6 +4,9 @@ import time import uuid +import requests +from firetail_toolbox.ft_logger import logger_wrapper + from openapi.validation import parse_resolve_and_validate_openapi_spec from static_analysis import LANGUAGE_ANALYSERS from utils import ( @@ -29,6 +32,7 @@ def handler(): # If API_SPEC_LOCATION is set then we upload the OpenAPI spec at that location collection_uuid = os.environ.get("COLLECTION_UUID") + org_uuid = os.environ.get("ORGANIZATION_UUID") api_spec_location = os.environ.get("API_SPEC_LOCATION") if api_spec_location is None: logger.info("API_SPEC_LOCATION is not set, skipping direct upload step.") @@ -115,7 +119,7 @@ def handler(): break for ex_id in external_uuids: - if has_findings_over_x(ex_id, firetail_api_token): + if has_findings_over_x(ex_id, org_uuid, firetail_api_token): raise "Error - This action found errors with your spec" @@ -140,8 +144,16 @@ def get_context(context): ) -def has_findings_over_x(ex_id: str): - pass +def has_findings_over_x(ex_id: str, org_uuid: str, api_token: str): + endpoint = f"/organisations/{org_uuid}/events/external-id/{ex_id}" + event_resp = requests.get(endpoint, headers={"x-ft-api-key": api_token, "Content-Type": "application/json"}) + if event_resp.status_code != 200: + logger_wrapper("ERROR", {"message": "Non 200 response from events", "resp": event_resp}) + thresholds = {"CRITICAL": 1, "HIGH": 1, "MEDIUM": 4, "LOW": 10} # note - skipping informational. + findings = event_resp.get("initialFindingSeverities") + for level, limit in thresholds.items(): + if findings.get(level, 0) > limit: + raise Exception(f"Findings breached limit: {findings}") if __name__ == "__main__": diff --git a/tests/requirements.txt b/tests/requirements.txt index 3b5f741..b9f8e03 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1 +1,2 @@ -responses \ No newline at end of file +responses +requests From a54030f35e9ed5c01e33576de685a4d20a8d015e Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 10:21:18 +0000 Subject: [PATCH 23/50] no py commons --- build_setup/requirements.txt | 3 +-- src/main_githubaction.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/build_setup/requirements.txt b/build_setup/requirements.txt index ff1bdcc..401e05e 100644 --- a/build_setup/requirements.txt +++ b/build_setup/requirements.txt @@ -5,5 +5,4 @@ dacite==1.8.1 tree_sitter==0.20.2 PyGithub==1.59.1 jsonschema==4.19.0 -requests -git+ssh://git@github.com/FireTail-io/python-commons-library.git@v1.0.118 +requests \ No newline at end of file diff --git a/src/main_githubaction.py b/src/main_githubaction.py index ba88954..d565190 100644 --- a/src/main_githubaction.py +++ b/src/main_githubaction.py @@ -5,7 +5,6 @@ import uuid import requests -from firetail_toolbox.ft_logger import logger_wrapper from openapi.validation import parse_resolve_and_validate_openapi_spec from static_analysis import LANGUAGE_ANALYSERS @@ -148,7 +147,7 @@ def has_findings_over_x(ex_id: str, org_uuid: str, api_token: str): endpoint = f"/organisations/{org_uuid}/events/external-id/{ex_id}" event_resp = requests.get(endpoint, headers={"x-ft-api-key": api_token, "Content-Type": "application/json"}) if event_resp.status_code != 200: - logger_wrapper("ERROR", {"message": "Non 200 response from events", "resp": event_resp}) + print("ERROR", {"message": "Non 200 response from events", "resp": event_resp}) thresholds = {"CRITICAL": 1, "HIGH": 1, "MEDIUM": 4, "LOW": 10} # note - skipping informational. findings = event_resp.get("initialFindingSeverities") for level, limit in thresholds.items(): From be30abf50452079664480c91399fddba49f0a574 Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 10:30:31 +0000 Subject: [PATCH 24/50] add ext id --- src/main_githubaction.py | 6 +++--- src/utils.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main_githubaction.py b/src/main_githubaction.py index d565190..c088a3d 100644 --- a/src/main_githubaction.py +++ b/src/main_githubaction.py @@ -118,7 +118,7 @@ def handler(): break for ex_id in external_uuids: - if has_findings_over_x(ex_id, org_uuid, firetail_api_token): + if findings_breach_threshold(ex_id, org_uuid, firetail_api_token): raise "Error - This action found errors with your spec" @@ -143,10 +143,10 @@ def get_context(context): ) -def has_findings_over_x(ex_id: str, org_uuid: str, api_token: str): +def findings_breach_threshold(ex_id: str, org_uuid: str, api_token: str): endpoint = f"/organisations/{org_uuid}/events/external-id/{ex_id}" event_resp = requests.get(endpoint, headers={"x-ft-api-key": api_token, "Content-Type": "application/json"}) - if event_resp.status_code != 200: + if event_resp.status_code != 200: # pragma: nocover print("ERROR", {"message": "Non 200 response from events", "resp": event_resp}) thresholds = {"CRITICAL": 1, "HIGH": 1, "MEDIUM": 4, "LOW": 10} # note - skipping informational. findings = event_resp.get("initialFindingSeverities") diff --git a/src/utils.py b/src/utils.py index 5a2df4e..259fab9 100644 --- a/src/utils.py +++ b/src/utils.py @@ -101,6 +101,7 @@ def upload_api_spec_to_firetail_collection( collection_uuid: str, firetail_api_url: str, firetail_api_token: str, + external_id: str, ): FIRETAIL_API_RESPONSE = requests.post( url=f"{firetail_api_url}/code_repository/spec", @@ -109,6 +110,7 @@ def upload_api_spec_to_firetail_collection( collection_uuid=collection_uuid, spec_data=openapi_spec, spec_type=get_spec_type(openapi_spec), + external_id=external_id, context=context, ) ), From 21de059d1c904b6a830f67556f5a933bb11cc5d2 Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 10:32:13 +0000 Subject: [PATCH 25/50] need to add to context for body --- src/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils.py b/src/utils.py index 259fab9..abb762b 100644 --- a/src/utils.py +++ b/src/utils.py @@ -45,6 +45,7 @@ class FireTailRequestBody: spec_data: dict spec_type: str context: GitHubContext | None = None + external_id: str def get_api_uuid_from_api_token(api_token: str) -> str: From f94a1e5bbb4e375318f6de8d7ec7559b3c6c0b24 Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 10:39:08 +0000 Subject: [PATCH 26/50] derp --- src/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.py b/src/utils.py index abb762b..fbfbdcf 100644 --- a/src/utils.py +++ b/src/utils.py @@ -44,8 +44,8 @@ class FireTailRequestBody: collection_uuid: str spec_data: dict spec_type: str - context: GitHubContext | None = None external_id: str + context: GitHubContext | None = None def get_api_uuid_from_api_token(api_token: str) -> str: From 3215060b2dcf83bac6339a816b2788baceca7124 Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 11:30:00 +0000 Subject: [PATCH 27/50] make configurable --- src/main_githubaction.py | 12 ++++++++++-- src/utils.py | 1 - 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main_githubaction.py b/src/main_githubaction.py index c088a3d..724789d 100644 --- a/src/main_githubaction.py +++ b/src/main_githubaction.py @@ -143,13 +143,21 @@ def get_context(context): ) +def get_thresholds() -> dict: + critical = os.environ.get("CRITICAL_FINDING_THRESHOLD", 1) + high = os.environ.get("HIGH_FINDING_THRESHOLD", 1) + medium = os.environ.get("MEDIUM_FINDING_THRESHOLD", 4) + low = os.environ.get("LOW_FINDING_THRESHOLD", 10) + return {"CRITICAL": critical, "HIGH": high, "MEDIUM": medium, "LOW": low} + + def findings_breach_threshold(ex_id: str, org_uuid: str, api_token: str): endpoint = f"/organisations/{org_uuid}/events/external-id/{ex_id}" event_resp = requests.get(endpoint, headers={"x-ft-api-key": api_token, "Content-Type": "application/json"}) if event_resp.status_code != 200: # pragma: nocover print("ERROR", {"message": "Non 200 response from events", "resp": event_resp}) - thresholds = {"CRITICAL": 1, "HIGH": 1, "MEDIUM": 4, "LOW": 10} # note - skipping informational. - findings = event_resp.get("initialFindingSeverities") + thresholds = get_thresholds() + findings = event_resp.get("initialFindingSeverities", {}) for level, limit in thresholds.items(): if findings.get(level, 0) > limit: raise Exception(f"Findings breached limit: {findings}") diff --git a/src/utils.py b/src/utils.py index fbfbdcf..9fb3654 100644 --- a/src/utils.py +++ b/src/utils.py @@ -72,7 +72,6 @@ def load_openapi_spec(api_spec_location: str) -> dict: except FileNotFoundError: raise Exception(f"Could not find OpenAPI spec at {api_spec_location}") if openapi_spec is None: - # TODO: a much more helpful error message here raise Exception(f"File at {api_spec_location} is not a valid OpenAPI spec") return openapi_spec From 2f822266c7550152a8e3ee6e84a416f43351438f Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 11:51:59 +0000 Subject: [PATCH 28/50] basic testing --- src/main_githubaction.py | 8 +++++--- tests/python/test_python_action.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 tests/python/test_python_action.py diff --git a/src/main_githubaction.py b/src/main_githubaction.py index 724789d..1966d57 100644 --- a/src/main_githubaction.py +++ b/src/main_githubaction.py @@ -17,6 +17,8 @@ upload_discovered_api_spec_to_firetail, ) +BASE_URL = os.environ.get("FIRETAIL_API_URL", "https://api.saas.eu-west-1.prod.firetail.app") + def handler(): firetail_api_token = os.environ.get("FIRETAIL_API_TOKEN") @@ -111,7 +113,7 @@ def handler(): # We don't have anything else to check, just return. return # We have external IDs now check for finding counts - wait_time = 60 # TODO: make this configurable + wait_time = os.environ.get("FINDING_TIMEOUT_SECONDS", 20) while True: # we loop until we have elapsed the timeout if (time.time() - last_time) > wait_time: @@ -152,12 +154,12 @@ def get_thresholds() -> dict: def findings_breach_threshold(ex_id: str, org_uuid: str, api_token: str): - endpoint = f"/organisations/{org_uuid}/events/external-id/{ex_id}" + endpoint = f"{BASE_URL}/organisations/{org_uuid}/events/external-id/{ex_id}" event_resp = requests.get(endpoint, headers={"x-ft-api-key": api_token, "Content-Type": "application/json"}) if event_resp.status_code != 200: # pragma: nocover print("ERROR", {"message": "Non 200 response from events", "resp": event_resp}) thresholds = get_thresholds() - findings = event_resp.get("initialFindingSeverities", {}) + findings = event_resp.json().get("initialFindingSeverities", {}) for level, limit in thresholds.items(): if findings.get(level, 0) > limit: raise Exception(f"Findings breached limit: {findings}") diff --git a/tests/python/test_python_action.py b/tests/python/test_python_action.py new file mode 100644 index 0000000..a826970 --- /dev/null +++ b/tests/python/test_python_action.py @@ -0,0 +1,16 @@ +import responses +from main_githubaction import findings_breach_threshold + + +@responses.activate +def test_findings_breach(): + responses.add( + responses.GET, + "https://api.saas.eu-west-1.prod.firetail.app/organisations/org_uuid/events/external-id/some-id", + json={}, + status=200, + ) + try: + findings_breach_threshold("some-id", "org_uuid", "api_token") + except Exception: + raise Exception("Should not raise exception") From 440d6615fe6908a82efe8ba73f41e592c7497161 Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 11:57:54 +0000 Subject: [PATCH 29/50] few more tests to handle responses --- src/main_githubaction.py | 1 + tests/python/test_python_action.py | 29 ++++++++++++++++++++++++++++- tests/requirements.txt | 1 + 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/main_githubaction.py b/src/main_githubaction.py index 1966d57..88af934 100644 --- a/src/main_githubaction.py +++ b/src/main_githubaction.py @@ -158,6 +158,7 @@ def findings_breach_threshold(ex_id: str, org_uuid: str, api_token: str): event_resp = requests.get(endpoint, headers={"x-ft-api-key": api_token, "Content-Type": "application/json"}) if event_resp.status_code != 200: # pragma: nocover print("ERROR", {"message": "Non 200 response from events", "resp": event_resp}) + return thresholds = get_thresholds() findings = event_resp.json().get("initialFindingSeverities", {}) for level, limit in thresholds.items(): diff --git a/tests/python/test_python_action.py b/tests/python/test_python_action.py index a826970..a3e7f7c 100644 --- a/tests/python/test_python_action.py +++ b/tests/python/test_python_action.py @@ -1,9 +1,10 @@ +import pytest import responses from main_githubaction import findings_breach_threshold @responses.activate -def test_findings_breach(): +def test_findings_breach_call(): responses.add( responses.GET, "https://api.saas.eu-west-1.prod.firetail.app/organisations/org_uuid/events/external-id/some-id", @@ -14,3 +15,29 @@ def test_findings_breach(): findings_breach_threshold("some-id", "org_uuid", "api_token") except Exception: raise Exception("Should not raise exception") + + +@responses.activate +def test_findings_call_breach(): + responses.add( + responses.GET, + "https://api.saas.eu-west-1.prod.firetail.app/organisations/org_uuid/events/external-id/some-id", + json={"initialFindingSeverities": {"CRITICAL": 200}}, + status=200, + ) + with pytest.raises(Exception): + findings_breach_threshold("some-id", "org_uuid", "api_token") + + +@responses.activate +def test_findings_call_non_200(): + responses.add( + responses.GET, + "https://api.saas.eu-west-1.prod.firetail.app/organisations/org_uuid/events/external-id/some-id", + json={"initialFindingSeverities": {"CRITICAL": 200}}, + status=200, + ) + try: + findings_breach_threshold("some-id", "org_uuid", "api_token") + except Exception: + raise Exception("Should not raise exception") diff --git a/tests/requirements.txt b/tests/requirements.txt index b9f8e03..a550dd2 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,2 +1,3 @@ +pytest responses requests From f4d5191658dc41de1b0473f5f475e2092d772f55 Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 12:01:48 +0000 Subject: [PATCH 30/50] fix response code --- tests/python/test_python_action.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/python/test_python_action.py b/tests/python/test_python_action.py index a3e7f7c..5773b16 100644 --- a/tests/python/test_python_action.py +++ b/tests/python/test_python_action.py @@ -1,5 +1,6 @@ import pytest import responses + from main_githubaction import findings_breach_threshold @@ -35,7 +36,7 @@ def test_findings_call_non_200(): responses.GET, "https://api.saas.eu-west-1.prod.firetail.app/organisations/org_uuid/events/external-id/some-id", json={"initialFindingSeverities": {"CRITICAL": 200}}, - status=200, + status=401, ) try: findings_breach_threshold("some-id", "org_uuid", "api_token") From a4f13d791ce0c6367eba8207a94340e8fdfb6688 Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 12:06:47 +0000 Subject: [PATCH 31/50] reuse var, raise exception --- src/main_githubaction.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main_githubaction.py b/src/main_githubaction.py index 88af934..7c83097 100644 --- a/src/main_githubaction.py +++ b/src/main_githubaction.py @@ -24,7 +24,6 @@ def handler(): firetail_api_token = os.environ.get("FIRETAIL_API_TOKEN") if firetail_api_token is None: raise Exception("Missing environment variable 'FIRETAIL_API_TOKEN") - firetail_api_url = os.environ.get("FIRETAIL_API_URL", "https://api.saas.eu-west-1.prod.firetail.app") external_uuids = [] last_time = time.time() context = os.environ.get("CONTEXT") @@ -43,14 +42,13 @@ def handler(): # If we have a CONTEXT then we can add the API_SPEC_LOCATION to the file_urls if context is not None: context.file_urls.append(api_spec_location) - openapi_spec = load_openapi_spec(api_spec_location) external_id = str(uuid.uuid4()) upload_api_spec_to_firetail_collection( openapi_spec=openapi_spec, context=context, collection_uuid=collection_uuid, - firetail_api_url=firetail_api_url, + firetail_api_url=BASE_URL, firetail_api_token=firetail_api_token, external_id=external_id, ) @@ -81,7 +79,7 @@ def handler(): source=full_path, openapi_spec=openapi_spec, api_uuid=get_api_uuid_from_api_token(firetail_api_token), - firetail_api_url=firetail_api_url, + firetail_api_url=BASE_URL, firetail_api_token=firetail_api_token, external_id=external_uuid, ) @@ -102,7 +100,7 @@ def handler(): source=openapi_spec_source, openapi_spec=openapi_spec, api_uuid=get_api_uuid_from_api_token(firetail_api_token), - firetail_api_url=firetail_api_url, + firetail_api_url=BASE_URL, firetail_api_token=firetail_api_token, external_id=external_uuid, ) @@ -121,7 +119,7 @@ def handler(): for ex_id in external_uuids: if findings_breach_threshold(ex_id, org_uuid, firetail_api_token): - raise "Error - This action found errors with your spec" + raise Exception("Error - This action found errors with your spec") def get_context(context): From a5110dfb751bc4ccbd5c7e92d72c9dcddaca8600 Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 14:09:03 +0000 Subject: [PATCH 32/50] start testing handler --- src/utils.py | 37 +++++++++++------------------- tests/python/test_python_action.py | 7 +++++- 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/src/utils.py b/src/utils.py index 9fb3654..2f4823e 100644 --- a/src/utils.py +++ b/src/utils.py @@ -2,7 +2,7 @@ import logging import time import uuid -from dataclasses import asdict, dataclass +from dataclasses import dataclass from typing import Callable, TypeVar import github @@ -39,15 +39,6 @@ class GitHubContext: file_urls: list[str] -@dataclass -class FireTailRequestBody: - collection_uuid: str - spec_data: dict - spec_type: str - external_id: str - context: GitHubContext | None = None - - def get_api_uuid_from_api_token(api_token: str) -> str: # Using the uuid lib here to make sure it's a valid UUID try: @@ -84,8 +75,7 @@ def respect_rate_limit(func: Callable[[], FuncReturnType], github_client: Github while True: try: return func() - - except github.RateLimitExceededException: + except github.RateLimitExceededException: # pragma: no cover sleep_duration = (github_client.get_rate_limit().core.reset - datetime.datetime.utcnow()).seconds + 1 logger.warning( f"Rate limited calling {func}, core rate limit resets at " @@ -103,21 +93,20 @@ def upload_api_spec_to_firetail_collection( firetail_api_token: str, external_id: str, ): - FIRETAIL_API_RESPONSE = requests.post( + request_body = { + "collection_uuid": collection_uuid, + "spec_data": openapi_spec, + "spec_type": get_spec_type(openapi_spec), + "external_id": external_id, + "context": context, + } + firetail_api_response = requests.post( url=f"{firetail_api_url}/code_repository/spec", - json=asdict( - FireTailRequestBody( - collection_uuid=collection_uuid, - spec_data=openapi_spec, - spec_type=get_spec_type(openapi_spec), - external_id=external_id, - context=context, - ) - ), + json=request_body, headers={"x-ft-api-key": firetail_api_token}, ) - if FIRETAIL_API_RESPONSE.status_code not in {201, 409}: - raise Exception(f"Failed to send FireTail API Spec. {FIRETAIL_API_RESPONSE.text}") + if firetail_api_response.status_code not in {201, 409}: + raise Exception(f"Failed to send FireTail API Spec. {firetail_api_response.text}") def upload_discovered_api_spec_to_firetail( diff --git a/tests/python/test_python_action.py b/tests/python/test_python_action.py index 5773b16..a4d4259 100644 --- a/tests/python/test_python_action.py +++ b/tests/python/test_python_action.py @@ -1,7 +1,7 @@ import pytest import responses -from main_githubaction import findings_breach_threshold +from main_githubaction import findings_breach_threshold, handler @responses.activate @@ -42,3 +42,8 @@ def test_findings_call_non_200(): findings_breach_threshold("some-id", "org_uuid", "api_token") except Exception: raise Exception("Should not raise exception") + + +def test_run_handler(): + with pytest.raises(Exception): + handler() From 8b0dfafb60a93cb961cc5564c326fe9d8dd57af8 Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 14:29:05 +0000 Subject: [PATCH 33/50] some os env test --- tests/python/test_python_action.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/python/test_python_action.py b/tests/python/test_python_action.py index a4d4259..e4e74e3 100644 --- a/tests/python/test_python_action.py +++ b/tests/python/test_python_action.py @@ -1,3 +1,4 @@ +import os import pytest import responses @@ -47,3 +48,8 @@ def test_findings_call_non_200(): def test_run_handler(): with pytest.raises(Exception): handler() + + +def test_base_env_var_set_no_context(): + os.environ["FIRETAIL_API_TOKEN"] = "token-token-token" + handler() From ec29b3d63a18641c4801877134b07cd34aa2a17d Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 14:51:29 +0000 Subject: [PATCH 34/50] test with os --- src/main_githubaction.py | 2 -- tests/python/test_python_action.py | 10 +++++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main_githubaction.py b/src/main_githubaction.py index 7c83097..907bb64 100644 --- a/src/main_githubaction.py +++ b/src/main_githubaction.py @@ -29,7 +29,6 @@ def handler(): context = os.environ.get("CONTEXT") if context: context = get_context(context) - # If API_SPEC_LOCATION is set then we upload the OpenAPI spec at that location collection_uuid = os.environ.get("COLLECTION_UUID") org_uuid = os.environ.get("ORGANIZATION_UUID") @@ -55,7 +54,6 @@ def handler(): last_time = time.time() external_uuids.append(external_id) logger.info(f"Successfully uploaded OpenAPI spec to Firetail: {api_spec_location}") - static_analysis_root_dir = os.environ.get("STATIC_ANALYSIS_ROOT_DIR", "/") static_analysis_languages = map( lambda v: v.strip(), os.environ.get("STATIC_ANALYSIS_LANGUAGES", "Python,Golang,Javascript").split(",") diff --git a/tests/python/test_python_action.py b/tests/python/test_python_action.py index e4e74e3..53ca699 100644 --- a/tests/python/test_python_action.py +++ b/tests/python/test_python_action.py @@ -3,6 +3,8 @@ import responses from main_githubaction import findings_breach_threshold, handler +from static_analysis.python.analyse_python import analyse_python +from unittest import mock @responses.activate @@ -50,6 +52,12 @@ def test_run_handler(): handler() -def test_base_env_var_set_no_context(): +def fake_analyser(): + return ({}, {}) + + +@mock.patch("main_githubaction.LANGUAGE_ANALYSERS", return_value={"Python": [fake_analyser]}) +def test_base_env_var_set_no_context(_): os.environ["FIRETAIL_API_TOKEN"] = "token-token-token" + os.environ["STATIC_ANALYSIS_ROOT_DIR"] = "src" handler() From d3cfdfd3ed281082f40d86d49886dda920289949 Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 15:07:50 +0000 Subject: [PATCH 35/50] clean up again - add to action yaml --- action.yml | 15 +++++++++++++++ src/static_analysis/python/analyse_python.py | 6 +++--- tests/python/test_python_action.py | 2 +- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/action.yml b/action.yml index 4a1b358..d21b8f8 100644 --- a/action.yml +++ b/action.yml @@ -5,6 +5,9 @@ inputs: FIRETAIL_API_TOKEN: description: "Your FireTail API token" required: true + ORGANIZATION_UUID: + description: "Your Firetail Organization UUID" + required: true FIRETAIL_API_URL: description: "Your FireTail API token" required: false @@ -26,6 +29,18 @@ inputs: description: "A comma separated list of languages to statically analyse (currently supported are Python, Golang and Javascript)" required: false default: "Python,Golang,Javascript" + CRITICAL_FINDING_THRESHOLD: + description: "Finding level for failing the action if there is more than this number of Critical findings" + default: "1" + HIGH_FINDING_THRESHOLD: + description: "Finding level for failing the action if there is more than this number of Finding findings" + default: "1" + MEDIUM_FINDING_THRESHOLD: + description: "Finding level for failing the action if there is more than this number of Medium findings" + default: "4" + LOW_FINDING_THRESHOLD: + description: "Finding level for failing the action if there is more than this number of Low findings" + default: "10" runs: using: "docker" image: "Dockerfile.githubaction" diff --git a/src/static_analysis/python/analyse_python.py b/src/static_analysis/python/analyse_python.py index 38d8c14..86ce9f1 100644 --- a/src/static_analysis/python/analyse_python.py +++ b/src/static_analysis/python/analyse_python.py @@ -20,12 +20,12 @@ def get_imports(module: ast.Module) -> list[str]: def analyse_python(file_path: str, get_file_contents: Callable[[], str]) -> tuple[set[str], dict[str, dict]]: if not file_path.endswith(".py"): - return (set(), {}) + return set(), {} try: parsed_module = ast.parse(get_file_contents()) - except SyntaxError: - return (set(), {}) + except SyntaxError: # pragma: no cover + return set(), {} imported_modules = get_imports(parsed_module) diff --git a/tests/python/test_python_action.py b/tests/python/test_python_action.py index 53ca699..561fc42 100644 --- a/tests/python/test_python_action.py +++ b/tests/python/test_python_action.py @@ -53,7 +53,7 @@ def test_run_handler(): def fake_analyser(): - return ({}, {}) + return set(), {} @mock.patch("main_githubaction.LANGUAGE_ANALYSERS", return_value={"Python": [fake_analyser]}) From 34d4b8f433d2d232f35c32f73bada9c0d2bab8d9 Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 15:51:24 +0000 Subject: [PATCH 36/50] stop hiding errors --- src/openapi/validation.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/openapi/validation.py b/src/openapi/validation.py index 4546133..8ba89cf 100644 --- a/src/openapi/validation.py +++ b/src/openapi/validation.py @@ -24,17 +24,9 @@ def resolve_and_validate_openapi_spec(file_contents: str) -> dict | None: def parse_resolve_and_validate_openapi_spec(file_path: str, get_file_contents: Callable[[], str]) -> dict | None: # First check it's a valid JSON/YAML file before passing it over to Prance if file_path.endswith(".json"): - try: - file_contents = json.loads(get_file_contents()) - except: # noqa: E722 - return None - + file_contents = json.loads(get_file_contents()) elif file_path.endswith((".yaml", ".yml")): - try: - file_contents = yaml.safe_load(get_file_contents()) - except: # noqa: E722 - return None - + file_contents = yaml.safe_load(get_file_contents()) else: return None From 64a51d354cc5a6134e51e253b395d078ece2ab10 Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 16:03:33 +0000 Subject: [PATCH 37/50] validate with relevant errors --- src/openapi/validation.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/openapi/validation.py b/src/openapi/validation.py index 8ba89cf..cf65ab0 100644 --- a/src/openapi/validation.py +++ b/src/openapi/validation.py @@ -13,11 +13,7 @@ def resolve_and_validate_openapi_spec(file_contents: str) -> dict | None: backend="openapi-spec-validator", lazy=True, ) - try: - parser.parse() - except: # noqa: E722 - # In the future, maybe we can provide some proper details here. - return None + parser.parse() return parser.specification From 5355ba8bb7ba75e00a21d970d8c81ee0aef30ace Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 16:39:12 +0000 Subject: [PATCH 38/50] fix --- src/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils.py b/src/utils.py index 2f4823e..03627bd 100644 --- a/src/utils.py +++ b/src/utils.py @@ -2,7 +2,7 @@ import logging import time import uuid -from dataclasses import dataclass +from dataclasses import dataclass, asdict from typing import Callable, TypeVar import github @@ -98,7 +98,7 @@ def upload_api_spec_to_firetail_collection( "spec_data": openapi_spec, "spec_type": get_spec_type(openapi_spec), "external_id": external_id, - "context": context, + "context": asdict(context), } firetail_api_response = requests.post( url=f"{firetail_api_url}/code_repository/spec", From 4cb441e13cd0a0613ad1da67ba7a6c37a88c182d Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 16:50:00 +0000 Subject: [PATCH 39/50] context with external id --- src/main_githubaction.py | 7 ++++--- src/utils.py | 7 +++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main_githubaction.py b/src/main_githubaction.py index 907bb64..5fd67ea 100644 --- a/src/main_githubaction.py +++ b/src/main_githubaction.py @@ -43,13 +43,13 @@ def handler(): context.file_urls.append(api_spec_location) openapi_spec = load_openapi_spec(api_spec_location) external_id = str(uuid.uuid4()) + context.external_id = external_id upload_api_spec_to_firetail_collection( openapi_spec=openapi_spec, context=context, collection_uuid=collection_uuid, firetail_api_url=BASE_URL, firetail_api_token=firetail_api_token, - external_id=external_id, ) last_time = time.time() external_uuids.append(external_id) @@ -73,13 +73,13 @@ def handler(): if openapi_spec is not None: logger.info(f"{full_path}: Detected OpenAPI spec, uploading to Firetail...") external_uuid = str(uuid.uuid4()) + context.external_id = external_id upload_discovered_api_spec_to_firetail( source=full_path, openapi_spec=openapi_spec, api_uuid=get_api_uuid_from_api_token(firetail_api_token), firetail_api_url=BASE_URL, firetail_api_token=firetail_api_token, - external_id=external_uuid, ) external_uuids.append(external_uuid) last_time = time.time() @@ -94,13 +94,13 @@ def handler(): for openapi_spec_source, openapi_spec in openapi_specs_from_analysis.items(): logger.info(f"{full_path}: Created OpenAPI spec via {language} static analysis...") external_uuid = str(uuid.uuid4()) + context.external_id = external_id upload_discovered_api_spec_to_firetail( source=openapi_spec_source, openapi_spec=openapi_spec, api_uuid=get_api_uuid_from_api_token(firetail_api_token), firetail_api_url=BASE_URL, firetail_api_token=firetail_api_token, - external_id=external_uuid, ) external_uuids.append(external_uuid) last_time = time.time() @@ -137,6 +137,7 @@ def get_context(context): runId=context.get("run_id"), timeTriggered=int(time.time() * 1000 * 1000), timeTriggeredUTCString=datetime.datetime.now(datetime.timezone.utc).isoformat(), + external_id=context.get("external_id"), file_urls=[], ) diff --git a/src/utils.py b/src/utils.py index 03627bd..94b8ca6 100644 --- a/src/utils.py +++ b/src/utils.py @@ -37,6 +37,7 @@ class GitHubContext: timeTriggered: int timeTriggeredUTCString: str file_urls: list[str] + external_id: str def get_api_uuid_from_api_token(api_token: str) -> str: @@ -91,13 +92,11 @@ def upload_api_spec_to_firetail_collection( collection_uuid: str, firetail_api_url: str, firetail_api_token: str, - external_id: str, ): request_body = { "collection_uuid": collection_uuid, "spec_data": openapi_spec, "spec_type": get_spec_type(openapi_spec), - "external_id": external_id, "context": asdict(context), } firetail_api_response = requests.post( @@ -110,7 +109,7 @@ def upload_api_spec_to_firetail_collection( def upload_discovered_api_spec_to_firetail( - source: str, openapi_spec: dict, api_uuid: str, firetail_api_url: str, firetail_api_token: str, external_id: str + source: str, openapi_spec: dict, api_uuid: str, firetail_api_url: str, firetail_api_token: str ): upload_api_spec_response = requests.post( f"{firetail_api_url}/discovery/api-repository/{api_uuid}/appspec", @@ -118,7 +117,7 @@ def upload_discovered_api_spec_to_firetail( "x-ft-api-key": firetail_api_token, "Content-Type": "application/json", }, - json={"source": source, "appspec": openapi_spec, "external_id": external_id}, + json={"source": source, "appspec": openapi_spec}, ) if upload_api_spec_response.status_code not in [201, 304]: From 917119f5b36042a106f287fb9bef9fb419ae0452 Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 17:03:39 +0000 Subject: [PATCH 40/50] thing --- src/main_githubaction.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main_githubaction.py b/src/main_githubaction.py index 5fd67ea..cac4d9c 100644 --- a/src/main_githubaction.py +++ b/src/main_githubaction.py @@ -73,7 +73,7 @@ def handler(): if openapi_spec is not None: logger.info(f"{full_path}: Detected OpenAPI spec, uploading to Firetail...") external_uuid = str(uuid.uuid4()) - context.external_id = external_id + context.external_id = external_uuid upload_discovered_api_spec_to_firetail( source=full_path, openapi_spec=openapi_spec, @@ -94,7 +94,7 @@ def handler(): for openapi_spec_source, openapi_spec in openapi_specs_from_analysis.items(): logger.info(f"{full_path}: Created OpenAPI spec via {language} static analysis...") external_uuid = str(uuid.uuid4()) - context.external_id = external_id + context.external_id = external_uuid upload_discovered_api_spec_to_firetail( source=openapi_spec_source, openapi_spec=openapi_spec, @@ -137,7 +137,7 @@ def get_context(context): runId=context.get("run_id"), timeTriggered=int(time.time() * 1000 * 1000), timeTriggeredUTCString=datetime.datetime.now(datetime.timezone.utc).isoformat(), - external_id=context.get("external_id"), + external_id=context.get("external_id", ""), file_urls=[], ) From d259e1164acf06118b6478538bc0e3d4f9bc9e79 Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 17:08:42 +0000 Subject: [PATCH 41/50] ft body thing again --- src/utils.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/utils.py b/src/utils.py index 94b8ca6..69dcc1a 100644 --- a/src/utils.py +++ b/src/utils.py @@ -86,6 +86,14 @@ def respect_rate_limit(func: Callable[[], FuncReturnType], github_client: Github time.sleep(sleep_duration) +@dataclass +class FireTailRequestBody: + collection_uuid: str + spec_data: dict + spec_type: str + context: GitHubContext | None = None + + def upload_api_spec_to_firetail_collection( openapi_spec: dict, context: GitHubContext | None, @@ -101,7 +109,14 @@ def upload_api_spec_to_firetail_collection( } firetail_api_response = requests.post( url=f"{firetail_api_url}/code_repository/spec", - json=request_body, + json=asdict( + FireTailRequestBody( + collection_uuid=collection_uuid, + spec_data=openapi_spec, + spec_type=get_spec_type(openapi_spec), + context=context, + ) + ), headers={"x-ft-api-key": firetail_api_token}, ) if firetail_api_response.status_code not in {201, 409}: From 56405668a71e3483f337d368971df41994dd9b26 Mon Sep 17 00:00:00 2001 From: ciaran Date: Mon, 22 Jan 2024 17:23:52 +0000 Subject: [PATCH 42/50] rm unused code --- src/utils.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/utils.py b/src/utils.py index 69dcc1a..7cd7153 100644 --- a/src/utils.py +++ b/src/utils.py @@ -101,12 +101,6 @@ def upload_api_spec_to_firetail_collection( firetail_api_url: str, firetail_api_token: str, ): - request_body = { - "collection_uuid": collection_uuid, - "spec_data": openapi_spec, - "spec_type": get_spec_type(openapi_spec), - "context": asdict(context), - } firetail_api_response = requests.post( url=f"{firetail_api_url}/code_repository/spec", json=asdict( From e620917ab6ba402b25f39e0cd30575fa35a9315c Mon Sep 17 00:00:00 2001 From: ciaran Date: Tue, 23 Jan 2024 10:24:59 +0000 Subject: [PATCH 43/50] catch anayluser issue --- src/main_githubaction.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main_githubaction.py b/src/main_githubaction.py index cac4d9c..f5af93c 100644 --- a/src/main_githubaction.py +++ b/src/main_githubaction.py @@ -90,7 +90,11 @@ def handler(): continue for language_analyser in language_analysers: - _, openapi_specs_from_analysis = language_analyser(full_path, file_contents) + try: + _, openapi_specs_from_analysis = language_analyser(full_path, file_contents) + except Exception as e: + logger.critical(f"{full_path}: Could not analyse, exception: {e}") + continue for openapi_spec_source, openapi_spec in openapi_specs_from_analysis.items(): logger.info(f"{full_path}: Created OpenAPI spec via {language} static analysis...") external_uuid = str(uuid.uuid4()) From 71c1712a8a30cd1091ff981dc0b875277fdd8d5f Mon Sep 17 00:00:00 2001 From: ciaran Date: Tue, 23 Jan 2024 10:33:09 +0000 Subject: [PATCH 44/50] add text --- src/main_githubaction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main_githubaction.py b/src/main_githubaction.py index f5af93c..1c802d0 100644 --- a/src/main_githubaction.py +++ b/src/main_githubaction.py @@ -158,7 +158,7 @@ def findings_breach_threshold(ex_id: str, org_uuid: str, api_token: str): endpoint = f"{BASE_URL}/organisations/{org_uuid}/events/external-id/{ex_id}" event_resp = requests.get(endpoint, headers={"x-ft-api-key": api_token, "Content-Type": "application/json"}) if event_resp.status_code != 200: # pragma: nocover - print("ERROR", {"message": "Non 200 response from events", "resp": event_resp}) + print("ERROR", {"message": "Non 200 response from events", "resp": event_resp, "resp_text": event_resp.text}) return thresholds = get_thresholds() findings = event_resp.json().get("initialFindingSeverities", {}) From 4ae5c20238282232339ac647d53a9e54151bd9ac Mon Sep 17 00:00:00 2001 From: ciaran Date: Tue, 23 Jan 2024 13:29:41 +0000 Subject: [PATCH 45/50] log details --- src/main_githubaction.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main_githubaction.py b/src/main_githubaction.py index 1c802d0..12c9788 100644 --- a/src/main_githubaction.py +++ b/src/main_githubaction.py @@ -161,6 +161,7 @@ def findings_breach_threshold(ex_id: str, org_uuid: str, api_token: str): print("ERROR", {"message": "Non 200 response from events", "resp": event_resp, "resp_text": event_resp.text}) return thresholds = get_thresholds() + print("Event resp json was -> ", event_resp.json()) findings = event_resp.json().get("initialFindingSeverities", {}) for level, limit in thresholds.items(): if findings.get(level, 0) > limit: From 824f29bcff438b22574b0c1ef04eb93ced0073cf Mon Sep 17 00:00:00 2001 From: ciaran Date: Wed, 24 Jan 2024 09:18:01 +0000 Subject: [PATCH 46/50] longer wait time for uuid getting --- src/main_githubaction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main_githubaction.py b/src/main_githubaction.py index 12c9788..cf3d97f 100644 --- a/src/main_githubaction.py +++ b/src/main_githubaction.py @@ -113,7 +113,7 @@ def handler(): # We don't have anything else to check, just return. return # We have external IDs now check for finding counts - wait_time = os.environ.get("FINDING_TIMEOUT_SECONDS", 20) + wait_time = os.environ.get("FINDING_TIMEOUT_SECONDS", 60) while True: # we loop until we have elapsed the timeout if (time.time() - last_time) > wait_time: From 76258061edb9fe2e32c807d34d34a11967df4096 Mon Sep 17 00:00:00 2001 From: rileyfiretail <107564215+rileyfiretail@users.noreply.github.com> Date: Tue, 13 Aug 2024 10:22:02 +0100 Subject: [PATCH 47/50] Update Dockerfile --- build_setup/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_setup/Dockerfile b/build_setup/Dockerfile index 84682d5..33a570e 100644 --- a/build_setup/Dockerfile +++ b/build_setup/Dockerfile @@ -10,7 +10,7 @@ RUN go tool cover -html coverage.out -o coverage.html FROM python:3.11-bullseye as build-tree-sitter WORKDIR /src RUN apt-get update -y && apt-get upgrade -y -RUN git clone https://github.com/tree-sitter/tree-sitter-javascript +RUN git clone https://github.com/tree-sitter/tree-sitter-javascript --branch v0.20.2 --single-branch RUN python3 -m pip install tree_sitter COPY analysers/tree-sitter/build.py build.py RUN python3 build.py @@ -56,4 +56,4 @@ RUN PYTHONPATH=${LAMBDA_TASK_ROOT} pytest --cov ${LAMBDA_TASK_ROOT} --cov-report FROM build-python-lambda as runtime-lambda RUN ls -la ${LAMBDA_TASK_ROOT} -CMD [ "main_awslambda.handler" ] \ No newline at end of file +CMD [ "main_awslambda.handler" ] From 20cb2c411539467b310fbe4a9b87500ef89c4ae5 Mon Sep 17 00:00:00 2001 From: rileyfiretail <107564215+rileyfiretail@users.noreply.github.com> Date: Tue, 13 Aug 2024 10:27:02 +0100 Subject: [PATCH 48/50] Update Dockerfile --- build_setup/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_setup/Dockerfile b/build_setup/Dockerfile index 33a570e..b13344b 100644 --- a/build_setup/Dockerfile +++ b/build_setup/Dockerfile @@ -11,7 +11,7 @@ FROM python:3.11-bullseye as build-tree-sitter WORKDIR /src RUN apt-get update -y && apt-get upgrade -y RUN git clone https://github.com/tree-sitter/tree-sitter-javascript --branch v0.20.2 --single-branch -RUN python3 -m pip install tree_sitter +RUN python3 -m pip install tree_sitter==0.20.2 COPY analysers/tree-sitter/build.py build.py RUN python3 build.py From cf86e98d74a9f22bd6a76f51708f94627f9ae048 Mon Sep 17 00:00:00 2001 From: rileyfiretail <107564215+rileyfiretail@users.noreply.github.com> Date: Tue, 13 Aug 2024 10:30:50 +0100 Subject: [PATCH 49/50] Update Dockerfile.githubaction --- Dockerfile.githubaction | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile.githubaction b/Dockerfile.githubaction index 48659d9..f49730d 100644 --- a/Dockerfile.githubaction +++ b/Dockerfile.githubaction @@ -8,8 +8,8 @@ RUN cd /src && go tool cover -html coverage.out -o coverage.html FROM python:3.11-bullseye as build-tree-sitter RUN apt-get update -y && apt-get upgrade -y -RUN mkdir /src && cd /src && git clone https://github.com/tree-sitter/tree-sitter-javascript -RUN python3 -m pip install tree_sitter +RUN mkdir /src && cd /src && git clone https://github.com/tree-sitter/tree-sitter-javascript --branch v0.20.2 --single-branch +RUN python3 -m pip install tree_sitter==0.20.2 COPY ./analysers/tree-sitter/build.py /src/build.py RUN cd /src && python3 build.py @@ -31,4 +31,4 @@ RUN cd /github-api-discovery && pytest --cov . --cov-report=xml:coverage.xml -vv FROM build-python as runtime RUN chmod +x /github-api-discovery/src/main_githubaction.py CMD ["/github-api-discovery/src/main_githubaction.py"] -ENTRYPOINT ["python"] \ No newline at end of file +ENTRYPOINT ["python"] From 68c9c59a959c0c18b59de8692e97f326c61aab10 Mon Sep 17 00:00:00 2001 From: rileyfiretail <107564215+rileyfiretail@users.noreply.github.com> Date: Tue, 13 Aug 2024 13:19:40 +0100 Subject: [PATCH 50/50] Update utils.py --- src/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.py b/src/utils.py index 7cd7153..c399965 100644 --- a/src/utils.py +++ b/src/utils.py @@ -129,7 +129,7 @@ def upload_discovered_api_spec_to_firetail( json={"source": source, "appspec": openapi_spec}, ) - if upload_api_spec_response.status_code not in [201, 304]: + if upload_api_spec_response.status_code not in [201, 200]: raise Exception(f"Failed to send API Spec to FireTail. {upload_api_spec_response.text}") logger.info(