From ab2517e987537b0bc556cd8ba0eedab5c61eff59 Mon Sep 17 00:00:00 2001 From: PhilipDeegan Date: Sun, 27 Oct 2024 16:39:13 +0100 Subject: [PATCH] github api retries --- .github/workflows/build_nix_python.yaml | 34 ++++ inc/maiken/github.hpp | 164 ++++++++++++++++++ mkn.yaml | 9 +- src/maiken/scm/github.cpp | 142 --------------- .../__pycache__/mock_api.cpython-312.pyc | Bin 0 -> 1201 bytes test/github/mock_api.py | 28 +++ test/github/requirements.txt | 3 + test/github/test.cpp | 40 +++++ test/github/test_github_api.sh | 39 +++++ 9 files changed, 316 insertions(+), 143 deletions(-) create mode 100644 .github/workflows/build_nix_python.yaml delete mode 100644 src/maiken/scm/github.cpp create mode 100644 test/github/__pycache__/mock_api.cpython-312.pyc create mode 100644 test/github/mock_api.py create mode 100644 test/github/requirements.txt create mode 100644 test/github/test.cpp create mode 100755 test/github/test_github_api.sh diff --git a/.github/workflows/build_nix_python.yaml b/.github/workflows/build_nix_python.yaml new file mode 100644 index 00000000..a6878ee0 --- /dev/null +++ b/.github/workflows/build_nix_python.yaml @@ -0,0 +1,34 @@ +name: ubuntu-python-latest + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +on: + pull_request: + branches: [ master ] + +jobs: + build: + name: Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + + strategy: + fail-fast: true + max-parallel: 4 + matrix: + python-version: ['3.11', '3.12'] + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + + - run: | + ./res/ci/nixish_setup.sh && make nix + chmod +x mkn && sudo cp mkn /usr/bin + python3 -m pip install -r test/github/requirements.txt + ./test/github/test_github_api.sh diff --git a/inc/maiken/github.hpp b/inc/maiken/github.hpp index 29a167af..304dfe21 100644 --- a/inc/maiken/github.hpp +++ b/inc/maiken/github.hpp @@ -34,16 +34,36 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "maiken.hpp" #ifdef _MKN_WITH_MKN_RAM_ #include "mkn/kul/yaml.hpp" +#include "mkn/ram/http.hpp" #include "mkn/ram/https.hpp" +#include +#include +#include + namespace maiken { +namespace github { + +static inline std::string URL = "api.github.com"; +static inline int port = 443; + +} // namespace github + +template class Github { private: static bool IS_SOLID(std::string const& r) { return r.find("://") != std::string::npos || r.find("@") != std::string::npos; } + auto static request(std::string const& path) { + if constexpr (https) + return mkn::ram::https::Get(github::URL, path, github::port); + else + return mkn::ram::http::Get(github::URL, path, github::port); + } + public: static bool GET_DEFAULT_BRANCH(std::string const& owner, std::string const& repo, std::string& branch); @@ -53,6 +73,150 @@ class Github { std::string& branch); static bool GET_LATEST(std::string const& repo, std::string& branch); }; + +template +bool Github::GET_DEFAULT_BRANCH(std::string const& owner, std::string const& repo, + std::string& branch) { + bool b = 0; + int retry = 3; + std::stringstream ss; + ss << "repos/" << owner << "/" << repo; + while (retry-- > 0) { + request(ss.str()) + .withHeaders({{"User-Agent", "Mozilla not a virus"}, {"Accept", "application/json"}}) + .withResponse([&b, &branch](auto const& r) { + if (r.status() == 200) // + try { + mkn::kul::yaml::String const yaml(r.body()); + KLOG(OTH) << "Github API default branch response: " << r.body(); + if (yaml.root() && yaml.root()["default_branch"]) { + branch = yaml.root()["default_branch"].Scalar(); + b = 1; + } + } catch (YAML::Exception const&) { + KLOG(ERR) << "maiken::Github::GET_DEFAULT_BRANCH invalid response received."; + } + }) + .send(); + if (b) return b; + KLOG(ERR) << "maiken::Github::GET_DEFAULT_BRANCH failed - retrying"; + using namespace std::chrono_literals; + std::this_thread::sleep_for(50ms); + } + return b; +} + +template +bool Github::GET_LATEST_RELEASE(std::string const& owner, std::string const& repo, + std::string& branch) { + bool b = 0; + int retry = 3; + std::stringstream ss; + ss << "repos/" << owner << "/" << repo << "/releases/latest"; + + while (retry-- > 0) { + request(ss.str()) + .withHeaders({{"User-Agent", "Mozilla not a virus"}, {"Accept", "application/json"}}) + .withResponse([&b, &branch](auto const& r) { + if (r.status() == 200) // + try { + mkn::kul::yaml::String const yaml(r.body()); + if (yaml.root()["tag_name"]) { + branch = yaml.root()["tag_name"].Scalar(); + b = 1; + } + } catch (YAML::Exception const&) { + KLOG(ERR) << "maiken::Github::GET_LATEST_RELEASE invalid response received."; + } + }) + .send(); + if (b) return b; + KLOG(ERR) << "maiken::Github::GET_LATEST_RELEASE failed - retrying"; + using namespace std::chrono_literals; + std::this_thread::sleep_for(50ms); + } + return b; +} + +template +bool Github::GET_LATEST_TAG(std::string const& owner, std::string const& repo, + std::string& branch) { + bool b = 0; + int retry = 3; + std::stringstream ss; + ss << "repos/" << owner << "/" << repo << "/git/tags"; + while (retry-- > 0) { + request(ss.str()) + .withHeaders({{"User-Agent", "Mozilla not a virus"}, {"Accept", "application/json"}}) + .withResponse([&b, &branch](auto const& r) { + if (r.status() == 200) // + try { + mkn::kul::yaml::String const yaml(r.body()); + if (yaml.root().Type() == 3) { + if (yaml.root()["ref"]) { + branch = yaml.root()["ref"].Scalar(); + b = 1; + } + } + } catch (YAML::Exception const&) { + KLOG(ERR) << "maiken::Github::GET_LATEST_TAG invalid response received."; + } + }) + .send(); + + if (b == 1) { + auto bits(mkn::kul::String::SPLIT(branch, "/")); + branch = bits[bits.size() - 1]; + return b; + } + KLOG(ERR) << "maiken::Github::GET_LATEST_TAG failed - retrying"; + using namespace std::chrono_literals; + std::this_thread::sleep_for(50ms); + } + return b; +} + +template +bool Github::GET_LATEST(std::string const& repo, std::string& branch) { +#ifndef _MKN_DISABLE_SCM_ + + std::vector> gets{ + &GET_DEFAULT_BRANCH, &GET_LATEST_RELEASE, &GET_LATEST_TAG}; + std::vector orders{0, 1, 2}; + if (_MKN_GIT_WITH_RAM_DEFAULT_CO_ACTION_ == 1) orders = {1, 2, 0}; + + std::vector repos; + if (IS_SOLID(repo)) + repos.push_back(repo); + else + for (std::string const& s : Settings::INSTANCE().remoteRepos()) repos.push_back(s + repo); + for (std::string const& s : repos) { + if (s.find("github.com") != std::string::npos) { + std::string owner = s.substr(s.find("github.com") + 10); + if (owner[0] != '/' && owner[0] != ':') { + KERR << "Repo \"" << s << "\" is invalid - skipping"; + continue; + } + owner.erase(0, 1); + if (owner.find("/") != std::string::npos) owner = owner.substr(0, owner.find("/")); + + if (owner.empty()) { + KERR << "Invalid attempt to perform github lookup"; + continue; + } + + for (auto const& order : orders) + if (gets[order](owner, repo, branch)) return 1; + } + } +#else + KEXIT(1, + "SCM disabled, cannot resolve dependency, check local paths and " + "configurations"); +#endif + return 0; +} + } // namespace maiken #endif //_MKN_WITH_MKN_RAM_ diff --git a/mkn.yaml b/mkn.yaml index 5b367cf9..e9c74e69 100644 --- a/mkn.yaml +++ b/mkn.yaml @@ -3,7 +3,7 @@ name: mkn version: master property: - DATE: 07-OCT-2024 + DATE: 27-OCT-2024 maiken_location: ${MKN_HOME}/app/mkn/${version} maiken_scm: https://github.com/mkn/mkn self.deps: mkn.kul @@ -85,3 +85,10 @@ profile: - name: lib_test dep: mkn&${maiken_location}(${maiken_scm})[mod] + + +- name: github + parent: headers + dep: parse.yaml + main: test/github/test.cpp + with: mkn.ram diff --git a/src/maiken/scm/github.cpp b/src/maiken/scm/github.cpp deleted file mode 100644 index 49c4f8d3..00000000 --- a/src/maiken/scm/github.cpp +++ /dev/null @@ -1,142 +0,0 @@ -/** -Copyright (c) 2022, Philip Deegan. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Philip Deegan nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ -#include "maiken/github.hpp" - -#ifdef _MKN_WITH_MKN_RAM_ - -bool maiken::Github::GET_DEFAULT_BRANCH(std::string const& owner, std::string const& repo, - std::string& branch) { - bool b = 0; - std::stringstream ss; - ss << "repos/" << owner << "/" << repo; - mkn::ram::https::Get("api.github.com", ss.str()) - .withHeaders({{"User-Agent", "Mozilla not a virus"}, {"Accept", "application/json"}}) - .withResponse([&b, &branch](auto const& r) { - if (r.status() == 200) { - mkn::kul::yaml::String yaml(r.body()); - KLOG(OTH) << "Github API default branch response: " << r.body(); - if (yaml.root() && yaml.root()["default_branch"]) { - branch = yaml.root()["default_branch"].Scalar(); - b = 1; - } - } - }) - .send(); - return b; -} - -bool maiken::Github::GET_LATEST_RELEASE(std::string const& owner, std::string const& repo, - std::string& branch) { - bool b = 0; - std::stringstream ss; - ss << "repos/" << owner << "/" << repo << "/releases/latest"; - mkn::ram::https::Get("api.github.com", ss.str()) - .withHeaders({{"User-Agent", "Mozilla not a virus"}, {"Accept", "application/json"}}) - .withResponse([&b, &branch](auto const& r) { - if (r.status() == 200) { - mkn::kul::yaml::String yaml(r.body()); - if (yaml.root()["tag_name"]) { - branch = yaml.root()["tag_name"].Scalar(); - b = 1; - } - } - }) - .send(); - return b; -} - -bool maiken::Github::GET_LATEST_TAG(std::string const& owner, std::string const& repo, - std::string& branch) { - bool b = 0; - std::stringstream ss; - ss << "repos/" << owner << "/" << repo << "/git/tags"; - mkn::ram::https::Get("api.github.com", ss.str()) - .withHeaders({{"User-Agent", "Mozilla not a virus"}, {"Accept", "application/json"}}) - .withResponse([&b, &branch](auto const& r) { - if (r.status() == 200) { - mkn::kul::yaml::String yaml(r.body()); - if (yaml.root().Type() == 3) { - if (yaml.root()["ref"]) { - branch = yaml.root()["ref"].Scalar(); - b = 1; - } - } - } - }) - .send(); - if (b == 1) { - auto bits(mkn::kul::String::SPLIT(branch, "/")); - branch = bits[bits.size() - 1]; - } - return b; -} - -bool maiken::Github::GET_LATEST(std::string const& repo, std::string& branch) { -#ifndef _MKN_DISABLE_SCM_ - - std::vector> gets{ - &GET_DEFAULT_BRANCH, &GET_LATEST_RELEASE, &GET_LATEST_TAG}; - std::vector orders{0, 1, 2}; - if (_MKN_GIT_WITH_RAM_DEFAULT_CO_ACTION_ == 1) orders = {1, 2, 0}; - - std::vector repos; - if (IS_SOLID(repo)) - repos.push_back(repo); - else - for (std::string const& s : Settings::INSTANCE().remoteRepos()) repos.push_back(s + repo); - for (std::string const& s : repos) { - if (s.find("github.com") != std::string::npos) { - std::string owner = s.substr(s.find("github.com") + 10); - if (owner[0] != '/' && owner[0] != ':') { - KERR << "Repo \"" << s << "\" is invalid - skipping"; - continue; - } - owner.erase(0, 1); - if (owner.find("/") != std::string::npos) owner = owner.substr(0, owner.find("/")); - - if (owner.empty()) { - KERR << "Invalid attempt to perform github lookup"; - continue; - } - - for (auto const& order : orders) - if (gets[order](owner, repo, branch)) return 1; - } - } -#else - KEXIT(1, - "SCM disabled, cannot resolve dependency, check local paths and " - "configurations"); -#endif - return 0; -} - -#endif /*_MKN_WITH_MKN_RAM_*/ diff --git a/test/github/__pycache__/mock_api.cpython-312.pyc b/test/github/__pycache__/mock_api.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dea79ea55bddfe7e78b6a623da9848d2929ae340 GIT binary patch literal 1201 zcmZ`&&ubGw6rS1LWH;ITXbVZD)}r+k#V+1e(1TEmsZbA+OW0DtZiGPAWLr_~Df z=&fgurTBjcf)sKo3qnB;?JYG3+LP~XlQg!;z`S|8-#6d;=FRTcY&K0`9KV{Yf1`x_ zz=!^jdr9{fB)h~S7IjF2Rw%_>a-@b_ktvY~qx(H-MX_W{wrM3{DOLj3BvW4J`+^s% zmE4DFI4t!&8SK`^u(bg@HN;MhVW$UdeTbbN!_ExY*&(*x%$4#dXs|>h^{FX>$1fHk zO+7IMd+u4xiE9!h1W(qQG&u7;VR&0E<8fX?wf>MG*@oFI zLRcac`Wy5yvO+$~TXc_>p`d~?d!&ot*?nW(YcRt%>ULl>OnZa5u!sOE1WW|@dVO=% zXn3`as_EMceltqr{_2*;Hw4dtMh2$PAsmxGA^EB2!==AMf#-K>infYZTvyjj+Zk;g z<<70!J>P*P|sVfeON z|IY*vi43!g-P|QjEPnC@@{x5F+|RMZ3`~)L5RxN3$MFc|QYKQ@;9}ttIci3l-?U6O zuxlLEBiZzQj(={XRqKwoYC0m4>+t>Iox}}}PY~;0x{5RHLBq2)9rlP{2Qq%F;uD09 zOey`HBy{#WnLAAqHMebt@}YL^VCK$&dN))KGlh@l&cjgaB#D&xF#UG=AT`^jH%=u= zC%Q6`<~o>lv;lgnP3IBKA$l9p+|b^8ZMuMH5m5ut;$ZL2bem2hm`89b&f?De1H>-< A$^ZZW literal 0 HcmV?d00001 diff --git a/test/github/mock_api.py b/test/github/mock_api.py new file mode 100644 index 00000000..db3540b5 --- /dev/null +++ b/test/github/mock_api.py @@ -0,0 +1,28 @@ +# needs fastapi + +from fastapi import FastAPI +from pydantic import BaseModel + + +app = FastAPI() +repo = {"default_branch": "default_branch"} +_globals = {"retries":1} + + +@app.get("/repos/owner/repo") +async def repo_works(): + return repo + + +@app.get("/repos/owner/repo_fail") +async def repo_fails(): + return {} + +@app.get("/repos/owner/repo_invalid_response") +async def repo_invalid_response(): + return "NOT JSON" + + +@app.get("/ping") +def ping(): + return 200 diff --git a/test/github/requirements.txt b/test/github/requirements.txt new file mode 100644 index 00000000..446a727a --- /dev/null +++ b/test/github/requirements.txt @@ -0,0 +1,3 @@ +pydantic +uvicorn +fastapi diff --git a/test/github/test.cpp b/test/github/test.cpp new file mode 100644 index 00000000..4abcf6ba --- /dev/null +++ b/test/github/test.cpp @@ -0,0 +1,40 @@ + + +#include "maiken/github.hpp" +#include "mkn/kul/assert.hpp" + +#include + +namespace maiken { +bool static const premain = []() { + github::URL = "localhost"; + github::port = 8000; + return true; +}(); + +} // namespace maiken + +int test_repo() { + std::string branch; + bool b = maiken::Github::GET_DEFAULT_BRANCH("owner", "repo", branch); + mkn::kul::abort_if(!b); + return !b; +} + +int test_repo_fail() { + std::string branch; + bool b = maiken::Github::GET_DEFAULT_BRANCH("owner", "repo_fail", branch); + mkn::kul::abort_if(b); + return b; +} + +int test_repo_invalid_response() { + std::string branch; + bool b = maiken::Github::GET_DEFAULT_BRANCH("owner", "repo_invalid_response", branch); + mkn::kul::abort_if(b); + return b; +} + +int main(int argc, char* argv[]) { // + return test_repo() + test_repo_fail() + test_repo_invalid_response(); +} diff --git a/test/github/test_github_api.sh b/test/github/test_github_api.sh new file mode 100755 index 00000000..4a8efcd3 --- /dev/null +++ b/test/github/test_github_api.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +CWD="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" && cd "$CWD" +set -ex + +export SERVER_URL="http://localhost:8000" + +function finish { + exit_code=$? + SERVER_PID=$(ps aux | grep -v grep | grep "uvicorn" | xargs | cut -d' ' -f2) + echo "killing server pid $SERVER_PID" + [[ -n "${SERVER_PID}" ]] && ((SERVER_PID > 0)) && kill -15 $SERVER_PID + exit $rv +} +trap finish EXIT INT + +( + cd ../.. + mkn clean build -p github -dtOa -fPIC +) + +( + export PYTHONPATH=$PWD + + # start server + uvicorn mock_api:app --workers 4 & + sleep 2 + + # wait for server + curl --retry-connrefused \ + --connect-timeout 10 \ + --retry 10 \ + --retry-delay 2 \ + "$SERVER_URL/ping" +) + +( + cd ../.. + mkn run -p github +)