github api retries

PhilipDeegan · Oct 27, 2024 · ab2517e · ab2517e
1 parent 15abf46
commit ab2517e
Show file tree

Hide file tree

Showing 9 changed files with 316 additions and 143 deletions.
diff --git a/.github/workflows/build_nix_python.yaml b/.github/workflows/build_nix_python.yaml
@@ -0,0 +1,34 @@
+name: ubuntu-python-latest
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    name: Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: true
+      max-parallel: 4
+      matrix:
+        python-version: ['3.11', '3.12']
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+        architecture: x64
+
+    - run: |
+        ./res/ci/nixish_setup.sh && make nix
+        chmod +x mkn && sudo cp mkn /usr/bin
+        python3 -m pip install -r test/github/requirements.txt
+        ./test/github/test_github_api.sh
diff --git a/inc/maiken/github.hpp b/inc/maiken/github.hpp
@@ -34,16 +34,36 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "maiken.hpp"
 #ifdef _MKN_WITH_MKN_RAM_
 #include "mkn/kul/yaml.hpp"
+#include "mkn/ram/http.hpp"
 #include "mkn/ram/https.hpp"
 
+#include <chrono>
+#include <string>
+#include <thread>
+
 namespace maiken {
 
+namespace github {
+
+static inline std::string URL = "api.github.com";
+static inline int port = 443;
+
+}  // namespace github
+
+template <bool https = true>
 class Github {
  private:
   static bool IS_SOLID(std::string const& r) {
     return r.find("://") != std::string::npos || r.find("@") != std::string::npos;
   }
 
+  auto static request(std::string const& path) {
+    if constexpr (https)
+      return mkn::ram::https::Get(github::URL, path, github::port);
+    else
+      return mkn::ram::http::Get(github::URL, path, github::port);
+  }
+
  public:
   static bool GET_DEFAULT_BRANCH(std::string const& owner, std::string const& repo,
                                  std::string& branch);
@@ -53,6 +73,150 @@ class Github {
                              std::string& branch);
   static bool GET_LATEST(std::string const& repo, std::string& branch);
 };
+
+template <bool https>
+bool Github<https>::GET_DEFAULT_BRANCH(std::string const& owner, std::string const& repo,
+                                       std::string& branch) {
+  bool b = 0;
+  int retry = 3;
+  std::stringstream ss;
+  ss << "repos/" << owner << "/" << repo;
+  while (retry-- > 0) {
+    request(ss.str())
+        .withHeaders({{"User-Agent", "Mozilla not a virus"}, {"Accept", "application/json"}})
+        .withResponse([&b, &branch](auto const& r) {
+          if (r.status() == 200)  //
+            try {
+              mkn::kul::yaml::String const yaml(r.body());
+              KLOG(OTH) << "Github API default branch response: " << r.body();
+              if (yaml.root() && yaml.root()["default_branch"]) {
+                branch = yaml.root()["default_branch"].Scalar();
+                b = 1;
+              }
+            } catch (YAML::Exception const&) {
+              KLOG(ERR) << "maiken::Github::GET_DEFAULT_BRANCH invalid response received.";
+            }
+        })
+        .send();
+    if (b) return b;
+    KLOG(ERR) << "maiken::Github::GET_DEFAULT_BRANCH failed - retrying";
+    using namespace std::chrono_literals;
+    std::this_thread::sleep_for(50ms);
+  }
+  return b;
+}
+
+template <bool https>
+bool Github<https>::GET_LATEST_RELEASE(std::string const& owner, std::string const& repo,
+                                       std::string& branch) {
+  bool b = 0;
+  int retry = 3;
+  std::stringstream ss;
+  ss << "repos/" << owner << "/" << repo << "/releases/latest";
+
+  while (retry-- > 0) {
+    request(ss.str())
+        .withHeaders({{"User-Agent", "Mozilla not a virus"}, {"Accept", "application/json"}})
+        .withResponse([&b, &branch](auto const& r) {
+          if (r.status() == 200)  //
+            try {
+              mkn::kul::yaml::String const yaml(r.body());
+              if (yaml.root()["tag_name"]) {
+                branch = yaml.root()["tag_name"].Scalar();
+                b = 1;
+              }
+            } catch (YAML::Exception const&) {
+              KLOG(ERR) << "maiken::Github::GET_LATEST_RELEASE invalid response received.";
+            }
+        })
+        .send();
+    if (b) return b;
+    KLOG(ERR) << "maiken::Github::GET_LATEST_RELEASE failed - retrying";
+    using namespace std::chrono_literals;
+    std::this_thread::sleep_for(50ms);
+  }
+  return b;
+}
+
+template <bool https>
+bool Github<https>::GET_LATEST_TAG(std::string const& owner, std::string const& repo,
+                                   std::string& branch) {
+  bool b = 0;
+  int retry = 3;
+  std::stringstream ss;
+  ss << "repos/" << owner << "/" << repo << "/git/tags";
+  while (retry-- > 0) {
+    request(ss.str())
+        .withHeaders({{"User-Agent", "Mozilla not a virus"}, {"Accept", "application/json"}})
+        .withResponse([&b, &branch](auto const& r) {
+          if (r.status() == 200)  //
+            try {
+              mkn::kul::yaml::String const yaml(r.body());
+              if (yaml.root().Type() == 3) {
+                if (yaml.root()["ref"]) {
+                  branch = yaml.root()["ref"].Scalar();
+                  b = 1;
+                }
+              }
+            } catch (YAML::Exception const&) {
+              KLOG(ERR) << "maiken::Github::GET_LATEST_TAG invalid response received.";
+            }
+        })
+        .send();
+
+    if (b == 1) {
+      auto bits(mkn::kul::String::SPLIT(branch, "/"));
+      branch = bits[bits.size() - 1];
+      return b;
+    }
+    KLOG(ERR) << "maiken::Github::GET_LATEST_TAG failed - retrying";
+    using namespace std::chrono_literals;
+    std::this_thread::sleep_for(50ms);
+  }
+  return b;
+}
+
+template <bool https>
+bool Github<https>::GET_LATEST(std::string const& repo, std::string& branch) {
+#ifndef _MKN_DISABLE_SCM_
+
+  std::vector<std::function<decltype(GET_DEFAULT_BRANCH)>> gets{
+      &GET_DEFAULT_BRANCH, &GET_LATEST_RELEASE, &GET_LATEST_TAG};
+  std::vector<size_t> orders{0, 1, 2};
+  if (_MKN_GIT_WITH_RAM_DEFAULT_CO_ACTION_ == 1) orders = {1, 2, 0};
+
+  std::vector<std::string> repos;
+  if (IS_SOLID(repo))
+    repos.push_back(repo);
+  else
+    for (std::string const& s : Settings::INSTANCE().remoteRepos()) repos.push_back(s + repo);
+  for (std::string const& s : repos) {
+    if (s.find("github.com") != std::string::npos) {
+      std::string owner = s.substr(s.find("github.com") + 10);
+      if (owner[0] != '/' && owner[0] != ':') {
+        KERR << "Repo \"" << s << "\" is invalid - skipping";
+        continue;
+      }
+      owner.erase(0, 1);
+      if (owner.find("/") != std::string::npos) owner = owner.substr(0, owner.find("/"));
+
+      if (owner.empty()) {
+        KERR << "Invalid attempt to perform github lookup";
+        continue;
+      }
+
+      for (auto const& order : orders)
+        if (gets[order](owner, repo, branch)) return 1;
+    }
+  }
+#else
+  KEXIT(1,
+        "SCM disabled, cannot resolve dependency, check local paths and "
+        "configurations");
+#endif
+  return 0;
+}
+
 }  // namespace maiken
 
 #endif  //_MKN_WITH_MKN_RAM_

diff --git a/mkn.yaml b/mkn.yaml
@@ -3,7 +3,7 @@
 name: mkn
 version: master
 property:
-  DATE: 07-OCT-2024
+  DATE: 27-OCT-2024
   maiken_location: ${MKN_HOME}/app/mkn/${version}
   maiken_scm: https://github.com/mkn/mkn
   self.deps: mkn.kul
@@ -85,3 +85,10 @@ profile:
 
 - name: lib_test
   dep: mkn&${maiken_location}(${maiken_scm})[mod]
+
+
+- name: github
+  parent: headers
+  dep: parse.yaml
+  main: test/github/test.cpp
+  with: mkn.ram
diff --git a/src/maiken/scm/github.cpp b/src/maiken/scm/github.cpp
diff --git a/test/github/__pycache__/mock_api.cpython-312.pyc b/test/github/__pycache__/mock_api.cpython-312.pyc