Skip to content

Commit

Permalink
Merge branch 'main' of https://github.com/fasiondog/hku_utils
Browse files Browse the repository at this point in the history
  • Loading branch information
@fasiondog
fasiondog committed Jul 31, 2024
2 parents 91bf110 + 3b73849 commit 5706ae7
Show file tree
Hide file tree
Showing 17 changed files with 861 additions and 119 deletions.
5 changes: 1 addition & 4 deletions .github/workflows/windows.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ jobs:
- uses: xmake-io/github-action-setup-xmake@v1
with:
xmake-version: 2.9.3
actions-cache-folder: '.xmake-cache'

- name: configure
shell: cmd
Expand All @@ -38,7 +39,3 @@ jobs:
run: |
xmake -b unit-test
- name: test
shell: cmd
run: |
xmake r unit-test
2 changes: 2 additions & 0 deletions config.h.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ ${define HKU_ENABLE_HTTP_CLIENT}
${define HKU_ENABLE_HTTP_CLIENT_SSL}
${define HKU_ENABLE_HTTP_CLIENT_ZIP}

${define HKU_ENABLE_NODE}

// clang-format on

#endif /* HKU_UTILS_CONFIG_H_*/
9 changes: 5 additions & 4 deletions copy_dependents.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ task("copy_dependents")
end
elseif type(pkg_path) == 'table' then
for i=1, #pkg_path do
local pos = string.find(pkg_path[i], "yh_utils")
local pos = string.find(pkg_path[i], "hku_utils")
if pos == nil then
pos = string.find(pkg_path[i], "opencv")
if pos == nil then
Expand All @@ -49,11 +49,11 @@ task("copy_dependents")
end
else
for _, filedir in ipairs(os.dirs(pkg_path[i] .. "/*")) do
local pos = string.find(filedir, "yihua")
local pos = string.find(filedir, "hikyuu")
if pos == nil then
os.trycp(filedir, destpath .. "/include")
else
os.trycp(filedir .. "/utils", destpath .. "/include/yihua")
os.trycp(filedir .. "/utilites", destpath .. "/include/hikyuu")
end
end
end
Expand All @@ -64,7 +64,8 @@ task("copy_dependents")
-- 拷贝依赖的库文件
os.trycp(pkg:installdir() .. "/lib/*", libdir)
if is_plat("windows") then
os.trycp(pkg:installdir() .. "/bin/*.dll", libdir)
-- os.trycp(pkg:installdir() .. "/bin/*.dll", libdir)
os.trycp(pkg:installdir() .. "/lib/*.dll", destpath .. '/bin')
end

:: continue ::
Expand Down
178 changes: 83 additions & 95 deletions hikyuu/utilities/http_client/HttpClient.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,18 @@
* Author: fasiondog
*/

#include "hikyuu/utilities/Log.h"
#include "hikyuu/utilities/os.h"
#include "HttpClient.h"

#if HKU_ENABLE_HTTP_CLIENT_ZIP
#include "gzip/compress.hpp"
#include "gzip/decompress.hpp"
#endif

#include <sstream>
#include "hikyuu/utilities/Log.h"
#include "hikyuu/utilities/os.h"
#include "url.h"

namespace hku {

HttpResponse::HttpResponse() {
Expand All @@ -31,6 +34,7 @@ void HttpResponse::reset() {
nng_http_res_free(m_res);
NNG_CHECK(nng_http_res_alloc(&m_res));
}
m_body.clear();
}

HttpResponse::HttpResponse(HttpResponse&& rhs) : m_res(rhs.m_res), m_body(std::move(rhs.m_body)) {
Expand Down Expand Up @@ -99,121 +103,105 @@ void HttpClient::_connect() {
}

HttpResponse HttpClient::request(const std::string& method, const std::string& path,
const HttpHeaders& headers, const char* body, size_t len,
const HttpParams& params, const HttpHeaders& headers,
const char* body, size_t body_len,
const std::string& content_type) {
HKU_CHECK(m_url.valid(), "Invalid url: {}", m_url.raw_url());

HttpResponse res;
try {
nng::http_req req(m_url);
req.set_method(method).set_uri(path).add_headers(m_default_headers).add_headers(headers);
if (body != nullptr) {
HKU_CHECK(len > 0, "Body is not null, but len is zero!");
req.add_header("Content-Type", content_type);

#if HKU_ENABLE_HTTP_CLIENT_ZIP
if (req.get_header("Content-Encoding") == "gzip") {
gzip::Compressor comp(Z_DEFAULT_COMPRESSION);
std::string output;
comp.compress(output, body, len);
req.copy_data(output.data(), output.size());
std::ostringstream buf;
bool first = true;
for (auto iter = params.cbegin(); iter != params.cend(); ++iter) {
if (first) {
buf << "?";
} else {
req.set_data(body, len);
buf << "&";
}
#else
req.del_header("Content-Encoding").set_data(body, len);
#endif
buf << iter->first << "=" << iter->second;
}

int count = 0;
while (count < 2) {
count++;
_connect();

// Send the request, and wait for that to finish.
m_conn.write_req(req, m_aio);
int rv = m_aio.wait().result();
if (NNG_ECLOSED == rv || NNG_ECONNSHUT == rv || NNG_ECONNREFUSED == rv) {
// HKU_DEBUG("rv: {}", nng_strerror(rv));
reset();
res.reset();
continue;
} else if (NNG_ETIMEDOUT == rv) {
throw HttpTimeoutException();
} else if (0 != rv) {
HKU_THROW("[NNG_ERROR] {} ", nng_strerror(rv));
}
std::string uri = buf.str();
uri = uri.empty() ? path : fmt::format("{}{}", path, uri);
res = _readResChunk(method, uri, headers, body, body_len, content_type);

m_conn.read_res(res.get(), m_aio);
rv = m_aio.wait().result();
if (0 == rv) {
break;
} else if (NNG_ETIMEDOUT == rv) {
throw HttpTimeoutException();
} else if (NNG_ECLOSED == rv || NNG_ECONNSHUT == rv || NNG_ECONNREFUSED == rv) {
// HKU_DEBUG("rv: {}", nng_strerror(rv));
reset();
res.reset();
} else {
HKU_THROW("[NNG_ERROR] {} ", nng_strerror(rv));
}
if (res.getHeader("Connection") == "close") {
HKU_WARN("Connect closed");
reset();
}

HKU_IF_RETURN(res.status() != NNG_HTTP_STATUS_OK, res);

std::string hdr = res.getHeader("Content-Length");
HKU_WARN_IF_RETURN(hdr.empty(), res, "Missing Content-Length header.");
} catch (const std::exception&) {
reset();
throw;
} catch (...) {
reset();
HKU_THROW_UNKNOWN;
}
return res;
}

size_t len = std::stoi(hdr);
HKU_IF_RETURN(len == 0, res);
HttpResponse HttpClient::_readResChunk(const std::string& method, const std::string& uri,
const HttpHeaders& headers, const char* body,
size_t body_len, const std::string& content_type) {
HttpResponse res;
nng::http_req req(m_url);
req.set_method(method).set_uri(uri).add_headers(m_default_headers).add_headers(headers);
if (body != nullptr) {
HKU_CHECK(body_len > 0, "Body is not null, but len is zero!");
req.add_header("Content-Type", content_type);

#if HKU_ENABLE_HTTP_CLIENT_ZIP
if (res.getHeader("Content-Encoding") == "gzip") {
nng_iov iov;
auto data = std::unique_ptr<char[]>(new char[len]);
iov.iov_len = len;
iov.iov_buf = data.get();
m_aio.set_iov(1, &iov);
m_conn.read_all(m_aio);
NNG_CHECK(m_aio.wait().result());

gzip::Decompressor decomp;
decomp.decompress(res.m_body, data.get(), len);

if (req.get_header("Content-Encoding") == "gzip") {
gzip::Compressor comp(Z_DEFAULT_COMPRESSION);
std::string output;
comp.compress(output, body, body_len);
req.copy_data(output.data(), output.size());
} else {
res._resizeBody(len);
nng_iov iov;
iov.iov_len = len;
iov.iov_buf = res.m_body.data();
m_aio.set_iov(1, &iov);
m_conn.read_all(m_aio);
NNG_CHECK(m_aio.wait().result());
req.set_data(body, body_len);
}
#else
HKU_WARN_IF(
res.getHeader("Content-Encoding") == "gzip",
"Automatic decompression is not supported. You need to decompress it yourself!");
res._resizeBody(len);
nng_iov iov;
iov.iov_len = len;
iov.iov_buf = res.m_body.data();
m_aio.set_iov(1, &iov);
m_conn.read_all(m_aio);
NNG_CHECK(m_aio.wait().result());

#endif // #if HKU_ENABLE_HTTP_CLIENT_ZIP
req.del_header("Content-Encoding").set_data(body, body_len);
#endif
}

if (res.getHeader("Connection") == "close") {
HKU_WARN("Connect closed");
int count = 0;
while (count < 2) {
count++;
_connect();

m_conn.transact(req.get(), res.get(), m_aio);
int rv = m_aio.wait().result();
if (0 == rv) {
break;
} else if (NNG_ETIMEDOUT == rv) {
throw HttpTimeoutException();
} else if (NNG_ECLOSED == rv || NNG_ECONNSHUT == rv || NNG_ECONNREFUSED == rv) {
// HKU_DEBUG("rv: {}", nng_strerror(rv));
reset();
res.reset();
} else {
HKU_THROW("[NNG_ERROR] {} ", nng_strerror(rv));
}
} catch (const std::exception&) {
reset();
throw;
} catch (...) {
reset();
HKU_THROW_UNKNOWN;
}

HKU_IF_RETURN(res.status() != NNG_HTTP_STATUS_OK, res);

void* data;
size_t len;
nng_http_res_get_data(res.get(), &data, &len);

#if HKU_ENABLE_HTTP_CLIENT_ZIP
if (res.getHeader("Content-Encoding") == "gzip") {
res.m_body = gzip::decompress((const char*)data, len);
} else {
res._resizeBody(len);
memcpy(res.m_body.data(), data, len);
}
#else
res._resizeBody(len);
memcpy(res.m_body.data(), data, len);
#endif

return res;
}

Expand Down
37 changes: 30 additions & 7 deletions hikyuu/utilities/http_client/HttpClient.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,29 +125,52 @@ class HKU_UTILS_API HttpClient {
void reset();

HttpResponse request(const std::string& method, const std::string& path,
const HttpHeaders& headers, const char* body, size_t len,
const std::string& content_type);
const HttpParams& params, const HttpHeaders& headers, const char* body,
size_t body_len, const std::string& content_type);

HttpResponse get(const std::string& path, const HttpHeaders& headers = {}) {
return request("GET", path, headers, nullptr, 0, "");
HttpResponse get(const std::string& path, const HttpHeaders& headers = HttpHeaders()) {
return request("GET", path, HttpParams(), headers, nullptr, 0, "");
}

HttpResponse get(const std::string& path, const HttpParams& params,
const HttpHeaders& headers) {
return request("GET", path, params, headers, nullptr, 0, "");
}

HttpResponse post(const std::string& path, const HttpParams& params, const HttpHeaders& headers,
const char* body, size_t len, const std::string& content_type) {
return request("POST", path, params, headers, body, len, content_type);
}

HttpResponse post(const std::string& path, const HttpHeaders& headers, const char* body,
size_t len, const std::string& content_type) {
return request("POST", path, headers, body, len, content_type);
return request("POST", path, HttpParams(), headers, body, len, content_type);
}

HttpResponse post(const std::string& path, const HttpParams& params, const HttpHeaders& headers,
const std::string& content, const std::string& content_type = "text/plaint") {
return post(path, params, headers, content.data(), content.size(), content_type);
}

HttpResponse post(const std::string& path, const HttpHeaders& headers,
const std::string& content, const std::string& content_type = "text/plaint") {
return post(path, headers, content.data(), content.size(), content_type);
return post(path, HttpParams(), headers, content, content_type);
}

HttpResponse post(const std::string& path, const HttpParams& params, const HttpHeaders& headers,
const json& body) {
return post(path, params, headers, body.dump(), "application/json");
}

HttpResponse post(const std::string& path, const HttpHeaders& headers, const json& body) {
return post(path, headers, body.dump(), "application/json");
return post(path, HttpParams(), headers, body);
}

private:
void _connect();
HttpResponse _readResChunk(const std::string& method, const std::string& uri,
const HttpHeaders& headers, const char* body, size_t body_len,
const std::string& content_type);

private:
HttpHeaders m_default_headers;
Expand Down
5 changes: 5 additions & 0 deletions hikyuu/utilities/http_client/nng_wrap.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ struct HttpTimeoutException : hku::exception {
};

using HttpHeaders = std::map<std::string, std::string>;
using HttpParams = std::map<std::string, std::string>;

} // namespace hku

Expand Down Expand Up @@ -463,6 +464,10 @@ class http_conn final {
nng_http_conn_read_all(m_conn, aio.get());
}

void transact(nng_http_req* req, nng_http_res* res, const aio& aio) {
nng_http_conn_transact(m_conn, req, res, aio.get());
}

private:
nng_http_conn* m_conn{nullptr};
};
Expand Down
Loading

0 comments on commit 5706ae7

Please sign in to comment.