feat(NODE-6540): Add c++ zstd compression API

.github/workflows/lint.yml

@@ -0,0 +1,38 @@
+name: Lint
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    name: ${{ matrix.lint-target }}
+    strategy:
+      matrix:
+        lint-target: ["c++", "typescript"]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Use Node.js LTS
+      uses: actions/setup-node@v4
+      with:
+        node-version: 'lts/*'
+        cache: 'npm'
+
+    - name: Install dependencies
+      shell: bash
+      run: npm i --ignore-scripts
+
+    - if: matrix.lint-target == 'c++'
+      shell: bash
+      run: |
+        npm run check:clang-format
+    - if: matrix.lint-target == 'typescript'
+      shell: bash
+      run: |
+        npm run check:eslint

.github/workflows/test.yml

@@ -24,8 +24,16 @@ jobs:
           cache: 'npm'
           registry-url: 'https://registry.npmjs.org'
 
-      - name: Build with Node.js ${{ matrix.node }} on ${{ matrix.os }}
-        run: npm install && npm run compile
+      - name: Install zstd
+        run: npm run install-zstd
+        shell: bash
+
+      - name: install dependencies
+        run: npm install --loglevel verbose --ignore-scripts
+        shell: bash
+
+      - name: Compile addon
+        run: npm run compile
         shell: bash
 
       - name: Test ${{ matrix.os }}

.gitignore

@@ -18,3 +18,4 @@ node_modules
 build
 
 npm-debug.log
+deps

addon/compress.h

@@ -0,0 +1,20 @@
+#include <vector>
+
+#include "compression_worker.h"  // CompressionResult
+#include "zstd.h"
+
+CompressionResult compress(const std::vector<uint8_t> data, size_t compression_level) {
+    size_t output_buffer_size = ZSTD_compressBound(data.size());
+    std::vector<uint8_t> output(output_buffer_size);
+
+    size_t result_code =
+        ZSTD_compress(output.data(), output.size(), data.data(), data.size(), compression_level);
+
+    if (ZSTD_isError(result_code)) {
+        return std::string(ZSTD_getErrorName(result_code));
+    }
+
+    output.resize(result_code);
+
+    return output;
+}

addon/compression_worker.h

@@ -0,0 +1,73 @@
+#ifndef COMPRESSION_WORKER_H
+#define COMPRESSION_WORKER_H
+#include <napi.h>
+
+#include <optional>
+#include <variant>
+
+using namespace Napi;
+
+using CompressionResult = std::variant<std::vector<uint8_t>, std::string>;
+
+// Implementation of the Overload pattern:
+// https://www.cppstories.com/2019/02/2lines3featuresoverload.html/
+template <class... Ts>
+struct overload : Ts... {
+    using Ts::operator()...;
+};
+template <class... Ts>
+overload(Ts...) -> overload<Ts...>;
+
+/**
+ * @brief An asynchronous Napi::Worker that can be with any function that produces
+ * CompressionResults.
+ * */
+class CompressionWorker : public Napi::AsyncWorker {
+   public:
+    CompressionWorker(const Napi::Env& env, std::function<CompressionResult()> worker)
+        : Napi::AsyncWorker{env, "Worker"}, m_deferred{env}, m_worker(worker), m_result{} {}
+
+    Napi::Promise GetPromise() {
+        return m_deferred.Promise();
+    }
+
+   protected:
+    void Execute() {
+        m_result = m_worker();
+    }
+
+    void OnOK() {
+        if (!m_result.has_value()) {
+            m_deferred.Reject(Napi::Error::New(Env(),
+                                               "zstd runtime error - async worker finished without "
+                                               "a compression or decompression result.")
+                                  .Value());
+            return;
+        }
+
+        auto result_visitor = overload{
+            [m_deferred = this->m_deferred](std::string error_message) {
+                auto error = Napi::Error::New(m_deferred.Env(), error_message);
+                m_deferred.Reject(error.Value());
+            },
+            [m_deferred = this->m_deferred](std::vector<uint8_t> result) {
+                Buffer<uint8_t> output =
+                    Buffer<uint8_t>::Copy(m_deferred.Env(), result.data(), result.size());
+
+                m_deferred.Resolve(output);
+            },
+        };
+        std::visit(result_visitor, *m_result);
+    }
+
+    void OnError(const Napi::Error& err) {
+        m_deferred.Reject(err.Value());
+    }
+
+   private:
+    Napi::Promise::Deferred m_deferred;
+    std::function<CompressionResult()> m_worker;
+    std::optional<CompressionResult> m_result;
+};
+
+#endif

addon/decompress.h

@@ -0,0 +1,55 @@
+#include <vector>
+
+#include "compression_worker.h"  // CompressionResult
+#include "zstd.h"
+
+CompressionResult decompress(const std::vector<uint8_t>& compressed) {
+    std::vector<uint8_t> decompressed;
+
+    using DCTX_Deleter = void (*)(ZSTD_DCtx*);
+
+    std::unique_ptr<ZSTD_DCtx, DCTX_Deleter> decompression_context(
+        ZSTD_createDCtx(), [](ZSTD_DCtx* ctx) { ZSTD_freeDCtx(ctx); });
+
+    ZSTD_inBuffer input = {compressed.data(), compressed.size(), 0};
+    std::vector<uint8_t> output_buffer(ZSTD_DStreamOutSize());
+    ZSTD_outBuffer output = {output_buffer.data(), output_buffer.size(), 0};
+
+    //   Source: https://facebook.github.io/zstd/zstd_manual.html#Chapter9
+    //
+    //   Use ZSTD_decompressStream() repetitively to consume your input.
+    //   The function will update both `pos` fields.
+    //   If `input.pos < input.size`, some input has not been consumed.
+    //   It's up to the caller to present again remaining data.
+    //   The function tries to flush all data decoded immediately, respecting output buffer size.
+    //   If `output.pos < output.size`, decoder has flushed everything it could.
+    //   But if `output.pos == output.size`, there might be some data left within internal buffers.,
+    //   In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
+    //   Note : with no additional input provided, amount of data flushed is necessarily <=
+    //   ZSTD_BLOCKSIZE_MAX.
+    //  @return : 0 when a frame is completely decoded and fully flushed,
+    //         or an error code, which can be tested using ZSTD_isError(),
+    //         or any other value > 0, which means there is still some decoding or flushing to do to
+    //         complete current frame :
+    //                                 the return value is a suggested next input size (just a hint
+    //                                 for better latency) that will never request more than the
+    //                                 remaining frame size.
+    auto inputRemains = [](ZSTD_inBuffer& input) { return input.pos < input.size; };
+    auto isOutputBufferFlushed = [](ZSTD_outBuffer& output) { return output.pos < output.size; };
+
+    while (inputRemains(input) || !isOutputBufferFlushed(output)) {
+        size_t const ret = ZSTD_decompressStream(decompression_context.get(), &output, &input);
+        if (ZSTD_isError(ret)) {
+            return std::string(ZSTD_getErrorName(ret));
+        }
+
+        for (size_t i = 0; i < output.pos; ++i) {
+            decompressed.push_back(output_buffer[i]);
+        }
+
+        // move the position back go 0, to indicate that we are ready for more data
+        output.pos = 0;
+    }
+
+    return decompressed;
+}

addon/napi_utils.h

@@ -0,0 +1,40 @@
+#ifndef NAPI_UTILS_H
+#define NAPI_UTILS_H
+
+#include <napi.h>
+
+using namespace Napi;
+
+/**
+ * @brief Get the Bytes From Uint8 Array object
+ *
+ * this function copies the bytes out of the Uint8Array.
+ */
+std::vector<uint8_t> getBytesFromUint8Array(const Uint8Array& source) {
+    const uint8_t* input_data = source.Data();
+    size_t total = source.ElementLength();
+    std::vector<uint8_t> data(total);
+
+    std::copy(input_data, input_data + total, data.data());
+
+    return data;
+}
+
+/**
+ * @brief Given an Napi::Value, this function returns the value as a Uint8Array, if the
+ * Value is a Uint8Array. Otherwise, this function throws.
+ *
+ * @param v - An Napi::Value
+ * @param argument_name - the name of the value, to use when constructing an error message.
+ * @return Napi::Uint8Array
+ */
+Uint8Array Uint8ArrayFromValue(Value v, std::string argument_name) {
+    if (!v.IsTypedArray() || v.As<TypedArray>().TypedArrayType() != napi_uint8_array) {
+        std::string error_message = "Parameter `" + argument_name + "` must be a Uint8Array.";
+        throw TypeError::New(v.Env(), error_message);
+    }
+
+    return v.As<Uint8Array>();
+}
+
+#endif

addon/zstd.cpp

@@ -1,14 +1,53 @@
+#include "zstd.h"
+
 #include <napi.h>
 
+#include <string>
+#include <vector>
+
+#include "compress.h"
+#include "compression_worker.h"
+#include "decompress.h"
+#include "napi_utils.h"
+
 using namespace Napi;
 
-Napi::String Compress(const Napi::CallbackInfo& info) {
-    auto string = Napi::String::New(info.Env(), "compress()");
-    return string;
+Napi::Promise Compress(const Napi::CallbackInfo& info) {
+    // Argument handling happens in JS
+    if (info.Length() != 2) {
+        std::string error_message = "Expected two arguments.";
+        throw TypeError::New(info.Env(), error_message);
+    }
+
+    Uint8Array to_compress = Uint8ArrayFromValue(info[0], "buffer");
+    std::vector<uint8_t> data = getBytesFromUint8Array(to_compress);
+
+    size_t compression_level = (size_t)info[1].ToNumber().Int32Value();
+
+    CompressionWorker* worker = new CompressionWorker(
+        info.Env(),
+        [data = std::move(data), compression_level] { return compress(data, compression_level); });
+
+    worker->Queue();
+
+    return worker->GetPromise();
 }
-Napi::String Decompress(const Napi::CallbackInfo& info) {
-    auto string = Napi::String::New(info.Env(), "decompress()");
-    return string;
+
+Napi::Promise Decompress(const CallbackInfo& info) {
+    // Argument handling happens in JS
+    if (info.Length() != 1) {
+        std::string error_message = "Expected one argument.";
+        throw TypeError::New(info.Env(), error_message);
+    }
+
+    Napi::Uint8Array compressed_data = Uint8ArrayFromValue(info[0], "buffer");
+    std::vector<uint8_t> data = getBytesFromUint8Array(compressed_data);
+    CompressionWorker* worker =
+        new CompressionWorker(info.Env(), [data = std::move(data)] { return decompress(data); });
+
+    worker->Queue();
+
+    return worker->GetPromise();
 }
 
 Napi::Object Init(Napi::Env env, Napi::Object exports) {

binding.gyp

@@ -4,25 +4,36 @@
     'type': 'loadable_module',
     'defines': ['ZSTD_STATIC_LINKING_ONLY'],
     'include_dirs': [
-        "<!(node -p \"require('node-addon-api').include_dir\")"
+        "<!(node -p \"require('node-addon-api').include_dir\")",
+        "<(module_root_dir)/deps/zstd/lib",
     ],
     'variables': {
       'ARCH': '<(host_arch)',
       'built_with_electron%': 0
     },
     'sources': [
-      'addon/zstd.cpp'
+      'addon/zstd.cpp',
+      'addon/compression_worker.h',
+      'addon/compress.h',
+      'addon/decompress.h',
+      'addon/napi_utils.h',
     ],
     'xcode_settings': {
       'GCC_ENABLE_CPP_EXCEPTIONS': 'YES',
       'CLANG_CXX_LIBRARY': 'libc++',
-      'MACOSX_DEPLOYMENT_TARGET': '10.12',
+      'MACOSX_DEPLOYMENT_TARGET': '11',
       'GCC_SYMBOLS_PRIVATE_EXTERN': 'YES', # -fvisibility=hidden
     },
     'cflags!': [ '-fno-exceptions' ],
     'cflags_cc!': [ '-fno-exceptions' ],
+    'cflags_cc': ['-std=c++17'],
     'msvs_settings': {
       'VCCLCompilerTool': { 'ExceptionHandling': 1 },
-    }
+    },
+	 'link_settings': {
+      'libraries': [
+        '<(module_root_dir)/deps/zstd/build/cmake/lib/libzstd.a',
+      ]
+    },
   }]
 }

etc/install-zstd.sh

@@ -0,0 +1,26 @@
+
+set -o xtrace
+
+clean_deps() {
+	rm -rf deps
+}
+
+download_zstd() {
+	rm -rf deps
+	mkdir -p deps/zstd
+
+	curl -L "https://github.com/facebook/zstd/releases/download/v1.5.6/zstd-1.5.6.tar.gz" \
+	 	| tar  -zxf - -C deps/zstd --strip-components 1
+}
+
+build_zstd() {
+	export MACOSX_DEPLOYMENT_TARGET=10.12
+	cd deps/zstd/build/cmake
+
+	cmake .
+	make
+}
+
+clean_deps
+download_zstd
+build_zstd