From 4b11d31e0f5eac729609a13007c5e5620c2dd634 Mon Sep 17 00:00:00 2001 From: Julia Naomi Date: Sat, 20 May 2023 13:40:39 -0500 Subject: [PATCH] implements submit-tx with put/delete/evict --- Makefile | 2 +- .../xtdb/protos/GrpcApi/SubmitTx-request.json | 9 +- project.clj | 2 +- resources/json.proto | 95 --- resources/transactions.proto | 17 +- src/com/google/protobuf.cljc | 714 ++++++++++++++++++ src/com/google/protobuf/GrpcApi/client.cljc | 43 ++ src/com/google/protobuf/GrpcApi/server.cljc | 28 + src/com/xtdb/protos.cljc | 71 +- src/gxtdb/adapters/json.clj | 13 + src/gxtdb/adapters/tx_log.clj | 38 +- src/gxtdb/service.clj | 21 +- src/gxtdb/utils.clj | 23 +- test/gxtdb/adapters/json_test.clj | 20 + test/gxtdb/adapters/tx_log_test.clj | 45 ++ test/gxtdb/service_test.clj | 13 +- test/gxtdb/utils_test.clj | 106 +++ 17 files changed, 1114 insertions(+), 146 deletions(-) delete mode 100644 resources/json.proto create mode 100644 src/com/google/protobuf.cljc create mode 100644 src/com/google/protobuf/GrpcApi/client.cljc create mode 100644 src/com/google/protobuf/GrpcApi/server.cljc create mode 100644 src/gxtdb/adapters/json.clj create mode 100644 test/gxtdb/adapters/json_test.clj create mode 100644 test/gxtdb/adapters/tx_log_test.clj create mode 100644 test/gxtdb/utils_test.clj diff --git a/Makefile b/Makefile index 8ab3125..08a6011 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ kibit: lint: fmt kibit proto: - protoc --clojure_out=grpc-client,grpc-server:src --proto_path=resources resources/service.proto + protoc --clojure_out=grpc-client,grpc-server:src --proto_path=resources resources/transactions.proto resources/service.proto all: proto lint diff --git a/kreya/grpc/com/xtdb/protos/GrpcApi/SubmitTx-request.json b/kreya/grpc/com/xtdb/protos/GrpcApi/SubmitTx-request.json index ce308e7..ff872e6 100644 --- a/kreya/grpc/com/xtdb/protos/GrpcApi/SubmitTx-request.json +++ b/kreya/grpc/com/xtdb/protos/GrpcApi/SubmitTx-request.json @@ -3,13 +3,8 @@ { "put": { "xtId": "id1", - "document": "Via Security" - } - }, - { - "put": { - "xtId": "id2", - "document": "Rustic Plastic Sausages Graphic Interface incubate" + "idType":"Keyword", + "document": {"key": "value"} } } ] diff --git a/project.clj b/project.clj index 7884540..a19a451 100644 --- a/project.clj +++ b/project.clj @@ -8,7 +8,7 @@ :plugins [[lein-cljfmt "0.9.2"] [lein-kibit "0.1.8"] [jonase/eastwood "1.4.0"]] - :dependencies [[org.clojure/clojure "1.10.3"] + :dependencies [[org.clojure/clojure "1.11.0"] [com.xtdb/xtdb-core "1.22.1"] [io.pedestal/pedestal.service "0.5.9"] [com.cognitect/anomalies "0.1.12"] diff --git a/resources/json.proto b/resources/json.proto deleted file mode 100644 index 8cdea1b..0000000 --- a/resources/json.proto +++ /dev/null @@ -1,95 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package com.xtdb.protos; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/protobuf/types/known/structpb"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "StructProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; - -// `Struct` represents a structured data value, consisting of fields -// which map to dynamically typed values. In some languages, `Struct` -// might be supported by a native representation. For example, in -// scripting languages like JS a struct is represented as an -// object. The details of that representation are described together -// with the proto support for the language. -// -// The JSON representation for `Struct` is JSON object. -message Struct { - // Unordered map of dynamically typed values. - map fields = 1; -} - -// `Value` represents a dynamically typed value which can be either -// null, a number, a string, a boolean, a recursive struct value, or a -// list of values. A producer of value is expected to set one of these -// variants. Absence of any variant indicates an error. -// -// The JSON representation for `Value` is JSON value. -message Value { - // The kind of value. - oneof kind { - // Represents a null value. - NullValue null_value = 1; - // Represents a double value. - double number_value = 2; - // Represents a string value. - string string_value = 3; - // Represents a boolean value. - bool bool_value = 4; - // Represents a structured value. - Struct struct_value = 5; - // Represents a repeated `Value`. - ListValue list_value = 6; - } -} - -// `NullValue` is a singleton enumeration to represent the null value for the -// `Value` type union. -// -// The JSON representation for `NullValue` is JSON `null`. -enum NullValue { - // Null value. - NULL_VALUE = 0; -} - -// `ListValue` is a wrapper around a repeated field of values. -// -// The JSON representation for `ListValue` is JSON array. -message ListValue { - // Repeated field of dynamically typed values. - repeated Value values = 1; -} \ No newline at end of file diff --git a/resources/transactions.proto b/resources/transactions.proto index dc5a80e..2cdfc8b 100644 --- a/resources/transactions.proto +++ b/resources/transactions.proto @@ -1,7 +1,8 @@ syntax = "proto3"; package com.xtdb.protos; -import "json.proto"; +import "google/protobuf/struct.proto"; + message Transaction { oneof transaction_type { @@ -11,17 +12,27 @@ message Transaction { } } +enum IdType { + Uuid = 0; + Keyword = 1; + String = 2; + Int = 3; +} + message Put { - string xt_id = 1; - com.xtdb.protos.Value document = 2; + IdType id_type = 1; + string xt_id = 2; + google.protobuf.Struct document = 3; } message Delete { string document_id = 1; + IdType id_type = 2; } message Evict { string document_id = 1; + IdType id_type = 2; } message SubmitRequest { diff --git a/src/com/google/protobuf.cljc b/src/com/google/protobuf.cljc new file mode 100644 index 0000000..04fc5e4 --- /dev/null +++ b/src/com/google/protobuf.cljc @@ -0,0 +1,714 @@ +;;;---------------------------------------------------------------------------------- +;;; Generated by protoc-gen-clojure. DO NOT EDIT +;;; +;;; Message Implementation of package com.google.protobuf +;;;---------------------------------------------------------------------------------- +(ns com.google.protobuf + (:require [protojure.protobuf.protocol :as pb] + [protojure.protobuf.serdes.core :as serdes.core] + [protojure.protobuf.serdes.complex :as serdes.complex] + [protojure.protobuf.serdes.utils :refer [tag-map]] + [protojure.protobuf.serdes.stream :as serdes.stream] + [com.google.protobuf :as com.google.protobuf] + [clojure.set :as set] + [clojure.spec.alpha :as s])) + +;;---------------------------------------------------------------------------------- +;;---------------------------------------------------------------------------------- +;; Forward declarations +;;---------------------------------------------------------------------------------- +;;---------------------------------------------------------------------------------- + +(declare cis->ListValue) +(declare ecis->ListValue) +(declare new-ListValue) +(declare cis->Empty) +(declare ecis->Empty) +(declare new-Empty) +(declare cis->Struct) +(declare ecis->Struct) +(declare new-Struct) +(declare cis->StatusResponse) +(declare ecis->StatusResponse) +(declare new-StatusResponse) +(declare cis->Evict) +(declare ecis->Evict) +(declare new-Evict) +(declare cis->Struct-FieldsEntry) +(declare ecis->Struct-FieldsEntry) +(declare new-Struct-FieldsEntry) +(declare cis->Put) +(declare ecis->Put) +(declare new-Put) +(declare cis->Delete) +(declare ecis->Delete) +(declare new-Delete) +(declare cis->SubmitRequest) +(declare ecis->SubmitRequest) +(declare new-SubmitRequest) +(declare cis->SubmitResponse) +(declare ecis->SubmitResponse) +(declare new-SubmitResponse) +(declare cis->Value) +(declare ecis->Value) +(declare new-Value) +(declare cis->Transaction) +(declare ecis->Transaction) +(declare new-Transaction) +(declare cis->OptionString) +(declare ecis->OptionString) +(declare new-OptionString) + +;;---------------------------------------------------------------------------------- +;;---------------------------------------------------------------------------------- +;; Enumerations +;;---------------------------------------------------------------------------------- +;;---------------------------------------------------------------------------------- + +;----------------------------------------------------------------------------- +; NullValue +;----------------------------------------------------------------------------- +(def NullValue-default :null-value) + +(def NullValue-val2label {0 :null-value}) + +(def NullValue-label2val (set/map-invert NullValue-val2label)) + +(defn cis->NullValue [is] + (let [val (serdes.core/cis->Enum is)] + (get NullValue-val2label val val))) + +(defn- get-NullValue [value] + {:pre [(or (int? value) (contains? NullValue-label2val value))]} + (get NullValue-label2val value value)) + +(defn write-NullValue + ([tag value os] (write-NullValue tag {:optimize false} value os)) + ([tag options value os] + (serdes.core/write-Enum tag options (get-NullValue value) os))) + +;;---------------------------------------------------------------------------------- +;;---------------------------------------------------------------------------------- +;; Value-kind's oneof Implementations +;;---------------------------------------------------------------------------------- +;;---------------------------------------------------------------------------------- + +(defn convert-Value-kind [origkeyval] + (cond + (get-in origkeyval [:kind :null-value]) origkeyval + (get-in origkeyval [:kind :number-value]) origkeyval + (get-in origkeyval [:kind :string-value]) origkeyval + (get-in origkeyval [:kind :bool-value]) origkeyval + (get-in origkeyval [:kind :struct-value]) (update-in origkeyval [:kind :struct-value] new-Struct) + (get-in origkeyval [:kind :list-value]) (update-in origkeyval [:kind :list-value] new-ListValue) + :default origkeyval)) + +(defn write-Value-kind [kind os] + (let [field (first kind) + k (when-not (nil? field) (key field)) + v (when-not (nil? field) (val field))] + (case k + :null-value (write-NullValue 1 {:optimize false} v os) + :number-value (serdes.core/write-Double 2 {:optimize false} v os) + :string-value (serdes.core/write-String 3 {:optimize false} v os) + :bool-value (serdes.core/write-Bool 4 {:optimize false} v os) + :struct-value (serdes.core/write-embedded 5 v os) + :list-value (serdes.core/write-embedded 6 v os) + nil))) + +;;---------------------------------------------------------------------------------- +;;---------------------------------------------------------------------------------- +;; Transaction-transaction-type's oneof Implementations +;;---------------------------------------------------------------------------------- +;;---------------------------------------------------------------------------------- + +(defn convert-Transaction-transaction-type [origkeyval] + (cond + (get-in origkeyval [:transaction-type :put]) (update-in origkeyval [:transaction-type :put] new-Put) + (get-in origkeyval [:transaction-type :delete]) (update-in origkeyval [:transaction-type :delete] new-Delete) + (get-in origkeyval [:transaction-type :evict]) (update-in origkeyval [:transaction-type :evict] new-Evict) + :default origkeyval)) + +(defn write-Transaction-transaction-type [transaction-type os] + (let [field (first transaction-type) + k (when-not (nil? field) (key field)) + v (when-not (nil? field) (val field))] + (case k + :put (serdes.core/write-embedded 1 v os) + :delete (serdes.core/write-embedded 2 v os) + :evict (serdes.core/write-embedded 3 v os) + nil))) + +;;---------------------------------------------------------------------------------- +;;---------------------------------------------------------------------------------- +;; OptionString-value's oneof Implementations +;;---------------------------------------------------------------------------------- +;;---------------------------------------------------------------------------------- + +(defn convert-OptionString-value [origkeyval] + (cond + (get-in origkeyval [:value :none]) (update-in origkeyval [:value :none] new-Empty) + (get-in origkeyval [:value :some]) origkeyval + :default origkeyval)) + +(defn write-OptionString-value [value os] + (let [field (first value) + k (when-not (nil? field) (key field)) + v (when-not (nil? field) (val field))] + (case k + :none (serdes.core/write-embedded 1 v os) + :some (serdes.core/write-String 2 {:optimize false} v os) + nil))) + +;;---------------------------------------------------------------------------------- +;;---------------------------------------------------------------------------------- +;; Message Implementations +;;---------------------------------------------------------------------------------- +;;---------------------------------------------------------------------------------- + +;----------------------------------------------------------------------------- +; ListValue +;----------------------------------------------------------------------------- +(defrecord ListValue-record [values] + pb/Writer + (serialize [this os] + (serdes.complex/write-repeated serdes.core/write-embedded 1 (:values this) os)) + pb/TypeReflection + (gettype [this] + "com.google.protobuf.ListValue")) + +(s/def ::ListValue-spec (s/keys :opt-un [])) +(def ListValue-defaults {:values []}) + +(defn cis->ListValue + "CodedInputStream to ListValue" + [is] + (map->ListValue-record (tag-map ListValue-defaults (fn [tag index] (case index 1 [:values (serdes.complex/cis->repeated ecis->Value is)] [index (serdes.core/cis->undefined tag is)])) is))) + +(defn ecis->ListValue + "Embedded CodedInputStream to ListValue" + [is] + (serdes.core/cis->embedded cis->ListValue is)) + +(defn new-ListValue + "Creates a new instance from a map, similar to map->ListValue except that + it properly accounts for nested messages, when applicable. + " + [init] + {:pre [(if (s/valid? ::ListValue-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::ListValue-spec init))))]} + (-> (merge ListValue-defaults init) + (cond-> (some? (get init :values)) (update :values #(map new-Value %))) + (map->ListValue-record))) + +(defn pb->ListValue + "Protobuf to ListValue" + [input] + (cis->ListValue (serdes.stream/new-cis input))) + +(def ^:protojure.protobuf.any/record ListValue-meta {:type "com.google.protobuf.ListValue" :decoder pb->ListValue}) + +;----------------------------------------------------------------------------- +; Empty +;----------------------------------------------------------------------------- +(defrecord Empty-record [] + pb/Writer + (serialize [this os]) + pb/TypeReflection + (gettype [this] + "com.google.protobuf.Empty")) + +(s/def ::Empty-spec (s/keys :opt-un [])) +(def Empty-defaults {}) + +(defn cis->Empty + "CodedInputStream to Empty" + [is] + (map->Empty-record (tag-map Empty-defaults (fn [tag index] (case index [index (serdes.core/cis->undefined tag is)])) is))) + +(defn ecis->Empty + "Embedded CodedInputStream to Empty" + [is] + (serdes.core/cis->embedded cis->Empty is)) + +(defn new-Empty + "Creates a new instance from a map, similar to map->Empty except that + it properly accounts for nested messages, when applicable. + " + [init] + {:pre [(if (s/valid? ::Empty-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::Empty-spec init))))]} + (map->Empty-record (merge Empty-defaults init))) + +(defn pb->Empty + "Protobuf to Empty" + [input] + (cis->Empty (serdes.stream/new-cis input))) + +(def ^:protojure.protobuf.any/record Empty-meta {:type "com.google.protobuf.Empty" :decoder pb->Empty}) + +;----------------------------------------------------------------------------- +; Struct +;----------------------------------------------------------------------------- +(defrecord Struct-record [fields] + pb/Writer + (serialize [this os] + (serdes.complex/write-map new-Struct-FieldsEntry 1 (:fields this) os)) + pb/TypeReflection + (gettype [this] + "com.google.protobuf.Struct")) + +(s/def ::Struct-spec (s/keys :opt-un [])) +(def Struct-defaults {:fields []}) + +(defn cis->Struct + "CodedInputStream to Struct" + [is] + (map->Struct-record (tag-map Struct-defaults (fn [tag index] (case index 1 [:fields (serdes.complex/cis->map ecis->Struct-FieldsEntry is)] [index (serdes.core/cis->undefined tag is)])) is))) + +(defn ecis->Struct + "Embedded CodedInputStream to Struct" + [is] + (serdes.core/cis->embedded cis->Struct is)) + +(defn new-Struct + "Creates a new instance from a map, similar to map->Struct except that + it properly accounts for nested messages, when applicable. + " + [init] + {:pre [(if (s/valid? ::Struct-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::Struct-spec init))))]} + (map->Struct-record (merge Struct-defaults init))) + +(defn pb->Struct + "Protobuf to Struct" + [input] + (cis->Struct (serdes.stream/new-cis input))) + +(def ^:protojure.protobuf.any/record Struct-meta {:type "com.google.protobuf.Struct" :decoder pb->Struct}) + +;----------------------------------------------------------------------------- +; StatusResponse +;----------------------------------------------------------------------------- +(defrecord StatusResponse-record [version index-version kv-store estimate-num-keys size revision consumer-state] + pb/Writer + (serialize [this os] + (serdes.core/write-String 1 {:optimize true} (:version this) os) + (serdes.core/write-Int32 2 {:optimize true} (:index-version this) os) + (serdes.core/write-String 3 {:optimize true} (:kv-store this) os) + (serdes.core/write-Int32 4 {:optimize true} (:estimate-num-keys this) os) + (serdes.core/write-Int64 5 {:optimize true} (:size this) os) + (serdes.core/write-embedded 6 (:revision this) os) + (serdes.core/write-embedded 7 (:consumer-state this) os)) + pb/TypeReflection + (gettype [this] + "com.google.protobuf.StatusResponse")) + +(s/def :com.google.protobuf.StatusResponse/version string?) +(s/def :com.google.protobuf.StatusResponse/index-version int?) +(s/def :com.google.protobuf.StatusResponse/kv-store string?) +(s/def :com.google.protobuf.StatusResponse/estimate-num-keys int?) +(s/def :com.google.protobuf.StatusResponse/size int?) + +(s/def ::StatusResponse-spec (s/keys :opt-un [:com.google.protobuf.StatusResponse/version :com.google.protobuf.StatusResponse/index-version :com.google.protobuf.StatusResponse/kv-store :com.google.protobuf.StatusResponse/estimate-num-keys :com.google.protobuf.StatusResponse/size])) +(def StatusResponse-defaults {:version "" :index-version 0 :kv-store "" :estimate-num-keys 0 :size 0}) + +(defn cis->StatusResponse + "CodedInputStream to StatusResponse" + [is] + (map->StatusResponse-record (tag-map StatusResponse-defaults (fn [tag index] (case index 1 [:version (serdes.core/cis->String is)] 2 [:index-version (serdes.core/cis->Int32 is)] 3 [:kv-store (serdes.core/cis->String is)] 4 [:estimate-num-keys (serdes.core/cis->Int32 is)] 5 [:size (serdes.core/cis->Int64 is)] 6 [:revision (ecis->OptionString is)] 7 [:consumer-state (ecis->OptionString is)] [index (serdes.core/cis->undefined tag is)])) is))) + +(defn ecis->StatusResponse + "Embedded CodedInputStream to StatusResponse" + [is] + (serdes.core/cis->embedded cis->StatusResponse is)) + +(defn new-StatusResponse + "Creates a new instance from a map, similar to map->StatusResponse except that + it properly accounts for nested messages, when applicable. + " + [init] + {:pre [(if (s/valid? ::StatusResponse-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::StatusResponse-spec init))))]} + (-> (merge StatusResponse-defaults init) + (cond-> (some? (get init :revision)) (update :revision new-OptionString)) + (cond-> (some? (get init :consumer-state)) (update :consumer-state new-OptionString)) + (map->StatusResponse-record))) + +(defn pb->StatusResponse + "Protobuf to StatusResponse" + [input] + (cis->StatusResponse (serdes.stream/new-cis input))) + +(def ^:protojure.protobuf.any/record StatusResponse-meta {:type "com.google.protobuf.StatusResponse" :decoder pb->StatusResponse}) + +;----------------------------------------------------------------------------- +; Evict +;----------------------------------------------------------------------------- +(defrecord Evict-record [document-id] + pb/Writer + (serialize [this os] + (serdes.core/write-String 1 {:optimize true} (:document-id this) os)) + pb/TypeReflection + (gettype [this] + "com.google.protobuf.Evict")) + +(s/def :com.google.protobuf.Evict/document-id string?) +(s/def ::Evict-spec (s/keys :opt-un [:com.google.protobuf.Evict/document-id])) +(def Evict-defaults {:document-id ""}) + +(defn cis->Evict + "CodedInputStream to Evict" + [is] + (map->Evict-record (tag-map Evict-defaults (fn [tag index] (case index 1 [:document-id (serdes.core/cis->String is)] [index (serdes.core/cis->undefined tag is)])) is))) + +(defn ecis->Evict + "Embedded CodedInputStream to Evict" + [is] + (serdes.core/cis->embedded cis->Evict is)) + +(defn new-Evict + "Creates a new instance from a map, similar to map->Evict except that + it properly accounts for nested messages, when applicable. + " + [init] + {:pre [(if (s/valid? ::Evict-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::Evict-spec init))))]} + (map->Evict-record (merge Evict-defaults init))) + +(defn pb->Evict + "Protobuf to Evict" + [input] + (cis->Evict (serdes.stream/new-cis input))) + +(def ^:protojure.protobuf.any/record Evict-meta {:type "com.google.protobuf.Evict" :decoder pb->Evict}) + +;----------------------------------------------------------------------------- +; Struct-FieldsEntry +;----------------------------------------------------------------------------- +(defrecord Struct-FieldsEntry-record [key value] + pb/Writer + (serialize [this os] + (serdes.core/write-String 1 {:optimize true} (:key this) os) + (serdes.core/write-embedded 2 (:value this) os)) + pb/TypeReflection + (gettype [this] + "com.google.protobuf.Struct-FieldsEntry")) + +(s/def :com.google.protobuf.Struct-FieldsEntry/key string?) + +(s/def ::Struct-FieldsEntry-spec (s/keys :opt-un [:com.google.protobuf.Struct-FieldsEntry/key])) +(def Struct-FieldsEntry-defaults {:key ""}) + +(defn cis->Struct-FieldsEntry + "CodedInputStream to Struct-FieldsEntry" + [is] + (map->Struct-FieldsEntry-record (tag-map Struct-FieldsEntry-defaults (fn [tag index] (case index 1 [:key (serdes.core/cis->String is)] 2 [:value (ecis->Value is)] [index (serdes.core/cis->undefined tag is)])) is))) + +(defn ecis->Struct-FieldsEntry + "Embedded CodedInputStream to Struct-FieldsEntry" + [is] + (serdes.core/cis->embedded cis->Struct-FieldsEntry is)) + +(defn new-Struct-FieldsEntry + "Creates a new instance from a map, similar to map->Struct-FieldsEntry except that + it properly accounts for nested messages, when applicable. + " + [init] + {:pre [(if (s/valid? ::Struct-FieldsEntry-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::Struct-FieldsEntry-spec init))))]} + (-> (merge Struct-FieldsEntry-defaults init) + (cond-> (some? (get init :value)) (update :value new-Value)) + (map->Struct-FieldsEntry-record))) + +(defn pb->Struct-FieldsEntry + "Protobuf to Struct-FieldsEntry" + [input] + (cis->Struct-FieldsEntry (serdes.stream/new-cis input))) + +(def ^:protojure.protobuf.any/record Struct-FieldsEntry-meta {:type "com.google.protobuf.Struct-FieldsEntry" :decoder pb->Struct-FieldsEntry}) + +;----------------------------------------------------------------------------- +; Put +;----------------------------------------------------------------------------- +(defrecord Put-record [xt-id document] + pb/Writer + (serialize [this os] + (serdes.core/write-String 1 {:optimize true} (:xt-id this) os) + (serdes.core/write-embedded 2 (:document this) os)) + pb/TypeReflection + (gettype [this] + "com.google.protobuf.Put")) + +(s/def :com.google.protobuf.Put/xt-id string?) + +(s/def ::Put-spec (s/keys :opt-un [:com.google.protobuf.Put/xt-id])) +(def Put-defaults {:xt-id ""}) + +(defn cis->Put + "CodedInputStream to Put" + [is] + (map->Put-record (tag-map Put-defaults (fn [tag index] (case index 1 [:xt-id (serdes.core/cis->String is)] 2 [:document (ecis->Value is)] [index (serdes.core/cis->undefined tag is)])) is))) + +(defn ecis->Put + "Embedded CodedInputStream to Put" + [is] + (serdes.core/cis->embedded cis->Put is)) + +(defn new-Put + "Creates a new instance from a map, similar to map->Put except that + it properly accounts for nested messages, when applicable. + " + [init] + {:pre [(if (s/valid? ::Put-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::Put-spec init))))]} + (-> (merge Put-defaults init) + (cond-> (some? (get init :document)) (update :document new-Value)) + (map->Put-record))) + +(defn pb->Put + "Protobuf to Put" + [input] + (cis->Put (serdes.stream/new-cis input))) + +(def ^:protojure.protobuf.any/record Put-meta {:type "com.google.protobuf.Put" :decoder pb->Put}) + +;----------------------------------------------------------------------------- +; Delete +;----------------------------------------------------------------------------- +(defrecord Delete-record [document-id] + pb/Writer + (serialize [this os] + (serdes.core/write-String 1 {:optimize true} (:document-id this) os)) + pb/TypeReflection + (gettype [this] + "com.google.protobuf.Delete")) + +(s/def :com.google.protobuf.Delete/document-id string?) +(s/def ::Delete-spec (s/keys :opt-un [:com.google.protobuf.Delete/document-id])) +(def Delete-defaults {:document-id ""}) + +(defn cis->Delete + "CodedInputStream to Delete" + [is] + (map->Delete-record (tag-map Delete-defaults (fn [tag index] (case index 1 [:document-id (serdes.core/cis->String is)] [index (serdes.core/cis->undefined tag is)])) is))) + +(defn ecis->Delete + "Embedded CodedInputStream to Delete" + [is] + (serdes.core/cis->embedded cis->Delete is)) + +(defn new-Delete + "Creates a new instance from a map, similar to map->Delete except that + it properly accounts for nested messages, when applicable. + " + [init] + {:pre [(if (s/valid? ::Delete-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::Delete-spec init))))]} + (map->Delete-record (merge Delete-defaults init))) + +(defn pb->Delete + "Protobuf to Delete" + [input] + (cis->Delete (serdes.stream/new-cis input))) + +(def ^:protojure.protobuf.any/record Delete-meta {:type "com.google.protobuf.Delete" :decoder pb->Delete}) + +;----------------------------------------------------------------------------- +; SubmitRequest +;----------------------------------------------------------------------------- +(defrecord SubmitRequest-record [tx-ops] + pb/Writer + (serialize [this os] + (serdes.complex/write-repeated serdes.core/write-embedded 1 (:tx-ops this) os)) + pb/TypeReflection + (gettype [this] + "com.google.protobuf.SubmitRequest")) + +(s/def ::SubmitRequest-spec (s/keys :opt-un [])) +(def SubmitRequest-defaults {:tx-ops []}) + +(defn cis->SubmitRequest + "CodedInputStream to SubmitRequest" + [is] + (map->SubmitRequest-record (tag-map SubmitRequest-defaults (fn [tag index] (case index 1 [:tx-ops (serdes.complex/cis->repeated ecis->Transaction is)] [index (serdes.core/cis->undefined tag is)])) is))) + +(defn ecis->SubmitRequest + "Embedded CodedInputStream to SubmitRequest" + [is] + (serdes.core/cis->embedded cis->SubmitRequest is)) + +(defn new-SubmitRequest + "Creates a new instance from a map, similar to map->SubmitRequest except that + it properly accounts for nested messages, when applicable. + " + [init] + {:pre [(if (s/valid? ::SubmitRequest-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::SubmitRequest-spec init))))]} + (-> (merge SubmitRequest-defaults init) + (cond-> (some? (get init :tx-ops)) (update :tx-ops #(map new-Transaction %))) + (map->SubmitRequest-record))) + +(defn pb->SubmitRequest + "Protobuf to SubmitRequest" + [input] + (cis->SubmitRequest (serdes.stream/new-cis input))) + +(def ^:protojure.protobuf.any/record SubmitRequest-meta {:type "com.google.protobuf.SubmitRequest" :decoder pb->SubmitRequest}) + +;----------------------------------------------------------------------------- +; SubmitResponse +;----------------------------------------------------------------------------- +(defrecord SubmitResponse-record [tx-time tx-id] + pb/Writer + (serialize [this os] + (serdes.core/write-String 1 {:optimize true} (:tx-time this) os) + (serdes.core/write-Int64 2 {:optimize true} (:tx-id this) os)) + pb/TypeReflection + (gettype [this] + "com.google.protobuf.SubmitResponse")) + +(s/def :com.google.protobuf.SubmitResponse/tx-time string?) +(s/def :com.google.protobuf.SubmitResponse/tx-id int?) +(s/def ::SubmitResponse-spec (s/keys :opt-un [:com.google.protobuf.SubmitResponse/tx-time :com.google.protobuf.SubmitResponse/tx-id])) +(def SubmitResponse-defaults {:tx-time "" :tx-id 0}) + +(defn cis->SubmitResponse + "CodedInputStream to SubmitResponse" + [is] + (map->SubmitResponse-record (tag-map SubmitResponse-defaults (fn [tag index] (case index 1 [:tx-time (serdes.core/cis->String is)] 2 [:tx-id (serdes.core/cis->Int64 is)] [index (serdes.core/cis->undefined tag is)])) is))) + +(defn ecis->SubmitResponse + "Embedded CodedInputStream to SubmitResponse" + [is] + (serdes.core/cis->embedded cis->SubmitResponse is)) + +(defn new-SubmitResponse + "Creates a new instance from a map, similar to map->SubmitResponse except that + it properly accounts for nested messages, when applicable. + " + [init] + {:pre [(if (s/valid? ::SubmitResponse-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::SubmitResponse-spec init))))]} + (map->SubmitResponse-record (merge SubmitResponse-defaults init))) + +(defn pb->SubmitResponse + "Protobuf to SubmitResponse" + [input] + (cis->SubmitResponse (serdes.stream/new-cis input))) + +(def ^:protojure.protobuf.any/record SubmitResponse-meta {:type "com.google.protobuf.SubmitResponse" :decoder pb->SubmitResponse}) + +;----------------------------------------------------------------------------- +; Value +;----------------------------------------------------------------------------- +(defrecord Value-record [kind] + pb/Writer + (serialize [this os] + (write-Value-kind (:kind this) os)) + pb/TypeReflection + (gettype [this] + "com.google.protobuf.Value")) + +(s/def ::Value-spec (s/keys :opt-un [])) +(def Value-defaults {}) + +(defn cis->Value + "CodedInputStream to Value" + [is] + (map->Value-record (tag-map Value-defaults (fn [tag index] (case index 1 [:kind {:null-value (cis->NullValue is)}] 2 [:kind {:number-value (serdes.core/cis->Double is)}] 3 [:kind {:string-value (serdes.core/cis->String is)}] 4 [:kind {:bool-value (serdes.core/cis->Bool is)}] 5 [:kind {:struct-value (ecis->Struct is)}] 6 [:kind {:list-value (ecis->ListValue is)}] [index (serdes.core/cis->undefined tag is)])) is))) + +(defn ecis->Value + "Embedded CodedInputStream to Value" + [is] + (serdes.core/cis->embedded cis->Value is)) + +(defn new-Value + "Creates a new instance from a map, similar to map->Value except that + it properly accounts for nested messages, when applicable. + " + [init] + {:pre [(if (s/valid? ::Value-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::Value-spec init))))]} + (-> (merge Value-defaults init) + (convert-Value-kind) + (map->Value-record))) + +(defn pb->Value + "Protobuf to Value" + [input] + (cis->Value (serdes.stream/new-cis input))) + +(def ^:protojure.protobuf.any/record Value-meta {:type "com.google.protobuf.Value" :decoder pb->Value}) + +;----------------------------------------------------------------------------- +; Transaction +;----------------------------------------------------------------------------- +(defrecord Transaction-record [transaction-type] + pb/Writer + (serialize [this os] + (write-Transaction-transaction-type (:transaction-type this) os)) + pb/TypeReflection + (gettype [this] + "com.google.protobuf.Transaction")) + +(s/def ::Transaction-spec (s/keys :opt-un [])) +(def Transaction-defaults {}) + +(defn cis->Transaction + "CodedInputStream to Transaction" + [is] + (map->Transaction-record (tag-map Transaction-defaults (fn [tag index] (case index 1 [:transaction-type {:put (ecis->Put is)}] 2 [:transaction-type {:delete (ecis->Delete is)}] 3 [:transaction-type {:evict (ecis->Evict is)}] [index (serdes.core/cis->undefined tag is)])) is))) + +(defn ecis->Transaction + "Embedded CodedInputStream to Transaction" + [is] + (serdes.core/cis->embedded cis->Transaction is)) + +(defn new-Transaction + "Creates a new instance from a map, similar to map->Transaction except that + it properly accounts for nested messages, when applicable. + " + [init] + {:pre [(if (s/valid? ::Transaction-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::Transaction-spec init))))]} + (-> (merge Transaction-defaults init) + (convert-Transaction-transaction-type) + (map->Transaction-record))) + +(defn pb->Transaction + "Protobuf to Transaction" + [input] + (cis->Transaction (serdes.stream/new-cis input))) + +(def ^:protojure.protobuf.any/record Transaction-meta {:type "com.google.protobuf.Transaction" :decoder pb->Transaction}) + +;----------------------------------------------------------------------------- +; OptionString +;----------------------------------------------------------------------------- +(defrecord OptionString-record [value] + pb/Writer + (serialize [this os] + (write-OptionString-value (:value this) os)) + pb/TypeReflection + (gettype [this] + "com.google.protobuf.OptionString")) + +(s/def ::OptionString-spec (s/keys :opt-un [])) +(def OptionString-defaults {}) + +(defn cis->OptionString + "CodedInputStream to OptionString" + [is] + (map->OptionString-record (tag-map OptionString-defaults (fn [tag index] (case index 1 [:value {:none (ecis->Empty is)}] 2 [:value {:some (serdes.core/cis->String is)}] [index (serdes.core/cis->undefined tag is)])) is))) + +(defn ecis->OptionString + "Embedded CodedInputStream to OptionString" + [is] + (serdes.core/cis->embedded cis->OptionString is)) + +(defn new-OptionString + "Creates a new instance from a map, similar to map->OptionString except that + it properly accounts for nested messages, when applicable. + " + [init] + {:pre [(if (s/valid? ::OptionString-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::OptionString-spec init))))]} + (-> (merge OptionString-defaults init) + (convert-OptionString-value) + (map->OptionString-record))) + +(defn pb->OptionString + "Protobuf to OptionString" + [input] + (cis->OptionString (serdes.stream/new-cis input))) + +(def ^:protojure.protobuf.any/record OptionString-meta {:type "com.google.protobuf.OptionString" :decoder pb->OptionString}) + diff --git a/src/com/google/protobuf/GrpcApi/client.cljc b/src/com/google/protobuf/GrpcApi/client.cljc new file mode 100644 index 0000000..f48f01a --- /dev/null +++ b/src/com/google/protobuf/GrpcApi/client.cljc @@ -0,0 +1,43 @@ +;;;---------------------------------------------------------------------------------- +;;; Generated by protoc-gen-clojure. DO NOT EDIT +;;; +;;; GRPC com.google.protobuf.GrpcApi Client Implementation +;;;---------------------------------------------------------------------------------- +(ns com.google.protobuf.GrpcApi.client + (:require [com.google.protobuf :refer :all] + [com.google.protobuf :as com.google.protobuf] + [clojure.core.async :as async] + [protojure.grpc.client.utils :refer [send-unary-params invoke-unary]] + [promesa.core :as p] + [protojure.grpc.client.api :as grpc])) + +;----------------------------------------------------------------------------- +; GRPC Client Implementation +;----------------------------------------------------------------------------- + +(def GrpcApi-service-name "com.xtdb.protos.GrpcApi") + +(defn Status + ([client params] (Status client {} params)) + ([client metadata params] + (let [input (async/chan 1) + output (async/chan 1) + desc {:service "com.xtdb.protos.GrpcApi" + :method "Status" + :input {:f com.google.protobuf/new-Empty :ch input} + :output {:f com.google.protobuf/pb->StatusResponse :ch output} + :metadata metadata}] + (p/then (send-unary-params input params) (fn [_] (invoke-unary client desc output)))))) + +(defn SubmitTx + ([client params] (SubmitTx client {} params)) + ([client metadata params] + (let [input (async/chan 1) + output (async/chan 1) + desc {:service "com.xtdb.protos.GrpcApi" + :method "SubmitTx" + :input {:f com.google.protobuf/new-SubmitRequest :ch input} + :output {:f com.google.protobuf/pb->SubmitResponse :ch output} + :metadata metadata}] + (p/then (send-unary-params input params) (fn [_] (invoke-unary client desc output)))))) + diff --git a/src/com/google/protobuf/GrpcApi/server.cljc b/src/com/google/protobuf/GrpcApi/server.cljc new file mode 100644 index 0000000..3efdeb8 --- /dev/null +++ b/src/com/google/protobuf/GrpcApi/server.cljc @@ -0,0 +1,28 @@ +;;;---------------------------------------------------------------------------------- +;;; Generated by protoc-gen-clojure. DO NOT EDIT +;;; +;;; GRPC com.google.protobuf.GrpcApi Service Implementation +;;;---------------------------------------------------------------------------------- +(ns com.google.protobuf.GrpcApi.server + (:require [com.google.protobuf :refer :all] + [com.google.protobuf :as com.google.protobuf])) + +;----------------------------------------------------------------------------- +; GRPC GrpcApi +;----------------------------------------------------------------------------- +(defprotocol Service + (Status [this param]) + (SubmitTx [this param])) + +(def GrpcApi-service-name "com.xtdb.protos.GrpcApi") + +(defn- Status-dispatch + [ctx request] + (Status ctx request)) +(defn- SubmitTx-dispatch + [ctx request] + (SubmitTx ctx request)) + +(def ^:const rpc-metadata + [{:pkg "com.xtdb.protos" :service "GrpcApi" :method "Status" :method-fn Status-dispatch :server-streaming false :client-streaming false :input pb->Empty :output new-StatusResponse} + {:pkg "com.xtdb.protos" :service "GrpcApi" :method "SubmitTx" :method-fn SubmitTx-dispatch :server-streaming false :client-streaming false :input pb->SubmitRequest :output new-SubmitResponse}]) diff --git a/src/com/xtdb/protos.cljc b/src/com/xtdb/protos.cljc index 4cf4740..4256bc2 100644 --- a/src/com/xtdb/protos.cljc +++ b/src/com/xtdb/protos.cljc @@ -48,6 +48,37 @@ (declare ecis->OptionString) (declare new-OptionString) +;;---------------------------------------------------------------------------------- +;;---------------------------------------------------------------------------------- +;; Enumerations +;;---------------------------------------------------------------------------------- +;;---------------------------------------------------------------------------------- + +;----------------------------------------------------------------------------- +; IdType +;----------------------------------------------------------------------------- +(def IdType-default :uuid) + +(def IdType-val2label {0 :uuid + 1 :keyword + 2 :string + 3 :int}) + +(def IdType-label2val (set/map-invert IdType-val2label)) + +(defn cis->IdType [is] + (let [val (serdes.core/cis->Enum is)] + (get IdType-val2label val val))) + +(defn- get-IdType [value] + {:pre [(or (int? value) (contains? IdType-label2val value))]} + (get IdType-label2val value value)) + +(defn write-IdType + ([tag value os] (write-IdType tag {:optimize false} value os)) + ([tag options value os] + (serdes.core/write-Enum tag options (get-IdType value) os))) + ;;---------------------------------------------------------------------------------- ;;---------------------------------------------------------------------------------- ;; Transaction-transaction-type's oneof Implementations @@ -193,22 +224,24 @@ ;----------------------------------------------------------------------------- ; Evict ;----------------------------------------------------------------------------- -(defrecord Evict-record [document-id] +(defrecord Evict-record [document-id id-type] pb/Writer (serialize [this os] - (serdes.core/write-String 1 {:optimize true} (:document-id this) os)) + (serdes.core/write-String 1 {:optimize true} (:document-id this) os) + (write-IdType 2 {:optimize true} (:id-type this) os)) pb/TypeReflection (gettype [this] "com.xtdb.protos.Evict")) (s/def :com.xtdb.protos.Evict/document-id string?) -(s/def ::Evict-spec (s/keys :opt-un [:com.xtdb.protos.Evict/document-id])) -(def Evict-defaults {:document-id ""}) +(s/def :com.xtdb.protos.Evict/id-type (s/or :keyword keyword? :int int?)) +(s/def ::Evict-spec (s/keys :opt-un [:com.xtdb.protos.Evict/document-id :com.xtdb.protos.Evict/id-type])) +(def Evict-defaults {:document-id "" :id-type IdType-default}) (defn cis->Evict "CodedInputStream to Evict" [is] - (map->Evict-record (tag-map Evict-defaults (fn [tag index] (case index 1 [:document-id (serdes.core/cis->String is)] [index (serdes.core/cis->undefined tag is)])) is))) + (map->Evict-record (tag-map Evict-defaults (fn [tag index] (case index 1 [:document-id (serdes.core/cis->String is)] 2 [:id-type (cis->IdType is)] [index (serdes.core/cis->undefined tag is)])) is))) (defn ecis->Evict "Embedded CodedInputStream to Evict" @@ -233,24 +266,26 @@ ;----------------------------------------------------------------------------- ; Put ;----------------------------------------------------------------------------- -(defrecord Put-record [xt-id document] +(defrecord Put-record [id-type xt-id document] pb/Writer (serialize [this os] - (serdes.core/write-String 1 {:optimize true} (:xt-id this) os) - (serdes.core/write-embedded 2 (:document this) os)) + (write-IdType 1 {:optimize true} (:id-type this) os) + (serdes.core/write-String 2 {:optimize true} (:xt-id this) os) + (serdes.core/write-embedded 3 (:document this) os)) pb/TypeReflection (gettype [this] "com.xtdb.protos.Put")) +(s/def :com.xtdb.protos.Put/id-type (s/or :keyword keyword? :int int?)) (s/def :com.xtdb.protos.Put/xt-id string?) -(s/def ::Put-spec (s/keys :opt-un [:com.xtdb.protos.Put/xt-id])) -(def Put-defaults {:xt-id ""}) +(s/def ::Put-spec (s/keys :opt-un [:com.xtdb.protos.Put/id-type :com.xtdb.protos.Put/xt-id])) +(def Put-defaults {:id-type IdType-default :xt-id ""}) (defn cis->Put "CodedInputStream to Put" [is] - (map->Put-record (tag-map Put-defaults (fn [tag index] (case index 1 [:xt-id (serdes.core/cis->String is)] 2 [:document (com.google.protobuf/ecis->Value is)] [index (serdes.core/cis->undefined tag is)])) is))) + (map->Put-record (tag-map Put-defaults (fn [tag index] (case index 1 [:id-type (cis->IdType is)] 2 [:xt-id (serdes.core/cis->String is)] 3 [:document (com.google.protobuf/ecis->Struct is)] [index (serdes.core/cis->undefined tag is)])) is))) (defn ecis->Put "Embedded CodedInputStream to Put" @@ -264,7 +299,7 @@ [init] {:pre [(if (s/valid? ::Put-spec init) true (throw (ex-info "Invalid input" (s/explain-data ::Put-spec init))))]} (-> (merge Put-defaults init) - (cond-> (some? (get init :document)) (update :document com.google.protobuf/new-Value)) + (cond-> (some? (get init :document)) (update :document com.google.protobuf/new-Struct)) (map->Put-record))) (defn pb->Put @@ -277,22 +312,24 @@ ;----------------------------------------------------------------------------- ; Delete ;----------------------------------------------------------------------------- -(defrecord Delete-record [document-id] +(defrecord Delete-record [document-id id-type] pb/Writer (serialize [this os] - (serdes.core/write-String 1 {:optimize true} (:document-id this) os)) + (serdes.core/write-String 1 {:optimize true} (:document-id this) os) + (write-IdType 2 {:optimize true} (:id-type this) os)) pb/TypeReflection (gettype [this] "com.xtdb.protos.Delete")) (s/def :com.xtdb.protos.Delete/document-id string?) -(s/def ::Delete-spec (s/keys :opt-un [:com.xtdb.protos.Delete/document-id])) -(def Delete-defaults {:document-id ""}) +(s/def :com.xtdb.protos.Delete/id-type (s/or :keyword keyword? :int int?)) +(s/def ::Delete-spec (s/keys :opt-un [:com.xtdb.protos.Delete/document-id :com.xtdb.protos.Delete/id-type])) +(def Delete-defaults {:document-id "" :id-type IdType-default}) (defn cis->Delete "CodedInputStream to Delete" [is] - (map->Delete-record (tag-map Delete-defaults (fn [tag index] (case index 1 [:document-id (serdes.core/cis->String is)] [index (serdes.core/cis->undefined tag is)])) is))) + (map->Delete-record (tag-map Delete-defaults (fn [tag index] (case index 1 [:document-id (serdes.core/cis->String is)] 2 [:id-type (cis->IdType is)] [index (serdes.core/cis->undefined tag is)])) is))) (defn ecis->Delete "Embedded CodedInputStream to Delete" diff --git a/src/gxtdb/adapters/json.clj b/src/gxtdb/adapters/json.clj new file mode 100644 index 0000000..efbc227 --- /dev/null +++ b/src/gxtdb/adapters/json.clj @@ -0,0 +1,13 @@ +(ns gxtdb.adapters.json + (:require [gxtdb.utils :as utils])) + +(defn value-record->edn [record] + (let [kind (:kind record) + key (-> kind keys first)] + (case key + :number-value (:number-value kind) + :bool-value (:bool-value kind) + :string-value (-> kind :string-value str) + :list-value (->> kind :list-value :values (mapv value-record->edn)) + :struct-value (->> kind :struct-value :fields (map (fn [[k v]] {(utils/str->keyword k) (value-record->edn v)})) (reduce into {})) + ""))) \ No newline at end of file diff --git a/src/gxtdb/adapters/tx_log.clj b/src/gxtdb/adapters/tx_log.clj index 8f00a38..94fffc9 100644 --- a/src/gxtdb/adapters/tx_log.clj +++ b/src/gxtdb/adapters/tx_log.clj @@ -1,11 +1,39 @@ (ns gxtdb.adapters.tx-log - (:require [gxtdb.utils :as utils])) + (:require [gxtdb.utils :as utils] + [xtdb.api :as xt] + [gxtdb.adapters.json :as json])) + +(defn ->evict [transaction] + (let [transaction (:evict transaction) + id (:document-id transaction) + id-type (:id-type transaction)] + [::xt/evict (utils/->id id-type id)])) + +(defn ->delete [transaction] + (let [transaction (:delete transaction) + id (:document-id transaction) + id-type (:id-type transaction)] + [::xt/delete (utils/->id id-type id)])) + +(defn ->put [transaction] + (let [transaction (:put transaction) + id (:xt-id transaction) + id-type (:id-type transaction) + document (:document transaction)] + [::xt/put (into {:xt/id (utils/->id id-type id)} (json/value-record->edn {:kind {:struct-value document}}))])) (defn ->tx-log [ops] - (let [transaction (-> ops :transaction-type) - transaction-type (-> transaction keys first)] - (println (str "---" "\n" transaction-type "\n" transaction "\n\n\n")))) + (let [transaction (:transaction-type ops) + transaction-type (-> transaction keys first) + xt-converted-type (->> transaction-type name (keyword ":xt"))] + (case transaction-type + :put (->put transaction) + :delete (->delete transaction) + :evict (->evict transaction) + :else (throw (str "Transaction type " xt-converted-type " not implemented"))))) (defn proto->tx-log [tx-ops] - (println (str "ARROZ" "\n\n\n" tx-ops "\n\n")) (mapv ->tx-log tx-ops)) + +(defn xtdb->proto [edn] + {:tx-time (-> edn :xtdb.api/tx-time utils/->inst-str) :tx-id (:xtdb.api/tx-id edn)}) \ No newline at end of file diff --git a/src/gxtdb/service.clj b/src/gxtdb/service.clj index 11775f2..f0c4991 100644 --- a/src/gxtdb/service.clj +++ b/src/gxtdb/service.clj @@ -13,7 +13,8 @@ ;; -- PROTOC-GEN-CLOJURE -- [protojure.pedestal.core :as protojure.pedestal] [protojure.pedestal.routes :as proutes] - [com.xtdb.protos.GrpcApi.server :as api])) + [com.xtdb.protos.GrpcApi.server :as api] + [gxtdb.adapters.tx-log :as tx-log])) (defn about-page [_request] @@ -45,18 +46,16 @@ (deftype GrpcAPI [xtdb-node] api/Service (SubmitTx [_this {{:keys [tx-ops]} :grpc-params}] - (let [tx-log (tx-log-adapter/proto->tx-log tx-ops)]) - {:status 200 - :body {:tx-time "time" :tx-id 3}}) + (let [tx-log (tx-log-adapter/proto->tx-log tx-ops) + xt-response (xt/submit-tx xtdb-node tx-log)] + {:status 200 + :body (tx-log-adapter/xtdb->proto xt-response)})) (Status [_this _request] (let [status (xt/status xtdb-node)] {:status 200 :body (status-adapter/edn->grpc status)}))) -;; Defines "/" and "/about" routes with their associated :get handlers. -;; The interceptors defined after the verb map (e.g., {:get home-page} -;; apply to / and its children (/about). (def common-interceptors [(body-params/body-params) http/html-body]) ;; Tabular routes @@ -64,19 +63,11 @@ ["/about" :get (conj common-interceptors `about-page)]}) ;; -- PROTOC-GEN-CLOJURE -- -;; Add the routes produced by Greeter->routes (defn grpc-routes [xtdb-node] (reduce conj routes (proutes/->tablesyntax {:rpc-metadata api/rpc-metadata :interceptors common-interceptors :callback-context (GrpcAPI. xtdb-node)}))) (defn service [xtdb-node] {:env :prod ::http/routes (grpc-routes xtdb-node) - - ;; -- PROTOC-GEN-CLOJURE -- - ;; We override the chain-provider with one provided by protojure.protobuf - ;; and based on the Undertow webserver. This provides the proper support - ;; for HTTP/2 trailers, which GRPCs rely on. A future version of pedestal - ;; may provide this support, in which case we can go back to using - ;; chain-providers from pedestal. ::http/type protojure.pedestal/config ::http/chain-provider protojure.pedestal/provider diff --git a/src/gxtdb/utils.clj b/src/gxtdb/utils.clj index 7cf57ea..e6905dd 100644 --- a/src/gxtdb/utils.clj +++ b/src/gxtdb/utils.clj @@ -3,6 +3,7 @@ [clojure.string :as str]) (:import java.text.SimpleDateFormat)) +#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defmacro tokenize [x] `(let [x# ~x] x#)) (defn assoc-some-str @@ -34,4 +35,24 @@ (defn edn-or-str [value] (if (str/starts-with? value ":") (keyword (subs value 1)) - value)) \ No newline at end of file + value)) + +(defn str->keyword [s] + (when-not (or (empty? s) (= s ":")) (-> s (str/replace #" " "-") keyword))) + +(defn ->id [id-type id] + (case id-type + :keyword (str->keyword id) + :string id + :int (parse-long id) + :uuid (let [uuid (parse-uuid id)] + (if (uuid? uuid) uuid id)) + (str->keyword id))) + +#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} +(defn record->map + [record] + (let [f #(if (record? %) (record->map %) %) + ks (keys record) + vs (map f (vals record))] + (zipmap ks vs))) \ No newline at end of file diff --git a/test/gxtdb/adapters/json_test.clj b/test/gxtdb/adapters/json_test.clj new file mode 100644 index 0000000..90bb251 --- /dev/null +++ b/test/gxtdb/adapters/json_test.clj @@ -0,0 +1,20 @@ +(ns gxtdb.adapters.json-test + #_{:clj-kondo/ignore [:refer-all]} + (:require [clojure.test :refer :all] + [gxtdb.adapters.json :as json])) + +(def edn {:kind {:struct-value {:fields + {"painting ids" {:kind {:list-value {:values [{:kind {:number-value 2.0}} {:kind {:number-value 3.0}} {:kind {:number-value 76.0}} {:kind {:number-value 3.0}}]}}}, + "age" {:kind {:number-value 200.0}}, + "is-artist?" {:kind {:bool-value true}}, + "full name" {:kind {:struct-value {:fields {"last" {:kind {:string-value "picasso"}}, "first" {:kind {:string-value "pablo"}}}}}}, + "name" {:kind {:string-value "pablo picasso"}}}}}}) +(def json {:painting-ids [2.0 3.0 76.0 3.0], + :age 200.0, + :is-artist? true, + :full-name {:last "picasso", :first "pablo"}, + :name "pablo picasso"}) + +(deftest json-value-test + (testing "convert proto to value" + (is (= (json/value-record->edn edn) json)))) \ No newline at end of file diff --git a/test/gxtdb/adapters/tx_log_test.clj b/test/gxtdb/adapters/tx_log_test.clj new file mode 100644 index 0000000..6d97a1c --- /dev/null +++ b/test/gxtdb/adapters/tx_log_test.clj @@ -0,0 +1,45 @@ +(ns gxtdb.adapters.tx-log-test + #_{:clj-kondo/ignore [:refer-all]} + (:require [clojure.test :refer :all] + [gxtdb.adapters.tx-log :as tx-log])) + +(def var->put-tx {:put {:xt-id "id 2", :document {:fields {"painting ids" {:kind {:list-value {:values [{:kind {:number-value 2.0}}, {:kind {:number-value 3.0}} {:kind {:number-value 76.0}} {:kind {:number-value 3.0}}]}}}, "age" {:kind {:number-value 200.0}}, "is-artist?" {:kind {:bool-value true}}, "full name" {:kind {:struct-value {:fields {"last" {:kind {:string-value "picasso"}}, "first" {:kind {:string-value "pablo"}}}}}}, "name" {:kind {:string-value "pablo picasso"}}}}}}) +(def expected-put-tx [:xtdb.api/put + {:xt/id :id-2, + :painting-ids [2.0 3.0 76.0 3.0], + :age 200.0, + :is-artist? true, + :full-name {:last "picasso", :first "pablo"}, + :name "pablo picasso"}]) + +(def var->delete-tx {:delete {:document-id "f2eed61a-1928-4d75-8620-debfc23eae8d", :id-type :uuid}}) +(def expected-delete-tx [:xtdb.api/delete #uuid "f2eed61a-1928-4d75-8620-debfc23eae8d"]) + +(def var->evict-tx {:evict {:document-id "f2eed61a-1928-4d75-8620-debfc23eae8d", :id-type :uuid}}) +(def expected-evict-tx [:xtdb.api/evict #uuid "f2eed61a-1928-4d75-8620-debfc23eae8d"]) + +(def var->tx-ops + [{:transaction-type + {:put {:id-type :keyword, :xt-id "id1", :document {:fields {"key" {:kind {:string-value "value"}}}}}}} + {:transaction-type + {:evict {:document-id "45", :id-type :int}}} + {:transaction-type + {:delete {:document-id "f2eed61a-1928-4d75-8620-debfc23eae8d", :id-type :uuid}}}]) + +(deftest tx-log-adapters-testing + (testing "Testing if put transactions parses correctly" + (is (= (tx-log/->put var->put-tx) expected-put-tx))) + (testing "Testing if delete transactions parses correctly" + (is (= (tx-log/->delete var->delete-tx) expected-delete-tx))) + (testing "Testing if evict transactions parses correctly" + (is (= (tx-log/->evict var->evict-tx) expected-evict-tx))) + (testing "Testing if proto tx-ops becomes xtdb datalog transaction" + (let [tx-ops (tx-log/proto->tx-log var->tx-ops)] + (is (= tx-ops + [[:xtdb.api/put {:xt/id :id1, :key "value"}] [:xtdb.api/evict 45] [:xtdb.api/delete #uuid "f2eed61a-1928-4d75-8620-debfc23eae8d"]]))))) + +(deftest xtdb-edn->proto-test + (testing "Testing if submit tx response can be parsed to proto" + (is (= + (tx-log/xtdb->proto {:xtdb.api/tx-id 0, :xtdb.api/tx-time #inst "2023-05-20T18:12:24.836-00:00"}) + {:tx-time "2023-05-20T13:12:24.836-05:00", :tx-id 0})))) \ No newline at end of file diff --git a/test/gxtdb/service_test.clj b/test/gxtdb/service_test.clj index b137adb..79dada0 100644 --- a/test/gxtdb/service_test.clj +++ b/test/gxtdb/service_test.clj @@ -1,3 +1,4 @@ +#_{:clj-kondo/ignore [:refer-all]} (ns gxtdb.service-test (:require [clojure.test :refer :all] [com.xtdb.protos.GrpcApi.client :refer :all] @@ -6,7 +7,8 @@ [protojure.grpc.client.providers.http2 :refer [connect]] [protojure.pedestal.core :as protojure.pedestal] [xtdb.api :as xt] - [gxtdb.service :as service])) + [gxtdb.service :as service] + [gxtdb.utils :as utils])) (def ^:dynamic *opts* {}) @@ -42,3 +44,12 @@ (:kv-store @(Status @(connect {:uri (str "http://localhost:" (:port @test-env))}) {})) "xtdb.mem_kv.MemKv")))) +(deftest submit-tx-test + (testing "Submit a put tx to xtdb-node" + (let [tx @(SubmitTx @(connect {:uri (str "http://localhost:" (:port @test-env))}) + {:tx-ops [{:transaction-type + {:put {:id-type :keyword, :xt-id "id1", :document {:fields {"key" {:kind {:string-value "value"}}}}}}}]})] + (is (inst? (-> tx :tx-time utils/->inst))) + (is (>= + (:tx-id tx) + 0))))) diff --git a/test/gxtdb/utils_test.clj b/test/gxtdb/utils_test.clj new file mode 100644 index 0000000..6a1fdbd --- /dev/null +++ b/test/gxtdb/utils_test.clj @@ -0,0 +1,106 @@ +#_{:clj-kondo/ignore [:refer-all]} +(ns gxtdb.utils-test + (:require [clojure.test :refer :all] + [gxtdb.utils :refer :all])) + +;; assoc-some-str tests +(deftest assoc-some-str-test + (testing "Should associate a key with a value when the value is not nil" + (let [input {:foo 42}] + (is (= (assoc-some-str input :bar "hello") + {:foo 42 :bar {:value {:some "hello"}}})))) + + (testing "Should associate a key with {:value {:none {}}} when the value is nil" + (let [input {:foo 42}] + (is (= (assoc-some-str input :bar nil) + {:foo 42 :bar {:value {:none {}}}}))))) + +;; assoc-with-fn tests +(deftest assoc-with-fn-test + (testing "Should associate a key in a map if the function applied to value is non-nil" + (let [input {:foo 42}] + (is (= (assoc-with-fn input :bar "hello" str) + {:foo 42 :bar "hello"})))) + + (testing "Should not associate a key in a map if the value is nil" + (let [input {:foo 42}] + (is (= (assoc-with-fn input :bar nil str) + {:foo 42}))))) + +;; nil->default tests +(deftest nil->default-test + (testing "Should associate a key with the default value when the value is nil" + (let [input {:foo nil}] + (is (= (nil->default input :foo nil "default") + {:foo "default"})))) + + (testing "Should not associate a key with the default value when the value is not nil" + (let [input {:foo "value"}] + (is (= (nil->default input :foo "default" :default) + {:foo "default"}))))) + +;; not-nil? tests +(deftest not-nil?-test + (testing "Should return true if the value is not nil" + (is (not-nil? 42))) + + (testing "Should return false if the value is nil" + (is (not (not-nil? nil))))) + +;; ->inst tests +(deftest ->inst-test + (testing "Should convert a valid string representation of an instant to an instance of java.time.Instant" + (let [input "2023-05-19T10:30:00Z"] + (is (inst? (->inst input))))) + + (testing "Should return nil if the input is an invalid string representation of an instant" + (let [input "invalid-instant"] + (is (nil? (->inst input)))))) + +;; ->inst-str tests +(deftest ->inst-str-test + (testing "Should convert an instance of java.time.Instant to a formatted string representation" + (let [input (->inst "2023-05-19T10:30:00Z")] + (is (string? (->inst-str input)))))) + +;; edn-or-str tests +(deftest edn-or-str-test + (testing "Should return a keyword when the value starts with a colon" + (is (= (edn-or-str ":keyword") :keyword))) + + (testing "Should return the original value when it doesn't start with a colon" + (is (= (edn-or-str "value") "value"))) + + (testing "Should handle empty string as input" + (is (= (edn-or-str "") "")))) + +;; str->keyword tests +(deftest str->keyword-test + (testing "Should convert a string to a keyword" + (is (= (str->keyword "my-keyword") :my-keyword))) + + (testing "Should handle whitespace in the string and replace it with hyphen" + (is (= (str->keyword "my keyword") :my-keyword))) + + (testing "Should handle empty string as input" + (is (= (str->keyword "") nil)))) + +;; ->id tests +(deftest ->id-test + (testing "Should convert a string ID to a keyword when id-type is :keyword" + (is (= (->id :keyword "my-id") :my-id))) + + (testing "Should parse a valid integer string and return the parsed integer when id-type is :int" + (is (= (->id :int "42") 42))) + + (testing "Should return the original string ID when id-type is :string" + (is (= (->id :string "my-id") "my-id"))) + + (testing "Should parse a valid UUID string and return the parsed UUID when id-type is :uuid" + (is (uuid? (->id :uuid "08d1fc6e-85f9-4a55-b0b9-3fd08963c375")))) + + (testing "Should return the original value when id-type is not recognized" + (is (= (->id :unknown "my-id") :my-id))) + + (testing "Should handle empty string as input" + (is (= (->id :keyword "") nil)))) \ No newline at end of file