diff --git a/.dockerignore b/.dockerignore index a116af6..4fa94a7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,5 @@ /pom.xml *jar -!opentelemetry-javaagent.jar /bin /lib /classes diff --git a/.gitignore b/.gitignore index e23b39f..b80326e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ /pom.xml *jar -!opentelemetry-javaagent.jar /bin /lib /classes diff --git a/Dockerfile b/Dockerfile index edcec3c..aa914a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ -FROM clojure:openjdk-17-lein-alpine +FROM clojure:temurin-22-lein-jammy WORKDIR /usr/src/app -RUN apk add --no-cache git +RUN apt-get update && \ + apt-get install -y git && \ + rm -rf /var/lib/apt/lists/* -RUN ln -s "/opt/openjdk-17/bin/java" "/bin/data-info" - -ENV OTEL_TRACES_EXPORTER none +RUN ln -s "/opt/java/openjdk/bin/java" "/bin/data-info" COPY project.clj /usr/src/app/ RUN lein deps @@ -14,10 +14,10 @@ RUN lein deps COPY conf/main/logback.xml /usr/src/app/ COPY . /usr/src/app -RUN lein uberjar && \ +RUN lein do clean, uberjar && \ cp target/data-info-standalone.jar . -ENTRYPOINT ["data-info", "-Dlogback.configurationFile=/etc/iplant/de/logging/data-info-logging.xml", "-javaagent:/usr/src/app/opentelemetry-javaagent.jar", "-Dotel.resource.attributes=service.name=data-info", "-cp", ".:data-info-standalone.jar", "data_info.core"] +ENTRYPOINT ["data-info", "-Dlogback.configurationFile=/etc/iplant/de/logging/data-info-logging.xml", "-cp", ".:data-info-standalone.jar", "data_info.core"] CMD ["--help"] ARG git_commit=unknown diff --git a/conf/test/logback.xml b/conf/test/logback-test.xml similarity index 100% rename from conf/test/logback.xml rename to conf/test/logback-test.xml diff --git a/opentelemetry-javaagent.jar b/opentelemetry-javaagent.jar deleted file mode 100644 index a152125..0000000 Binary files a/opentelemetry-javaagent.jar and /dev/null differ diff --git a/project.clj b/project.clj index f0d03cc..a7b6f8e 100644 --- a/project.clj +++ b/project.clj @@ -14,44 +14,42 @@ :url "http://iplantcollaborative.org/sites/default/files/iPLANT-LICENSE.txt"} :manifest {"Git-Ref" ~(git-ref)} :uberjar-name "data-info-standalone.jar" - :dependencies [[org.clojure/clojure "1.11.1"] - [cheshire "5.12.0" + :dependencies [[org.clojure/clojure "1.11.3"] + [cheshire "5.13.0" :exclusions [[com.fasterxml.jackson.dataformat/jackson-dataformat-cbor] [com.fasterxml.jackson.dataformat/jackson-dataformat-smile] [com.fasterxml.jackson.core/jackson-annotations] [com.fasterxml.jackson.core/jackson-databind] [com.fasterxml.jackson.core/jackson-core]]] [com.cemerick/url "0.1.1" :exclusions [com.cemerick/clojurescript.test]] - [dire "0.5.4"] + [org.cyverse/dire "0.5.5-SNAPSHOT"] [me.raynes/fs "1.4.6"] - [org.apache.tika/tika-core "2.9.1"] + [org.apache.tika/tika-core "2.9.2" :exclusions [org.slf4j/slf4j-api]] [net.sf.opencsv/opencsv "2.3"] - [de.ubercode.clostache/clostache "1.4.0" :exclusions [org.clojure/core.incubator]] - [org.cyverse/clj-irods "0.3.5"] - [org.cyverse/clj-icat-direct "2.9.5" - :exclusions [[org.slf4j/slf4j-log4j12] + [org.cyverse/clj-irods "0.3.6"] + [org.cyverse/clj-icat-direct "2.9.6" + :exclusions [[org.slf4j/slf4j-api] + [org.slf4j/slf4j-log4j12] [log4j]]] - [org.cyverse/clj-jargon "3.1.0" - :exclusions [[org.slf4j/slf4j-log4j12] + [org.cyverse/clj-jargon "3.1.1" + :exclusions [[org.slf4j/slf4j-api] + [org.slf4j/slf4j-log4j12] [log4j]]] [javax.servlet/servlet-api "2.5"] - [metosin/compojure-api "1.1.13"] - [ring/ring-jetty-adapter "1.11.0"] ;; update this when underlying ring version changes, probably - [org.slf4j/slf4j-api "1.7.31"] - [org.cyverse/otel "0.2.5"] - [org.cyverse/clojure-commons "3.0.7"] - [org.cyverse/common-cli "2.8.1"] - [org.cyverse/common-cfg "2.8.2"] - [org.cyverse/common-swagger-api "3.4.4"] - [org.cyverse/heuristomancer "2.8.6"] - [org.cyverse/kameleon "3.0.8"] - [org.cyverse/metadata-client "3.1.1"] - [org.cyverse/async-tasks-client "0.0.4"] - [org.cyverse/metadata-files "2.1.0"] - [org.cyverse/oai-ore "1.0.3"] - [org.cyverse/service-logging "2.8.3"] - [org.cyverse/event-messages "0.0.1"] - [com.novemberain/langohr "3.5.1"] + [metosin/compojure-api "1.1.14"] + [ring/ring-jetty-adapter "1.12.2" :exclusions [org.slf4j/slf4j-api]] + [org.cyverse/clojure-commons "3.0.9-SNAPSHOT"] + [org.cyverse/common-cli "2.8.2"] + [org.cyverse/common-cfg "2.8.3"] + [org.cyverse/common-swagger-api "3.4.5"] + [org.cyverse/heuristomancer "2.8.7"] + [org.cyverse/kameleon "3.0.10"] + [org.cyverse/metadata-client "3.1.2"] + [org.cyverse/async-tasks-client "0.0.5"] + [org.cyverse/metadata-files "2.1.1"] + [org.cyverse/oai-ore "1.0.4"] + [org.cyverse/service-logging "2.8.4"] + [com.novemberain/langohr "5.4.0"] [slingshot "0.12.2"]] :eastwood {:exclude-namespaces [data-info.routes.schemas.tickets data-info.routes.schemas.stats @@ -59,12 +57,11 @@ data-info.routes.schemas.trash :test-paths] :linters [:wrong-arity :wrong-ns-form :wrong-pre-post :wrong-tag :misplaced-docstrings]} - :plugins [[jonase/eastwood "1.4.2"] + :plugins [[jonase/eastwood "1.4.3"] [lein-ancient "0.7.0"] - [test2junit "1.1.3"]] + [test2junit "1.4.4"]] :profiles {:dev {:plugins [[lein-ring "0.12.5"]] - :resource-paths ["conf/test"] - :jvm-opts ["-Dotel.javaagent.enabled=false"]} + :resource-paths ["conf/test"]} :repl {:source-paths ["repl"]} :uberjar {:aot :all}} :main ^:skip-aot data-info.core @@ -73,4 +70,4 @@ :port 31360 :auto-reload? false} :uberjar-exclusions [#".*[.]SF" #"LICENSE" #"NOTICE"] - :jvm-opts ["-Dlogback.configurationFile=/etc/iplant/de/logging/data-info-logging.xml" "-javaagent:./opentelemetry-javaagent.jar" "-Dotel.resource.attributes=service.name=data-info"]) + :jvm-opts ["-Dlogback.configurationFile=/etc/iplant/de/logging/data-info-logging.xml"]) diff --git a/src/data_info/amqp.clj b/src/data_info/amqp.clj index a8dc978..2e8f2f3 100644 --- a/src/data_info/amqp.clj +++ b/src/data_info/amqp.clj @@ -28,14 +28,6 @@ (handler channel metadata msg) (log/error (format "[amqp/message-router] [%s] [%s] unroutable" routing-key (String. msg)))))) -(defn connect - [exchange-cfg queue-cfg handlers] - (let [channel (lch/open (rmq/connect {:uri (config/amqp-uri)}))] - (log/info (format "[amqp/connect] [%s]" (config/amqp-uri))) - (declare-exchange channel exchange-cfg) - (declare-queue channel exchange-cfg queue-cfg (keys handlers)) - (lc/blocking-subscribe channel (:name queue-cfg) (partial message-router handlers)))) - (defn publish-msg [routing-key msg] (try+ diff --git a/src/data_info/clients/async_tasks.clj b/src/data_info/clients/async_tasks.clj index c3b743e..f6063de 100644 --- a/src/data_info/clients/async_tasks.clj +++ b/src/data_info/clients/async_tasks.clj @@ -6,7 +6,6 @@ [clojure.string :as string] [clj-irods.core :refer [make-threadpool]] [clj-irods.cache-tools :as c] - [otel.otel :as otel] [async-tasks-client.core :as async-tasks-client])) ;; https://stackoverflow.com/questions/12068640/retrying-something-3-times-before-throwing-an-exception-in-clojure @@ -27,16 +26,14 @@ (defn- retry-via-threadpool [pool retries handler f & args] - (let [ag (agent nil) - s (otel/current-span)] - (send-via pool ag (fn [_nil] (c/otel-with-subspan [s] - (try+ - (apply retry-with-handler retries handler f args) - (catch Object o - {::error o}))))) + (let [ag (agent nil)] + (send-via pool ag (fn [_nil] (try+ + (apply retry-with-handler retries handler f args) + (catch Object o + {::error o})))) (delay (if-let [err (::error @ag)] - (throw+ err) - @ag)))) + (throw+ err) + @ag)))) (defn get-by-id [id] @@ -68,49 +65,50 @@ (defn run-async-thread [async-task-id thread-function prefix] - (otel/with-span [outer-span ["run-async-thread" {:kind :producer :attributes {"async-task-id" (str async-task-id)}}]] - (let [^Runnable task-thread (fn [] - (with-open [_ (otel/span-scope outer-span)] - (otel/with-span [s ["async thread" {:kind :consumer :attributes {"async-task-id" (str async-task-id)}}]] - (thread-function async-task-id))))] - (.start (Thread. task-thread (str prefix "-" (string/replace async-task-id #".*/tasks/" ""))))) - async-task-id)) + (let [^Runnable task-thread (fn [] (thread-function async-task-id))] + (.start (Thread. task-thread (str prefix "-" (string/replace async-task-id #".*/tasks/" ""))))) + async-task-id) + +(defn create-update-fn [async-task-id pool] + (fn [path action] + (log/info "Updating async task:" async-task-id ":" path action) + ;; we use the thread pool version here to be non-blocking, but elsewhere we use it so things happen in order (but + ;; deref the result to wait for the result before continuing) + (retry-via-threadpool + pool 3 + (fn [e t] (log/error (:throwable t) "failed updating async task")) + add-status async-task-id {:status "running" + :detail (format "[%s] %s: %s" (config/service-identifier) path (name action))}))) (defn paths-async-thread ([async-task-id jargon-fn end-fn] (paths-async-thread async-task-id jargon-fn end-fn true)) ([async-task-id jargon-fn end-fn use-client-user?] - (otel/with-span [s ["paths-async-thread" {:attributes {"async-task-id" (str async-task-id)}}]] - (let [pool (make-threadpool (str async-task-id "-async-tasks-update") 1) - {:keys [username] :as async-task} (get-by-id async-task-id) - update-fn (fn [path action] - (otel/with-span [s ["update-fn"]] - (log/info "Updating async task:" async-task-id ":" path action) - ;; we use the thread pool version here to be non-blocking, but elsewhere we use it so things happen in order (but deref the result to wait for the result before continuing) - (retry-via-threadpool pool 3 - (fn [e t] (log/error (:throwable t) "failed updating async task")) - add-status async-task-id {:status "running" :detail (format "[%s] %s: %s" (config/service-identifier) path (name action))})))] - (try+ - (deref (retry-via-threadpool pool 3 - (fn [e t] (log/error (:throwable t) "failed updating async task with started status")) - add-status async-task-id {:status "started"})) - (if use-client-user? - (irods/with-jargon-exceptions :client-user username [cm] - (otel/with-span [s ["jargon-fn" {:attributes {"client-user" username}}]] - (jargon-fn cm async-task update-fn))) - (irods/with-jargon-exceptions [cm] - (otel/with-span [s ["jargon-fn"]] - (jargon-fn cm async-task update-fn)))) - ;; For the completed statuses we want a lot of retries because the presence or absence of an end date controls locking behavior - (deref (retry-via-threadpool pool 100 - (fn [e t] (log/error (:throwable t) "failed updating async task with completed status")) - add-completed-status async-task-id {:status "completed" :detail (str "[" (config/service-identifier) "]")})) - (otel/with-span [s ["end-fn"]] - (end-fn async-task false)) - (catch Object _ - (log/error (:throwable &throw-context) "failed processing async task" async-task-id) - (deref (retry-via-threadpool pool 100 - (fn [e t] (log/error (:throwable t) "failed updating async task with completed status")) - add-completed-status async-task-id {:status "failed" :detail (format "[%s] %s" (config/service-identifier) (pr-str (:throwable &throw-context)))})) - (end-fn async-task true)))) - (log/info "Finished processing async task " async-task-id)))) + (let [pool (make-threadpool (str async-task-id "-async-tasks-update") 1) + {:keys [username] :as async-task} (get-by-id async-task-id) + update-fn (create-update-fn async-task-id pool)] + (try+ + (deref (retry-via-threadpool pool 3 + (fn [e t] (log/error (:throwable t) "failed updating async task with started status")) + add-status async-task-id {:status "started"})) + (if use-client-user? + (irods/with-jargon-exceptions :client-user username [cm] + (jargon-fn cm async-task update-fn)) + (irods/with-jargon-exceptions [cm] (jargon-fn cm async-task update-fn))) + ;; For the completed statuses we want a lot of retries because the presence or absence of an end date controls + ;; locking behavior + (deref (retry-via-threadpool + pool 100 + (fn [e t] (log/error (:throwable t) "failed updating async task with completed status")) + add-completed-status async-task-id {:status "completed" :detail (str "[" (config/service-identifier) "]")})) + (end-fn async-task false) + (catch Object _ + (log/error (:throwable &throw-context) "failed processing async task" async-task-id) + (deref (retry-via-threadpool + pool 100 + (fn [e t] (log/error (:throwable t) "failed updating async task with completed status")) + add-completed-status async-task-id + {:status "failed" + :detail (format "[%s] %s" (config/service-identifier) (pr-str (:throwable &throw-context)))})) + (end-fn async-task true)))) + (log/info "Finished processing async task " async-task-id))) diff --git a/src/data_info/core.clj b/src/data_info/core.clj index 5d3bbff..0025f1a 100644 --- a/src/data_info/core.clj +++ b/src/data_info/core.clj @@ -5,7 +5,6 @@ [data-info.routes :as routes] [data-info.util.config :as config] [data-info.amqp :as amqp] - [data-info.events :as events] [me.raynes.fs :as fs] [common-cli.core :as ccli] [service-logging.thread-context :as tc] @@ -69,13 +68,6 @@ (icat/configure-icat)) -(defn listen-for-events - [] - (let [exchange-cfg (events/exchange-config) - queue-cfg (events/queue-config)] - (amqp/connect exchange-cfg queue-cfg {"events.data-info.ping" events/ping-handler}))) - - (defn- cli-options [] [["-c" "--config PATH" "Path to the config file" @@ -101,5 +93,4 @@ (ccli/exit 1 "The config file is not readable.")) (load-configuration-from-file (:config options)) (icat/configure-icat) - (.start (Thread. listen-for-events)) (run-jetty)))) diff --git a/src/data_info/events.clj b/src/data_info/events.clj deleted file mode 100644 index cd83f37..0000000 --- a/src/data_info/events.clj +++ /dev/null @@ -1,29 +0,0 @@ -(ns data-info.events - (:require [clojure.tools.logging :as log] - [data-info.util.config :as config] - [data-info.amqp :as amqp] - [langohr.basic :as lb]) - (:import [org.cyverse.events.ping PingMessages$Pong] - [com.google.protobuf.util JsonFormat])) - -(defn exchange-config - [] - {:name (config/exchange-name) - :durable (config/exchange-durable?) - :auto-delete (config/exchange-auto-delete?)}) - -(defn queue-config - [] - {:name (config/queue-name) - :durable (config/queue-durable?) - :auto-delete (config/queue-auto-delete?)}) - -(defn ping-handler - [channel {:keys [delivery-tag routing-key]} msg] - (lb/ack channel delivery-tag) - (log/info (format "[events/ping-handler] [%s] [%s]" routing-key (String. msg))) - (lb/publish channel (config/exchange-name) "events.data-info.pong" - (.print (JsonFormat/printer) - (.. (PingMessages$Pong/newBuilder) - (setPongFrom "data-info") - (build))))) diff --git a/src/data_info/routes.clj b/src/data_info/routes.clj index 9f3e736..5a0c35f 100644 --- a/src/data_info/routes.clj +++ b/src/data_info/routes.clj @@ -29,29 +29,29 @@ (defapi app (swagger-routes - {:ui config/docs-uri - :options {:ui {:supported-submit-methods ["get", "post", "put", "delete", "patch", "head"] - :validator-url nil}} - :data {:info {:title "Discovery Environment Data Info API" - :description "Documentation for the Discovery Environment Data Info REST API" - :version "2.8.0"} - :tags [{:name "service-info", :description "Service Information"} - {:name "data-by-id", :description "Data Operations (by ID)"} - {:name "data", :description "Data Operations"} - {:name "tickets", :description "Ticket Operations"} - {:name "bulk", :description "Bulk Operations"} - {:name "navigation", :description "Navigation"} - {:name "users", :description "User Operations"} - {:name "filetypes", :description "File Type Metadata"} - {:name "groups", :description "Group Operations"}]}}) - (middleware - [add-user-to-context - wrap-query-params - wrap-lcase-params - params/wrap-keyword-params - [wrap-exceptions cx/exception-handlers] - util/req-logger - log-validation-errors] + {:ui config/docs-uri + :options {:ui {:supported-submit-methods ["get", "post", "put", "delete", "patch", "head"] + :validator-url nil}} + :data {:info {:title "Discovery Environment Data Info API" + :description "Documentation for the Discovery Environment Data Info REST API" + :version "2.8.0"} + :tags [{:name "service-info", :description "Service Information"} + {:name "data-by-id", :description "Data Operations (by ID)"} + {:name "data", :description "Data Operations"} + {:name "tickets", :description "Ticket Operations"} + {:name "bulk", :description "Bulk Operations"} + {:name "navigation", :description "Navigation"} + {:name "users", :description "User Operations"} + {:name "filetypes", :description "File Type Metadata"} + {:name "groups", :description "Group Operations"}]}}) + (context "/" [] + :middleware [add-user-to-context + wrap-query-params + wrap-lcase-params + params/wrap-keyword-params + [wrap-exceptions cx/exception-handlers] + util/req-logger + log-validation-errors] status-routes/status data-routes/data-operations rename-routes/rename-routes diff --git a/src/data_info/routes/avus.clj b/src/data_info/routes/avus.clj index cb340b2..3f138be 100644 --- a/src/data_info/routes/avus.clj +++ b/src/data_info/routes/avus.clj @@ -1,7 +1,6 @@ (ns data-info.routes.avus (:use [common-swagger-api.routes] [common-swagger-api.schema] - [otel.middleware :only [otel-middleware]] [data-info.routes.schemas.common] [data-info.routes.schemas.avus]) (:require [data-info.services.metadata :as meta] @@ -13,7 +12,6 @@ :tags ["data-by-id"] (GET "/metadata" [:as {uri :uri}] - :middleware [otel-middleware] :query [{:keys [user]} StandardUserQueryParams] :return AVUGetResult :summary "List AVUs (administrative)" @@ -29,7 +27,6 @@ (svc/trap uri meta/admin-metadata-get data-id)) (PATCH "/metadata" [:as {uri :uri}] - :middleware [otel-middleware] :query [{:keys [user]} StandardUserQueryParams] :body [body (describe AddMetadataRequest "The iRODS and Metadata AVUs to add")] :return AVUChangeResult @@ -50,7 +47,6 @@ :tags ["data-by-id"] (GET "/metadata" [:as {uri :uri}] - :middleware [otel-middleware] :query [{:keys [user]} StandardUserQueryParams] :return AVUGetResult :summary "List AVUs" @@ -66,7 +62,6 @@ (svc/trap uri meta/metadata-get user data-id :system false)) (PATCH "/metadata" [:as {uri :uri}] - :middleware [otel-middleware] :query [{:keys [user]} StandardUserQueryParams] :body [body (describe AddMetadataRequest "The iRODS and Metadata AVUs to add")] :return AVUChangeResult @@ -84,7 +79,6 @@ (svc/trap uri meta/metadata-add user data-id body)) (PUT "/metadata" [:as {uri :uri}] - :middleware [otel-middleware] :query [{:keys [user]} StandardUserQueryParams] :body [body (describe MetadataListing "A list of AVUs to set for this file. @@ -105,7 +99,6 @@ (svc/trap uri meta/metadata-set user data-id body)) (POST "/metadata/copy" [:as {uri :uri}] - :middleware [otel-middleware] :query [{:keys [user]} StandardUserQueryParams] :body [{:keys [destination_ids]} (describe MetadataCopyRequest "The destination data items.")] :return MetadataCopyResult @@ -120,7 +113,6 @@ (svc/trap uri meta/metadata-copy user data-id destination_ids)) (POST "/metadata/csv-parser" [:as {uri :uri}] - :middleware [otel-middleware] :query [params MetadataCSVParseParams] :return MetadataCSVParseResult :summary "Add Batch Metadata from CSV File" diff --git a/src/data_info/routes/data.clj b/src/data_info/routes/data.clj index a257451..762749a 100644 --- a/src/data_info/routes/data.clj +++ b/src/data_info/routes/data.clj @@ -1,7 +1,6 @@ (ns data-info.routes.data (:use [common-swagger-api.routes] [common-swagger-api.schema] - [otel.middleware :only [otel-middleware]] [data-info.routes.schemas.common] [data-info.routes.schemas.data] [data-info.routes.schemas.stats]) @@ -24,7 +23,6 @@ :tags ["data"] (GET "/uuid" [:as {uri :uri}] - :middleware [otel-middleware] :query [params PathToUUIDParams] :return PathToUUIDReturn :summary "Get the UUID for a path" @@ -33,7 +31,6 @@ (svc/trap uri uuids/do-simple-uuid-for-path params)) (GET "/path/:zone/*" [:as {{zone :zone path :*} :params uri :uri}] - :middleware [otel-middleware] :query [params FolderListingParams] :no-doc true (ce/trap uri entry/dispatch-path-to-resource zone path params)) @@ -60,7 +57,7 @@ (POST "/" [:as {uri :uri}] :query [params FileUploadQueryParams] :multipart-params [file :- s/Any] - :middleware [otel-middleware write/wrap-multipart-create] + :middleware [write/wrap-multipart-create] :return FileStat :summary "Upload a file" :description (str @@ -69,7 +66,6 @@ (svc/trap uri write/do-upload params file)) (POST "/directories" [:as {uri :uri}] - :middleware [otel-middleware] :tags ["bulk"] :query [params StandardUserQueryParams] :body [body (describe Paths "The paths to create.")] @@ -91,13 +87,11 @@ with characters in a runtime-configurable parameter. Currently, this parameter l ; add both versions to catch multiple types of path passing (GET "/manifest/*" [:as {{path :*} :params uri :uri}] - :middleware [otel-middleware] :query [{:keys [user]} StandardUserQueryParams] :no-doc true (svc/trap uri manifest/do-manifest user (str "/" path))) (GET "/manifest/:path" [:as {uri :uri}] - :middleware [otel-middleware] :query [{:keys [user]} StandardUserQueryParams] :path-params [path :- String] :return Manifest @@ -108,13 +102,11 @@ with characters in a runtime-configurable parameter. Currently, this parameter l (svc/trap uri manifest/do-manifest user (str "/" path))) (GET "/chunks/*" [:as {{path :*} :params uri :uri}] - :middleware [otel-middleware] :query [params ChunkParams] :no-doc true (svc/trap uri page-file/do-read-chunk params (str "/" path))) (GET "/chunks/:path" [:as {uri :uri}] - :middleware [otel-middleware] :query [params ChunkParams] :path-params [path :- String] :return ChunkReturn @@ -126,13 +118,11 @@ with characters in a runtime-configurable parameter. Currently, this parameter l (svc/trap uri page-file/do-read-chunk params (str "/" path))) (GET "/chunks-tabular/*" [:as {{path :*} :params uri :uri}] - :middleware [otel-middleware] :query [params TabularChunkParams] :no-doc true (svc/trap uri page-tabular/do-read-csv-chunk params (str "/" path))) (GET "/chunks-tabular/:path" [:as {uri :uri}] - :middleware [otel-middleware] :query [params TabularChunkParams] :path-params [path :- String] :return (doc-only TabularChunkReturn TabularChunkDoc) @@ -148,7 +138,6 @@ with characters in a runtime-configurable parameter. Currently, this parameter l :tags ["data-by-id"] (HEAD "/" [:as {uri :uri}] - :middleware [otel-middleware] :query [{:keys [user]} StandardUserQueryParams] :responses {200 {:description "User has read permissions for given data item."} 403 {:description "User does not have read permissions for given data item."} @@ -161,7 +150,7 @@ with characters in a runtime-configurable parameter. Currently, this parameter l (PUT "/" [:as {uri :uri}] :query [params StandardUserQueryParams] :multipart-params [file :- s/Any] - :middleware [otel-middleware write/wrap-multipart-overwrite] + :middleware [write/wrap-multipart-overwrite] :return FileStat :summary "Overwrite Contents" :description (str @@ -170,7 +159,6 @@ with characters in a runtime-configurable parameter. Currently, this parameter l (svc/trap uri write/do-upload params file)) (GET "/manifest" [:as {uri :uri}] - :middleware [otel-middleware] :query [{:keys [user]} StandardUserQueryParams] :return Manifest :summary "Return file manifest" @@ -180,7 +168,6 @@ with characters in a runtime-configurable parameter. Currently, this parameter l (svc/trap uri manifest/do-manifest-uuid user data-id)) (GET "/chunks" [:as {uri :uri}] - :middleware [otel-middleware] :query [params ChunkParams] :return ChunkReturn :summary "Get File Chunk" @@ -191,7 +178,6 @@ with characters in a runtime-configurable parameter. Currently, this parameter l (svc/trap uri page-file/do-read-chunk-uuid params data-id)) (GET "/chunks-tabular" [:as {uri :uri}] - :middleware [otel-middleware] :query [params TabularChunkParams] :return (doc-only TabularChunkReturn TabularChunkDoc) :summary "Get Tabular File Chunk" @@ -202,7 +188,6 @@ with characters in a runtime-configurable parameter. Currently, this parameter l (svc/trap uri page-tabular/do-read-csv-chunk-uuid params data-id)) (POST "/metadata/save" [:as {uri :uri}] - :middleware [otel-middleware] :query [params StandardUserQueryParams] :body [body (describe MetadataSaveRequest "The metadata save request.")] :return FileStat @@ -221,7 +206,6 @@ with characters in a runtime-configurable parameter. Currently, this parameter l (svc/trap uri meta/do-metadata-save data-id params body)) (POST "/ore/save" [:as {uri :uri}] - :middleware [otel-middleware] :query [params StandardUserQueryParams] :summary "Generating an OAI-ORE File for a Data Set" :description (str diff --git a/src/data_info/routes/exists.clj b/src/data_info/routes/exists.clj index b2cd1b1..86957ea 100644 --- a/src/data_info/routes/exists.clj +++ b/src/data_info/routes/exists.clj @@ -1,6 +1,5 @@ (ns data-info.routes.exists (:use [common-swagger-api.schema] - [otel.middleware :only [otel-middleware]] [ring.util.http-response :only [ok]]) (:require [common-swagger-api.schema.data.exists :as schema] [data-info.services.exists :as exists])) @@ -12,7 +11,6 @@ :tags ["bulk"] (POST "/" [] - :middleware [otel-middleware] :query [params StandardUserQueryParams] :body [body schema/ExistenceRequest] :responses schema/ExistenceResponses diff --git a/src/data_info/routes/filetypes.clj b/src/data_info/routes/filetypes.clj index 65605ae..6c277cc 100644 --- a/src/data_info/routes/filetypes.clj +++ b/src/data_info/routes/filetypes.clj @@ -1,14 +1,12 @@ (ns data-info.routes.filetypes (:use [common-swagger-api.schema] [common-swagger-api.schema.filetypes] - [otel.middleware :only [otel-middleware]] [data-info.routes.schemas.common]) (:require [data-info.services.filetypes :as filetypes] [data-info.util.service :as svc])) (defroutes filetypes-operations (GET "/file-types" [:as {uri :uri}] - :middleware [otel-middleware] :tags ["filetypes"] :return TypesList :summary "List File Types" @@ -16,7 +14,6 @@ (svc/trap uri filetypes/do-type-list)) (PUT "/data/:data-id/type" [:as {uri :uri}] - :middleware [otel-middleware] :tags ["data-by-id"] :query [params StandardUserQueryParams] :path-params [data-id :- DataIdPathParam] diff --git a/src/data_info/routes/groups.clj b/src/data_info/routes/groups.clj index 42857b0..6bc866e 100644 --- a/src/data_info/routes/groups.clj +++ b/src/data_info/routes/groups.clj @@ -1,7 +1,6 @@ (ns data-info.routes.groups (:use [clojure-commons.error-codes] [common-swagger-api.schema] - [otel.middleware :only [otel-middleware]] [data-info.routes.schemas.users]) (:require [data-info.services.groups :as groups] [common-swagger-api.schema.data :as data-schema] @@ -47,7 +46,6 @@ :tags ["groups"] (POST "/" [] - :middleware [otel-middleware] :query [params StandardUserQueryParams] :body [body Group] :responses (merge CommonResponses @@ -65,7 +63,6 @@ :path-params [group-name :- (describe NonBlankString "The name of an iRODS group, with or without zone qualification")] (GET "/" [] - :middleware [otel-middleware] :query [params StandardUserQueryParams] :responses (merge CommonResponses {500 {:schema GetUpdateErrorResponses @@ -77,7 +74,6 @@ (ok (groups/get-group params group-name))) (PUT "/" [] - :middleware [otel-middleware] :query [params StandardUserQueryParams] :body [body GroupMembers] :responses (merge CommonResponses @@ -92,7 +88,6 @@ (ok (groups/update-group-members params body group-name))) (DELETE "/" [] - :middleware [otel-middleware] :query [params StandardUserQueryParams] :responses (merge CommonResponses {500 {:schema DeleteErrorResponses diff --git a/src/data_info/routes/navigation.clj b/src/data_info/routes/navigation.clj index 0e85830..05b1a0a 100644 --- a/src/data_info/routes/navigation.clj +++ b/src/data_info/routes/navigation.clj @@ -1,6 +1,5 @@ (ns data-info.routes.navigation (:use [common-swagger-api.schema] - [otel.middleware :only [otel-middleware]] [data-info.routes.schemas.common :only [get-error-code-block]] [ring.util.http-response :only [ok]]) (:require [common-swagger-api.schema.data.navigation :as schema] @@ -15,7 +14,6 @@ :tags ["navigation"] (GET "/base-paths" [:as {uri :uri}] - :middleware [otel-middleware] :query [{:keys [user]} StandardUserQueryParams] :return schema/UserBasePaths :summary "Get User's Base Paths" @@ -26,7 +24,6 @@ (svc/trap uri root/user-base-paths user)) (GET "/home" [:as {uri :uri}] - :middleware [otel-middleware] :query [params StandardUserQueryParams] :return schema/RootListing :summary "Get User's Home Dir" @@ -38,7 +35,6 @@ (svc/trap uri home/do-homedir params)) (GET "/root" [:as {uri :uri}] - :middleware [otel-middleware] :query [{:keys [user]} StandardUserQueryParams] :responses schema/NavigationRootResponses :summary schema/NavigationRootSummary @@ -46,7 +42,6 @@ (ok (root/do-root-listing user))) (GET "/path/:zone/*" [:as {{path :*} :params uri :uri}] - :middleware [otel-middleware] :path-params [zone :- String] :query [params StandardUserQueryParams] :responses schema/NavigationResponses diff --git a/src/data_info/routes/path_lists.clj b/src/data_info/routes/path_lists.clj index df79ab2..fa315d9 100644 --- a/src/data_info/routes/path_lists.clj +++ b/src/data_info/routes/path_lists.clj @@ -1,6 +1,5 @@ (ns data-info.routes.path-lists (:use [common-swagger-api.schema] - [otel.middleware :only [otel-middleware]] [data-info.routes.schemas.common] [data-info.routes.schemas.path-lists] [data-info.routes.schemas.stats]) @@ -12,7 +11,6 @@ :tags ["bulk"] (POST "/" [:as {uri :uri}] - :middleware [otel-middleware] :query [params PathListSaveQueryParams] :body [body (describe Paths "The folder or file paths to process for the HT Path List file contents.")] :responses {200 {:schema FileStat diff --git a/src/data_info/routes/permissions.clj b/src/data_info/routes/permissions.clj index 4f38fd6..578ebce 100644 --- a/src/data_info/routes/permissions.clj +++ b/src/data_info/routes/permissions.clj @@ -1,6 +1,5 @@ (ns data-info.routes.permissions (:use [common-swagger-api.schema] - [otel.middleware :only [otel-middleware]] [data-info.routes.schemas.common] [data-info.routes.schemas.permissions]) (:require [data-info.services.users :as users] @@ -13,7 +12,6 @@ :tags ["bulk"] (POST "/" [:as {uri :uri}] - :middleware [otel-middleware] :query [params StandardUserQueryParams] :body [body (describe Paths "The paths to gather permissions information on.")] :return PermissionsResponse @@ -33,7 +31,6 @@ (context "/permissions" [] (GET "/" [:as {uri :uri}] - :middleware [otel-middleware] :query [params StandardUserQueryParams] :return DataItemPermissionsResponse :summary "List Data Item Permissions" @@ -43,7 +40,6 @@ (svc/trap uri perms/list-permissions params data-id)) (PUT "/:share-with/:permission" [:as {uri :uri}] - :middleware [otel-middleware] :path-params [share-with :- (describe NonBlankString "The user to grant permissions to.") permission :- (describe PermissionEnum "The permission level to grant.")] :query [params StandardUserQueryParams] @@ -55,7 +51,6 @@ (svc/trap uri perms/add-permission params data-id share-with permission)) (DELETE "/:unshare-with" [:as {uri :uri}] - :middleware [otel-middleware] :path-params [unshare-with :- (describe NonBlankString "The user whose permissions will be revoked.")] :query [params StandardUserQueryParams] :return DataItemPermissionsResponse diff --git a/src/data_info/routes/rename.clj b/src/data_info/routes/rename.clj index 0541ff2..45ef229 100644 --- a/src/data_info/routes/rename.clj +++ b/src/data_info/routes/rename.clj @@ -1,6 +1,5 @@ (ns data-info.routes.rename (:use [common-swagger-api.schema] - [otel.middleware :only [otel-middleware]] [data-info.routes.schemas.common] [data-info.routes.schemas.rename]) (:require [data-info.services.rename :as rename] @@ -8,7 +7,6 @@ (defroutes rename-routes (POST "/mover" [:as {uri :uri}] - :middleware [otel-middleware] :tags ["bulk"] :query [params StandardUserQueryParams] :body [body (describe MultiRenameRequest "The paths to rename and their destination.")] @@ -24,7 +22,6 @@ :tags ["data-by-id"] (PUT "/name" [:as {uri :uri}] - :middleware [otel-middleware] :query [params StandardUserQueryParams] :body [body (describe Filename "The new name of the data item.")] :return RenameResult @@ -36,7 +33,6 @@ (svc/trap uri rename/do-rename-uuid params body data-id)) (PUT "/dir" [:as {uri :uri}] - :middleware [otel-middleware] :query [params StandardUserQueryParams] :body [body (describe Dirname "The new directory name of the data item.")] :return RenameResult @@ -48,7 +44,6 @@ (svc/trap uri rename/do-move-uuid params body data-id)) (PUT "/children/dir" [:as {uri :uri}] - :middleware [otel-middleware] :query [params StandardUserQueryParams] :body [body (describe Dirname "The new directory name of the data items.")] :return MultiRenameResult diff --git a/src/data_info/routes/sharing.clj b/src/data_info/routes/sharing.clj index 2869daf..6569a0c 100644 --- a/src/data_info/routes/sharing.clj +++ b/src/data_info/routes/sharing.clj @@ -1,6 +1,5 @@ (ns data-info.routes.sharing (:use [common-swagger-api.schema] - [otel.middleware :only [otel-middleware]] [data-info.routes.schemas.common] [data-info.routes.schemas.sharing]) (:require [data-info.services.sharing :as sharing] @@ -8,7 +7,6 @@ (defroutes sharing-routes (POST "/anonymizer" [:as {uri :uri}] - :middleware [otel-middleware] :tags ["bulk"] :query [params StandardUserQueryParams] :body [body (describe Paths "The paths to make readable by the anonymous user.")] diff --git a/src/data_info/routes/stats.clj b/src/data_info/routes/stats.clj index 36aef01..d364eaa 100644 --- a/src/data_info/routes/stats.clj +++ b/src/data_info/routes/stats.clj @@ -1,6 +1,5 @@ (ns data-info.routes.stats (:use [common-swagger-api.schema] - [otel.middleware :only [otel-middleware]] [data-info.routes.schemas.common] [data-info.routes.schemas.stats] [ring.util.http-response :only [ok]]) @@ -16,7 +15,6 @@ :tags ["bulk"] (POST "/" [] - :middleware [otel-middleware] :query [params StatQueryParams] :body [body data-schema/OptionalPathsOrDataIds] :responses schema/StatResponses @@ -29,7 +27,6 @@ :tags ["bulk"] (POST "/" [] - :middleware [otel-middleware] :query [params FilteredStatQueryParams] :body [body data-schema/OptionalPathsOrDataIds] :responses (merge CommonResponses diff --git a/src/data_info/routes/status.clj b/src/data_info/routes/status.clj index c2f173d..c590d29 100644 --- a/src/data_info/routes/status.clj +++ b/src/data_info/routes/status.clj @@ -1,6 +1,5 @@ (ns data-info.routes.status (:use [common-swagger-api.schema] - [otel.middleware :only [otel-middleware]] [data-info.routes.schemas.status]) (:require [clojure-commons.service :as commons-service] [clojure-commons.error-codes :as ce] @@ -12,7 +11,6 @@ (defroutes status (GET "/" [:as {:keys [uri server-name server-port]}] - :middleware [otel-middleware] :query [{:keys [expecting]} StatusParams] :return DataInfoStatusResponse :tags ["service-info"] @@ -24,7 +22,6 @@ :iRODS (status/irods-running?))))) (GET "/admin/config" [:as {:keys [uri]}] - :middleware [otel-middleware] :return (describe s/Any "A map of configuration keys to values.") :tags ["service-info"] :summary "Configuration Information" diff --git a/src/data_info/routes/tickets.clj b/src/data_info/routes/tickets.clj index e872280..d0b64f7 100644 --- a/src/data_info/routes/tickets.clj +++ b/src/data_info/routes/tickets.clj @@ -1,6 +1,5 @@ (ns data-info.routes.tickets (:use [common-swagger-api.schema] - [otel.middleware :only [otel-middleware]] [data-info.routes.schemas.tickets] [ring.util.http-response :only [ok]]) (:require [common-swagger-api.schema.data :as data-schema] @@ -14,7 +13,6 @@ :tags ["tickets"] (POST "/" [] - :middleware [otel-middleware] :query [params AddTicketQueryParams] :body [body data-schema/Paths] :responses schema/AddTicketResponses @@ -23,7 +21,6 @@ (ok (tickets/do-add-tickets params body)))) (POST "/ticket-lister" [] - :middleware [otel-middleware] :tags ["tickets"] :query [params StandardUserQueryParams] :body [body data-schema/Paths] @@ -33,7 +30,6 @@ (ok (tickets/do-list-tickets params body))) (POST "/ticket-deleter" [] - :middleware [otel-middleware] :tags ["tickets"] :query [params DeleteTicketQueryParams] :body [body schema/Tickets] diff --git a/src/data_info/routes/trash.clj b/src/data_info/routes/trash.clj index 7e854e6..86589be 100644 --- a/src/data_info/routes/trash.clj +++ b/src/data_info/routes/trash.clj @@ -1,7 +1,6 @@ (ns data-info.routes.trash (:use [common-swagger-api.schema] [common-swagger-api.schema.data :only [OptionalPaths]] - [otel.middleware :only [otel-middleware]] [data-info.routes.schemas.common] [data-info.routes.schemas.trash]) (:require [data-info.services.trash :as trash] @@ -9,7 +8,6 @@ (defroutes trash (DELETE "/trash" [:as {uri :uri}] - :middleware [otel-middleware] :tags ["data"] :query [params StandardUserQueryParams] :return Trash @@ -19,7 +17,6 @@ (svc/trap uri trash/do-delete-trash params)) (POST "/deleter" [:as {uri :uri}] - :middleware [otel-middleware] :tags ["bulk"] :query [params StandardUserQueryParams] :body [body (describe Paths "The paths to move to the trash")] @@ -32,7 +29,6 @@ (svc/trap uri trash/do-delete params body)) (POST "/restorer" [:as {uri :uri}] - :middleware [otel-middleware] :tags ["bulk"] :query [params StandardUserQueryParams] :body [body (describe OptionalPaths "The paths to restore, or an empty or missing list to restore the whole trash")] @@ -49,7 +45,6 @@ :tags ["data-by-id"] (DELETE "/" [:as {uri :uri}] - :middleware [otel-middleware] :query [params StandardUserQueryParams] :return TrashPaths :summary "Delete Data Item" @@ -60,7 +55,6 @@ (svc/trap uri trash/do-delete-uuid params data-id)) (DELETE "/children" [:as {uri :uri}] - :middleware [otel-middleware] :query [params StandardUserQueryParams] :return TrashPaths :summary "Delete Data Item Contents" diff --git a/src/data_info/routes/users.clj b/src/data_info/routes/users.clj index fc730ce..1cd6446 100644 --- a/src/data_info/routes/users.clj +++ b/src/data_info/routes/users.clj @@ -1,6 +1,5 @@ (ns data-info.routes.users (:use [common-swagger-api.schema] - [otel.middleware :only [otel-middleware]] [data-info.routes.schemas.common] [data-info.routes.schemas.users]) (:require [data-info.services.users :as users] @@ -12,7 +11,6 @@ :tags ["users"] (GET "/:username/groups" [:as {uri :uri}] - :middleware [otel-middleware] :path-params [username :- (describe String "The username whose groups should be listed")] :query [{:keys [user]} StandardUserQueryParams] :return UserGroupsReturn diff --git a/src/data_info/services/manifest.clj b/src/data_info/services/manifest.clj index d41bdff..33b0f1e 100644 --- a/src/data_info/services/manifest.clj +++ b/src/data_info/services/manifest.clj @@ -6,7 +6,6 @@ [clojure-commons.file-utils :as ft] [clj-irods.core :as rods] [clj-irods.validate :refer [validate]] - [otel.otel :as otel] [dire.core :refer [with-pre-hook! with-post-hook!]] [data-info.util.validators :as validators] [data-info.util.irods :as irods] @@ -19,28 +18,26 @@ (defn- extract-urls [irods user fpath] - (otel/with-span [s ["extract-urls"]] - (let [readable (anon-readable? irods fpath)] - (future (if @readable [(format-anon-files-url fpath)] []))))) + (let [readable (anon-readable? irods fpath)] + (future (if @readable [(format-anon-files-url fpath)] [])))) (defn- manifest [user path-or-uuid uuid?] - (otel/with-span [s ["manifest"]] - (irods/with-irods-exceptions {:use-icat-transaction false} irods - (validate irods [:user-exists user (cfg/irods-zone)]) - (let [path (ft/rm-last-slash - (if uuid? - @(rods/uuid->path irods path-or-uuid) - path-or-uuid))] - (validate irods - [:path-exists path user (cfg/irods-zone)] - [:path-is-file path user (cfg/irods-zone)] - [:path-readable path user (cfg/irods-zone)]) - (let [urls (extract-urls irods user path) - info-type (rods/info-type irods user (cfg/irods-zone) path)] - {:content-type (irods/detect-media-type (:jargon irods) path) - :infoType (or @info-type "unknown") - :urls @urls}))))) + (irods/with-irods-exceptions {:use-icat-transaction false} irods + (validate irods [:user-exists user (cfg/irods-zone)]) + (let [path (ft/rm-last-slash + (if uuid? + @(rods/uuid->path irods path-or-uuid) + path-or-uuid))] + (validate irods + [:path-exists path user (cfg/irods-zone)] + [:path-is-file path user (cfg/irods-zone)] + [:path-readable path user (cfg/irods-zone)]) + (let [urls (extract-urls irods user path) + info-type (rods/info-type irods user (cfg/irods-zone) path)] + {:content-type (irods/detect-media-type (:jargon irods) path) + :infoType (or @info-type "unknown") + :urls @urls})))) (defn do-manifest-uuid [user data-id] diff --git a/src/data_info/services/page_file.clj b/src/data_info/services/page_file.clj index 4f41684..cfd9389 100644 --- a/src/data_info/services/page_file.clj +++ b/src/data_info/services/page_file.clj @@ -6,7 +6,6 @@ [clojure-commons.file-utils :as ft] [clj-irods.core :as rods] [clj-irods.validate :refer [validate]] - [otel.otel :as otel] [dire.core :refer [with-pre-hook! with-post-hook!]] [data-info.services.uuids :as uuids] [data-info.util.config :as cfg] @@ -16,25 +15,24 @@ (defn- read-file-chunk "Reads a chunk of a file starting at 'position' and reading a chunk of length 'chunk-size'." [user path-or-uuid position chunk-size uuid?] - (otel/with-span [s ["read-file-chunk"]] - (irods/with-irods-exceptions {:use-icat-transaction false} irods - (future (force (:jargon irods))) + (irods/with-irods-exceptions {:use-icat-transaction false} irods + (future (force (:jargon irods))) + (validate irods + [:user-exists user (cfg/irods-zone)]) + (let [path (ft/rm-last-slash + (if uuid? + @(rods/uuid->path irods path-or-uuid) + path-or-uuid))] (validate irods - [:user-exists user (cfg/irods-zone)]) - (let [path (ft/rm-last-slash - (if uuid? - @(rods/uuid->path irods path-or-uuid) - path-or-uuid))] - (validate irods - [:path-exists path user (cfg/irods-zone)] - [:path-is-file path user (cfg/irods-zone)] - [:path-readable path user (cfg/irods-zone)]) - {:path path - :user user - :start (str position) - :chunk-size (str chunk-size) - :file-size (str @(rods/file-size irods user (cfg/irods-zone) path)) - :chunk (read-at-position @(:jargon irods) path position chunk-size)})))) + [:path-exists path user (cfg/irods-zone)] + [:path-is-file path user (cfg/irods-zone)] + [:path-readable path user (cfg/irods-zone)]) + {:path path + :user user + :start (str position) + :chunk-size (str chunk-size) + :file-size (str @(rods/file-size irods user (cfg/irods-zone) path)) + :chunk (read-at-position @(:jargon irods) path position chunk-size)}))) (defn do-read-chunk-uuid [{user :user position :position chunk-size :size} data-id] diff --git a/src/data_info/services/page_tabular.clj b/src/data_info/services/page_tabular.clj index bbe0342..b2f34df 100644 --- a/src/data_info/services/page_tabular.clj +++ b/src/data_info/services/page_tabular.clj @@ -10,7 +10,6 @@ [clojure-commons.file-utils :as ft] [clj-irods.core :as rods] [clj-irods.validate :refer [validate]] - [otel.otel :as otel] [dire.core :refer [with-pre-hook! with-post-hook!]] [data-info.services.uuids :as uuids] [data-info.util.config :as cfg] @@ -24,8 +23,7 @@ (defn read-csv-stream [^String separator ^InputStream stream] - (otel/with-span [s ["read-csv-stream"]] - (.readAll (CSVReader. stream (.charAt separator 0))))) + (.readAll (CSVReader. stream (.charAt separator 0)))) (defn- chunk-start [page chunk-size] @@ -53,11 +51,10 @@ (defn- trim-chunk [^String chunk chunk-size page pages] - (otel/with-span [s ["trim-chunk"]] - (log/debug "untrimmed chunk:" chunk) - (let [lstart (seek-prev-line chunk (chunk-start page chunk-size)) - lend (end-pos chunk page pages)] - (.substring chunk lstart lend)))) + (log/debug "untrimmed chunk:" chunk) + (let [lstart (seek-prev-line chunk (chunk-start page chunk-size)) + lend (end-pos chunk page pages)] + (.substring chunk lstart lend))) (defn- fix-record [record] @@ -67,13 +64,12 @@ (defn- read-csv [^String csv-str ^String separator] - (otel/with-span [s ["read-csv"]] - (if-not (string/blank? csv-str) - (let [ba (java.io.ByteArrayInputStream. (.getBytes csv-str)) - isr (java.io.InputStreamReader. ba "UTF-8")] - (map fix-record (mapv #(zipmap (mapv str (range (count %1))) %1) - (mapv vec (read-csv-stream separator isr))))) - [{}]))) + (if-not (string/blank? csv-str) + (let [ba (java.io.ByteArrayInputStream. (.getBytes csv-str)) + isr (java.io.InputStreamReader. ba "UTF-8")] + (map fix-record (mapv #(zipmap (mapv str (range (count %1))) %1) + (mapv vec (read-csv-stream separator isr))))) + [{}])) (defn- num-pages [chunk-size file-size] @@ -94,46 +90,45 @@ we shouldn't try to parse partial rows. We scan forward from the starting position to find the first line-ending and then scan backwards from the last position for the last line-ending." [user path-or-uuid page chunk-size separator uuid?] - (otel/with-span [s ["read-csv-chunk"]] - (irods/with-irods-exceptions {:use-icat-transaction false} irods - (log/warn "[read-csv-chunk]" user path-or-uuid page chunk-size separator) - (future (force (:jargon irods))) + (irods/with-irods-exceptions {:use-icat-transaction false} irods + (log/warn "[read-csv-chunk]" user path-or-uuid page chunk-size separator) + (future (force (:jargon irods))) + (validate irods + [:user-exists user (cfg/irods-zone)]) + (let [path (ft/rm-last-slash + (if uuid? + @(rods/uuid->path irods path-or-uuid) + path-or-uuid))] (validate irods - [:user-exists user (cfg/irods-zone)]) - (let [path (ft/rm-last-slash - (if uuid? - @(rods/uuid->path irods path-or-uuid) - path-or-uuid))] - (validate irods - [:path-exists path user (cfg/irods-zone)] - [:path-is-file path user (cfg/irods-zone)] - [:path-readable path user (cfg/irods-zone)]) - (let [page (dec page) - start-pg (if (= page 0) 0 (dec page)) - full-chunk-size (calc-chunk-size page chunk-size) - fsize (deref (rods/file-size irods user (cfg/irods-zone) path)) - pages (num-pages chunk-size fsize) - position (start-pos page chunk-size) - load-pos (start-pos start-pg chunk-size)] - (log/debug "reading from" load-pos "for" full-chunk-size) - - (when-not (<= page pages) - (throw+ {:error_code "ERR_INVALID_PAGE" - :page (str page) - :number-pages (str pages)})) - - (let [^String chunk (trim-chunk (read-at-position @(:jargon irods) path load-pos full-chunk-size) chunk-size page pages) - the-csv (read-csv chunk separator)] - (log/debug "trimmed chunk for page" (inc page) ":" chunk) - (log/debug "parsed csv" the-csv) - {:path path - :page (str (inc page)) - :number-pages (str pages) - :user user - :max-cols (str (reduce #(if (>= %1 %2) %1 %2) (map count the-csv))) - :chunk-size (str (count (.getBytes chunk))) - :file-size (str fsize) - :csv the-csv})))))) + [:path-exists path user (cfg/irods-zone)] + [:path-is-file path user (cfg/irods-zone)] + [:path-readable path user (cfg/irods-zone)]) + (let [page (dec page) + start-pg (if (= page 0) 0 (dec page)) + full-chunk-size (calc-chunk-size page chunk-size) + fsize (deref (rods/file-size irods user (cfg/irods-zone) path)) + pages (num-pages chunk-size fsize) + position (start-pos page chunk-size) + load-pos (start-pos start-pg chunk-size)] + (log/debug "reading from" load-pos "for" full-chunk-size) + + (when-not (<= page pages) + (throw+ {:error_code "ERR_INVALID_PAGE" + :page (str page) + :number-pages (str pages)})) + + (let [^String chunk (trim-chunk (read-at-position @(:jargon irods) path load-pos full-chunk-size) chunk-size page pages) + the-csv (read-csv chunk separator)] + (log/debug "trimmed chunk for page" (inc page) ":" chunk) + (log/debug "parsed csv" the-csv) + {:path path + :page (str (inc page)) + :number-pages (str pages) + :user user + :max-cols (str (reduce #(if (>= %1 %2) %1 %2) (map count the-csv))) + :chunk-size (str (count (.getBytes chunk))) + :file-size (str fsize) + :csv the-csv}))))) (with-pre-hook! #'read-csv-chunk (fn [user path-or-uuid page chunk-size separator uuid?] diff --git a/src/data_info/services/path_lists.clj b/src/data_info/services/path_lists.clj index dc04f43..88ed1eb 100644 --- a/src/data_info/services/path_lists.clj +++ b/src/data_info/services/path_lists.clj @@ -8,7 +8,6 @@ [clj-irods.core :as rods] [clj-irods.validate :refer [validate]] [clj-jargon.permissions :as perms] - [otel.otel :as otel] [data-info.services.filetypes :as filetypes] [data-info.services.stat :as stat] [data-info.services.stat.common :refer [process-filters]] @@ -40,29 +39,27 @@ (defn- folder-listing "Fetches a folder's contents, listing files, folders, or both depending on the given parameters." [user info-types folders-only? recursive? path] - (otel/with-span [s ["folder-listing"]] - (map fmt-entry - (icat/paged-folder-listing - :user user - :zone (cfg/irods-zone) - :folder-path path - :info-types info-types - :entity-type (filter-entity-type recursive? folders-only?) - :sort-column :full-path - :sort-direction :asc - :limit nil - :offset 0)))) + (map fmt-entry + (icat/paged-folder-listing + :user user + :zone (cfg/irods-zone) + :folder-path path + :info-types info-types + :entity-type (filter-entity-type recursive? folders-only?) + :sort-column :full-path + :sort-direction :asc + :limit nil + :offset 0))) (defn- list-item-with-subitems [user info-types folders-only? recursive? {:keys [path] :as data-item}] "If given a file, returns that file as the only item in a list. If given a folder, returns that folder in a list, and the list may include the folder's subfolder/files depending on the given parameters." - (otel/with-span [s ["list-item-with-subitems"]] - (let [sub-listing (when (and recursive? (stat-is-dir? data-item)) - (mapcat (partial list-item-with-subitems user info-types folders-only? recursive?) - (folder-listing user info-types folders-only? recursive? path)))] - (concat [data-item] sub-listing)))) + (let [sub-listing (when (and recursive? (stat-is-dir? data-item)) + (mapcat (partial list-item-with-subitems user info-types folders-only? recursive?) + (folder-listing user info-types folders-only? recursive? path)))] + (concat [data-item] sub-listing))) (defn- keep-top-level-file? "A filter predicate to keep only files with an info-type that matches one in the given `info-types` list, @@ -97,24 +94,23 @@ "Filters the given paths and returns a string of these paths appended to an HT Path List header. Throws an error if the filtering params result in no matching paths." [irods user path-list-file-identifier name-pattern info-types folders-only? recursive? paths] - (otel/with-span [s ["paths->path-list"]] - (let [zone-from-path (fn [path] (first (remove empty? (string/split path #"/")))) - {folder-paths true file-paths false} (group-by #(= @(rods/object-type irods user (zone-from-path %) %) :dir) paths) - files (->> file-paths - (map (partial get-top-level-file-stats irods user)) - (filter (keep-top-level-file? info-types))) - folders (->> folder-paths - (mapcat (partial folder-listing user info-types folders-only? recursive?)) - (mapcat (partial list-item-with-subitems user info-types folders-only? recursive?))) - filtered-paths (->> (concat files folders) - (filter (partial keep-data-item? name-pattern folders-only?)) - (map :path))] - - (when (empty? filtered-paths) - (throw+ {:error_code ERR_NOT_FOUND - :reason "No paths matched the request."})) - - (string/join "\n" (concat [path-list-file-identifier] filtered-paths))))) + (let [zone-from-path (fn [path] (first (remove empty? (string/split path #"/")))) + {folder-paths true file-paths false} (group-by #(= @(rods/object-type irods user (zone-from-path %) %) :dir) paths) + files (->> file-paths + (map (partial get-top-level-file-stats irods user)) + (filter (keep-top-level-file? info-types))) + folders (->> folder-paths + (mapcat (partial folder-listing user info-types folders-only? recursive?)) + (mapcat (partial list-item-with-subitems user info-types folders-only? recursive?))) + filtered-paths (->> (concat files folders) + (filter (partial keep-data-item? name-pattern folders-only?)) + (map :path))] + + (when (empty? filtered-paths) + (throw+ {:error_code ERR_NOT_FOUND + :reason "No paths matched the request."})) + + (string/join "\n" (concat [path-list-file-identifier] filtered-paths)))) (defn- info-type->file-identifier "Returns the appropriate Path List file identifier for the given `path-list-info-type`." @@ -145,20 +141,19 @@ [{:keys [user dest path-list-info-type name-pattern info-type folders-only recursive] :or {path-list-info-type (cfg/ht-path-list-info-type)}} {:keys [paths]}] - (otel/with-span [s ["create-path-list"]] - (irods/with-irods-exceptions {:jargon-opts {:client-user user}} irods - (validate-request-paths irods user dest paths) - - (let [path-list-contents (paths->path-list irods - user - (info-type->file-identifier path-list-info-type) - name-pattern - info-type - folders-only - recursive - paths) - path-list-file-stat (with-in-str path-list-contents (copy-stream @(:jargon irods) *in* user dest))] - (irods/with-jargon-exceptions [admin-cm] - (filetypes/add-type-to-validated-path admin-cm dest path-list-info-type)) - (rods/invalidate irods dest) - {:file (stat/decorate-stat irods user (cfg/irods-zone) path-list-file-stat (process-filters nil nil))})))) + (irods/with-irods-exceptions {:jargon-opts {:client-user user}} irods + (validate-request-paths irods user dest paths) + + (let [path-list-contents (paths->path-list irods + user + (info-type->file-identifier path-list-info-type) + name-pattern + info-type + folders-only + recursive + paths) + path-list-file-stat (with-in-str path-list-contents (copy-stream @(:jargon irods) *in* user dest))] + (irods/with-jargon-exceptions [admin-cm] + (filetypes/add-type-to-validated-path admin-cm dest path-list-info-type)) + (rods/invalidate irods dest) + {:file (stat/decorate-stat irods user (cfg/irods-zone) path-list-file-stat (process-filters nil nil))}))) diff --git a/src/data_info/services/rename.clj b/src/data_info/services/rename.clj index 8ebcc7e..0c9ed07 100644 --- a/src/data_info/services/rename.clj +++ b/src/data_info/services/rename.clj @@ -14,8 +14,7 @@ [data-info.util.config :as cfg] [data-info.util.irods :as irods] [data-info.util.logging :as dul] - [data-info.util.validators :as validators] - [otel.otel :as otel])) + [data-info.util.validators :as validators])) (defn- source->dest [source-path dest-path] @@ -44,34 +43,33 @@ ;; Locked dest + '/' prefix of new dest, we're moving stuff into new dest with two processes ;; New dest + '/' prefix of locked source, we might be moving stuff into locked source while it's being moved out ;; New dest + '/' prefix of locked dest, we're moving stuff into locked dest with two processes - (otel/with-span [s ["validate-unlocked"]] - (let [far-future "9999-12-31T23:59:59Z" - eligible-async-task-types ["data-move" "data-rename" "data-delete" "data-delete-trash" "data-restore"] - eligible-tasks (async-tasks/get-by-filter {:type eligible-async-task-types - :include_null_end true - :end_date_since far-future}) - add-destination-to-basenames (fn [destination sources] - (map #(ft/path-join destination %) (map ft/basename sources))) - extract-paths (fn [{:keys [data type]}] - (condp = type - "data-move" (concat (add-destination-to-basenames (:destination data) (:sources data)) - (:sources data)) - "data-rename" (map #(get data %) [:destination :source]) - "data-delete" (concat (:paths data) (vals (:trash-paths data))) - "data-delete-trash" (:trash-paths data) - "data-restore" (concat (:paths data) - (map :restored-path (vals (:restoration-paths data)))) - nil)) - locked-paths (reduce conj #{} (mapcat extract-paths eligible-tasks)) - path-matches (fn [path] (or - (get locked-paths path) - (some #(string/starts-with? (ft/add-trailing-slash %) path) locked-paths) ;; locked path + '/' prefix of new path - (some #(string/starts-with? (ft/add-trailing-slash path) %) locked-paths) ;; new path + '/' prefix of locked path - )) - matching-paths (filterv path-matches paths)] - (if (seq matching-paths) - (throw+ {:error_code error/ERR_CONFLICT - :paths matching-paths})))))) + (let [far-future "9999-12-31T23:59:59Z" + eligible-async-task-types ["data-move" "data-rename" "data-delete" "data-delete-trash" "data-restore"] + eligible-tasks (async-tasks/get-by-filter {:type eligible-async-task-types + :include_null_end true + :end_date_since far-future}) + add-destination-to-basenames (fn [destination sources] + (map #(ft/path-join destination %) (map ft/basename sources))) + extract-paths (fn [{:keys [data type]}] + (condp = type + "data-move" (concat (add-destination-to-basenames (:destination data) (:sources data)) + (:sources data)) + "data-rename" (map #(get data %) [:destination :source]) + "data-delete" (concat (:paths data) (vals (:trash-paths data))) + "data-delete-trash" (:trash-paths data) + "data-restore" (concat (:paths data) + (map :restored-path (vals (:restoration-paths data)))) + nil)) + locked-paths (reduce conj #{} (mapcat extract-paths eligible-tasks)) + path-matches (fn [path] (or + (get locked-paths path) + (some #(string/starts-with? (ft/add-trailing-slash %) path) locked-paths) ;; locked path + '/' prefix of new path + (some #(string/starts-with? (ft/add-trailing-slash path) %) locked-paths) ;; new path + '/' prefix of locked path + )) + matching-paths (filterv path-matches paths)] + (if (seq matching-paths) + (throw+ {:error_code error/ERR_CONFLICT + :paths matching-paths}))))) (defn- move-paths-thread [async-task-id] @@ -111,49 +109,47 @@ (defn- move-paths "As 'user', moves objects in 'sources' into the directory in 'dest', establishing an asynchronous task and processing in another thread." [user sources dest] - (otel/with-span [s ["move-paths"]] - (let [all-paths (apply merge (mapv #(hash-map (source->dest %1 dest) %1) sources)) - dest-paths (keys all-paths) - sources (mapv ft/rm-last-slash sources) - dest (ft/rm-last-slash dest)] - (validate-unlocked sources dest-paths) - (irods/with-jargon-exceptions :client-user user [cm] - (validators/user-exists cm user) - (validators/all-paths-exist cm sources) - (validators/all-paths-exist cm [dest]) - (validators/path-is-dir cm dest) - (validators/user-owns-paths cm user sources) - (validators/path-writeable cm user dest) - (validators/no-paths-exist cm dest-paths)) - (let [async-task-id (async-tasks/run-async-thread - (new-task "data-move" user {:sources sources :destination dest}) - move-paths-thread "data-move")] - {:user user :sources sources :dest dest :async-task-id async-task-id})))) + (let [all-paths (apply merge (mapv #(hash-map (source->dest %1 dest) %1) sources)) + dest-paths (keys all-paths) + sources (mapv ft/rm-last-slash sources) + dest (ft/rm-last-slash dest)] + (validate-unlocked sources dest-paths) + (irods/with-jargon-exceptions :client-user user [cm] + (validators/user-exists cm user) + (validators/all-paths-exist cm sources) + (validators/all-paths-exist cm [dest]) + (validators/path-is-dir cm dest) + (validators/user-owns-paths cm user sources) + (validators/path-writeable cm user dest) + (validators/no-paths-exist cm dest-paths)) + (let [async-task-id (async-tasks/run-async-thread + (new-task "data-move" user {:sources sources :destination dest}) + move-paths-thread "data-move")] + {:user user :sources sources :dest dest :async-task-id async-task-id}))) (defn- rename-path "As 'user', move 'source' to 'dest', establishing an asynchronous task and processing in another thread." [user source dest] - (otel/with-span [s ["rename-path"]] - (let [source (ft/rm-last-slash source) - dest (ft/rm-last-slash dest) - src-base (ft/basename source) - dest-base (ft/basename dest)] - (if (= source dest) - {:source source :dest dest :user user} - (do - (validate-unlocked [source dest]) - (irods/with-jargon-exceptions :client-user user [cm] - (validators/user-exists cm user) - (validators/all-paths-exist cm [source (ft/dirname dest)]) - (validators/path-is-dir cm (ft/dirname dest)) - (validators/user-owns-path cm user source) - (if-not (= (ft/dirname source) (ft/dirname dest)) - (validators/path-writeable cm user (ft/dirname dest))) - (validators/path-not-exists cm dest)) - (let [async-task-id (async-tasks/run-async-thread - (new-task "data-rename" user {:source source :destination dest}) - rename-path-thread "data-rename")] - {:user user :source source :dest dest :async-task-id async-task-id})))))) + (let [source (ft/rm-last-slash source) + dest (ft/rm-last-slash dest) + src-base (ft/basename source) + dest-base (ft/basename dest)] + (if (= source dest) + {:source source :dest dest :user user} + (do + (validate-unlocked [source dest]) + (irods/with-jargon-exceptions :client-user user [cm] + (validators/user-exists cm user) + (validators/all-paths-exist cm [source (ft/dirname dest)]) + (validators/path-is-dir cm (ft/dirname dest)) + (validators/user-owns-path cm user source) + (if-not (= (ft/dirname source) (ft/dirname dest)) + (validators/path-writeable cm user (ft/dirname dest))) + (validators/path-not-exists cm dest)) + (let [async-task-id (async-tasks/run-async-thread + (new-task "data-rename" user {:source source :destination dest}) + rename-path-thread "data-rename")] + {:user user :source source :dest dest :async-task-id async-task-id}))))) (defn- rename-uuid "Rename by UUID: given a user, a source file UUID, and a new name, rename within the same folder." diff --git a/src/data_info/services/sharing.clj b/src/data_info/services/sharing.clj index aad7cba..4503a24 100644 --- a/src/data_info/services/sharing.clj +++ b/src/data_info/services/sharing.clj @@ -7,7 +7,6 @@ [clojure-commons.file-utils :as ft] [clj-irods.core :as rods] [cemerick.url :as url] - [otel.otel :as otel] [dire.core :refer [with-pre-hook! with-post-hook!]] [data-info.util.logging :as dul] [data-info.util.paths :as paths] diff --git a/src/data_info/services/stat.clj b/src/data_info/services/stat.clj index 8731419..d551ba5 100644 --- a/src/data_info/services/stat.clj +++ b/src/data_info/services/stat.clj @@ -1,6 +1,5 @@ (ns data-info.services.stat (:require [dire.core :refer [with-pre-hook! with-post-hook!]] - [otel.otel :as otel] [clj-irods.core :as rods] [clj-irods.validate :refer [validate]] [clojure-commons.file-utils :as ft] @@ -30,52 +29,47 @@ (defn- merge-counts [stat-map irods user zone path included-keys] (if (and (needs-any-key? included-keys :file-count :dir-count) (is-dir? stat-map)) - (otel/with-span [s ["merge-counts"]] - (assoc-if-selected stat-map included-keys - :file-count @(rods/number-of-files-in-folder irods user zone path) - :dir-count @(rods/number-of-folders-in-folder irods user zone path))) + (assoc-if-selected stat-map included-keys + :file-count @(rods/number-of-files-in-folder irods user zone path) + :dir-count @(rods/number-of-folders-in-folder irods user zone path)) stat-map)) (defn- merge-shares [stat-map irods user path included-keys] (if (and (needs-key? included-keys :share-count) (owns? stat-map)) - (otel/with-span [s ["merge-shares"]] - (assoc stat-map :share-count (count-shares irods user path))) + (assoc stat-map :share-count (count-shares irods user path)) stat-map)) (defn- merge-type-info [stat-map irods user zone path included-keys & {:keys [validate?] :or {validate? true}}] (if (and (needs-any-key? included-keys :infoType :content-type) (not (is-dir? stat-map))) - (otel/with-span [s ["merge-type-info"]] - (assoc-if-selected stat-map included-keys - :infoType (get-types irods user zone path :validate? validate?) - :content-type (irods/detect-media-type @(:jargon irods) path))) + (assoc-if-selected stat-map included-keys + :infoType (get-types irods user zone path :validate? validate?) + :content-type (irods/detect-media-type @(:jargon irods) path)) stat-map)) (defn decorate-stat [irods user zone {:keys [path] :as stat} included-keys & {:keys [validate?] :or {validate? true}}] - (otel/with-span [s ["decorate-stat"]] - (-> stat - (assoc-if-selected included-keys - :id @(rods/uuid irods user zone path) - :permission @(rods/permission irods user zone path)) - (merge-label user path included-keys) - (merge-type-info irods user zone path included-keys :validate? validate?) - (merge-shares irods user path included-keys) - (merge-counts irods user zone path included-keys) - (select-keys included-keys)))) + (-> stat + (assoc-if-selected included-keys + :id @(rods/uuid irods user zone path) + :permission @(rods/permission irods user zone path)) + (merge-label user path included-keys) + (merge-type-info irods user zone path included-keys :validate? validate?) + (merge-shares irods user path included-keys) + (merge-counts irods user zone path included-keys) + (select-keys included-keys))) (defn path-stat [irods user path & {:keys [filter-include filter-exclude validate?] :or {filter-include nil filter-exclude nil validate? true}}] - (otel/with-span [s ["path-stat" {:attributes {"path" path}}]] - (let [path (ft/rm-last-slash path) - included-keys (process-filters filter-include filter-exclude)] - (when validate? (validate irods [:path-exists path user (cfg/irods-zone)])) - (let [base-stat (if (needs-any-key? included-keys :type :date-created :date-modified :file-size :md5) - @(rods/stat irods user (cfg/irods-zone) path) - {:path path})] - (decorate-stat irods user (cfg/irods-zone) base-stat included-keys :validate? validate?))))) + (let [path (ft/rm-last-slash path) + included-keys (process-filters filter-include filter-exclude)] + (when validate? (validate irods [:path-exists path user (cfg/irods-zone)])) + (let [base-stat (if (needs-any-key? included-keys :type :date-created :date-modified :file-size :md5) + @(rods/stat irods user (cfg/irods-zone) path) + {:path path})] + (decorate-stat irods user (cfg/irods-zone) base-stat included-keys :validate? validate?)))) (defn- remove-missing-paths "Removes non-existent paths from a list of item paths." diff --git a/src/data_info/services/stat/jargon.clj b/src/data_info/services/stat/jargon.clj index aecba33..4bb3846 100644 --- a/src/data_info/services/stat/jargon.clj +++ b/src/data_info/services/stat/jargon.clj @@ -11,8 +11,7 @@ [data-info.services.uuids :as uuids] [data-info.util.config :as cfg] [data-info.util.irods :as irods] - [data-info.util.validators :as validators] - [otel.otel :as otel])) + [data-info.util.validators :as validators])) (defn- get-types "Gets the file type associated with path." @@ -28,10 +27,9 @@ (defn- merge-type-info [stat-map cm user path included-keys & {:keys [validate?] :or {validate? true}}] (if (and (needs-any-key? included-keys :infoType :content-type) (not (is-dir? stat-map))) - (otel/with-span [s ["merge-type-info"]] - (assoc-if-selected stat-map included-keys - :infoType (get-types cm user path :validate? validate?) - :content-type (irods/detect-media-type cm path))) + (assoc-if-selected stat-map included-keys + :infoType (get-types cm user path :validate? validate?) + :content-type (irods/detect-media-type cm path)) stat-map)) (defn- count-shares @@ -43,48 +41,43 @@ (defn- merge-shares [stat-map cm user path included-keys] (if (and (needs-key? included-keys :share-count) (owns? stat-map)) - (otel/with-span [s ["merge-shares"]] - (assoc stat-map :share-count (count-shares cm user path))) + (assoc stat-map :share-count (count-shares cm user path)) stat-map)) (defn- merge-counts [stat-map cm user path included-keys] (if (and (needs-any-key? included-keys :file-count :dir-count) (is-dir? stat-map)) - (otel/with-span [s ["merge-counts"]] - (assoc-if-selected stat-map included-keys - :file-count (icat/number-of-files-in-folder user (cfg/irods-zone) path) - :dir-count (icat/number-of-folders-in-folder user (cfg/irods-zone) path))) + (assoc-if-selected stat-map included-keys + :file-count (icat/number-of-files-in-folder user (cfg/irods-zone) path) + :dir-count (icat/number-of-folders-in-folder user (cfg/irods-zone) path)) stat-map)) (defn decorate-stat [cm user {:keys [path] :as stat} included-keys & {:keys [validate?] :or {validate? true}}] - (otel/with-span [s ["decorate-stat"]] - (-> stat - (assoc-if-selected included-keys - :id (-> (meta/get-attribute cm path uuid/uuid-attr) first :value) - :permission (perm/permission-for cm user path)) - (merge-label user path included-keys) - (merge-type-info cm user path included-keys :validate? validate?) - (merge-shares cm user path included-keys) - (merge-counts cm user path included-keys) - (select-keys included-keys)))) + (-> stat + (assoc-if-selected included-keys + :id (-> (meta/get-attribute cm path uuid/uuid-attr) first :value) + :permission (perm/permission-for cm user path)) + (merge-label user path included-keys) + (merge-type-info cm user path included-keys :validate? validate?) + (merge-shares cm user path included-keys) + (merge-counts cm user path included-keys) + (select-keys included-keys))) (defn path-stat [cm user path & {:keys [filter-include filter-exclude validate?] :or {filter-include nil filter-exclude nil validate? true}}] - (otel/with-span [s ["path-stat" {:attributes {"path" path}}]] - (let [path (ft/rm-last-slash path) - included-keys (process-filters filter-include filter-exclude)] - (log/debug "[path-stat] user:" user "path:" path) - (when validate? (validators/path-exists cm path)) - (let [base-stat (if (needs-any-key? included-keys :type :date-created :date-modified :file-size :md5) - (info/stat cm path) - {:path path})] - (decorate-stat cm user base-stat included-keys :validate? validate?))))) + (let [path (ft/rm-last-slash path) + included-keys (process-filters filter-include filter-exclude)] + (log/debug "[path-stat] user:" user "path:" path) + (when validate? (validators/path-exists cm path)) + (let [base-stat (if (needs-any-key? included-keys :type :date-created :date-modified :file-size :md5) + (info/stat cm path) + {:path path})] + (decorate-stat cm user base-stat included-keys :validate? validate?)))) (defn uuid-stat [cm user uuid & {:keys [filter-include filter-exclude] :or {filter-include nil filter-exclude nil}}] - (otel/with-span [s ["uuid-stat"]] - (log/debug "[uuid-stat] user:" user "uuid:" uuid) - (let [path (uuids/path-for-uuid cm user uuid)] - (path-stat cm user path :filter-include filter-include :filter-exclude filter-exclude)))) + (log/debug "[uuid-stat] user:" user "uuid:" uuid) + (let [path (uuids/path-for-uuid cm user uuid)] + (path-stat cm user path :filter-include filter-include :filter-exclude filter-exclude))) diff --git a/src/data_info/services/status.clj b/src/data_info/services/status.clj index 4c69a9b..42dee40 100644 --- a/src/data_info/services/status.clj +++ b/src/data_info/services/status.clj @@ -3,18 +3,16 @@ [clj-jargon.init :as init] [clj-jargon.item-info :as item] [clojure-commons.error-codes :as ce] - [otel.otel :as otel] [data-info.util.config :as cfg])) (defn ^Boolean irods-running? "Determines whether or not iRODS is running." [] - (otel/with-span [s ["irods-running?"]] - (try - (init/with-jargon (cfg/jargon-cfg) [cm] - (item/exists? cm (:home cm))) - (catch Exception e - (log/error "Error performing iRODS status check:") - (log/error (ce/format-exception e)) - false)))) + (try + (init/with-jargon (cfg/jargon-cfg) [cm] + (item/exists? cm (:home cm))) + (catch Exception e + (log/error "Error performing iRODS status check:") + (log/error (ce/format-exception e)) + false))) diff --git a/src/data_info/services/tickets.clj b/src/data_info/services/tickets.clj index 2706f32..41c3344 100644 --- a/src/data_info/services/tickets.clj +++ b/src/data_info/services/tickets.clj @@ -5,7 +5,7 @@ (:require [clojure.tools.logging :as log] [clojure.string :as string] [clojure-commons.file-utils :as ft] - [clostache.parser :as stache] + [clojure-commons.template :refer [render]] [dire.core :refer [with-pre-hook! with-post-hook!]] [data-info.util.logging :as dul] [data-info.util.config :as cfg] @@ -29,9 +29,9 @@ (defn- render-ticket-tmpl [cm ticket-map tmpl] - (stache/render tmpl {:url (cfg/kifshare-external-url) - :ticket-id (:ticket-id ticket-map) - :filename (ft/basename (:path ticket-map))})) + (render tmpl {:url (cfg/kifshare-external-url) + :ticket-id (:ticket-id ticket-map) + :filename (ft/basename (:path ticket-map))})) (defn- url-join [url path] diff --git a/src/data_info/services/uuids.clj b/src/data_info/services/uuids.clj index 884f9eb..ea30930 100644 --- a/src/data_info/services/uuids.clj +++ b/src/data_info/services/uuids.clj @@ -7,7 +7,6 @@ [clj-irods.core :as rods] [clj-irods.validate :refer [validate]] [clojure-commons.error-codes :as error] - [otel.otel :as otel] [data-info.util.irods :as irods] [data-info.util.config :as cfg]) (:import [java.util UUID] @@ -24,13 +23,12 @@ Returns: It returns a path." ([^IPersistentMap cm ^String user ^UUID uuid] - (otel/with-span [s ["path-for-uuid" {:attributes {"uuid" (str uuid)}}]] - (if-let [path (uuid/get-path cm uuid)] - path - (throw+ {:error_code error/ERR_DOES_NOT_EXIST :uuid uuid})))) + (if-let [path (uuid/get-path cm uuid)] + path + (throw+ {:error_code error/ERR_DOES_NOT_EXIST :uuid uuid}))) ([^String user ^UUID uuid] (irods/with-jargon-exceptions [cm] - (path-for-uuid cm user uuid)))) + (path-for-uuid cm user uuid)))) (defn do-simple-uuid-for-path [{:keys [user path]}] diff --git a/src/data_info/services/write.clj b/src/data_info/services/write.clj index 3b3e831..3cc3988 100644 --- a/src/data_info/services/write.clj +++ b/src/data_info/services/write.clj @@ -10,7 +10,6 @@ [clj-irods.validate :refer [validate]] [heuristomancer.core :as hm] [ring.middleware.multipart-params :as multipart] - [otel.otel :as otel] [data-info.services.stat :as stat] [data-info.services.stat.common :refer [process-filters]] [data-info.services.uuids :as uuids] @@ -36,77 +35,70 @@ (defn- get-info-type [istream-ref] - (otel/with-span [s ["get-info-type"]] - (.mark @istream-ref (cfg/type-detect-read-amount)) - (let [data (hm/sip (reset-on-close-istream @istream-ref) (cfg/type-detect-read-amount))] - (future - (otel/with-span [s ["identify-sample"]] - (let [result (hm/identify-sample data)] - (if-not (nil? result) - (name result) - "unknown"))))))) + (.mark @istream-ref (cfg/type-detect-read-amount)) + (let [data (hm/sip (reset-on-close-istream @istream-ref) (cfg/type-detect-read-amount))] + (future + (let [result (hm/identify-sample data)] + (if-not (nil? result) + (name result) + "unknown"))))) (defn- set-info-type [cm path info-type] - (otel/with-span [s ["set-info-type"]] - (let [existing (meta/get-attribute cm path (cfg/type-detect-type-attribute) :known-type :file)] - (if (seq existing) - (:value (first existing) "") - (do (log/info "adding type" info-type " to file " path) - (meta/add-metadata cm path (cfg/type-detect-type-attribute) info-type "ipc-data-info-detected" :known-type :file) - info-type))))) + (let [existing (meta/get-attribute cm path (cfg/type-detect-type-attribute) :known-type :file)] + (if (seq existing) + (:value (first existing) "") + (do (log/info "adding type" info-type " to file " path) + (meta/add-metadata cm path (cfg/type-detect-type-attribute) info-type "ipc-data-info-detected" :known-type :file) + info-type)))) (defn- save-file-contents "Save an istream to a destination. Relies on upstream functions to validate." [irods istream-raw user dest-path set-owner?] - (otel/with-span [s ["save-file-contents"]] - (let [istream-ref (delay (io/input-stream istream-raw)) - media-type (irods/detect-media-type (:jargon irods) dest-path istream-ref) - info-type (get-info-type istream-ref) - base-stat (ops/copy-stream @(:jargon irods) @istream-ref user dest-path :set-owner? set-owner?) - final-info-type (irods/with-jargon-exceptions [admin-cm] (set-info-type admin-cm dest-path @info-type))] - (log/info "Detected info-type:" @info-type ", final type:" final-info-type) - (rods/invalidate irods dest-path) - (assoc - (stat/decorate-stat irods user (cfg/irods-zone) base-stat (process-filters nil [:content-type :infoType]) :validate? false) - :infoType final-info-type - :content-type media-type)))) + (let [istream-ref (delay (io/input-stream istream-raw)) + media-type (irods/detect-media-type (:jargon irods) dest-path istream-ref) + info-type (get-info-type istream-ref) + base-stat (ops/copy-stream @(:jargon irods) @istream-ref user dest-path :set-owner? set-owner?) + final-info-type (irods/with-jargon-exceptions [admin-cm] (set-info-type admin-cm dest-path @info-type))] + (log/info "Detected info-type:" @info-type ", final type:" final-info-type) + (rods/invalidate irods dest-path) + (assoc + (stat/decorate-stat irods user (cfg/irods-zone) base-stat (process-filters nil [:content-type :infoType]) :validate? false) + :infoType final-info-type + :content-type media-type))) (defn- create-at-path "Create a new file at dest-path from istream. Error if the path exists or if the destination directory does not exist or is not writeable." [irods istream user dest-path] - (otel/with-span [s ["create-at-path"]] - (let [dest-dir (ft/dirname dest-path)] - (validate irods - [:path-not-exists dest-path user (cfg/irods-zone)] - [:path-exists dest-dir user (cfg/irods-zone)] - [:path-writeable dest-dir user (cfg/irods-zone)]) - (save-file-contents irods istream user dest-path true)))) + (let [dest-dir (ft/dirname dest-path)] + (validate irods + [:path-not-exists dest-path user (cfg/irods-zone)] + [:path-exists dest-dir user (cfg/irods-zone)] + [:path-writeable dest-dir user (cfg/irods-zone)]) + (save-file-contents irods istream user dest-path true))) (defn- overwrite-path "Save new contents for the file at dest-path from istream. Error if there is no file at that path or the user lacks write permissions thereupon." [irods istream user dest-path] - (otel/with-span [s ["overwrite-path"]] - (validate irods - [:path-exists dest-path user (cfg/irods-zone)] - [:path-is-file dest-path user (cfg/irods-zone)] - [:path-readable dest-path user (cfg/irods-zone)]) - (save-file-contents irods istream user dest-path false))) + (validate irods + [:path-exists dest-path user (cfg/irods-zone)] + [:path-is-file dest-path user (cfg/irods-zone)] + [:path-readable dest-path user (cfg/irods-zone)]) + (save-file-contents irods istream user dest-path false)) (defn- multipart-create-handler "When partially applied, creates a storage handler for ring.middleware.multipart-params/multipart-params-request which stores the file in iRODS." [user dest-dir {istream :stream filename :filename}] - (otel/with-span [s ["multipart-create-handler"]] - (validators/good-pathname filename) - (irods/with-irods-exceptions {:jargon-opts {:client-user user}} irods - (validate irods [:user-exists user (cfg/irods-zone)]) - (let [dest-path (ft/path-join dest-dir filename)] - (create-at-path irods istream user dest-path))))) + (validators/good-pathname filename) + (irods/with-irods-exceptions {:jargon-opts {:client-user user}} irods + (validate irods [:user-exists user (cfg/irods-zone)]) + (let [dest-path (ft/path-join dest-dir filename)] + (create-at-path irods istream user dest-path)))) (defn wrap-multipart-create "Middleware which saves a new file from a multipart request." @@ -118,13 +110,12 @@ "When partially applied, creates a storage handler for ring.middleware.multipart-params/multipart-params-request which overwrites the file in iRODS." [user path-or-uuid uuid? {istream :stream}] - (otel/with-span [s ["multipart-overwrite-handler"]] - (irods/with-irods-exceptions {:jargon-opts {:client-user user}} irods - (validate irods [:user-exists user (cfg/irods-zone)]) - (let [path (ft/rm-last-slash (if uuid? - @(rods/uuid->path irods path-or-uuid) - path-or-uuid))] - (overwrite-path irods istream user path))))) + (irods/with-irods-exceptions {:jargon-opts {:client-user user}} irods + (validate irods [:user-exists user (cfg/irods-zone)]) + (let [path (ft/rm-last-slash (if uuid? + @(rods/uuid->path irods path-or-uuid) + path-or-uuid))] + (overwrite-path irods istream user path)))) (defn wrap-multipart-overwrite "Middleware which overwrites a file's contents from a multipart request." diff --git a/src/data_info/util/irods.clj b/src/data_info/util/irods.clj index a8e92b1..1a44cc4 100644 --- a/src/data_info/util/irods.clj +++ b/src/data_info/util/irods.clj @@ -7,7 +7,6 @@ [clj-jargon.init :as init] [clj-jargon.item-ops :as ops] [clj-jargon.metadata :as meta] - [otel.otel :as otel] [clojure-commons.error-codes :as error] [clojure-commons.file-utils :as file] [data-info.util.config :as cfg]) @@ -72,9 +71,8 @@ [cm ^String path & [istream-ref]] (let [^InputStream istream (if istream-ref @istream-ref (ops/input-stream cm path))] (try+ - (otel/with-span [s ["Tika detect (InputStream)"]] - (.detect (Tika.) istream)) - (finally (when-not istream-ref (otel/with-span [s ["close istream"]] (.close istream))))))) + (.detect (Tika.) istream) + (finally (when-not istream-ref (.close istream)))))) (defn ^String detect-media-type "detects the media type of a given file @@ -86,12 +84,11 @@ Returns: It returns the media type." ([cm ^String path & [istream-ref]] - (otel/with-span [s ["detect-media-type"]] - (let [path-type (.detect (Tika.) (file/basename path))] - (if (or (= path-type "application/octet-stream") - (= path-type "text/plain")) - (detect-media-type-from-contents (if (delay? cm) @cm cm) path istream-ref) - path-type)))) + (let [path-type (.detect (Tika.) (file/basename path))] + (if (or (= path-type "application/octet-stream") + (= path-type "text/plain")) + (detect-media-type-from-contents (if (delay? cm) @cm cm) path istream-ref) + path-type))) ([^String path] (with-jargon-exceptions :lazy true [cm]