From 2d392997cde79da08c28adbcdaebafdfbce3a08c Mon Sep 17 00:00:00 2001 From: "Stuart B. Wilkins" Date: Sun, 13 Sep 2020 12:13:24 -0400 Subject: [PATCH 1/9] Initial Checkin of S3 Plugin --- ADApp/Db/Makefile | 1 + ADApp/Db/NDFileTIFFS3.template | 7 ++ ADApp/Db/NDFileTIFFS3_settings.req | 1 + ADApp/commonDriverMakefile | 14 ++++ ADApp/pluginSrc/Makefile | 13 ++++ ADApp/pluginSrc/NDFileTIFFS3.cpp | 101 +++++++++++++++++++++++++++++ ADApp/pluginSrc/NDFileTIFFS3.dbd | 1 + ADApp/pluginSrc/NDFileTIFFS3.h | 33 ++++++++++ 8 files changed, 171 insertions(+) create mode 100755 ADApp/Db/NDFileTIFFS3.template create mode 100644 ADApp/Db/NDFileTIFFS3_settings.req create mode 100755 ADApp/pluginSrc/NDFileTIFFS3.cpp create mode 100644 ADApp/pluginSrc/NDFileTIFFS3.dbd create mode 100755 ADApp/pluginSrc/NDFileTIFFS3.h diff --git a/ADApp/Db/Makefile b/ADApp/Db/Makefile index 79162b39c..0f45ae32b 100644 --- a/ADApp/Db/Makefile +++ b/ADApp/Db/Makefile @@ -52,6 +52,7 @@ DB += NDAttrPlotAttr.template DB += NDAttrPlotData.template DB += NDAttrPlot.template DB += NDCodec.template +DB += NDFileTIFFS3.template #---------------------------------------------------- # If .db template is not named *.template add diff --git a/ADApp/Db/NDFileTIFFS3.template b/ADApp/Db/NDFileTIFFS3.template new file mode 100755 index 000000000..6719c4271 --- /dev/null +++ b/ADApp/Db/NDFileTIFFS3.template @@ -0,0 +1,7 @@ +#=================================================================# +# Template file: NDFileTIFFS3.template +# Database for NDFileTIFFS3 driver, which saves NDArray data +# in the TIFF file format + +include "NDFileTIFF.template" +include "NDPluginBase.template" diff --git a/ADApp/Db/NDFileTIFFS3_settings.req b/ADApp/Db/NDFileTIFFS3_settings.req new file mode 100644 index 000000000..44b2729d5 --- /dev/null +++ b/ADApp/Db/NDFileTIFFS3_settings.req @@ -0,0 +1 @@ +file "NDPluginFile_settings.req", P=$(P), R=$(R) diff --git a/ADApp/commonDriverMakefile b/ADApp/commonDriverMakefile index 67c14780b..3815535ec 100644 --- a/ADApp/commonDriverMakefile +++ b/ADApp/commonDriverMakefile @@ -262,6 +262,20 @@ ifdef ADPLUGINCENTROIDS PROD_LIBS += NDPluginCentroids centroids endif +ifeq ($(WITH_TIFFS3),YES) + $(DBD_NAME)_DBD += NDFileTIFFS3.dbd + ifeq ($(TIFF_EXTERNAL),NO) + PROD_LIBS += tiff + else + ifdef TIFF_LIB + tiff_DIR = $(TIFF_LIB) + PROD_LIBS += tiff + else + PROD_SYS_LIBS += tiff + endif + endif +endif + # Required modules $(DBD_NAME)_DBD += asyn.dbd PROD_LIBS += asyn diff --git a/ADApp/pluginSrc/Makefile b/ADApp/pluginSrc/Makefile index ebf5bdd06..d466b2056 100644 --- a/ADApp/pluginSrc/Makefile +++ b/ADApp/pluginSrc/Makefile @@ -189,6 +189,19 @@ ifeq ($(WITH_BITSHUFFLE), YES) USR_CXXFLAGS += -DHAVE_BITSHUFFLE endif +ifeq ($(WITH_TIFFS3),YES) + DBD += NDFileTIFFS3.dbd + INC += NDFileTIFFS3.h + LIB_SRCS += NDFileTIFFS3.cpp + ifeq ($(SHARED_LIBRARIES),NO) + # This flag is used to indicate that the TIFF library was built statically + USR_CXXFLAGS_WIN32 += -DLIBTIFF_STATIC + endif + ifdef TIFF_INCLUDE + USR_INCLUDES += $(addprefix -I, $(TIFF_INCLUDE)) + endif +endif + ifdef BLOSC_INCLUDE USR_INCLUDES += $(addprefix -I, $(BLOSC_INCLUDE)) endif diff --git a/ADApp/pluginSrc/NDFileTIFFS3.cpp b/ADApp/pluginSrc/NDFileTIFFS3.cpp new file mode 100755 index 000000000..6af119e70 --- /dev/null +++ b/ADApp/pluginSrc/NDFileTIFFS3.cpp @@ -0,0 +1,101 @@ +/* NDFileTIFFS3.cpp + * Writes NDArrays to TIFF files on S3 Storage + * + * Stuart B. Wilkins + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include "NDPluginFile.h" +#include "NDFileTIFFS3.h" + + +/** Constructor for NDFileTIFFS3; all parameters are simply passed to NDPluginFile::NDPluginFile. + * \param[in] portName The name of the asyn port driver to be created. + * \param[in] queueSize The number of NDArrays that the input queue for this plugin can hold when + * NDPluginDriverBlockingCallbacks=0. Larger queues can decrease the number of dropped arrays, + * at the expense of more NDArray buffers being allocated from the underlying driver's NDArrayPool. + * \param[in] blockingCallbacks Initial setting for the NDPluginDriverBlockingCallbacks flag. + * 0=callbacks are queued and executed by the callback thread; 1 callbacks execute in the thread + * of the driver doing the callbacks. + * \param[in] NDArrayPort Name of asyn port driver for initial source of NDArray callbacks. + * \param[in] NDArrayAddr asyn port driver address for initial source of NDArray callbacks. + * \param[in] priority The thread priority for the asyn port driver thread if ASYN_CANBLOCK is set in asynFlags. + * \param[in] stackSize The stack size for the asyn port driver thread if ASYN_CANBLOCK is set in asynFlags. + */ +NDFileTIFFS3::NDFileTIFFS3(const char *portName, int queueSize, int blockingCallbacks, + const char *NDArrayPort, int NDArrayAddr, + int priority, int stackSize) + /* Invoke the base class constructor. + * We allocate 2 NDArrays of unlimited size in the NDArray pool. + * This driver can block (because writing a file can be slow), and it is not multi-device. + * Set autoconnect to 1. priority and stacksize can be 0, which will use defaults. */ + : NDFileTIFF(portName, queueSize, blockingCallbacks, + NDArrayPort, NDArrayAddr, priority, stackSize) +{ + //static const char *functionName = "NDFileTIFF"; + + /* Set the plugin type string */ + setStringParam(NDPluginDriverPluginType, "NDFileTIFFS3"); + this->supportsMultipleArrays = 0; + +} + +/* Configuration routine. Called directly, or from the iocsh */ + +extern "C" int NDFileTIFFS3Configure(const char *portName, int queueSize, int blockingCallbacks, + const char *NDArrayPort, int NDArrayAddr, + int priority, int stackSize) +{ + // Stack size must be a minimum of 40000 on vxWorks because of automatic variables in NDFileTIFF::openFile() + #ifdef vxWorks + if (stackSize < 40000) stackSize = 40000; + #endif + NDFileTIFFS3 *pPlugin = new NDFileTIFFS3(portName, queueSize, blockingCallbacks, NDArrayPort, NDArrayAddr, + priority, stackSize); + return pPlugin->start(); +} + + +/* EPICS iocsh shell commands */ + +static const iocshArg initArg0 = { "portName",iocshArgString}; +static const iocshArg initArg1 = { "frame queue size",iocshArgInt}; +static const iocshArg initArg2 = { "blocking callbacks",iocshArgInt}; +static const iocshArg initArg3 = { "NDArray Port",iocshArgString}; +static const iocshArg initArg4 = { "NDArray Addr",iocshArgInt}; +static const iocshArg initArg5 = { "priority",iocshArgInt}; +static const iocshArg initArg6 = { "stack size",iocshArgInt}; +static const iocshArg * const initArgs[] = {&initArg0, + &initArg1, + &initArg2, + &initArg3, + &initArg4, + &initArg5, + &initArg6}; +static const iocshFuncDef initFuncDef = {"NDFileTIFFS3Configure",7,initArgs}; +static void initCallFunc(const iocshArgBuf *args) +{ + NDFileTIFFS3Configure(args[0].sval, args[1].ival, args[2].ival, args[3].sval, args[4].ival, args[5].ival, args[6].ival); +} + +extern "C" void NDFileTIFFS3Register(void) +{ + iocshRegister(&initFuncDef,initCallFunc); +} + +extern "C" { +epicsExportRegistrar(NDFileTIFFS3Register); +} diff --git a/ADApp/pluginSrc/NDFileTIFFS3.dbd b/ADApp/pluginSrc/NDFileTIFFS3.dbd new file mode 100644 index 000000000..4a35c66c3 --- /dev/null +++ b/ADApp/pluginSrc/NDFileTIFFS3.dbd @@ -0,0 +1 @@ +registrar("NDFileTIFFS3Register") diff --git a/ADApp/pluginSrc/NDFileTIFFS3.h b/ADApp/pluginSrc/NDFileTIFFS3.h new file mode 100755 index 000000000..32c082a0f --- /dev/null +++ b/ADApp/pluginSrc/NDFileTIFFS3.h @@ -0,0 +1,33 @@ +/* + * NDFileTIFF.h + * Writes NDArrays to TIFF files. + * John Hammonds + * April 17, 2009 + */ + +#ifndef DRV_NDFileTIFFS3_H +#define DRV_NDFileTIFFS3_H + +#include "NDFileTIFF.h" + +/** Writes NDArrays in the TIFF file format. To an Amazon S3 + Tagged Image File Format is a file format for storing images. The format was originally created by Aldus corporation and is + currently developed by Adobe Systems Incorporated. This plugin was developed using the libtiff library to write the file. + The current version is only capable of writing 2-D images with one image per file. + */ + +class epicsShareClass NDFileTIFFS3 : public NDFileTIFF { + public: + NDFileTIFFS3(const char *portName, int queueSize, int blockingCallbacks, + const char *NDArrayPort, int NDArrayAddr, + int priority, int stackSize); +// +// /* The methods that this class implements */ +// virtual asynStatus openFile(const char *fileName, NDFileOpenMode_t openMode, NDArray *pArray); +// virtual asynStatus readFile(NDArray **pArray); +// virtual asynStatus writeFile(NDArray *pArray); +// virtual asynStatus closeFile(); +// +}; + +#endif From da3bb8afaa07f726bd495598c2c323df91600ece Mon Sep 17 00:00:00 2001 From: "Stuart B. Wilkins" Date: Sun, 13 Sep 2020 17:45:59 -0400 Subject: [PATCH 2/9] Added AWS Libraries --- ADApp/commonDriverMakefile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ADApp/commonDriverMakefile b/ADApp/commonDriverMakefile index 3815535ec..5ea8d9fc3 100644 --- a/ADApp/commonDriverMakefile +++ b/ADApp/commonDriverMakefile @@ -264,14 +264,20 @@ endif ifeq ($(WITH_TIFFS3),YES) $(DBD_NAME)_DBD += NDFileTIFFS3.dbd - ifeq ($(TIFF_EXTERNAL),NO) + ifeq ($(TIFFS3_EXTERNAL),NO) PROD_LIBS += tiff else - ifdef TIFF_LIB + ifdef TIFFS3_LIB tiff_DIR = $(TIFF_LIB) PROD_LIBS += tiff else PROD_SYS_LIBS += tiff + PROD_SYS_LIBS += tiffxx + PROD_SYS_LIBS += aws-cpp-sdk-s3 + PROD_SYS_LIBS += aws-cpp-sdk-core + PROD_SYS_LIBS += aws-c-event-stream + PROD_SYS_LIBS += aws-c-common + PROD_SYS_LIBS += aws-checksums endif endif endif From 64698a335fa75333702ff99c422a3b0d58c465b1 Mon Sep 17 00:00:00 2001 From: "Stuart B. Wilkins" Date: Sun, 13 Sep 2020 17:46:30 -0400 Subject: [PATCH 3/9] Correct Variable Name for S3 --- ADApp/pluginSrc/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ADApp/pluginSrc/Makefile b/ADApp/pluginSrc/Makefile index d466b2056..3c2622dfa 100644 --- a/ADApp/pluginSrc/Makefile +++ b/ADApp/pluginSrc/Makefile @@ -197,8 +197,8 @@ ifeq ($(WITH_TIFFS3),YES) # This flag is used to indicate that the TIFF library was built statically USR_CXXFLAGS_WIN32 += -DLIBTIFF_STATIC endif - ifdef TIFF_INCLUDE - USR_INCLUDES += $(addprefix -I, $(TIFF_INCLUDE)) + ifdef TIFFS3_INCLUDE + USR_INCLUDES += $(addprefix -I, $(TIFFS3_INCLUDE)) endif endif From a998090ce39f01840aa4c3e86a902cb6eb899465 Mon Sep 17 00:00:00 2001 From: "Stuart B. Wilkins" Date: Sun, 13 Sep 2020 17:46:53 -0400 Subject: [PATCH 4/9] Moved to protected for class inh. --- ADApp/pluginSrc/NDFileTIFF.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ADApp/pluginSrc/NDFileTIFF.h b/ADApp/pluginSrc/NDFileTIFF.h index 6173b1f56..48369e86e 100644 --- a/ADApp/pluginSrc/NDFileTIFF.h +++ b/ADApp/pluginSrc/NDFileTIFF.h @@ -33,13 +33,12 @@ class epicsShareClass NDFileTIFF : public NDPluginFile { virtual asynStatus writeFile(NDArray *pArray); virtual asynStatus closeFile(); -private: +protected: TIFF *tiff; NDColorMode_t colorMode; - int *pAttributeId; NDAttributeList *pFileAttributes; int numAttributes_; - + int *pAttributeId; }; #endif From 2f0ba71ddc3624b1d596885f5070192f336317b3 Mon Sep 17 00:00:00 2001 From: "Stuart B. Wilkins" Date: Sun, 13 Sep 2020 17:47:15 -0400 Subject: [PATCH 5/9] Working S3 Version --- ADApp/pluginSrc/NDFileTIFFS3.cpp | 363 +++++++++++++++++++++++++++++++ ADApp/pluginSrc/NDFileTIFFS3.h | 13 +- 2 files changed, 374 insertions(+), 2 deletions(-) diff --git a/ADApp/pluginSrc/NDFileTIFFS3.cpp b/ADApp/pluginSrc/NDFileTIFFS3.cpp index 6af119e70..a0fd1364d 100755 --- a/ADApp/pluginSrc/NDFileTIFFS3.cpp +++ b/ADApp/pluginSrc/NDFileTIFFS3.cpp @@ -18,9 +18,355 @@ #include #include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + #include "NDPluginFile.h" +#include "NDFileTIFF.h" #include "NDFileTIFFS3.h" +#define STRING_BUFFER_SIZE 2048 + +static const char *driverName = "NDFileTIFF"; + +static const int TIFFTAG_NDTIMESTAMP = 65000; +static const int TIFFTAG_UNIQUEID = 65001; +static const int TIFFTAG_EPICSTSSEC = 65002; +static const int TIFFTAG_EPICSTSNSEC = 65003; +static const int TIFFTAG_FIRST_ATTRIBUTE = 65010; +static const int TIFFTAG_LAST_ATTRIBUTE = 65535; + +#define NUM_CUSTOM_TIFF_TAGS (4 + TIFFTAG_LAST_ATTRIBUTE - TIFFTAG_FIRST_ATTRIBUTE + 1) + +static TIFFFieldInfo tiffFieldInfo[NUM_CUSTOM_TIFF_TAGS] = { + {TIFFTAG_NDTIMESTAMP, 1, 1, TIFF_DOUBLE,FIELD_CUSTOM, 1, 0, (char *)"NDTimeStamp"}, + {TIFFTAG_UNIQUEID, 1, 1, TIFF_LONG,FIELD_CUSTOM, 1, 0, (char *)"NDUniqueId"}, + {TIFFTAG_EPICSTSSEC, 1, 1, TIFF_LONG,FIELD_CUSTOM, 1, 0, (char *)"EPICSTSSec"}, + {TIFFTAG_EPICSTSNSEC, 1, 1, TIFF_LONG,FIELD_CUSTOM, 1, 0, (char *)"EPICSTSNsec"} +}; + +static void registerCustomTIFFTags(TIFF *tif) +{ + /* Install the extended Tag field info */ + TIFFMergeFieldInfo(tif, tiffFieldInfo, sizeof(tiffFieldInfo)/sizeof(tiffFieldInfo[0])); +} + +static void augmentLibTiffWithCustomTags() { + static bool first_time = true; + if (!first_time) return; + first_time = false; + TIFFSetTagExtender(registerCustomTIFFTags); +} + +/** Opens a TIFF file. + * \param[in] fileName The name of the file to open. + * \param[in] openMode Mask defining how the file should be opened; bits are + * NDFileModeRead, NDFileModeWrite, NDFileModeAppend, NDFileModeMultiple + * \param[in] pArray A pointer to an NDArray; this is used to determine the array and attribute properties. + */ +asynStatus NDFileTIFFS3::openFile(const char *fileName, NDFileOpenMode_t openMode, NDArray *pArray) +{ + /* When we create TIFF variables and dimensions, we get back an + * ID for each one. */ + static const char *functionName = "openFile"; + size_t sizeX, sizeY, rowsPerStrip; + int bitsPerSample=8, sampleFormat=SAMPLEFORMAT_INT, samplesPerPixel, photoMetric, planarConfig; + int colorMode=NDColorModeMono; + NDAttribute *pAttribute = NULL; + char tagString[STRING_BUFFER_SIZE] = {0}; + char attrString[STRING_BUFFER_SIZE] = {0}; + char tagName[STRING_BUFFER_SIZE] = {0}; + int i; + TIFFFieldInfo fieldInfo = {0, 1, 1, TIFF_ASCII, FIELD_CUSTOM, 1, 0, tagName}; + + for (i=TIFFTAG_FIRST_ATTRIBUTE; i<=TIFFTAG_LAST_ATTRIBUTE; i++) { + sprintf(tagName, "Attribute_%d", i-TIFFTAG_FIRST_ATTRIBUTE+1); + fieldInfo.field_tag = i; + tiffFieldInfo[4+i-TIFFTAG_FIRST_ATTRIBUTE] = fieldInfo; + } + + augmentLibTiffWithCustomTags(); + + /* Suppress error and warning messages from the TIFF library */ + TIFFSetErrorHandler(NULL); + TIFFSetWarningHandler(NULL); + + /* We don't support opening an existing file for appending yet */ + if (openMode != NDFileModeWrite) { + return(asynError); + } + + // Now do AMAZON S3 Stuff + awsStream = Aws::MakeShared(""); + strncpy(keyName, fileName, 256); + + if ((this->tiff = TIFFStreamOpen("TIFF", (std::ostream*)awsStream.get())) == NULL) + { + asynPrint(this->pasynUserSelf, ASYN_TRACE_ERROR, + "%s:%s error opening file %s\n", + driverName, functionName, fileName); + return (asynError); + } + asynPrint(pasynUserSelf, ASYN_TRACE_FLOW, + "%s::%s opened file %s\n", + driverName, functionName, fileName); + + /* We do some special treatment based on colorMode */ + pAttribute = pArray->pAttributeList->find("ColorMode"); + if (pAttribute) pAttribute->getValue(NDAttrInt32, &colorMode); + + switch (pArray->dataType) { + case NDInt8: + sampleFormat = SAMPLEFORMAT_INT; + bitsPerSample = 8; + break; + case NDUInt8: + sampleFormat = SAMPLEFORMAT_UINT; + bitsPerSample = 8; + break; + case NDInt16: + sampleFormat = SAMPLEFORMAT_INT; + bitsPerSample = 16; + break; + case NDUInt16: + sampleFormat = SAMPLEFORMAT_UINT; + bitsPerSample = 16; + break; + case NDInt32: + sampleFormat = SAMPLEFORMAT_INT; + bitsPerSample = 32; + break; + case NDUInt32: + sampleFormat = SAMPLEFORMAT_UINT; + bitsPerSample = 32; + break; + case NDInt64: + sampleFormat = SAMPLEFORMAT_INT; + bitsPerSample = 64; + break; + case NDUInt64: + sampleFormat = SAMPLEFORMAT_UINT; + bitsPerSample = 64; + break; + case NDFloat32: + sampleFormat = SAMPLEFORMAT_IEEEFP; + bitsPerSample = 32; + break; + case NDFloat64: + sampleFormat = SAMPLEFORMAT_IEEEFP; + bitsPerSample = 64; + break; + } + if (pArray->ndims == 1) { + sizeX = pArray->dims[0].size; + sizeY = 1; + rowsPerStrip = sizeY; + samplesPerPixel = 1; + photoMetric = PHOTOMETRIC_MINISBLACK; + planarConfig = PLANARCONFIG_CONTIG; + this->colorMode = NDColorModeMono; + } else if (pArray->ndims == 2) { + sizeX = pArray->dims[0].size; + sizeY = pArray->dims[1].size; + rowsPerStrip = sizeY; + samplesPerPixel = 1; + photoMetric = PHOTOMETRIC_MINISBLACK; + planarConfig = PLANARCONFIG_CONTIG; + this->colorMode = NDColorModeMono; + } else if ((pArray->ndims == 3) && (pArray->dims[0].size == 3) && (colorMode == NDColorModeRGB1)) { + sizeX = pArray->dims[1].size; + sizeY = pArray->dims[2].size; + rowsPerStrip = sizeY; + samplesPerPixel = 3; + photoMetric = PHOTOMETRIC_RGB; + planarConfig = PLANARCONFIG_CONTIG; + this->colorMode = NDColorModeRGB1; + } else if ((pArray->ndims == 3) && (pArray->dims[1].size == 3) && (colorMode == NDColorModeRGB2)) { + sizeX = pArray->dims[0].size; + sizeY = pArray->dims[2].size; + rowsPerStrip = 1; + samplesPerPixel = 3; + photoMetric = PHOTOMETRIC_RGB; + planarConfig = PLANARCONFIG_SEPARATE; + this->colorMode = NDColorModeRGB2; + } else if ((pArray->ndims == 3) && (pArray->dims[2].size == 3) && (colorMode == NDColorModeRGB3)) { + sizeX = pArray->dims[0].size; + sizeY = pArray->dims[1].size; + rowsPerStrip = sizeY; + samplesPerPixel = 3; + photoMetric = PHOTOMETRIC_RGB; + planarConfig = PLANARCONFIG_SEPARATE; + this->colorMode = NDColorModeRGB3; + } else { + asynPrint(this->pasynUserSelf, ASYN_TRACE_ERROR, + "%s:%s: unsupported array structure\n", + driverName, functionName); + return(asynError); + } + + TIFFSetField(this->tiff, TIFFTAG_NDTIMESTAMP, pArray->timeStamp); + TIFFSetField(this->tiff, TIFFTAG_UNIQUEID, pArray->uniqueId); + TIFFSetField(this->tiff, TIFFTAG_EPICSTSSEC, pArray->epicsTS.secPastEpoch); + TIFFSetField(this->tiff, TIFFTAG_EPICSTSNSEC, pArray->epicsTS.nsec); + TIFFSetField(this->tiff, TIFFTAG_BITSPERSAMPLE, bitsPerSample); + TIFFSetField(this->tiff, TIFFTAG_SAMPLEFORMAT, sampleFormat); + TIFFSetField(this->tiff, TIFFTAG_SAMPLESPERPIXEL, samplesPerPixel); + TIFFSetField(this->tiff, TIFFTAG_PHOTOMETRIC, photoMetric); + TIFFSetField(this->tiff, TIFFTAG_PLANARCONFIG, planarConfig); + TIFFSetField(this->tiff, TIFFTAG_IMAGEWIDTH, (epicsUInt32)sizeX); + TIFFSetField(this->tiff, TIFFTAG_IMAGELENGTH, (epicsUInt32)sizeY); + TIFFSetField(this->tiff, TIFFTAG_ROWSPERSTRIP, (epicsUInt32)rowsPerStrip); + + this->pFileAttributes->clear(); + this->getAttributes(this->pFileAttributes); + pArray->pAttributeList->copy(this->pFileAttributes); + + pAttribute = this->pFileAttributes->find("Model"); + if (pAttribute) { + pAttribute->getValue(NDAttrString, tagString, sizeof(tagString)-1); + TIFFSetField(this->tiff, TIFFTAG_MODEL, tagString); + } else { + TIFFSetField(this->tiff, TIFFTAG_MODEL, "Unknown"); + } + + pAttribute = this->pFileAttributes->find("Manufacturer"); + if (pAttribute) { + pAttribute->getValue(NDAttrString, tagString); + TIFFSetField(this->tiff, TIFFTAG_MAKE, tagString, sizeof(tagString)-1); + } else { + TIFFSetField(this->tiff, TIFFTAG_MAKE, "Unknown"); + } + + TIFFSetField(this->tiff, TIFFTAG_SOFTWARE, "EPICS areaDetector"); + + // If the attribute TIFFImageDescription exists use it to set the TIFFTAG_IMAGEDESCRIPTION + pAttribute = this->pFileAttributes->find("TIFFImageDescription"); + if (pAttribute) { + pAttribute->getValue(NDAttrString, tagString, sizeof(tagString)-1); + TIFFSetField(this->tiff, TIFFTAG_IMAGEDESCRIPTION, tagString); + } + + int count = 0; + int tagId = TIFFTAG_FIRST_ATTRIBUTE; + + numAttributes_ = this->pFileAttributes->count(); + asynPrint(this->pasynUserSelf, ASYN_TRACEIO_DRIVER, + "%s:%s this->pFileAttributes->count(): %d\n", + driverName, functionName, numAttributes_); + + asynPrint(this->pasynUserSelf, ASYN_TRACEIO_DRIVER, + "%s:%s Looping over attributes...\n", + driverName, functionName); + + pAttribute = this->pFileAttributes->next(NULL); + while (pAttribute) { + const char *attributeName = pAttribute->getName(); + //const char *attributeDescription = pAttribute->getDescription(); + const char *attributeSource = pAttribute->getSource(); + + asynPrint(this->pasynUserSelf, ASYN_TRACEIO_DRIVER, + "%s:%s : attribute: %s, source: %s\n", + driverName, functionName, attributeName, attributeSource); + + NDAttrDataType_t attrDataType; + size_t attrSize; + NDAttrValue value; + pAttribute->getValueInfo(&attrDataType, &attrSize); + memset(tagString, 0, sizeof(tagString)); + + switch (attrDataType) { + case NDAttrInt8: + case NDAttrUInt8: + case NDAttrInt16: + case NDAttrUInt16: + case NDAttrInt32: + case NDAttrUInt32: + case NDAttrInt64: + case NDAttrUInt64: { + pAttribute->getValue(attrDataType, &value.i64); + epicsSnprintf(tagString, sizeof(tagString)-1, "%s:%lld", attributeName, value.i64); + break; + } + case NDAttrFloat32: { + pAttribute->getValue(attrDataType, &value.f32); + epicsSnprintf(tagString, sizeof(tagString)-1, "%s:%f", attributeName, value.f32); + break; + } + case NDAttrFloat64: { + pAttribute->getValue(attrDataType, &value.f64); + epicsSnprintf(tagString, sizeof(tagString)-1, "%s:%f", attributeName, value.f64); + break; + } + case NDAttrString: { + memset(attrString, 0, sizeof(tagString)-1); + pAttribute->getValue(attrDataType, attrString, sizeof(attrString)-1); + epicsSnprintf(tagString, sizeof(tagString)-1, "%s:%s", attributeName, attrString); + break; + } + case NDAttrUndefined: + break; + default: + asynPrint(this->pasynUserSelf, ASYN_TRACE_ERROR, + "%s:%s error, unknown attrDataType=%d\n", + driverName, functionName, attrDataType); + return asynError; + break; + } + + if (attrDataType != NDAttrUndefined) { + asynPrint(this->pasynUserSelf, ASYN_TRACEIO_DRIVER, + "%s:%s : tagId: %d, tagString: %s\n", + driverName, functionName, tagId, tagString); + TIFFSetField(this->tiff, tagId, tagString); + ++count; + ++tagId; + if ((tagId > TIFFTAG_LAST_ATTRIBUTE) || (count > numAttributes_)) { + asynPrint(this->pasynUserSelf, ASYN_TRACE_ERROR, + "%s:%s error, Too many tags/attributes for file. tagId: %d, count: %d\n", + driverName, functionName, tagId, count); + break; + } + } + pAttribute = this->pFileAttributes->next(pAttribute); + } + + return(asynSuccess); +} + +asynStatus NDFileTIFFS3::closeFile() +{ + NDFileTIFF::closeFile(); + + Aws::S3::Model::PutObjectRequest objectRequest; + objectRequest.SetBucket(""); + objectRequest.SetKey(keyName); + objectRequest.SetBody(awsStream); + + Aws::S3::Model::PutObjectOutcome outcome = + s3Client->PutObject(objectRequest); + + if (!outcome.IsSuccess()) + { + asynPrint(this->pasynUserSelf, ASYN_TRACE_ERROR, + "AWS S3 Error : %s\n", outcome.GetError().GetMessage().c_str()); + } + + return asynSuccess; +} /** Constructor for NDFileTIFFS3; all parameters are simply passed to NDPluginFile::NDPluginFile. * \param[in] portName The name of the asyn port driver to be created. @@ -51,6 +397,23 @@ NDFileTIFFS3::NDFileTIFFS3(const char *portName, int queueSize, int blockingCall setStringParam(NDPluginDriverPluginType, "NDFileTIFFS3"); this->supportsMultipleArrays = 0; + // Aws::Utils::Logging::InitializeAWSLogging( + // Aws::MakeShared( + // "NDFileTIFFS3", Aws::Utils::Logging::LogLevel::Error, "NDFileTIFFS3_")); + + Aws::InitAPI(options); + + Aws::Client::ClientConfiguration config; + config.endpointOverride = "https://dtn01.sdcc.bnl.gov:8000"; + config.verifySSL = false; + + s3Client = std::make_shared(config); +} + +NDFileTIFFS3::~NDFileTIFFS3() +{ + ShutdownAPI(options); + // Aws::Utils::Logging::ShutdownAWSLogging(); } /* Configuration routine. Called directly, or from the iocsh */ diff --git a/ADApp/pluginSrc/NDFileTIFFS3.h b/ADApp/pluginSrc/NDFileTIFFS3.h index 32c082a0f..8dda2a3cf 100755 --- a/ADApp/pluginSrc/NDFileTIFFS3.h +++ b/ADApp/pluginSrc/NDFileTIFFS3.h @@ -17,13 +17,22 @@ */ class epicsShareClass NDFileTIFFS3 : public NDFileTIFF { - public: + public: NDFileTIFFS3(const char *portName, int queueSize, int blockingCallbacks, const char *NDArrayPort, int NDArrayAddr, int priority, int stackSize); + + virtual ~NDFileTIFFS3(); + virtual asynStatus openFile(const char *fileName, NDFileOpenMode_t openMode, NDArray *pArray); + virtual asynStatus closeFile(); + + private: + Aws::SDKOptions options; + std::shared_ptr awsStream; + std::shared_ptr s3Client; + char keyName[256]; // // /* The methods that this class implements */ -// virtual asynStatus openFile(const char *fileName, NDFileOpenMode_t openMode, NDArray *pArray); // virtual asynStatus readFile(NDArray **pArray); // virtual asynStatus writeFile(NDArray *pArray); // virtual asynStatus closeFile(); From 6524eb45e5da822e645ad0266362ca28cc53855f Mon Sep 17 00:00:00 2001 From: "Stuart B. Wilkins" Date: Sun, 13 Sep 2020 18:21:09 -0400 Subject: [PATCH 6/9] Fixed IOC Shell to add debug and endpoint --- ADApp/pluginSrc/NDFileTIFFS3.cpp | 74 ++++++++++++++++++++++++++------ ADApp/pluginSrc/NDFileTIFFS3.h | 2 + 2 files changed, 62 insertions(+), 14 deletions(-) diff --git a/ADApp/pluginSrc/NDFileTIFFS3.cpp b/ADApp/pluginSrc/NDFileTIFFS3.cpp index a0fd1364d..eff3f69da 100755 --- a/ADApp/pluginSrc/NDFileTIFFS3.cpp +++ b/ADApp/pluginSrc/NDFileTIFFS3.cpp @@ -112,7 +112,7 @@ asynStatus NDFileTIFFS3::openFile(const char *fileName, NDFileOpenMode_t openMod // Now do AMAZON S3 Stuff awsStream = Aws::MakeShared(""); - strncpy(keyName, fileName, 256); + strncpy(keyName, fileName, 255); if ((this->tiff = TIFFStreamOpen("TIFF", (std::ostream*)awsStream.get())) == NULL) { @@ -383,7 +383,7 @@ asynStatus NDFileTIFFS3::closeFile() */ NDFileTIFFS3::NDFileTIFFS3(const char *portName, int queueSize, int blockingCallbacks, const char *NDArrayPort, int NDArrayAddr, - int priority, int stackSize) + const char *endpoint, int awslog, int priority, int stackSize) /* Invoke the base class constructor. * We allocate 2 NDArrays of unlimited size in the NDArray pool. * This driver can block (because writing a file can be slow), and it is not multi-device. @@ -397,14 +397,51 @@ NDFileTIFFS3::NDFileTIFFS3(const char *portName, int queueSize, int blockingCall setStringParam(NDPluginDriverPluginType, "NDFileTIFFS3"); this->supportsMultipleArrays = 0; - // Aws::Utils::Logging::InitializeAWSLogging( - // Aws::MakeShared( - // "NDFileTIFFS3", Aws::Utils::Logging::LogLevel::Error, "NDFileTIFFS3_")); + if (awslog) + { + Aws::Utils::Logging::LogLevel level; + switch (awslog) + { + case 0: + level = Aws::Utils::Logging::LogLevel::Off; + break; + case 1: + level = Aws::Utils::Logging::LogLevel::Fatal; + break; + case 2: + level = Aws::Utils::Logging::LogLevel::Error; + break; + case 3: + level = Aws::Utils::Logging::LogLevel::Warn; + break; + case 4: + level = Aws::Utils::Logging::LogLevel::Info; + break; + case 5: + level = Aws::Utils::Logging::LogLevel::Debug; + break; + case 6: + level = Aws::Utils::Logging::LogLevel::Trace; + break; + default: + level = Aws::Utils::Logging::LogLevel::Off; + break; + } + Aws::Utils::Logging::InitializeAWSLogging( + Aws::MakeShared( + "NDFileTIFFS3", level, "NDFileTIFFS3_")); + awsLogging = true; + } else { + awsLogging = false; + } Aws::InitAPI(options); Aws::Client::ClientConfiguration config; - config.endpointOverride = "https://dtn01.sdcc.bnl.gov:8000"; + if (endpoint && endpoint[0]) + { + config.endpointOverride = endpoint; + } config.verifySSL = false; s3Client = std::make_shared(config); @@ -413,21 +450,24 @@ NDFileTIFFS3::NDFileTIFFS3(const char *portName, int queueSize, int blockingCall NDFileTIFFS3::~NDFileTIFFS3() { ShutdownAPI(options); - // Aws::Utils::Logging::ShutdownAWSLogging(); + if (awsLogging) + { + Aws::Utils::Logging::ShutdownAWSLogging(); + } } /* Configuration routine. Called directly, or from the iocsh */ extern "C" int NDFileTIFFS3Configure(const char *portName, int queueSize, int blockingCallbacks, const char *NDArrayPort, int NDArrayAddr, - int priority, int stackSize) + const char *endpoint, int awslog, int priority, int stackSize) { // Stack size must be a minimum of 40000 on vxWorks because of automatic variables in NDFileTIFF::openFile() #ifdef vxWorks if (stackSize < 40000) stackSize = 40000; #endif NDFileTIFFS3 *pPlugin = new NDFileTIFFS3(portName, queueSize, blockingCallbacks, NDArrayPort, NDArrayAddr, - priority, stackSize); + endpoint, awslog, priority, stackSize); return pPlugin->start(); } @@ -439,19 +479,25 @@ static const iocshArg initArg1 = { "frame queue size",iocshArgInt}; static const iocshArg initArg2 = { "blocking callbacks",iocshArgInt}; static const iocshArg initArg3 = { "NDArray Port",iocshArgString}; static const iocshArg initArg4 = { "NDArray Addr",iocshArgInt}; -static const iocshArg initArg5 = { "priority",iocshArgInt}; -static const iocshArg initArg6 = { "stack size",iocshArgInt}; +static const iocshArg initArg5 = { "endpoint",iocshArgString}; +static const iocshArg initArg6 = { "awslog",iocshArgInt}; +static const iocshArg initArg7 = { "priority",iocshArgInt}; +static const iocshArg initArg8 = { "stack size",iocshArgInt}; static const iocshArg * const initArgs[] = {&initArg0, &initArg1, &initArg2, &initArg3, &initArg4, &initArg5, - &initArg6}; -static const iocshFuncDef initFuncDef = {"NDFileTIFFS3Configure",7,initArgs}; + &initArg6, + &initArg7, + &initArg8}; +static const iocshFuncDef initFuncDef = {"NDFileTIFFS3Configure",9,initArgs}; static void initCallFunc(const iocshArgBuf *args) { - NDFileTIFFS3Configure(args[0].sval, args[1].ival, args[2].ival, args[3].sval, args[4].ival, args[5].ival, args[6].ival); + NDFileTIFFS3Configure(args[0].sval, args[1].ival, args[2].ival, args[3].sval, + args[4].ival, args[5].sval, args[6].ival, args[7].ival, + args[8].ival); } extern "C" void NDFileTIFFS3Register(void) diff --git a/ADApp/pluginSrc/NDFileTIFFS3.h b/ADApp/pluginSrc/NDFileTIFFS3.h index 8dda2a3cf..1d2d7b175 100755 --- a/ADApp/pluginSrc/NDFileTIFFS3.h +++ b/ADApp/pluginSrc/NDFileTIFFS3.h @@ -20,6 +20,7 @@ class epicsShareClass NDFileTIFFS3 : public NDFileTIFF { public: NDFileTIFFS3(const char *portName, int queueSize, int blockingCallbacks, const char *NDArrayPort, int NDArrayAddr, + const char *endpoint, int awslog, int priority, int stackSize); virtual ~NDFileTIFFS3(); @@ -31,6 +32,7 @@ class epicsShareClass NDFileTIFFS3 : public NDFileTIFF { std::shared_ptr awsStream; std::shared_ptr s3Client; char keyName[256]; + bool awsLogging = false; // // /* The methods that this class implements */ // virtual asynStatus readFile(NDArray **pArray); From 4f0cd6e3ea581b644fe344f29d819611a97a0357 Mon Sep 17 00:00:00 2001 From: "Stuart B. Wilkins" Date: Mon, 14 Sep 2020 08:13:23 -0400 Subject: [PATCH 7/9] Move to ASYNC upload to S3 --- ADApp/pluginSrc/NDFileTIFFS3.cpp | 35 ++++++++++++++++++++++++-------- ADApp/pluginSrc/NDFileTIFFS3.h | 16 +++++++++++++++ 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/ADApp/pluginSrc/NDFileTIFFS3.cpp b/ADApp/pluginSrc/NDFileTIFFS3.cpp index eff3f69da..308851722 100755 --- a/ADApp/pluginSrc/NDFileTIFFS3.cpp +++ b/ADApp/pluginSrc/NDFileTIFFS3.cpp @@ -347,6 +347,28 @@ asynStatus NDFileTIFFS3::openFile(const char *fileName, NDFileOpenMode_t openMod return(asynSuccess); } +void PutObjectAsyncFinished(const Aws::S3::S3Client* s3Client, + const Aws::S3::Model::PutObjectRequest& request, + const Aws::S3::Model::PutObjectOutcome& outcome, + const std::shared_ptr& context) +{ + // NOTE: This just seems wrong, but we have the shared_ptr passed as + // const so we can't cast dynamically. Perhaps it's ok given that its only + // used in this function? + + NDFileTIFFS3_AWSContext *ctx = (NDFileTIFFS3_AWSContext*)context.get(); + + if (!outcome.IsSuccess()) { + std::cerr << "AWS S3 Error uploading : " << + outcome.GetError().GetMessage() << std::endl; + } + // else { + // std::cerr << "AWS S3 Uploaded : " << + // context->GetUUID() << std::endl; + // } +} + + asynStatus NDFileTIFFS3::closeFile() { NDFileTIFF::closeFile(); @@ -356,14 +378,11 @@ asynStatus NDFileTIFFS3::closeFile() objectRequest.SetKey(keyName); objectRequest.SetBody(awsStream); - Aws::S3::Model::PutObjectOutcome outcome = - s3Client->PutObject(objectRequest); - - if (!outcome.IsSuccess()) - { - asynPrint(this->pasynUserSelf, ASYN_TRACE_ERROR, - "AWS S3 Error : %s\n", outcome.GetError().GetMessage().c_str()); - } + std::shared_ptr context = + Aws::MakeShared("PutObjectAllocationTag"); + context->SetUUID(keyName); + context->SetTIFFS3(this); + s3Client->PutObjectAsync(objectRequest, PutObjectAsyncFinished, context); return asynSuccess; } diff --git a/ADApp/pluginSrc/NDFileTIFFS3.h b/ADApp/pluginSrc/NDFileTIFFS3.h index 1d2d7b175..3d8551944 100755 --- a/ADApp/pluginSrc/NDFileTIFFS3.h +++ b/ADApp/pluginSrc/NDFileTIFFS3.h @@ -41,4 +41,20 @@ class epicsShareClass NDFileTIFFS3 : public NDFileTIFF { // }; +class NDFileTIFFS3_AWSContext : public Aws::Client::AsyncCallerContext { + public: + NDFileTIFFS3_AWSContext() + : Aws::Client::AsyncCallerContext() {} + + void SetTIFFS3(NDFileTIFFS3 *c) { + tiffS3 = c; + } + NDFileTIFFS3* GetTIFFS3(void) { + return tiffS3; + } + private: + NDFileTIFFS3 *tiffS3; +}; + + #endif From 4da3f220cacb3d857a43e843637673a501de0368 Mon Sep 17 00:00:00 2001 From: "Stuart B. Wilkins" Date: Mon, 14 Sep 2020 12:04:05 -0400 Subject: [PATCH 8/9] Move to const cast --- ADApp/pluginSrc/NDFileTIFFS3.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ADApp/pluginSrc/NDFileTIFFS3.cpp b/ADApp/pluginSrc/NDFileTIFFS3.cpp index 308851722..b9694f5ef 100755 --- a/ADApp/pluginSrc/NDFileTIFFS3.cpp +++ b/ADApp/pluginSrc/NDFileTIFFS3.cpp @@ -352,11 +352,12 @@ void PutObjectAsyncFinished(const Aws::S3::S3Client* s3Client, const Aws::S3::Model::PutObjectOutcome& outcome, const std::shared_ptr& context) { - // NOTE: This just seems wrong, but we have the shared_ptr passed as - // const so we can't cast dynamically. Perhaps it's ok given that its only - // used in this function? - - NDFileTIFFS3_AWSContext *ctx = (NDFileTIFFS3_AWSContext*)context.get(); + std::shared_ptr __ctx(context); + std::shared_ptr _ctx = + std::dynamic_pointer_cast(__ctx); + std::shared_ptr ctx = std::const_pointer_cast(_ctx); + + std::cerr << ctx->GetTIFFS3() << std::endl; if (!outcome.IsSuccess()) { std::cerr << "AWS S3 Error uploading : " << From 1a6cc516705e762e10e262990aadfc3b2f3cd2af Mon Sep 17 00:00:00 2001 From: "Stuart B. Wilkins" Date: Wed, 16 Sep 2020 07:45:39 -0400 Subject: [PATCH 9/9] Rebuild to have static member of class called --- ADApp/pluginSrc/NDFileTIFFS3.cpp | 22 ++++++++++++---------- ADApp/pluginSrc/NDFileTIFFS3.h | 9 +++++++++ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/ADApp/pluginSrc/NDFileTIFFS3.cpp b/ADApp/pluginSrc/NDFileTIFFS3.cpp index b9694f5ef..b4bf17fea 100755 --- a/ADApp/pluginSrc/NDFileTIFFS3.cpp +++ b/ADApp/pluginSrc/NDFileTIFFS3.cpp @@ -347,7 +347,7 @@ asynStatus NDFileTIFFS3::openFile(const char *fileName, NDFileOpenMode_t openMod return(asynSuccess); } -void PutObjectAsyncFinished(const Aws::S3::S3Client* s3Client, +void NDFileTIFFS3::PutObjectAsyncFinished(const Aws::S3::S3Client* s3Client, const Aws::S3::Model::PutObjectRequest& request, const Aws::S3::Model::PutObjectOutcome& outcome, const std::shared_ptr& context) @@ -356,17 +356,19 @@ void PutObjectAsyncFinished(const Aws::S3::S3Client* s3Client, std::shared_ptr _ctx = std::dynamic_pointer_cast(__ctx); std::shared_ptr ctx = std::const_pointer_cast(_ctx); - - std::cerr << ctx->GetTIFFS3() << std::endl; + ctx->GetTIFFS3()->putObjectFinished(outcome); +} + +void NDFileTIFFS3::putObjectFinished(const Aws::S3::Model::PutObjectOutcome& outcome) +{ + const char* functionName = "NDFileTIFFS3::putObjectFinished"; if (!outcome.IsSuccess()) { - std::cerr << "AWS S3 Error uploading : " << - outcome.GetError().GetMessage() << std::endl; + asynPrint(this->pasynUserSelf, ASYN_TRACE_ERROR, + "%s:%s error, AWS S3 Error : %s\n", + driverName, functionName, outcome.GetError().GetMessage().c_str()); } - // else { - // std::cerr << "AWS S3 Uploaded : " << - // context->GetUUID() << std::endl; - // } + this->callParamCallbacks(); } @@ -383,7 +385,7 @@ asynStatus NDFileTIFFS3::closeFile() Aws::MakeShared("PutObjectAllocationTag"); context->SetUUID(keyName); context->SetTIFFS3(this); - s3Client->PutObjectAsync(objectRequest, PutObjectAsyncFinished, context); + s3Client->PutObjectAsync(objectRequest, NDFileTIFFS3::PutObjectAsyncFinished, context); return asynSuccess; } diff --git a/ADApp/pluginSrc/NDFileTIFFS3.h b/ADApp/pluginSrc/NDFileTIFFS3.h index 3d8551944..c9be3af63 100755 --- a/ADApp/pluginSrc/NDFileTIFFS3.h +++ b/ADApp/pluginSrc/NDFileTIFFS3.h @@ -27,7 +27,16 @@ class epicsShareClass NDFileTIFFS3 : public NDFileTIFF { virtual asynStatus openFile(const char *fileName, NDFileOpenMode_t openMode, NDArray *pArray); virtual asynStatus closeFile(); + // Note: The following need to be static to deal with the callback being out of the + // scope of the class + static void PutObjectAsyncFinished(const Aws::S3::S3Client* s3Client, + const Aws::S3::Model::PutObjectRequest& request, + const Aws::S3::Model::PutObjectOutcome& outcome, + const std::shared_ptr& context); + void putObjectFinished(const Aws::S3::Model::PutObjectOutcome& outcome); + private: + Aws::SDKOptions options; std::shared_ptr awsStream; std::shared_ptr s3Client;