diff --git a/dss_client/include/dss.h b/dss_client/include/dss.h index b535b9a..6b22ee8 100644 --- a/dss_client/include/dss.h +++ b/dss_client/include/dss.h @@ -1,6 +1,9 @@ #ifndef DSS_H #define DSS_H +#define FAILURE -1 +#define END_OF_LIST -2 + #ifdef __cplusplus extern "C" { #endif @@ -8,9 +11,12 @@ typedef void* DSSClient; DSSClient DSSClientInit(char *ip, char* user, char* passwd, char* uuid, int endpoints_per_cluster); int GetObjectBuffer(DSSClient c, void* key, int key_len, unsigned char* buffer, long int buffer_size); int GetObject(DSSClient c, void* key, int key_len, char* dst_file); +int PutObject(DSSClient c, void* key, int key_len, char* src_file); int PutObjectBuffer(DSSClient c, void* key, int key_len, unsigned char* buffer, long int content_length); int DeleteObject(DSSClient c, void* key, int key_len); - +int ListObjects(DSSClient c, char* prefix, char* delimit, char* obj_keys, int cur_pg); +int DeleteAll(DSSClient c, char* prefix, char* delimit); +int GetPageSize(); #ifdef __cplusplus } #endif diff --git a/dss_client/include/dss_client.hpp b/dss_client/include/dss_client.hpp index f2b17b6..92f5ed6 100644 --- a/dss_client/include/dss_client.hpp +++ b/dss_client/include/dss_client.hpp @@ -206,6 +206,7 @@ namespace dss { uint32_t page_size = DSS_PAGINATION_DEFAULT); std::set&& ListObjects(const std::string& prefix, const std::string& delimiter); std::set ListBuckets(); + std::unique_ptr list_objs; private: diff --git a/dss_client/src/dss_client.cpp b/dss_client/src/dss_client.cpp index b424f13..c969ee8 100644 --- a/dss_client/src/dss_client.cpp +++ b/dss_client/src/dss_client.cpp @@ -1308,6 +1308,23 @@ namespace dss { return ret; } + extern "C" int PutObject(DSSClient c, void* key_name, int key_len, char* src_file) + { + Client *client = (Client*) c; + if (client == nullptr || (char*) key_name == nullptr){ + return -1; + } + std::string key_str ((char*) key_name, key_len); + int ret = -1; + + try { + ret = client->PutObject(key_str.c_str(), src_file); + } catch(...) { + printf("Exception caught in PutObject - %s\n", key_str.c_str()); + } + return ret; + } + extern "C" int DeleteObject(DSSClient c, void* key_name, int key_len) { Client *client = (Client*) c; @@ -1323,7 +1340,90 @@ namespace dss { printf("Exception caught in DeleteObject for %s\n", key_str.c_str()); } return ret; + } + + extern "C" int ListObjects(DSSClient c, char* prefix, char* delimit, char* keys, int cur_pg) + { + Client *client = (Client*) c; + if (client == nullptr){ + printf("the DSS client cannot be a null pointer.\n"); + return FAILURE; + } + if (keys == nullptr){ + printf("the buffer to store keys cannot be a null pointer.\n"); + return FAILURE; + } + if (cur_pg == -1){ // for the first page of list + client->list_objs = client->GetObjects(prefix, delimit); + } + if (client->list_objs->GetObjKeys() < 0) { // reached the end of the pages for LIST + return END_OF_LIST; + } + std::string keys_concat = ""; + for (std::string key: client->list_objs->GetPage()){ + keys_concat += key + "\n"; // seperate each key by line assuming there is no "\n" in the object name or directory name or delimiter + } + if (keys_concat == "") return END_OF_LIST; + keys[keys_concat.length()] = '\0'; + strncpy(keys, keys_concat.c_str(), keys_concat.length()); + if (keys[keys_concat.length()] != '\0') { + printf("the buffer overflow for storing keys\n"); + return FAILURE; + } + return cur_pg + 1; + } + extern "C" int DeleteAll(DSSClient c, char* prefix, char* delimit) + { + Client *client = (Client*) c; + if (client == nullptr){ + printf("the DSS client cannot be a null pointer.\n"); + return FAILURE; + } + if (prefix == nullptr){ + printf("the prefix cannot be a null pointer.\n"); + return FAILURE; + } + if (delimit == nullptr){ + printf("the delimiter cannot be a null pointer.\n"); + return FAILURE; + } + uint32_t pg_size = DSS_PAGINATION_DEFAULT; + int max_key_len = 1024; + char* keys = (char*) malloc(sizeof(char) * max_key_len * pg_size); + int pg; + while (true){ + pg = ListObjects(client, prefix, delimit, keys, -1); // list the very first page + if (pg == FAILURE){ + printf("ListObjects failed for prefix=\"%s\" and delimit=\"%s\".\n", prefix, delimit); + printf("DeleteAll failed for prefix=\"%s\" and delimit=\"%s\".\n", prefix, delimit); + free(keys); + return FAILURE; + } + if (pg == END_OF_LIST) break; + std::stringstream all_keys(keys); + std::string each_key; + while (std::getline(all_keys, each_key, '\n')){ // retrieve each key by line + try { + if (client->DeleteObject(each_key.c_str()) < 0 ){ // delete the very first page listed + printf("DeleteObject failed for %s.\n", each_key.c_str()); + free(keys); + return FAILURE; + } + // printf("Deleting %s\n", each_key.c_str()); + } catch(...) { + printf("Exception caught in DeleteObject for %s.\n", each_key.c_str()); + free(keys); + return FAILURE; + } + } + } + free(keys); + return 0; } + extern "C" int GetPageSize() { + return DSS_PAGINATION_DEFAULT; + } + } // namespace dss diff --git a/dss_client/test/test_dss_c_interface.c b/dss_client/test/test_dss_c_interface.c index 3bd676a..7a4289d 100644 --- a/dss_client/test/test_dss_c_interface.c +++ b/dss_client/test/test_dss_c_interface.c @@ -5,6 +5,7 @@ #include #include #include +#include #include "dss.h" @@ -18,6 +19,45 @@ int hexdump(unsigned char* buff, int size) return 0; } +int putRecursive(DSSClient c, char *basePath) +{ + int i; + char path[1000]; + struct dirent *dp; + DIR *dir = opendir(basePath); + + if (!dir){ + return 0; + } + + while ((dp = readdir(dir)) != NULL) + { + if (strcmp(dp->d_name, ".") != 0 && strcmp(dp->d_name, "..") != 0) + { + strcpy(path, basePath); + strcat(path, "/"); + strcat(path, dp->d_name); + // printf("%s\n", path); + FILE* fp = fopen(path, "r"); + if (fp == NULL) { + return 0; + } + fseek(fp, 0L, SEEK_END); + // calculating the size of the file + long int fsize = ftell(fp); + if (dp->d_type == DT_REG && fsize > 0){ // PutObject does not support empty files + if (PutObject(c, (void*) path, strlen(path), path) < 0 ){ + printf("PutObject - %s failed.\n", path); + return -1; + } + } + if (putRecursive(c, path) < 0) return -1; + } + } + closedir(dir); + return 0; +} + int main(int argc, char* argv[]) { DSSClient *c; int fd = -1; @@ -25,6 +65,9 @@ int main(int argc, char* argv[]) { int size = 1024*1024; unsigned char* buff; unsigned char* buff1; + int cur_page = -1; + int max_key_len = 1024; + char* keys_list; char obj_name[256]; char uuid[] = "12345"; @@ -38,8 +81,9 @@ int main(int argc, char* argv[]) { printf("Invalid endpoint URL\n"); return -1; } - if (argc == 5) + if (argc >= 5) size = atoi(argv[4]) * 1024; + buff = (unsigned char*)calloc(1, size); buff1 = (unsigned char*)calloc(1, size); @@ -77,10 +121,50 @@ int main(int argc, char* argv[]) { ret = DeleteObject(c, (void*) obj_name, strlen(obj_name)); printf("Object testfile1 deleted\n"); + // upload objects with a directory structure to test LIST + if (putRecursive(c, "/etc") < 0 ){ + printf("Uploading directory failed.\n"); + goto out; + } + //test the LIST + keys_list = (char*) malloc(sizeof(char) * max_key_len * GetPageSize()); + if (keys_list == NULL) { + printf("malloc failed for allocating buffer to store the LIST results\n"); + goto out; + } + while (1){ + cur_page = ListObjects(c, "", "", keys_list, cur_page); // List all objects with prefix "" + if (cur_page == FAILURE){ + printf("ListObjects failed.\n"); + goto out; + } + if (cur_page == END_OF_LIST) break; + printf("page index = %d:\n[%s]\n", cur_page, keys_list); + } + //test the DeleteAll + ret = DeleteAll(c, "etc", ""); + if (ret < 0) { + printf("DeleteAll failed.\n"); + goto out; + } + //retest the LIST after DeleteAll + printf("After DeleteAll:\n"); + cur_page = -1; + while (1){ + cur_page = ListObjects(c, "", "", keys_list, cur_page); // List all objects with prefix="" + if (cur_page == FAILURE){ + printf("ListObjects failed.\n"); + goto out; + } + if (cur_page == END_OF_LIST) break; + printf("page index = %d:\n[%s]\n", cur_page, keys_list); + } + out: if (fd != -1) close(fd); free(buff); free(buff1); + free(keys_list); return ret; }