From 30cddcab8e7535705b375f3306cb58c58f4ec494 Mon Sep 17 00:00:00 2001 From: Zhe Wang Date: Thu, 21 Nov 2024 13:02:14 -0800 Subject: [PATCH 1/2] init --- fdbclient/S3Client.actor.cpp | 260 ++++++++++++++++++ fdbclient/S3Cp.actor.cpp | 207 +------------- fdbclient/include/fdbclient/S3Client.actor.h | 42 +++ .../workloads/BulkLoadingS3Client.actor.cpp | 56 ++++ tests/CMakeLists.txt | 1 + tests/fast/BulkLoadingS3Client.toml | 6 + 6 files changed, 370 insertions(+), 202 deletions(-) create mode 100644 fdbclient/S3Client.actor.cpp create mode 100644 fdbclient/include/fdbclient/S3Client.actor.h create mode 100644 fdbserver/workloads/BulkLoadingS3Client.actor.cpp create mode 100644 tests/fast/BulkLoadingS3Client.toml diff --git a/fdbclient/S3Client.actor.cpp b/fdbclient/S3Client.actor.cpp new file mode 100644 index 00000000000..8cff6c6ca1c --- /dev/null +++ b/fdbclient/S3Client.actor.cpp @@ -0,0 +1,260 @@ +/* + * S3Cp.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#endif + +#include "fdbclient/BuildFlags.h" +#include "fdbclient/BackupContainerFileSystem.h" +#include "fdbclient/FDBTypes.h" +#include "fdbclient/IKnobCollection.h" +#include "fdbclient/Knobs.h" +#include "fdbclient/versions.h" +#include "fdbclient/S3BlobStore.h" +#include "fdbclient/S3Client.actor.h" +#include "flow/Platform.h" +#include "flow/ArgParseUtil.h" +#include "flow/FastRef.h" +#include "flow/Trace.h" +#include "flow/flow.h" +#include "flow/TLSConfig.actor.h" + +#include "flow/actorcompiler.h" // has to be last include + +extern const char* getSourceVersion(); + +// Copy files and directories to and from s3. +// Has a main function so can exercise the actor from the command line. Uses +// the S3BlobStoreEndpoint to interact with s3. The s3url is of the form +// expected by S3BlobStoreEndpoint: +// blobstore://:@/resource?bucket=, etc. +// See the section 'Backup URls' in the backup documentation, +// https://apple.github.io/foundationdb/backups.html, for more information. +// TODO: Handle prefix as a parameter on the URL so can strip the first part +// of the resource from the blobstore URL. + +// TLS and blob credentials for backups and setup for these credentials. +// Copied from fdbbackup/BackupTLSConfig.* and renamed S3CpTLSConfig. +struct S3CpTLSConfig { + std::string tlsCertPath, tlsKeyPath, tlsCAPath, tlsPassword, tlsVerifyPeers; + std::vector blobCredentials; + + // Returns if TLS setup is successful + bool setupTLS(); + + // Sets up blob crentials. Add the file specified by FDB_BLOB_CREDENTIALS as well. + // Note this must be called after g_network is set up. + void setupBlobCredentials(); +}; + +void S3CpTLSConfig::setupBlobCredentials() { + // Add blob credentials files from the environment to the list collected from the command line. + const char* blobCredsFromENV = getenv("FDB_BLOB_CREDENTIALS"); + if (blobCredsFromENV != nullptr) { + StringRef t((uint8_t*)blobCredsFromENV, strlen(blobCredsFromENV)); + do { + StringRef file = t.eat(":"); + if (file.size() != 0) + blobCredentials.push_back(file.toString()); + } while (t.size() != 0); + } + + // Update the global blob credential files list + std::vector* pFiles = (std::vector*)g_network->global(INetwork::enBlobCredentialFiles); + if (pFiles != nullptr) { + for (auto& f : blobCredentials) { + pFiles->push_back(f); + } + } +} + +bool S3CpTLSConfig::setupTLS() { + if (tlsCertPath.size()) { + try { + setNetworkOption(FDBNetworkOptions::TLS_CERT_PATH, tlsCertPath); + } catch (Error& e) { + std::cerr << "ERROR: cannot set TLS certificate path to " << tlsCertPath << " (" << e.what() << ")\n"; + return false; + } + } + + if (tlsCAPath.size()) { + try { + setNetworkOption(FDBNetworkOptions::TLS_CA_PATH, tlsCAPath); + } catch (Error& e) { + std::cerr << "ERROR: cannot set TLS CA path to " << tlsCAPath << " (" << e.what() << ")\n"; + return false; + } + } + if (tlsKeyPath.size()) { + try { + if (tlsPassword.size()) + setNetworkOption(FDBNetworkOptions::TLS_PASSWORD, tlsPassword); + + setNetworkOption(FDBNetworkOptions::TLS_KEY_PATH, tlsKeyPath); + } catch (Error& e) { + std::cerr << "ERROR: cannot set TLS key path to " << tlsKeyPath << " (" << e.what() << ")\n"; + return false; + } + } + if (tlsVerifyPeers.size()) { + try { + setNetworkOption(FDBNetworkOptions::TLS_VERIFY_PEERS, tlsVerifyPeers); + } catch (Error& e) { + std::cerr << "ERROR: cannot set TLS peer verification to " << tlsVerifyPeers << " (" << e.what() << ")\n"; + return false; + } + } + return true; +} + +// Get the endpoint for the given s3url. +// Populates parameters and resource with parse of s3url. +static Reference getEndpoint(std::string s3url, + std::string& resource, + S3BlobStoreEndpoint::ParametersT& parameters) { + std::string error; + Reference endpoint = + S3BlobStoreEndpoint::fromString(s3url, {}, &resource, &error, ¶meters); + if (resource.empty()) { + TraceEvent(SevError, "S3ClientEmptyResource").detail("s3url", s3url); + throw backup_invalid_url(); + } + for (auto c : resource) { + if (!isalnum(c) && c != '_' && c != '-' && c != '.' && c != '/') { + TraceEvent(SevError, "S3ClientIllegalCharacter").detail("s3url", s3url); + throw backup_invalid_url(); + } + } + if (error.size()) { + TraceEvent(SevError, "S3ClientGetEndpointError").detail("s3url", s3url).detail("error", error); + throw backup_invalid_url(); + } + return endpoint; +} + +// Copy filepath to bucket at resource in s3. +ACTOR Future copyUpFile(Reference endpoint, + std::string bucket, + std::string resource, + std::string filepath) { + // Reading an SST file fully into memory is pretty obnoxious. They are about 16MB on + // average. Streaming would require changing this s3blobstore interface. + // Make 32MB the max size for now even though its arbitrary and way to big. + state std::string content = readFileBytes(filepath, 1024 * 1024 * 32); + wait(endpoint->writeEntireFile(bucket, resource, content)); + TraceEvent("S3ClientUpload") + .detail("filepath", filepath) + .detail("bucket", bucket) + .detail("resource", resource) + .detail("size", content.size()); + return Void(); +} + +// Copy the file from the local filesystem up to s3. +ACTOR Future copyUpFile(std::string filepath, std::string s3url) { + std::string resource; + S3BlobStoreEndpoint::ParametersT parameters; + Reference endpoint = getEndpoint(s3url, resource, parameters); + wait(copyUpFile(endpoint, parameters["bucket"], resource, filepath)); + return Void(); +} + +// Copy the directory content from the local filesystem up to s3. +ACTOR Future copyUpDirectory(std::string dirpath, std::string s3url) { + state std::string resource; + S3BlobStoreEndpoint::ParametersT parameters; + state Reference endpoint = getEndpoint(s3url, resource, parameters); + state std::string bucket = parameters["bucket"]; + state std::vector files; + platform::findFilesRecursively(dirpath, files); + TraceEvent("S3ClientUploadDirStart") + .detail("filecount", files.size()) + .detail("bucket", bucket) + .detail("resource", resource); + for (const auto& file : files) { + std::string filepath = file; + std::string s3path = resource + "/" + file.substr(dirpath.size() + 1); + wait(copyUpFile(endpoint, bucket, s3path, filepath)); + } + TraceEvent("S3ClientUploadDirEnd").detail("bucket", bucket).detail("resource", resource); + return Void(); +} + +// Copy filepath to bucket at resource in s3. +ACTOR Future copyDownFile(Reference endpoint, + std::string bucket, + std::string resource, + std::string filepath) { + std::string content = wait(endpoint->readEntireFile(bucket, resource)); + auto parent = std::filesystem::path(filepath).parent_path(); + if (parent != "" && !std::filesystem::exists(parent)) { + std::filesystem::create_directories(parent); + } + writeFile(filepath, content); + TraceEvent("S3ClientDownload") + .detail("filepath", filepath) + .detail("bucket", bucket) + .detail("resource", resource) + .detail("size", content.size()); + return Void(); +} + +// Copy the file from s3 down to the local filesystem. +// Overwrites existing file. +ACTOR Future copyDownFile(std::string s3url, std::string filepath) { + std::string resource; + S3BlobStoreEndpoint::ParametersT parameters; + Reference endpoint = getEndpoint(s3url, resource, parameters); + wait(copyDownFile(endpoint, parameters["bucket"], resource, filepath)); + return Void(); +} + +// Copy down the directory content from s3 to the local filesystem. +ACTOR Future copyDownDirectory(std::string s3url, std::string dirpath) { + state std::string resource; + S3BlobStoreEndpoint::ParametersT parameters; + state Reference endpoint = getEndpoint(s3url, resource, parameters); + state std::string bucket = parameters["bucket"]; + S3BlobStoreEndpoint::ListResult items = wait(endpoint->listObjects(parameters["bucket"], resource)); + state std::vector objects = items.objects; + TraceEvent("S3ClientDownloadDirStart") + .detail("filecount", objects.size()) + .detail("bucket", bucket) + .detail("resource", resource); + for (const auto& object : objects) { + std::string filepath = dirpath + "/" + object.name.substr(resource.size()); + std::string s3path = object.name; + wait(copyDownFile(endpoint, bucket, s3path, filepath)); + } + TraceEvent("S3ClientDownloadDirEnd").detail("bucket", bucket).detail("resource", resource); + return Void(); +} diff --git a/fdbclient/S3Cp.actor.cpp b/fdbclient/S3Cp.actor.cpp index c8fdcf80771..7c39cb7b56f 100644 --- a/fdbclient/S3Cp.actor.cpp +++ b/fdbclient/S3Cp.actor.cpp @@ -39,6 +39,7 @@ #include "fdbclient/Knobs.h" #include "fdbclient/versions.h" #include "fdbclient/S3BlobStore.h" +#include "fdbclient/S3Client.actor.h" #include "flow/Platform.h" #include "flow/ArgParseUtil.h" #include "flow/FastRef.h" @@ -64,204 +65,6 @@ extern const char* getSourceVersion(); namespace s3cp { const std::string BLOBSTORE_PREFIX = "blobstore://"; -// TLS and blob credentials for backups and setup for these credentials. -// Copied from fdbbackup/BackupTLSConfig.* and renamed S3CpTLSConfig. -struct S3CpTLSConfig { - std::string tlsCertPath, tlsKeyPath, tlsCAPath, tlsPassword, tlsVerifyPeers; - std::vector blobCredentials; - - // Returns if TLS setup is successful - bool setupTLS(); - - // Sets up blob crentials. Add the file specified by FDB_BLOB_CREDENTIALS as well. - // Note this must be called after g_network is set up. - void setupBlobCredentials(); -}; - -void S3CpTLSConfig::setupBlobCredentials() { - // Add blob credentials files from the environment to the list collected from the command line. - const char* blobCredsFromENV = getenv("FDB_BLOB_CREDENTIALS"); - if (blobCredsFromENV != nullptr) { - StringRef t((uint8_t*)blobCredsFromENV, strlen(blobCredsFromENV)); - do { - StringRef file = t.eat(":"); - if (file.size() != 0) - blobCredentials.push_back(file.toString()); - } while (t.size() != 0); - } - - // Update the global blob credential files list - std::vector* pFiles = (std::vector*)g_network->global(INetwork::enBlobCredentialFiles); - if (pFiles != nullptr) { - for (auto& f : blobCredentials) { - pFiles->push_back(f); - } - } -} - -bool S3CpTLSConfig::setupTLS() { - if (tlsCertPath.size()) { - try { - setNetworkOption(FDBNetworkOptions::TLS_CERT_PATH, tlsCertPath); - } catch (Error& e) { - std::cerr << "ERROR: cannot set TLS certificate path to " << tlsCertPath << " (" << e.what() << ")\n"; - return false; - } - } - - if (tlsCAPath.size()) { - try { - setNetworkOption(FDBNetworkOptions::TLS_CA_PATH, tlsCAPath); - } catch (Error& e) { - std::cerr << "ERROR: cannot set TLS CA path to " << tlsCAPath << " (" << e.what() << ")\n"; - return false; - } - } - if (tlsKeyPath.size()) { - try { - if (tlsPassword.size()) - setNetworkOption(FDBNetworkOptions::TLS_PASSWORD, tlsPassword); - - setNetworkOption(FDBNetworkOptions::TLS_KEY_PATH, tlsKeyPath); - } catch (Error& e) { - std::cerr << "ERROR: cannot set TLS key path to " << tlsKeyPath << " (" << e.what() << ")\n"; - return false; - } - } - if (tlsVerifyPeers.size()) { - try { - setNetworkOption(FDBNetworkOptions::TLS_VERIFY_PEERS, tlsVerifyPeers); - } catch (Error& e) { - std::cerr << "ERROR: cannot set TLS peer verification to " << tlsVerifyPeers << " (" << e.what() << ")\n"; - return false; - } - } - return true; -} - -// Get the endpoint for the given s3url. -// Populates parameters and resource with parse of s3url. -static Reference getEndpoint(std::string s3url, - std::string& resource, - S3BlobStoreEndpoint::ParametersT& parameters) { - std::string error; - Reference endpoint = - S3BlobStoreEndpoint::fromString(s3url, {}, &resource, &error, ¶meters); - if (resource.empty()) { - TraceEvent(SevError, "EmptyResource").detail("s3url", s3url); - throw backup_invalid_url(); - } - for (auto c : resource) { - if (!isalnum(c) && c != '_' && c != '-' && c != '.' && c != '/') { - TraceEvent(SevError, "IllegalCharacter").detail("s3url", s3url); - throw backup_invalid_url(); - } - } - if (error.size()) { - TraceEvent(SevError, "GetEndpointError").detail("s3url", s3url).detail("error", error); - throw backup_invalid_url(); - } - return endpoint; -} - -// Copy filepath to bucket at resource in s3. -ACTOR Future copy_up_file(Reference endpoint, - std::string bucket, - std::string resource, - std::string filepath) { - // Reading an SST file fully into memory is pretty obnoxious. They are about 16MB on - // average. Streaming would require changing this s3blobstore interface. - // Make 32MB the max size for now even though its arbitrary and way to big. - state std::string content = readFileBytes(filepath, 1024 * 1024 * 32); - wait(endpoint->writeEntireFile(bucket, resource, content)); - TraceEvent("Upload") - .detail("filepath", filepath) - .detail("bucket", bucket) - .detail("resource", resource) - .detail("size", content.size()); - return Void(); -} - -// Copy the file from the local filesystem up to s3. -ACTOR Future copy_up_file(std::string filepath, std::string s3url) { - std::string resource; - S3BlobStoreEndpoint::ParametersT parameters; - Reference endpoint = getEndpoint(s3url, resource, parameters); - wait(copy_up_file(endpoint, parameters["bucket"], resource, filepath)); - return Void(); -} - -// Copy the directory content from the local filesystem up to s3. -ACTOR Future copy_up_directory(std::string dirpath, std::string s3url) { - state std::string resource; - S3BlobStoreEndpoint::ParametersT parameters; - state Reference endpoint = getEndpoint(s3url, resource, parameters); - state std::string bucket = parameters["bucket"]; - state std::vector files; - platform::findFilesRecursively(dirpath, files); - TraceEvent("UploadDirStart") - .detail("filecount", files.size()) - .detail("bucket", bucket) - .detail("resource", resource); - for (const auto& file : files) { - std::string filepath = file; - std::string s3path = resource + "/" + file.substr(dirpath.size() + 1); - wait(copy_up_file(endpoint, bucket, s3path, filepath)); - } - TraceEvent("UploadDirEnd").detail("bucket", bucket).detail("resource", resource); - return Void(); -} - -// Copy filepath to bucket at resource in s3. -ACTOR Future copy_down_file(Reference endpoint, - std::string bucket, - std::string resource, - std::string filepath) { - std::string content = wait(endpoint->readEntireFile(bucket, resource)); - auto parent = std::filesystem::path(filepath).parent_path(); - if (parent != "" && !std::filesystem::exists(parent)) { - std::filesystem::create_directories(parent); - } - writeFile(filepath, content); - TraceEvent("Download") - .detail("filepath", filepath) - .detail("bucket", bucket) - .detail("resource", resource) - .detail("size", content.size()); - return Void(); -} - -// Copy the file from s3 down to the local filesystem. -// Overwrites existing file. -ACTOR Future copy_down_file(std::string s3url, std::string filepath) { - std::string resource; - S3BlobStoreEndpoint::ParametersT parameters; - Reference endpoint = getEndpoint(s3url, resource, parameters); - wait(copy_down_file(endpoint, parameters["bucket"], resource, filepath)); - return Void(); -} - -// Copy down the directory content from s3 to the local filesystem. -ACTOR Future copy_down_directory(std::string s3url, std::string dirpath) { - state std::string resource; - S3BlobStoreEndpoint::ParametersT parameters; - state Reference endpoint = getEndpoint(s3url, resource, parameters); - state std::string bucket = parameters["bucket"]; - S3BlobStoreEndpoint::ListResult items = wait(endpoint->listObjects(parameters["bucket"], resource)); - state std::vector objects = items.objects; - TraceEvent("DownloadDirStart") - .detail("filecount", objects.size()) - .detail("bucket", bucket) - .detail("resource", resource); - for (const auto& object : objects) { - std::string filepath = dirpath + "/" + object.name.substr(resource.size()); - std::string s3path = object.name; - wait(copy_down_file(endpoint, bucket, s3path, filepath)); - } - TraceEvent("DownloadDirEnd").detail("bucket", bucket).detail("resource", resource); - return Void(); -} - enum { OPT_BLOB_CREDENTIALS, OPT_TRACE, @@ -478,15 +281,15 @@ static int parseCommandLine(Reference param, CSimpleOpt* args) { ACTOR Future run(Reference params) { if (params->whichIsBlobstoreURL == 1) { if (std::filesystem::is_directory(params->src)) { - wait(copy_up_directory(params->src, params->tgt)); + wait(copyUpDirectory(params->src, params->tgt)); } else { - wait(copy_up_file(params->src, params->tgt)); + wait(copyUpFile(params->src, params->tgt)); } } else { if (std::filesystem::is_directory(params->tgt)) { - wait(copy_down_directory(params->src, params->tgt)); + wait(copyDownDirectory(params->src, params->tgt)); } else { - wait(copy_down_file(params->src, params->tgt)); + wait(copyDownFile(params->src, params->tgt)); } } return Void(); diff --git a/fdbclient/include/fdbclient/S3Client.actor.h b/fdbclient/include/fdbclient/S3Client.actor.h new file mode 100644 index 00000000000..02c0a2993ad --- /dev/null +++ b/fdbclient/include/fdbclient/S3Client.actor.h @@ -0,0 +1,42 @@ +/* + * S3Client.actor.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_S3CLIENT_ACTOR_G_H) +#define FDBCLIENT_S3CLIENT_ACTOR_G_H +#include "fdbclient/S3Client.actor.g.h" +#elif !defined(FDBCLIENT_S3CLIENT_ACTOR_H) +#define FDBCLIENT_S3CLIENT_ACTOR_H + +#include "fdbclient/Knobs.h" +#include "fdbclient/SystemData.h" +#include "flow/actorcompiler.h" // This must be the last #include. + +ACTOR Future copyDownDirectory(std::string s3url, std::string dirpath); + +ACTOR Future copyDownFile(std::string s3url, std::string filepath); + +ACTOR Future copyUpDirectory(std::string dirpath, std::string s3url); + +ACTOR Future copyUpFile(std::string filepath, std::string s3url); + +#include "flow/unactorcompiler.h" +#endif diff --git a/fdbserver/workloads/BulkLoadingS3Client.actor.cpp b/fdbserver/workloads/BulkLoadingS3Client.actor.cpp new file mode 100644 index 00000000000..69477c7272c --- /dev/null +++ b/fdbserver/workloads/BulkLoadingS3Client.actor.cpp @@ -0,0 +1,56 @@ +/* + * BulkLoadingS3Client.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbclient/NativeAPI.actor.h" +// #include "fdbclient/S3Client.actor.h" +#include "fdbserver/workloads/workloads.actor.h" +#include "flow/actorcompiler.h" // This must be the last #include. + +struct BulkLoadingS3Client : TestWorkload { + static constexpr auto NAME = "BulkLoadingS3ClientWorkload"; + const bool enabled; + bool pass; + + BulkLoadingS3Client(WorkloadContext const& wcx) : TestWorkload(wcx), enabled(true), pass(true) {} + + Future setup(Database const& cx) override { return Void(); } + + Future start(Database const& cx) override { return _start(this, cx); } + + Future check(Database const& cx) override { return true; } + + void getMetrics(std::vector& m) override {} + + ACTOR Future _start(BulkLoadingS3Client* self, Database cx) { + if (self->clientId != 0) { + // Our simulation test can trigger multiple same workloads at the same time + // Only run one time workload in the simulation + return Void(); + } + + /* Add S3 function call here */ + /* Need to have an alive seaweed running with simulation */ + TraceEvent("BulkLoadingS3ClientWorkloadStart"); + // Run with ../build_output/bin/fdbserver -r simulation -f + // ../src/foundationdb/tests/fast/BulkLoadingS3Client.toml + } +}; + +WorkloadFactory BulkLoadingS3ClientFactory; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a827ff71066..78a376305ef 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -153,6 +153,7 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES rare/BlobRestoreToVersion.toml) add_fdb_test(TEST_FILES rare/BlobRestoreTenantMode.toml) add_fdb_test(TEST_FILES fast/BulkLoading.toml) + add_fdb_test(TEST_FILES fast/BulkLoadingS3Client.toml) add_fdb_test(TEST_FILES fast/CacheTest.toml) add_fdb_test(TEST_FILES fast/CloggedSideband.toml) add_fdb_test(TEST_FILES fast/CompressionUtilsUnit.toml IGNORE) diff --git a/tests/fast/BulkLoadingS3Client.toml b/tests/fast/BulkLoadingS3Client.toml new file mode 100644 index 00000000000..2e13a523b99 --- /dev/null +++ b/tests/fast/BulkLoadingS3Client.toml @@ -0,0 +1,6 @@ +[[test]] +testTitle = 'BulkLoadingS3ClientWorkload' +useDB = true # TODO(BulkLoad): remove if do not use FDB database in the test + + [[test.workload]] + testName = 'BulkLoadingS3ClientWorkload' From 48cb7493b974d8a855f65dd004ab83114ae7717f Mon Sep 17 00:00:00 2001 From: Zhe Wang Date: Thu, 21 Nov 2024 13:47:08 -0800 Subject: [PATCH 2/2] make s3 client work with simulation --- fdbclient/S3Client.actor.cpp | 30 +------------------ fdbclient/S3Cp.actor.cpp | 2 +- fdbclient/include/fdbclient/S3Client.actor.h | 27 +++++++++++++++-- .../workloads/BulkLoadingS3Client.actor.cpp | 6 +++- 4 files changed, 32 insertions(+), 33 deletions(-) diff --git a/fdbclient/S3Client.actor.cpp b/fdbclient/S3Client.actor.cpp index 8cff6c6ca1c..3805a949cdb 100644 --- a/fdbclient/S3Client.actor.cpp +++ b/fdbclient/S3Client.actor.cpp @@ -1,5 +1,5 @@ /* - * S3Cp.actor.cpp + * S3Client.actor.cpp * * This source file is part of the FoundationDB open source project * @@ -32,49 +32,21 @@ #include #endif -#include "fdbclient/BuildFlags.h" #include "fdbclient/BackupContainerFileSystem.h" #include "fdbclient/FDBTypes.h" #include "fdbclient/IKnobCollection.h" #include "fdbclient/Knobs.h" #include "fdbclient/versions.h" -#include "fdbclient/S3BlobStore.h" #include "fdbclient/S3Client.actor.h" #include "flow/Platform.h" -#include "flow/ArgParseUtil.h" #include "flow/FastRef.h" #include "flow/Trace.h" #include "flow/flow.h" -#include "flow/TLSConfig.actor.h" #include "flow/actorcompiler.h" // has to be last include extern const char* getSourceVersion(); -// Copy files and directories to and from s3. -// Has a main function so can exercise the actor from the command line. Uses -// the S3BlobStoreEndpoint to interact with s3. The s3url is of the form -// expected by S3BlobStoreEndpoint: -// blobstore://:@/resource?bucket=, etc. -// See the section 'Backup URls' in the backup documentation, -// https://apple.github.io/foundationdb/backups.html, for more information. -// TODO: Handle prefix as a parameter on the URL so can strip the first part -// of the resource from the blobstore URL. - -// TLS and blob credentials for backups and setup for these credentials. -// Copied from fdbbackup/BackupTLSConfig.* and renamed S3CpTLSConfig. -struct S3CpTLSConfig { - std::string tlsCertPath, tlsKeyPath, tlsCAPath, tlsPassword, tlsVerifyPeers; - std::vector blobCredentials; - - // Returns if TLS setup is successful - bool setupTLS(); - - // Sets up blob crentials. Add the file specified by FDB_BLOB_CREDENTIALS as well. - // Note this must be called after g_network is set up. - void setupBlobCredentials(); -}; - void S3CpTLSConfig::setupBlobCredentials() { // Add blob credentials files from the environment to the list collected from the command line. const char* blobCredsFromENV = getenv("FDB_BLOB_CREDENTIALS"); diff --git a/fdbclient/S3Cp.actor.cpp b/fdbclient/S3Cp.actor.cpp index 7c39cb7b56f..a006ef88208 100644 --- a/fdbclient/S3Cp.actor.cpp +++ b/fdbclient/S3Cp.actor.cpp @@ -136,7 +136,7 @@ struct Params : public ReferenceCounted { Optional proxy; bool log_enabled = false; std::string log_dir, trace_format, trace_log_group; - s3cp::S3CpTLSConfig tlsConfig; + S3CpTLSConfig tlsConfig; std::vector> knobs; std::string src; std::string tgt; diff --git a/fdbclient/include/fdbclient/S3Client.actor.h b/fdbclient/include/fdbclient/S3Client.actor.h index 02c0a2993ad..60a2fd4b729 100644 --- a/fdbclient/include/fdbclient/S3Client.actor.h +++ b/fdbclient/include/fdbclient/S3Client.actor.h @@ -26,10 +26,33 @@ #elif !defined(FDBCLIENT_S3CLIENT_ACTOR_H) #define FDBCLIENT_S3CLIENT_ACTOR_H -#include "fdbclient/Knobs.h" -#include "fdbclient/SystemData.h" +#include "fdbclient/S3BlobStore.h" #include "flow/actorcompiler.h" // This must be the last #include. +// Copy files and directories to and from s3. +// Has a main function so can exercise the actor from the command line. Uses +// the S3BlobStoreEndpoint to interact with s3. The s3url is of the form +// expected by S3BlobStoreEndpoint: +// blobstore://:@/resource?bucket=, etc. +// See the section 'Backup URls' in the backup documentation, +// https://apple.github.io/foundationdb/backups.html, for more information. +// TODO: Handle prefix as a parameter on the URL so can strip the first part +// of the resource from the blobstore URL. + +// TLS and blob credentials for backups and setup for these credentials. +// Copied from fdbbackup/BackupTLSConfig.* and renamed S3CpTLSConfig. +struct S3CpTLSConfig { + std::string tlsCertPath, tlsKeyPath, tlsCAPath, tlsPassword, tlsVerifyPeers; + std::vector blobCredentials; + + // Returns if TLS setup is successful + bool setupTLS(); + + // Sets up blob crentials. Add the file specified by FDB_BLOB_CREDENTIALS as well. + // Note this must be called after g_network is set up. + void setupBlobCredentials(); +}; + ACTOR Future copyDownDirectory(std::string s3url, std::string dirpath); ACTOR Future copyDownFile(std::string s3url, std::string filepath); diff --git a/fdbserver/workloads/BulkLoadingS3Client.actor.cpp b/fdbserver/workloads/BulkLoadingS3Client.actor.cpp index 69477c7272c..d48d5eaf488 100644 --- a/fdbserver/workloads/BulkLoadingS3Client.actor.cpp +++ b/fdbserver/workloads/BulkLoadingS3Client.actor.cpp @@ -48,8 +48,12 @@ struct BulkLoadingS3Client : TestWorkload { /* Add S3 function call here */ /* Need to have an alive seaweed running with simulation */ TraceEvent("BulkLoadingS3ClientWorkloadStart"); - // Run with ../build_output/bin/fdbserver -r simulation -f + wait(delay(0.1)); // code place holder for passing ACTOR compiling + + // Run this workload with ../build_output/bin/fdbserver -r simulation -f // ../src/foundationdb/tests/fast/BulkLoadingS3Client.toml + + return Void(); } };