Skip to content

Commit

Permalink
Merge bitcoin#31000: bench: add support for custom data directory
Browse files Browse the repository at this point in the history
fa66e08 bench: add support for custom data directory (furszy)
ad9c2cc test, bench: specialize working directory name (furszy)

Pull request description:

  Expands the benchmark framework with the existing `-testdatadir` arg,
  enabling the ability to change the benchmark data directory.

  This is useful for running benchmarks on different storage devices, and
  not just under the OS `/tmp/` directory.

  A good use case is bitcoin#28574, where we are benchmarking the wallet
  migration process on an HDD.

ACKs for top commit:
  maflcko:
    re-ACK fa66e08
  achow101:
    ACK fa66e08
  tdb3:
    re ACK fa66e08
  hodlinator:
    re-ACK fa66e08
  pablomartin4btc:
    re-ACK fa66e08

Tree-SHA512: 4e87206c07e26fe193c07074ae9eb0cc9c70a58aeea8cf27d18fb5425d77e4b00dbe0e6d6a75c17b427744e9066458b9a84e5ef7b0420f02a4fccb9c5ef4dacc
  • Loading branch information
achow101 committed Nov 14, 2024
2 parents 36f5eff + fa66e08 commit 4228259
Show file tree
Hide file tree
Showing 5 changed files with 53 additions and 12 deletions.
30 changes: 28 additions & 2 deletions src/bench/bench.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,26 @@ using util::Join;

const std::function<void(const std::string&)> G_TEST_LOG_FUN{};

const std::function<std::vector<const char*>()> G_TEST_COMMAND_LINE_ARGUMENTS{};
/**
* Retrieves the available test setup command line arguments that may be used
* in the benchmark. They will be used only if the benchmark utilizes a
* 'BasicTestingSetup' or any child of it.
*/
static std::function<std::vector<const char*>()> g_bench_command_line_args{};
const std::function<std::vector<const char*>()> G_TEST_COMMAND_LINE_ARGUMENTS = []() {
return g_bench_command_line_args();
};

const std::function<std::string()> G_TEST_GET_FULL_NAME{};
/**
* Retrieve the name of the currently in-use benchmark.
* This is applicable only to benchmarks that utilize the unit test
* framework context setup (e.g. ones using 'MakeNoLogFileContext<TestingSetup>()').
* It places the datadir of each benchmark run within their respective benchmark name.
*/
static std::string g_running_benchmark_name;
const std::function<std::string()> G_TEST_GET_FULL_NAME = []() {
return g_running_benchmark_name;
};

namespace {

Expand Down Expand Up @@ -94,6 +111,14 @@ void BenchRunner::RunAll(const Args& args)
std::cout << "Running with -sanity-check option, output is being suppressed as benchmark results will be useless." << std::endl;
}

// Load inner test setup args
g_bench_command_line_args = [&args]() {
std::vector<const char*> ret;
ret.reserve(args.setup_args.size());
for (const auto& arg : args.setup_args) ret.emplace_back(arg.c_str());
return ret;
};

std::vector<ankerl::nanobench::Result> benchmarkResults;
for (const auto& [name, bench_func] : benchmarks()) {
const auto& [func, priority_level] = bench_func;
Expand All @@ -117,6 +142,7 @@ void BenchRunner::RunAll(const Args& args)
bench.output(nullptr);
}
bench.name(name);
g_running_benchmark_name = name;
if (args.min_time > 0ms) {
// convert to nanos before dividing to reduce rounding errors
std::chrono::nanoseconds min_time_ns = args.min_time;
Expand Down
1 change: 1 addition & 0 deletions src/bench/bench.h
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ struct Args {
fs::path output_json;
std::string regex_filter;
uint8_t priority;
std::vector<std::string> setup_args;
};

class BenchRunner
Expand Down
15 changes: 15 additions & 0 deletions src/bench/bench_bitcoin.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#include <tinyformat.h>
#include <util/fs.h>
#include <util/string.h>
#include <test/util/setup_common.h>

#include <chrono>
#include <cstdint>
Expand All @@ -27,6 +28,7 @@ static const std::string DEFAULT_PRIORITY{"all"};
static void SetupBenchArgs(ArgsManager& argsman)
{
SetupHelpOptions(argsman);
SetupCommonTestArgs(argsman);

argsman.AddArg("-asymptote=<n1,n2,n3,...>", "Test asymptotic growth of the runtime of an algorithm, if supported by the benchmark", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
argsman.AddArg("-filter=<regex>", strprintf("Regular expression filter to select benchmark by name (default: %s)", DEFAULT_BENCH_FILTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
Expand Down Expand Up @@ -60,6 +62,18 @@ static uint8_t parsePriorityLevel(const std::string& str) {
return levels;
}

static std::vector<std::string> parseTestSetupArgs(const ArgsManager& argsman)
{
// Parses unit test framework arguments supported by the benchmark framework.
std::vector<std::string> args;
static std::vector<std::string> AVAILABLE_ARGS = {"-testdatadir"};
for (const std::string& arg_name : AVAILABLE_ARGS) {
auto op_arg = argsman.GetArg(arg_name);
if (op_arg) args.emplace_back(strprintf("%s=%s", arg_name, *op_arg));
}
return args;
}

int main(int argc, char** argv)
{
ArgsManager argsman;
Expand Down Expand Up @@ -131,6 +145,7 @@ int main(int argc, char** argv)
args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER);
args.sanity_check = argsman.GetBoolArg("-sanity-check", false);
args.priority = parsePriorityLevel(argsman.GetArg("-priority-level", DEFAULT_PRIORITY));
args.setup_args = parseTestSetupArgs(argsman);

benchmark::BenchRunner::RunAll(args);

Expand Down
16 changes: 6 additions & 10 deletions src/test/util/setup_common.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,6 @@ using node::VerifyLoadedChainstate;
const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr;

constexpr inline auto TEST_DIR_PATH_ELEMENT{"test_common bitcoin"}; // Includes a space to catch possible path escape issues.
/** Random context to get unique temp data dirs. Separate from m_rng, which can be seeded from a const env var */
static FastRandomContext g_rng_temp_path;

struct NetworkSetup
{
Expand All @@ -87,8 +85,7 @@ struct NetworkSetup
};
static NetworkSetup g_networksetup_instance;

/** Register test-only arguments */
static void SetupUnitTestArgs(ArgsManager& argsman)
void SetupCommonTestArgs(ArgsManager& argsman)
{
argsman.AddArg("-testdatadir", strprintf("Custom data directory (default: %s<random_string>)", fs::PathToString(fs::temp_directory_path() / TEST_DIR_PATH_ELEMENT / "")),
ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST);
Expand Down Expand Up @@ -127,7 +124,7 @@ BasicTestingSetup::BasicTestingSetup(const ChainType chainType, TestOpts opts)
gArgs.ClearPathCache();
{
SetupServerArgs(*m_node.args);
SetupUnitTestArgs(*m_node.args);
SetupCommonTestArgs(*m_node.args);
std::string error;
if (!m_node.args->ParseParameters(arguments.size(), arguments.data(), error)) {
m_node.args->ClearArgs();
Expand All @@ -139,10 +136,10 @@ BasicTestingSetup::BasicTestingSetup(const ChainType chainType, TestOpts opts)
// data directories use a random name that doesn't overlap with other tests.
SeedRandomForTest(SeedRand::FIXED_SEED);

const std::string test_name{G_TEST_GET_FULL_NAME ? G_TEST_GET_FULL_NAME() : ""};
if (!m_node.args->IsArgSet("-testdatadir")) {
// By default, the data directory has a random name
const auto rand_str{g_rng_temp_path.rand256().ToString()};
m_path_root = fs::temp_directory_path() / TEST_DIR_PATH_ELEMENT / rand_str;
const auto now{TicksSinceEpoch<std::chrono::nanoseconds>(SystemClock::now())};
m_path_root = fs::temp_directory_path() / TEST_DIR_PATH_ELEMENT / test_name / util::ToString(now);
TryCreateDirectories(m_path_root);
} else {
// Custom data directory
Expand All @@ -151,8 +148,7 @@ BasicTestingSetup::BasicTestingSetup(const ChainType chainType, TestOpts opts)
if (root_dir.empty()) ExitFailure("-testdatadir argument is empty, please specify a path");

root_dir = fs::absolute(root_dir);
const std::string test_path{G_TEST_GET_FULL_NAME ? G_TEST_GET_FULL_NAME() : ""};
m_path_lock = root_dir / TEST_DIR_PATH_ELEMENT / fs::PathFromString(test_path);
m_path_lock = root_dir / TEST_DIR_PATH_ELEMENT / fs::PathFromString(test_name);
m_path_root = m_path_lock / "datadir";

// Try to obtain the lock; if unsuccessful don't disturb the existing test.
Expand Down
3 changes: 3 additions & 0 deletions src/test/util/setup_common.h
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ extern const std::function<std::string()> G_TEST_GET_FULL_NAME;

static constexpr CAmount CENT{1000000};

/** Register common test args. Shared across binaries that rely on the test framework. */
void SetupCommonTestArgs(ArgsManager& argsman);

struct TestOpts {
std::vector<const char*> extra_args{};
bool coins_db_in_memory{true};
Expand Down

0 comments on commit 4228259

Please sign in to comment.