From eba567c230361118ed94d159fd6088131ff10082 Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Wed, 14 Aug 2024 13:18:21 +0200 Subject: [PATCH] refactor: factor out remaining options from options.h --- cmake/libdwarfs.cmake | 2 +- include/dwarfs/history.h | 2 +- include/dwarfs/history_config.h | 30 ++++++ include/dwarfs/utility/rewrite_filesystem.h | 5 +- include/dwarfs/utility/rewrite_options.h | 43 ++++++++ include/dwarfs/writer/file_order_options.h | 43 ++++++++ include/dwarfs/writer/filesystem_writer.h | 23 ++-- .../dwarfs/writer/filesystem_writer_options.h | 35 ++++++ include/dwarfs/writer/fragment_order_parser.h | 2 +- include/dwarfs/writer/inode_options.h | 43 ++++++++ include/dwarfs/writer/scanner.h | 4 +- .../{options.h => writer/scanner_options.h} | 60 +---------- src/util.cpp | 1 - src/utility/filesystem_extractor.cpp | 1 - src/utility/rewrite_filesystem.cpp | 1 + .../file_order_options.cpp} | 6 +- src/writer/filesystem_writer.cpp | 14 +++ src/writer/internal/entry.cpp | 1 - src/writer/internal/file_scanner.cpp | 1 - src/writer/internal/global_entry_data.cpp | 2 +- src/writer/internal/inode_manager.cpp | 2 +- src/writer/internal/inode_ordering.cpp | 2 +- src/writer/scanner.cpp | 2 +- test/badfs_test.cpp | 1 - test/compat_test.cpp | 12 ++- test/dwarfs_benchmark.cpp | 3 +- test/dwarfs_test.cpp | 101 +++++++++--------- test/utils_test.cpp | 1 - tools/src/mkdwarfs_main.cpp | 10 +- 29 files changed, 312 insertions(+), 141 deletions(-) create mode 100644 include/dwarfs/history_config.h create mode 100644 include/dwarfs/utility/rewrite_options.h create mode 100644 include/dwarfs/writer/file_order_options.h create mode 100644 include/dwarfs/writer/filesystem_writer_options.h create mode 100644 include/dwarfs/writer/inode_options.h rename include/dwarfs/{options.h => writer/scanner_options.h} (56%) rename src/{options.cpp => writer/file_order_options.cpp} (93%) diff --git a/cmake/libdwarfs.cmake b/cmake/libdwarfs.cmake index b7a09981a..12fa09a92 100644 --- a/cmake/libdwarfs.cmake +++ b/cmake/libdwarfs.cmake @@ -35,7 +35,6 @@ add_library( src/logger.cpp src/mmap.cpp src/option_map.cpp - src/options.cpp src/os_access_generic.cpp src/pcm_sample_transformer.cpp src/performance_monitor.cpp @@ -92,6 +91,7 @@ add_library( src/writer/compression_metadata_requirements.cpp src/writer/console_writer.cpp src/writer/entry_factory.cpp + src/writer/file_order_options.cpp src/writer/filesystem_block_category_resolver.cpp src/writer/filesystem_writer.cpp src/writer/filter_debug.cpp diff --git a/include/dwarfs/history.h b/include/dwarfs/history.h index 3cc505307..9ee4e018b 100644 --- a/include/dwarfs/history.h +++ b/include/dwarfs/history.h @@ -30,7 +30,7 @@ #include -#include +#include namespace dwarfs { diff --git a/include/dwarfs/history_config.h b/include/dwarfs/history_config.h new file mode 100644 index 000000000..b7b7ad60b --- /dev/null +++ b/include/dwarfs/history_config.h @@ -0,0 +1,30 @@ +/* vim:set ts=2 sw=2 sts=2 et: */ +/** + * \author Marcus Holland-Moritz (github@mhxnet.de) + * \copyright Copyright (c) Marcus Holland-Moritz + * + * This file is part of dwarfs. + * + * dwarfs is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * dwarfs is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with dwarfs. If not, see . + */ + +#pragma once + +namespace dwarfs { + +struct history_config { + bool with_timestamps{false}; +}; + +} // namespace dwarfs diff --git a/include/dwarfs/utility/rewrite_filesystem.h b/include/dwarfs/utility/rewrite_filesystem.h index c4ce65345..082046f43 100644 --- a/include/dwarfs/utility/rewrite_filesystem.h +++ b/include/dwarfs/utility/rewrite_filesystem.h @@ -25,9 +25,6 @@ namespace dwarfs { class logger; -// TODO: move to writer namespace -struct rewrite_options; - namespace reader { class filesystem_v2; @@ -43,6 +40,8 @@ class filesystem_writer; namespace utility { +struct rewrite_options; + void rewrite_filesystem(logger& lgr, dwarfs::reader::filesystem_v2 const& fs, dwarfs::writer::filesystem_writer& writer, dwarfs::writer::category_resolver const& cat_resolver, diff --git a/include/dwarfs/utility/rewrite_options.h b/include/dwarfs/utility/rewrite_options.h new file mode 100644 index 000000000..979f08acd --- /dev/null +++ b/include/dwarfs/utility/rewrite_options.h @@ -0,0 +1,43 @@ +/* vim:set ts=2 sw=2 sts=2 et: */ +/** + * \author Marcus Holland-Moritz (github@mhxnet.de) + * \copyright Copyright (c) Marcus Holland-Moritz + * + * This file is part of dwarfs. + * + * dwarfs is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * dwarfs is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with dwarfs. If not, see . + */ + +#pragma once + +#include +#include +#include +#include + +#include + +namespace dwarfs::utility { + +struct rewrite_options { + bool recompress_block{false}; + bool recompress_metadata{false}; + std::unordered_set recompress_categories; + bool recompress_categories_exclude{false}; + bool enable_history{true}; + std::optional> command_line_arguments; + history_config history; +}; + +} // namespace dwarfs::utility diff --git a/include/dwarfs/writer/file_order_options.h b/include/dwarfs/writer/file_order_options.h new file mode 100644 index 000000000..ba32366b4 --- /dev/null +++ b/include/dwarfs/writer/file_order_options.h @@ -0,0 +1,43 @@ +/* vim:set ts=2 sw=2 sts=2 et: */ +/** + * \author Marcus Holland-Moritz (github@mhxnet.de) + * \copyright Copyright (c) Marcus Holland-Moritz + * + * This file is part of dwarfs. + * + * dwarfs is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * dwarfs is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with dwarfs. If not, see . + */ + +#pragma once + +#include + +namespace dwarfs::writer { + +// TODO: rename? -> inode_order_mode / fragment_order_mode +enum class file_order_mode { NONE, PATH, REVPATH, SIMILARITY, NILSIMSA }; + +// TODO: rename? -> inode_order_options / fragment_order_options +struct file_order_options { + static constexpr int const kDefaultNilsimsaMaxChildren{16384}; + static constexpr int const kDefaultNilsimsaMaxClusterSize{16384}; + + file_order_mode mode{file_order_mode::NONE}; + int nilsimsa_max_children{kDefaultNilsimsaMaxChildren}; + int nilsimsa_max_cluster_size{kDefaultNilsimsaMaxClusterSize}; +}; + +std::ostream& operator<<(std::ostream& os, file_order_mode mode); + +} // namespace dwarfs::writer diff --git a/include/dwarfs/writer/filesystem_writer.h b/include/dwarfs/writer/filesystem_writer.h index bdd1b59e7..51c870fb1 100644 --- a/include/dwarfs/writer/filesystem_writer.h +++ b/include/dwarfs/writer/filesystem_writer.h @@ -24,7 +24,6 @@ #include #include -#include #include namespace dwarfs { @@ -34,6 +33,8 @@ class thread_pool; namespace writer { +struct filesystem_writer_options; + class writer_progress; namespace internal { @@ -44,12 +45,20 @@ class filesystem_writer_detail; class filesystem_writer { public: - filesystem_writer( - std::ostream& os, logger& lgr, thread_pool& pool, writer_progress& prog, - block_compressor const& schema_bc, block_compressor const& metadata_bc, - block_compressor const& history_bc, - filesystem_writer_options const& options = filesystem_writer_options(), - std::istream* header = nullptr); + filesystem_writer(std::ostream& os, logger& lgr, thread_pool& pool, + writer_progress& prog, block_compressor const& bc); + + filesystem_writer(std::ostream& os, logger& lgr, thread_pool& pool, + writer_progress& prog, block_compressor const& schema_bc, + block_compressor const& metadata_bc, + block_compressor const& history_bc); + + filesystem_writer(std::ostream& os, logger& lgr, thread_pool& pool, + writer_progress& prog, block_compressor const& schema_bc, + block_compressor const& metadata_bc, + block_compressor const& history_bc, + filesystem_writer_options const& options, + std::istream* header = nullptr); ~filesystem_writer(); filesystem_writer(filesystem_writer&&); diff --git a/include/dwarfs/writer/filesystem_writer_options.h b/include/dwarfs/writer/filesystem_writer_options.h new file mode 100644 index 000000000..c52f99c0b --- /dev/null +++ b/include/dwarfs/writer/filesystem_writer_options.h @@ -0,0 +1,35 @@ +/* vim:set ts=2 sw=2 sts=2 et: */ +/** + * \author Marcus Holland-Moritz (github@mhxnet.de) + * \copyright Copyright (c) Marcus Holland-Moritz + * + * This file is part of dwarfs. + * + * dwarfs is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * dwarfs is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with dwarfs. If not, see . + */ + +#pragma once + +#include + +namespace dwarfs::writer { + +struct filesystem_writer_options { + size_t max_queue_size{64 << 20}; + size_t worst_case_block_size{4 << 20}; + bool remove_header{false}; + bool no_section_index{false}; +}; + +} // namespace dwarfs::writer diff --git a/include/dwarfs/writer/fragment_order_parser.h b/include/dwarfs/writer/fragment_order_parser.h index c28425437..9dc2f9ce6 100644 --- a/include/dwarfs/writer/fragment_order_parser.h +++ b/include/dwarfs/writer/fragment_order_parser.h @@ -24,7 +24,7 @@ #include #include -#include +#include namespace dwarfs::writer { diff --git a/include/dwarfs/writer/inode_options.h b/include/dwarfs/writer/inode_options.h new file mode 100644 index 000000000..f0466e9ef --- /dev/null +++ b/include/dwarfs/writer/inode_options.h @@ -0,0 +1,43 @@ +/* vim:set ts=2 sw=2 sts=2 et: */ +/** + * \author Marcus Holland-Moritz (github@mhxnet.de) + * \copyright Copyright (c) Marcus Holland-Moritz + * + * This file is part of dwarfs. + * + * dwarfs is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * dwarfs is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with dwarfs. If not, see . + */ + +#pragma once + +#include +#include +#include +#include + +#include +#include + +namespace dwarfs::writer { + +class categorizer_manager; + +struct inode_options { + std::optional max_similarity_scan_size; + std::shared_ptr categorizer_mgr; + writer::categorized_option fragment_order{ + file_order_options()}; +}; + +} // namespace dwarfs::writer diff --git a/include/dwarfs/writer/scanner.h b/include/dwarfs/writer/scanner.h index 413be6c32..1ca1e5498 100644 --- a/include/dwarfs/writer/scanner.h +++ b/include/dwarfs/writer/scanner.h @@ -29,8 +29,6 @@ namespace dwarfs { -struct scanner_options; - class file_access; class filesystem_writer; class logger; @@ -39,6 +37,8 @@ class thread_pool; namespace writer { +struct scanner_options; + class entry_filter; class entry_transformer; class entry_factory; diff --git a/include/dwarfs/options.h b/include/dwarfs/writer/scanner_options.h similarity index 56% rename from include/dwarfs/options.h rename to include/dwarfs/writer/scanner_options.h index 653117e15..1dfa0ff5e 100644 --- a/include/dwarfs/options.h +++ b/include/dwarfs/writer/scanner_options.h @@ -22,58 +22,20 @@ #pragma once #include +#include #include -#include -#include #include #include -#include -#include #include #include -#include +#include +#include -namespace dwarfs { +namespace dwarfs::writer { -namespace writer { - -class categorizer_manager; class entry_interface; -} // namespace writer - -struct history_config { - bool with_timestamps{false}; -}; - -struct filesystem_writer_options { - size_t max_queue_size{64 << 20}; - size_t worst_case_block_size{4 << 20}; - bool remove_header{false}; - bool no_section_index{false}; -}; - -// TODO: rename -enum class file_order_mode { NONE, PATH, REVPATH, SIMILARITY, NILSIMSA }; - -// TODO: rename -struct file_order_options { - static constexpr int const kDefaultNilsimsaMaxChildren{16384}; - static constexpr int const kDefaultNilsimsaMaxClusterSize{16384}; - - file_order_mode mode{file_order_mode::NONE}; - int nilsimsa_max_children{kDefaultNilsimsaMaxChildren}; - int nilsimsa_max_cluster_size{kDefaultNilsimsaMaxClusterSize}; -}; - -struct inode_options { - std::optional max_similarity_scan_size; - std::shared_ptr categorizer_mgr; - writer::categorized_option fragment_order{ - file_order_options()}; -}; - struct scanner_options { std::optional file_hash_algorithm{"xxh3-128"}; std::optional uid; @@ -104,16 +66,4 @@ struct scanner_options { history_config history; }; -struct rewrite_options { - bool recompress_block{false}; - bool recompress_metadata{false}; - std::unordered_set recompress_categories; - bool recompress_categories_exclude{false}; - bool enable_history{true}; - std::optional> command_line_arguments; - history_config history; -}; - -std::ostream& operator<<(std::ostream& os, file_order_mode mode); - -} // namespace dwarfs +} // namespace dwarfs::writer diff --git a/src/util.cpp b/src/util.cpp index 609d8043b..eac79ada5 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -50,7 +50,6 @@ #include #include -#include #include extern "C" int dwarfs_wcwidth(int ucs); diff --git a/src/utility/filesystem_extractor.cpp b/src/utility/filesystem_extractor.cpp index d871ab2a4..f5f182df3 100644 --- a/src/utility/filesystem_extractor.cpp +++ b/src/utility/filesystem_extractor.cpp @@ -43,7 +43,6 @@ #include #include #include -#include #include #include #include diff --git a/src/utility/rewrite_filesystem.cpp b/src/utility/rewrite_filesystem.cpp index b0aad7995..6d257f7b3 100644 --- a/src/utility/rewrite_filesystem.cpp +++ b/src/utility/rewrite_filesystem.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include diff --git a/src/options.cpp b/src/writer/file_order_options.cpp similarity index 93% rename from src/options.cpp rename to src/writer/file_order_options.cpp index d3310b971..d5932c1e9 100644 --- a/src/options.cpp +++ b/src/writer/file_order_options.cpp @@ -25,9 +25,9 @@ #include #include -#include +#include -namespace dwarfs { +namespace dwarfs::writer { std::ostream& operator<<(std::ostream& os, file_order_mode mode) { std::string modestr{"unknown"}; @@ -53,4 +53,4 @@ std::ostream& operator<<(std::ostream& os, file_order_mode mode) { return os << modestr; } -} // namespace dwarfs +} // namespace dwarfs::writer diff --git a/src/writer/filesystem_writer.cpp b/src/writer/filesystem_writer.cpp index 792234b5e..ffda259a8 100644 --- a/src/writer/filesystem_writer.cpp +++ b/src/writer/filesystem_writer.cpp @@ -41,6 +41,7 @@ #include #include #include +#include #include #include @@ -1102,6 +1103,19 @@ void filesystem_writer_::write_section_index() { } // namespace internal +filesystem_writer::filesystem_writer(std::ostream& os, logger& lgr, + thread_pool& pool, writer_progress& prog, + block_compressor const& bc) + : filesystem_writer(os, lgr, pool, prog, bc, bc, bc) {} + +filesystem_writer::filesystem_writer(std::ostream& os, logger& lgr, + thread_pool& pool, writer_progress& prog, + block_compressor const& schema_bc, + block_compressor const& metadata_bc, + block_compressor const& history_bc) + : filesystem_writer(os, lgr, pool, prog, schema_bc, metadata_bc, history_bc, + filesystem_writer_options{}) {} + filesystem_writer::filesystem_writer(std::ostream& os, logger& lgr, thread_pool& pool, writer_progress& prog, block_compressor const& schema_bc, diff --git a/src/writer/internal/entry.cpp b/src/writer/internal/entry.cpp index b2e2a88b0..29756d2ce 100644 --- a/src/writer/internal/entry.cpp +++ b/src/writer/internal/entry.cpp @@ -29,7 +29,6 @@ #include #include #include -#include #include #include diff --git a/src/writer/internal/file_scanner.cpp b/src/writer/internal/file_scanner.cpp index ba5987558..ee93f0a0b 100644 --- a/src/writer/internal/file_scanner.cpp +++ b/src/writer/internal/file_scanner.cpp @@ -36,7 +36,6 @@ #include #include #include -#include #include #include diff --git a/src/writer/internal/global_entry_data.cpp b/src/writer/internal/global_entry_data.cpp index c6464bfc2..304a6ef2d 100644 --- a/src/writer/internal/global_entry_data.cpp +++ b/src/writer/internal/global_entry_data.cpp @@ -24,7 +24,7 @@ #include #include -#include +#include #include diff --git a/src/writer/internal/inode_manager.cpp b/src/writer/internal/inode_manager.cpp index 0ed7ab279..c58cee551 100644 --- a/src/writer/internal/inode_manager.cpp +++ b/src/writer/internal/inode_manager.cpp @@ -45,10 +45,10 @@ #include #include #include -#include #include #include #include +#include #include #include diff --git a/src/writer/internal/inode_ordering.cpp b/src/writer/internal/inode_ordering.cpp index a81e5b9c0..c305e607a 100644 --- a/src/writer/internal/inode_ordering.cpp +++ b/src/writer/internal/inode_ordering.cpp @@ -22,7 +22,7 @@ #include #include -#include +#include #include #include diff --git a/src/writer/scanner.cpp b/src/writer/scanner.cpp index ac3b812a9..cc49c11a5 100644 --- a/src/writer/scanner.cpp +++ b/src/writer/scanner.cpp @@ -43,7 +43,6 @@ #include #include #include -#include #include #include #include @@ -54,6 +53,7 @@ #include #include #include +#include #include #include diff --git a/test/badfs_test.cpp b/test/badfs_test.cpp index 68c5bbb81..c030eacac 100644 --- a/test/badfs_test.cpp +++ b/test/badfs_test.cpp @@ -26,7 +26,6 @@ #include #include -#include #include #include diff --git a/test/compat_test.cpp b/test/compat_test.cpp index 1581f5030..4e317eaff 100644 --- a/test/compat_test.cpp +++ b/test/compat_test.cpp @@ -47,9 +47,11 @@ #include #include #include +#include #include #include #include +#include #include #include "mmap_mock.h" @@ -1116,7 +1118,7 @@ TEST_P(rewrite, filesystem_rewrite) { test::os_access_mock os; auto filename = std::string(TEST_DATA_DIR "/compat-v") + version + ".dwarfs"; - rewrite_options opts; + utility::rewrite_options opts; opts.recompress_block = recompress_block; opts.recompress_metadata = recompress_metadata; @@ -1156,7 +1158,7 @@ TEST_P(rewrite, filesystem_rewrite) { { std::istringstream hdr_iss(format_sh); - filesystem_writer_options fsw_opts; + writer::filesystem_writer_options fsw_opts; writer::filesystem_writer fsw(rewritten, lgr, pool, prog, bc, bc, bc, fsw_opts, &hdr_iss); fsw.add_default_compressor(bc); @@ -1183,7 +1185,7 @@ TEST_P(rewrite, filesystem_rewrite) { { std::istringstream hdr_iss("D"); - filesystem_writer_options fsw_opts; + writer::filesystem_writer_options fsw_opts; writer::filesystem_writer fsw(rewritten2, lgr, pool, prog, bc, bc, bc, fsw_opts, &hdr_iss); fsw.add_default_compressor(bc); @@ -1219,7 +1221,7 @@ TEST_P(rewrite, filesystem_rewrite) { std::ostringstream rewritten4; { - filesystem_writer_options fsw_opts; + writer::filesystem_writer_options fsw_opts; fsw_opts.remove_header = true; writer::filesystem_writer fsw(rewritten4, lgr, pool, prog, bc, bc, bc, fsw_opts); @@ -1239,7 +1241,7 @@ TEST_P(rewrite, filesystem_rewrite) { std::ostringstream rewritten5; { - filesystem_writer_options fsw_opts; + writer::filesystem_writer_options fsw_opts; fsw_opts.no_section_index = true; writer::filesystem_writer fsw(rewritten5, lgr, pool, prog, bc, bc, bc, fsw_opts); diff --git a/test/dwarfs_benchmark.cpp b/test/dwarfs_benchmark.cpp index 7a0028192..a00fdd011 100644 --- a/test/dwarfs_benchmark.cpp +++ b/test/dwarfs_benchmark.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -96,7 +97,7 @@ void PackParamsDirs(::benchmark::internal::Benchmark* b) { std::string make_filesystem(::benchmark::State const& state) { writer::segmenter_factory::config cfg; - scanner_options options; + writer::scanner_options options; cfg.blockhash_window_size.set_default(12); cfg.window_increment_shift.set_default(1); diff --git a/test/dwarfs_test.cpp b/test/dwarfs_test.cpp index ea2761314..c382c93f6 100644 --- a/test/dwarfs_test.cpp +++ b/test/dwarfs_test.cpp @@ -49,10 +49,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include @@ -78,7 +80,7 @@ std::string build_dwarfs(logger& lgr, std::shared_ptr input, std::string const& compression, writer::segmenter::config const& cfg = writer::segmenter::config(), - scanner_options const& options = scanner_options(), + writer::scanner_options const& options = writer::scanner_options(), writer::writer_progress* prog = nullptr, std::shared_ptr ftd = nullptr, std::optional> input_list = @@ -126,24 +128,22 @@ build_dwarfs(logger& lgr, std::shared_ptr input, return oss.str(); } -void basic_end_to_end_test(std::string const& compressor, - unsigned block_size_bits, file_order_mode file_order, - bool with_devices, bool with_specials, bool set_uid, - bool set_gid, bool set_time, bool keep_all_times, - bool enable_nlink, bool pack_chunk_table, - bool pack_directories, bool pack_shared_files_table, - bool pack_names, bool pack_names_index, - bool pack_symlinks, bool pack_symlinks_index, - bool plain_names_table, bool plain_symlinks_table, - bool access_fail, size_t readahead, - std::optional file_hash_algo) { +void basic_end_to_end_test( + std::string const& compressor, unsigned block_size_bits, + writer::file_order_mode file_order, bool with_devices, bool with_specials, + bool set_uid, bool set_gid, bool set_time, bool keep_all_times, + bool enable_nlink, bool pack_chunk_table, bool pack_directories, + bool pack_shared_files_table, bool pack_names, bool pack_names_index, + bool pack_symlinks, bool pack_symlinks_index, bool plain_names_table, + bool plain_symlinks_table, bool access_fail, size_t readahead, + std::optional file_hash_algo) { writer::segmenter::config cfg; - scanner_options options; + writer::scanner_options options; cfg.blockhash_window_size = 10; cfg.block_size_bits = block_size_bits; - file_order_options order_opts; + writer::file_order_options order_opts; order_opts.mode = file_order; options.file_hash_algorithm = file_hash_algo; @@ -195,8 +195,8 @@ void basic_end_to_end_test(std::string const& compressor, auto image_size = fsimage.size(); auto mm = std::make_shared(std::move(fsimage)); - bool similarity = file_order == file_order_mode::SIMILARITY || - file_order == file_order_mode::NILSIMSA; + bool similarity = file_order == writer::file_order_mode::SIMILARITY || + file_order == writer::file_order_mode::NILSIMSA; size_t const num_fail_empty = access_fail ? 1 : 0; @@ -576,8 +576,9 @@ std::vector const compressions{ } // namespace class compression_test - : public testing::TestWithParam>> { + : public testing::TestWithParam< + std::tuple>> { DWARFS_SLOW_FIXTURE }; @@ -618,7 +619,7 @@ TEST_P(scanner_test, end_to_end) { auto [with_devices, with_specials, set_uid, set_gid, set_time, keep_all_times, enable_nlink, access_fail, file_hash_algo] = GetParam(); - basic_end_to_end_test(compressions[0], 15, file_order_mode::NONE, + basic_end_to_end_test(compressions[0], 15, writer::file_order_mode::NONE, with_devices, with_specials, set_uid, set_gid, set_time, keep_all_times, enable_nlink, true, true, true, true, true, true, true, false, false, access_fail, 0, @@ -626,29 +627,31 @@ TEST_P(scanner_test, end_to_end) { } TEST_P(hashing_test, end_to_end) { - basic_end_to_end_test(compressions[0], 15, file_order_mode::NONE, true, true, + basic_end_to_end_test(compressions[0], 15, writer::file_order_mode::NONE, true, true, true, true, true, true, true, true, true, - true, true, true, false, false, false, 0, GetParam()); + true, true, true, true, true, false, false, false, 0, + GetParam()); } TEST_P(packing_test, end_to_end) { auto [pack_chunk_table, pack_directories, pack_shared_files_table, pack_names, pack_names_index, pack_symlinks, pack_symlinks_index] = GetParam(); - basic_end_to_end_test(compressions[0], 15, file_order_mode::NONE, true, true, - false, false, false, false, false, pack_chunk_table, - pack_directories, pack_shared_files_table, pack_names, - pack_names_index, pack_symlinks, pack_symlinks_index, - false, false, false, 0, default_file_hash_algo); + basic_end_to_end_test( + compressions[0], 15, writer::file_order_mode::NONE, true, true, false, + false, false, false, false, pack_chunk_table, pack_directories, + pack_shared_files_table, pack_names, pack_names_index, pack_symlinks, + pack_symlinks_index, false, false, false, 0, default_file_hash_algo); } TEST_P(plain_tables_test, end_to_end) { auto [plain_names_table, plain_symlinks_table] = GetParam(); - basic_end_to_end_test(compressions[0], 15, file_order_mode::NONE, true, true, - false, false, false, false, false, false, false, false, - false, false, false, false, plain_names_table, - plain_symlinks_table, false, 0, default_file_hash_algo); + basic_end_to_end_test(compressions[0], 15, writer::file_order_mode::NONE, + true, true, false, false, false, false, false, false, + false, false, false, false, false, false, + plain_names_table, plain_symlinks_table, false, 0, + default_file_hash_algo); } TEST_P(packing_test, regression_empty_fs) { @@ -656,7 +659,7 @@ TEST_P(packing_test, regression_empty_fs) { pack_names_index, pack_symlinks, pack_symlinks_index] = GetParam(); writer::segmenter::config cfg; - scanner_options options; + writer::scanner_options options; cfg.blockhash_window_size = 8; cfg.block_size_bits = 10; @@ -704,12 +707,14 @@ TEST_P(packing_test, regression_empty_fs) { INSTANTIATE_TEST_SUITE_P( dwarfs, compression_test, - ::testing::Combine( - ::testing::ValuesIn(compressions), ::testing::Values(12, 15, 20, 28), - ::testing::Values(file_order_mode::NONE, file_order_mode::PATH, - file_order_mode::REVPATH, file_order_mode::NILSIMSA, - file_order_mode::SIMILARITY), - ::testing::Values(std::nullopt, "xxh3-128"))); + ::testing::Combine(::testing::ValuesIn(compressions), + ::testing::Values(12, 15, 20, 28), + ::testing::Values(writer::file_order_mode::NONE, + writer::file_order_mode::PATH, + writer::file_order_mode::REVPATH, + writer::file_order_mode::NILSIMSA, + writer::file_order_mode::SIMILARITY), + ::testing::Values(std::nullopt, "xxh3-128"))); INSTANTIATE_TEST_SUITE_P( dwarfs, scanner_test, @@ -860,7 +865,7 @@ INSTANTIATE_TEST_SUITE_P(dwarfs, compression_regression, class file_scanner : public testing::TestWithParam< - std::tuple>> { + std::tuple>> { DWARFS_SLOW_FIXTURE }; @@ -870,9 +875,9 @@ TEST_P(file_scanner, inode_ordering) { test::test_logger lgr; auto bmcfg = writer::segmenter::config(); - auto opts = scanner_options(); + auto opts = writer::scanner_options(); - file_order_options order_opts; + writer::file_order_options order_opts; order_opts.mode = order_mode; opts.file_hash_algorithm = file_hash_algo; @@ -918,10 +923,10 @@ TEST_P(file_scanner, inode_ordering) { INSTANTIATE_TEST_SUITE_P( dwarfs, file_scanner, - ::testing::Combine(::testing::Values(file_order_mode::PATH, - file_order_mode::REVPATH, - file_order_mode::SIMILARITY, - file_order_mode::NILSIMSA), + ::testing::Combine(::testing::Values(writer::file_order_mode::PATH, + writer::file_order_mode::REVPATH, + writer::file_order_mode::SIMILARITY, + writer::file_order_mode::NILSIMSA), ::testing::Values(std::nullopt, "xxh3-128"))); class filter_test @@ -969,7 +974,7 @@ class filter_test std::ostringstream oss; - scanner_options options; + writer::scanner_options options; options.remove_empty_dirs = false; options.debug_filter_function = [&](bool exclude, writer::entry_interface const& ei) { @@ -1006,7 +1011,7 @@ TEST_P(filter_test, filesystem) { writer::segmenter::config cfg; - scanner_options options; + writer::scanner_options options; options.remove_empty_dirs = true; auto fsimage = build_dwarfs(lgr, input, "null", cfg, options, nullptr, @@ -1087,9 +1092,9 @@ TEST(file_scanner, input_list) { test::test_logger lgr; auto bmcfg = writer::segmenter::config(); - auto opts = scanner_options(); + auto opts = writer::scanner_options(); - file_order_options order_opts; + writer::file_order_options order_opts; opts.inode.fragment_order.set_default(order_opts); auto input = test::os_access_mock::create_test_instance(); diff --git a/test/utils_test.cpp b/test/utils_test.cpp index 497a81531..9148b73e3 100644 --- a/test/utils_test.cpp +++ b/test/utils_test.cpp @@ -28,7 +28,6 @@ #include #include -#include #include #include diff --git a/tools/src/mkdwarfs_main.cpp b/tools/src/mkdwarfs_main.cpp index 0a88f09e8..72ca634d0 100644 --- a/tools/src/mkdwarfs_main.cpp +++ b/tools/src/mkdwarfs_main.cpp @@ -65,7 +65,6 @@ #include #include #include -#include #include #include #include @@ -77,6 +76,7 @@ #include #include #include +#include #include #include #include @@ -84,10 +84,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -404,7 +406,7 @@ int mkdwarfs_main(int argc, sys_char** argv, iolayer const& iol) { writer::fragment_order_parser order_parser; block_compressor_parser compressor_parser; - scanner_options options; + writer::scanner_options options; logger_options logopts; auto order_desc = "inode fragments order (" + order_parser.choices() + ")"; @@ -846,7 +848,7 @@ int mkdwarfs_main(int argc, sys_char** argv, iolayer const& iol) { path = iol.os->canonical(path); bool recompress = vm.count("recompress"); - rewrite_options rw_opts; + utility::rewrite_options rw_opts; if (recompress) { std::unordered_map const modes{ {"all", 3}, @@ -1067,7 +1069,7 @@ int mkdwarfs_main(int argc, sys_char** argv, iolayer const& iol) { ? 2000ms : 200ms; - filesystem_writer_options fswopts; + writer::filesystem_writer_options fswopts; fswopts.max_queue_size = mem_limit; fswopts.worst_case_block_size = UINT64_C(1) << sf_config.block_size_bits; fswopts.remove_header = remove_header;