From e4ca1aa469f8d734d8862174316ce6258b93a0c0 Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Mon, 18 Dec 2023 19:12:09 +0100 Subject: [PATCH] test(filter): add some more filter rule tests --- CMakeLists.txt | 1 + include/dwarfs/filter_debug.h | 43 +++++++ src/dwarfs/filter_debug.cpp | 55 +++++++++ src/mkdwarfs_main.cpp | 37 +----- test/dwarfs.cpp | 222 +++++++++++++++++++++++++++++----- test/filter_test_data.cpp | 14 +++ 6 files changed, 308 insertions(+), 64 deletions(-) create mode 100644 include/dwarfs/filter_debug.h create mode 100644 src/dwarfs/filter_debug.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 6cc63cd8b..ea4e50774 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -384,6 +384,7 @@ list( src/dwarfs/filesystem_extractor.cpp src/dwarfs/filesystem_v2.cpp src/dwarfs/filesystem_writer.cpp + src/dwarfs/filter_debug.cpp src/dwarfs/fragment_chunkable.cpp src/dwarfs/fragment_order_parser.cpp src/dwarfs/fstypes.cpp diff --git a/include/dwarfs/filter_debug.h b/include/dwarfs/filter_debug.h new file mode 100644 index 000000000..fa54b214a --- /dev/null +++ b/include/dwarfs/filter_debug.h @@ -0,0 +1,43 @@ +/* vim:set ts=2 sw=2 sts=2 et: */ +/** + * \author Marcus Holland-Moritz (github@mhxnet.de) + * \copyright Copyright (c) Marcus Holland-Moritz + * + * This file is part of dwarfs. + * + * dwarfs is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * dwarfs is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with dwarfs. If not, see . + */ + +#pragma once + +#include + +namespace dwarfs { + +class entry; + +enum class debug_filter_mode { + OFF, + INCLUDED, + INCLUDED_FILES, + EXCLUDED, + EXCLUDED_FILES, + FILES, + ALL +}; + +void debug_filter_output(std::ostream& os, bool exclude, entry const* pe, + debug_filter_mode mode); + +} // namespace dwarfs diff --git a/src/dwarfs/filter_debug.cpp b/src/dwarfs/filter_debug.cpp new file mode 100644 index 000000000..aa9ffbf1a --- /dev/null +++ b/src/dwarfs/filter_debug.cpp @@ -0,0 +1,55 @@ +/* vim:set ts=2 sw=2 sts=2 et: */ +/** + * \author Marcus Holland-Moritz (github@mhxnet.de) + * \copyright Copyright (c) Marcus Holland-Moritz + * + * This file is part of dwarfs. + * + * dwarfs is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * dwarfs is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with dwarfs. If not, see . + */ + +#include + +#include "dwarfs/entry.h" +#include "dwarfs/filter_debug.h" + +namespace dwarfs { + +void debug_filter_output(std::ostream& os, bool exclude, entry const* pe, + debug_filter_mode mode) { + if (exclude ? mode == debug_filter_mode::INCLUDED or + mode == debug_filter_mode::INCLUDED_FILES + : mode == debug_filter_mode::EXCLUDED or + mode == debug_filter_mode::EXCLUDED_FILES) { + return; + } + + bool const files_only = mode == debug_filter_mode::FILES or + mode == debug_filter_mode::INCLUDED_FILES or + mode == debug_filter_mode::EXCLUDED_FILES; + + if (files_only and pe->type() == entry::E_DIR) { + return; + } + + char const* prefix = ""; + + if (mode == debug_filter_mode::FILES or mode == debug_filter_mode::ALL) { + prefix = exclude ? "- " : "+ "; + } + + os << prefix << pe->unix_dpath() << "\n"; +} + +} // namespace dwarfs diff --git a/src/mkdwarfs_main.cpp b/src/mkdwarfs_main.cpp index d5f572307..e4da9926a 100644 --- a/src/mkdwarfs_main.cpp +++ b/src/mkdwarfs_main.cpp @@ -62,6 +62,7 @@ #include "dwarfs/filesystem_block_category_resolver.h" #include "dwarfs/filesystem_v2.h" #include "dwarfs/filesystem_writer.h" +#include "dwarfs/filter_debug.h" #include "dwarfs/fragment_order_parser.h" #include "dwarfs/integral_value_parser.h" #include "dwarfs/logger.h" @@ -85,16 +86,6 @@ namespace dwarfs { namespace { -enum class debug_filter_mode { - OFF, - INCLUDED, - INCLUDED_FILES, - EXCLUDED, - EXCLUDED_FILES, - FILES, - ALL -}; - const std::map progress_modes{ {"none", console_writer::NONE}, {"simple", console_writer::SIMPLE}, @@ -131,32 +122,6 @@ const std::map time_resolutions{ constexpr size_t min_block_size_bits{10}; constexpr size_t max_block_size_bits{30}; -void debug_filter_output(std::ostream& os, bool exclude, entry const* pe, - debug_filter_mode mode) { - if (exclude ? mode == debug_filter_mode::INCLUDED or - mode == debug_filter_mode::INCLUDED_FILES - : mode == debug_filter_mode::EXCLUDED or - mode == debug_filter_mode::EXCLUDED_FILES) { - return; - } - - bool const files_only = mode == debug_filter_mode::FILES or - mode == debug_filter_mode::INCLUDED_FILES or - mode == debug_filter_mode::EXCLUDED_FILES; - - if (files_only and pe->type() == entry::E_DIR) { - return; - } - - char const* prefix = ""; - - if (mode == debug_filter_mode::FILES or mode == debug_filter_mode::ALL) { - prefix = exclude ? "- " : "+ "; - } - - os << prefix << pe->unix_dpath() << "\n"; -} - struct level_defaults { unsigned block_size_bits; std::string_view data_compression; diff --git a/test/dwarfs.cpp b/test/dwarfs.cpp index 322e7ca24..9166c1000 100644 --- a/test/dwarfs.cpp +++ b/test/dwarfs.cpp @@ -41,6 +41,7 @@ #include "dwarfs/file_type.h" #include "dwarfs/filesystem_v2.h" #include "dwarfs/filesystem_writer.h" +#include "dwarfs/filter_debug.h" #include "dwarfs/logger.h" #include "dwarfs/mmif.h" #include "dwarfs/options.h" @@ -817,44 +818,163 @@ INSTANTIATE_TEST_SUITE_P( file_order_mode::NILSIMSA), ::testing::Values(std::nullopt, "xxh3-128"))); -class filter : public testing::TestWithParam {}; +class filter_test + : public testing::TestWithParam { + public: + test::test_logger lgr; + std::shared_ptr scr; + std::shared_ptr input; + + void SetUp() override { + scr = std::make_shared(lgr); + scr->set_root_path(""); + + input = std::make_shared(); + + for (auto const& [stat, name] : dwarfs::test::test_dirtree()) { + auto path = name.substr(name.size() == 5 ? 5 : 6); + + switch (stat.type()) { + case posix_file_type::regular: + input->add(path, stat, + [size = stat.size] { return test::loremipsum(size); }); + break; + case posix_file_type::symlink: + input->add(path, stat, test::loremipsum(stat.size)); + break; + default: + input->add(path, stat); + break; + } + } + } -TEST_P(filter, filesystem) { - auto spec = GetParam(); + void set_filter_rules(test::filter_test_data const& spec) { + std::istringstream iss(spec.filter()); + scr->add_filter_rules(iss); + } - segmenter::config cfg; - scanner_options options; + std::string get_filter_debug_output(test::filter_test_data const& spec, + debug_filter_mode mode) { + set_filter_rules(spec); - options.remove_empty_dirs = true; + std::ostringstream oss; - test::test_logger lgr; + scanner_options options; + options.remove_empty_dirs = false; + options.debug_filter_function = [&](bool exclude, entry const* pe) { + debug_filter_output(oss, exclude, pe, mode); + }; - auto scr = std::make_shared(lgr); + progress prog([](const progress&, bool) {}, 1000); + worker_group wg("worker", 1); + auto sf = std::make_shared(lgr, prog, + segmenter_factory::config{}); + scanner s(lgr, wg, sf, entry_factory::create(), input, scr, options); - scr->set_root_path(""); - { - std::istringstream iss(spec.filter()); - scr->add_filter_rules(iss); + block_compressor bc("null"); + std::ostringstream null; + filesystem_writer fsw(null, lgr, wg, prog, bc, bc, bc); + s.scan(fsw, std::filesystem::path("/"), prog); + + return oss.str(); } - auto input = std::make_shared(); + std::string get_expected_filter_output(test::filter_test_data const& spec, + debug_filter_mode mode) { + std::string expected; + auto expected_files = spec.expected_files(); + + auto check_included = [&](auto const& stat, std::string const& path, + std::string_view prefix = "") { + if (stat.type() == posix_file_type::directory) { + expected += fmt::format("{}/{}/\n", prefix, path); + } else if (expected_files.count(path)) { + expected += fmt::format("{}/{}\n", prefix, path); + } + }; + + auto check_included_files = [&](auto const& stat, std::string const& path, + std::string_view prefix = "") { + if (stat.type() != posix_file_type::directory && + expected_files.count(path)) { + expected += fmt::format("{}/{}\n", prefix, path); + } + }; + + auto check_excluded = [&](auto const& stat, std::string const& path, + std::string_view prefix = "") { + if (stat.type() != posix_file_type::directory && + expected_files.count(path) == 0) { + expected += fmt::format("{}/{}\n", prefix, path); + } + }; + + auto check_excluded_files = [&](auto const& stat, std::string const& path, + std::string_view prefix = "") { + if (stat.type() != posix_file_type::directory && + expected_files.count(path) == 0) { + expected += fmt::format("{}/{}\n", prefix, path); + } + }; + + for (auto const& [stat, name] : dwarfs::test::test_dirtree()) { + std::string path(name.substr(name.size() == 5 ? 5 : 6)); + + if (path.empty()) { + continue; + } + + switch (mode) { + case debug_filter_mode::INCLUDED_FILES: + check_included_files(stat, path); + break; + + case debug_filter_mode::INCLUDED: + check_included(stat, path); + break; + + case debug_filter_mode::EXCLUDED_FILES: + check_excluded_files(stat, path); + break; + + case debug_filter_mode::EXCLUDED: + check_excluded(stat, path); + break; + + case debug_filter_mode::FILES: + check_included_files(stat, path, "+ "); + check_excluded_files(stat, path, "- "); + break; + + case debug_filter_mode::ALL: + check_included(stat, path, "+ "); + check_excluded(stat, path, "- "); + break; - for (auto const& [stat, name] : dwarfs::test::test_dirtree()) { - auto path = name.substr(name.size() == 5 ? 5 : 6); - - switch (stat.type()) { - case posix_file_type::regular: - input->add(path, stat, - [size = stat.size] { return test::loremipsum(size); }); - break; - case posix_file_type::symlink: - input->add(path, stat, test::loremipsum(stat.size)); - break; - default: - input->add(path, stat); - break; + case debug_filter_mode::OFF: + throw std::logic_error("invalid debug filter mode"); + } } + + return expected; + } + + void TearDown() override { + scr.reset(); + input.reset(); } +}; + +TEST_P(filter_test, filesystem) { + auto spec = GetParam(); + + set_filter_rules(spec); + + segmenter::config cfg; + + scanner_options options; + options.remove_empty_dirs = true; auto fsimage = build_dwarfs(lgr, input, "null", cfg, options, nullptr, scr); @@ -874,7 +994,53 @@ TEST_P(filter, filesystem) { EXPECT_EQ(spec.expected_files(), got); } -INSTANTIATE_TEST_SUITE_P(dwarfs, filter, +TEST_P(filter_test, debug_filter_function_included) { + auto spec = GetParam(); + auto output = get_filter_debug_output(spec, debug_filter_mode::INCLUDED); + auto expected = get_expected_filter_output(spec, debug_filter_mode::INCLUDED); + EXPECT_EQ(expected, output); +} + +TEST_P(filter_test, debug_filter_function_included_files) { + auto spec = GetParam(); + auto output = + get_filter_debug_output(spec, debug_filter_mode::INCLUDED_FILES); + auto expected = + get_expected_filter_output(spec, debug_filter_mode::INCLUDED_FILES); + EXPECT_EQ(expected, output); +} + +TEST_P(filter_test, debug_filter_function_excluded) { + auto spec = GetParam(); + auto output = get_filter_debug_output(spec, debug_filter_mode::EXCLUDED); + auto expected = get_expected_filter_output(spec, debug_filter_mode::EXCLUDED); + EXPECT_EQ(expected, output); +} + +TEST_P(filter_test, debug_filter_function_excluded_files) { + auto spec = GetParam(); + auto output = + get_filter_debug_output(spec, debug_filter_mode::EXCLUDED_FILES); + auto expected = + get_expected_filter_output(spec, debug_filter_mode::EXCLUDED_FILES); + EXPECT_EQ(expected, output); +} + +TEST_P(filter_test, debug_filter_function_all) { + auto spec = GetParam(); + auto output = get_filter_debug_output(spec, debug_filter_mode::ALL); + auto expected = get_expected_filter_output(spec, debug_filter_mode::ALL); + EXPECT_EQ(expected, output); +} + +TEST_P(filter_test, debug_filter_function_files) { + auto spec = GetParam(); + auto output = get_filter_debug_output(spec, debug_filter_mode::FILES); + auto expected = get_expected_filter_output(spec, debug_filter_mode::FILES); + EXPECT_EQ(expected, output); +} + +INSTANTIATE_TEST_SUITE_P(dwarfs, filter_test, ::testing::ValuesIn(dwarfs::test::get_filter_tests())); TEST(file_scanner, input_list) { diff --git a/test/filter_test_data.cpp b/test/filter_test_data.cpp index a7f6e29e8..1ae151e6d 100644 --- a/test/filter_test_data.cpp +++ b/test/filter_test_data.cpp @@ -60,6 +60,20 @@ R"( "usr/lib64/gconv/ISO8859-16.so", "lib", "lib/libpcprofile.so", +}}, +{ + "IncludeSomeObjects", +R"( +- gcc/**.o ++ *.o +- * +)", { + "", + "usr", + "usr/lib", + "usr/lib/Mcrt1.o", + "usr/lib64", + "usr/lib64/gcrt1.o", }}, // clang-format on };