From 0d56a95ea706e9bb4bd723a17f61fbe0f2fbb55d Mon Sep 17 00:00:00 2001 From: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> Date: Tue, 3 Oct 2023 15:53:48 -0400 Subject: [PATCH 1/7] integrate reproducible build with CI Build & Test workflow --- .github/workflows/build.yaml | 70 +++++++++++++------ .../workflows/performance_harness_run.yaml | 5 +- .../workflows/ph_backward_compatibility.yaml | 4 +- 3 files changed, 54 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1a03de2215..7a9fd1a8d0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -39,7 +39,9 @@ jobs: contents: read with: runs-on: '["self-hosted", "enf-x86-beefy"]' - platform-files: .cicd/platforms + platform-files: | + .cicd/platforms + tools/reproducible.Dockerfile:builder build-base: name: Run Build Workflow @@ -77,13 +79,13 @@ jobs: echo eos-system-contracts-ref=${{inputs.override-eos-system-contracts}} >> $GITHUB_OUTPUT fi - dev-package: - name: Build leap-dev package + package: + name: Build deb packages needs: [platform-cache, build-base] strategy: fail-fast: false matrix: - platform: [ubuntu20, ubuntu22] + platform: [ubuntu20, ubuntu22, reproducible] runs-on: ubuntu-latest container: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}} steps: @@ -94,41 +96,55 @@ jobs: uses: actions/download-artifact@v3 with: name: ${{matrix.platform}}-build - - name: Build dev package + - name: Build packages run: | zstdcat build.tar.zst | tar x cd build cpack + ../tools/tweak-deb.sh leap_*.deb - name: Install dev package + if: matrix.platform != 'reproducible' run: | apt-get update && apt-get upgrade -y apt-get install -y ./build/leap_*.deb ./build/leap-dev*.deb - name: Test using TestHarness + if: matrix.platform != 'reproducible' run: | python3 -c "from TestHarness import Cluster" - name: Upload dev package uses: actions/upload-artifact@v3 + if: matrix.platform != 'reproducible' with: name: leap-dev-${{matrix.platform}}-amd64 path: build/leap-dev*.deb + - name: Upload leap package + uses: actions/upload-artifact@v3 + if: matrix.platform == 'reproducible' + with: + name: leap-deb-amd64 + path: build/leap_*.deb tests: - name: Tests + name: Tests (${{matrix.cfg.name}}) needs: [platform-cache, build-base] strategy: fail-fast: false matrix: - platform: [ubuntu20, ubuntu22] + include: + - cfg: {name: 'ubuntu20', base: 'ubuntu20', builddir: 'ubuntu20'} + - cfg: {name: 'ubuntu22', base: 'ubuntu22', builddir: 'ubuntu22'} + - cfg: {name: 'ubuntu20repro', base: 'ubuntu20', builddir: 'reproducible'} + - cfg: {name: 'ubuntu22repro', base: 'ubuntu22', builddir: 'reproducible'} runs-on: ["self-hosted", "enf-x86-hightier"] container: - image: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}} + image: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.cfg.base].image}} options: --security-opt seccomp=unconfined steps: - uses: actions/checkout@v3 - name: Download builddir uses: actions/download-artifact@v3 with: - name: ${{matrix.platform}}-build + name: ${{matrix.cfg.builddir}}-build - name: Run Parallel Tests run: | # https://github.com/actions/runner/issues/2033 -- need this because of full version label test looking at git revs @@ -140,66 +156,74 @@ jobs: run: awk 'BEGIN {err = 1} /bmi2/ && /adx/ {err = 0} END {exit err}' /proc/cpuinfo np-tests: - name: NP Tests + name: NP Tests (${{matrix.cfg.name}}) needs: [platform-cache, build-base] strategy: fail-fast: false matrix: - platform: [ubuntu20, ubuntu22] + include: + - cfg: {name: 'ubuntu20', base: 'ubuntu20', builddir: 'ubuntu20'} + - cfg: {name: 'ubuntu22', base: 'ubuntu22', builddir: 'ubuntu22'} + - cfg: {name: 'ubuntu20repro', base: 'ubuntu20', builddir: 'reproducible'} + - cfg: {name: 'ubuntu22repro', base: 'ubuntu22', builddir: 'reproducible'} runs-on: ["self-hosted", "enf-x86-midtier"] steps: - uses: actions/checkout@v3 - name: Download builddir uses: actions/download-artifact@v3 with: - name: ${{matrix.platform}}-build + name: ${{matrix.cfg.builddir}}-build - name: Run tests in parallel containers uses: ./.github/actions/parallel-ctest-containers with: - container: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}} + container: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.cfg.base].image}} error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd", "build/TestLogs"]' - log-tarball-prefix: ${{matrix.platform}} + log-tarball-prefix: ${{matrix.cfg.name}} tests-label: nonparallelizable_tests test-timeout: 420 - name: Upload logs from failed tests uses: actions/upload-artifact@v3 if: failure() with: - name: ${{matrix.platform}}-np-logs + name: ${{matrix.cfg.name}}-np-logs path: '*-logs.tar.gz' lr-tests: - name: LR Tests + name: LR Tests (${{matrix.cfg.name}}) needs: [platform-cache, build-base] strategy: fail-fast: false matrix: - platform: [ubuntu20, ubuntu22] + include: + - cfg: {name: 'ubuntu20', base: 'ubuntu20', builddir: 'ubuntu20'} + - cfg: {name: 'ubuntu22', base: 'ubuntu22', builddir: 'ubuntu22'} + - cfg: {name: 'ubuntu20repro', base: 'ubuntu20', builddir: 'reproducible'} + - cfg: {name: 'ubuntu22repro', base: 'ubuntu22', builddir: 'reproducible'} runs-on: ["self-hosted", "enf-x86-lowtier"] steps: - uses: actions/checkout@v3 - name: Download builddir uses: actions/download-artifact@v3 with: - name: ${{matrix.platform}}-build + name: ${{matrix.cfg.builddir}}-build - name: Run tests in parallel containers uses: ./.github/actions/parallel-ctest-containers with: - container: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}} + container: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.cfg.base].image}} error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd", "build/TestLogs"]' - log-tarball-prefix: ${{matrix.platform}} + log-tarball-prefix: ${{matrix.cfg.name}} tests-label: long_running_tests test-timeout: 1800 - name: Upload logs from failed tests uses: actions/upload-artifact@v3 if: failure() with: - name: ${{matrix.platform}}-lr-logs + name: ${{matrix.cfg.name}}-lr-logs path: '*-logs.tar.gz' libtester-tests: name: libtester tests - needs: [platform-cache, build-base, v, dev-package] + needs: [platform-cache, build-base, v, package] strategy: fail-fast: false matrix: @@ -290,7 +314,7 @@ jobs: all-passing: name: All Required Tests Passed - needs: [dev-package, tests, np-tests, libtester-tests] + needs: [tests, np-tests, libtester-tests] runs-on: ubuntu-latest steps: - run: true diff --git a/.github/workflows/performance_harness_run.yaml b/.github/workflows/performance_harness_run.yaml index ce037555a7..b9d84bf74d 100644 --- a/.github/workflows/performance_harness_run.yaml +++ b/.github/workflows/performance_harness_run.yaml @@ -9,6 +9,7 @@ on: options: - ubuntu20 - ubuntu22 + - reproducible override-test-params: description: 'Override perf harness params' type: string @@ -66,7 +67,9 @@ jobs: contents: read with: runs-on: '["self-hosted", "enf-x86-beefy"]' - platform-files: .cicd/platforms + platform-files: | + .cicd/platforms + tools/reproducible.Dockerfile:builder reuse-build: name: Reuse leap build diff --git a/.github/workflows/ph_backward_compatibility.yaml b/.github/workflows/ph_backward_compatibility.yaml index 4633108821..4f27c29098 100644 --- a/.github/workflows/ph_backward_compatibility.yaml +++ b/.github/workflows/ph_backward_compatibility.yaml @@ -20,7 +20,9 @@ jobs: packages: write with: runs-on: '["self-hosted", "enf-x86-beefy"]' - platform-files: .cicd/platforms + platform-files: | + .cicd/platforms + tools/reproducible.Dockerfile:builder build-base: name: Run Build Workflow From 39ae21c6df99c4c9c2fe583a411b20c51327f792 Mon Sep 17 00:00:00 2001 From: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> Date: Wed, 4 Oct 2023 10:48:02 -0400 Subject: [PATCH 2/7] place 'manual' reproducible source dir in same location as CI --- tools/reproducible.Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/reproducible.Dockerfile b/tools/reproducible.Dockerfile index 667913a892..59a19f9d51 100644 --- a/tools/reproducible.Dockerfile +++ b/tools/reproducible.Dockerfile @@ -97,10 +97,12 @@ FROM builder AS build ARG LEAP_BUILD_JOBS -COPY / /src -RUN cmake -S src -B build -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -GNinja && \ +# Yuck: This places the source at the same location as leap's CI (build.yaml, build_base.yaml). Unfortunately this location only matches +# when build.yaml etc are being run from a repository named leap. +COPY / /__w/leap/leap +RUN cmake -S /__w/leap/leap -B build -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -GNinja && \ cmake --build build -t package -- ${LEAP_BUILD_JOBS:+-j$LEAP_BUILD_JOBS} && \ - src/tools/tweak-deb.sh build/leap_*.deb + /__w/leap/leap/tools/tweak-deb.sh build/leap_*.deb FROM scratch AS exporter COPY --from=build /build/*.deb /build/*.tar.* / From f06b2f9488fcc89a97f0258185a0e8501463c4c1 Mon Sep 17 00:00:00 2001 From: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> Date: Thu, 19 Oct 2023 13:58:53 -0400 Subject: [PATCH 3/7] don't change PH workflows for now --- .github/workflows/performance_harness_run.yaml | 5 +---- .github/workflows/ph_backward_compatibility.yaml | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/performance_harness_run.yaml b/.github/workflows/performance_harness_run.yaml index b9d84bf74d..ce037555a7 100644 --- a/.github/workflows/performance_harness_run.yaml +++ b/.github/workflows/performance_harness_run.yaml @@ -9,7 +9,6 @@ on: options: - ubuntu20 - ubuntu22 - - reproducible override-test-params: description: 'Override perf harness params' type: string @@ -67,9 +66,7 @@ jobs: contents: read with: runs-on: '["self-hosted", "enf-x86-beefy"]' - platform-files: | - .cicd/platforms - tools/reproducible.Dockerfile:builder + platform-files: .cicd/platforms reuse-build: name: Reuse leap build diff --git a/.github/workflows/ph_backward_compatibility.yaml b/.github/workflows/ph_backward_compatibility.yaml index 4f27c29098..4633108821 100644 --- a/.github/workflows/ph_backward_compatibility.yaml +++ b/.github/workflows/ph_backward_compatibility.yaml @@ -20,9 +20,7 @@ jobs: packages: write with: runs-on: '["self-hosted", "enf-x86-beefy"]' - platform-files: | - .cicd/platforms - tools/reproducible.Dockerfile:builder + platform-files: .cicd/platforms build-base: name: Run Build Workflow From 4ed36f9ecb4698962c6254539c756b0bf9ac27b5 Mon Sep 17 00:00:00 2001 From: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> Date: Tue, 24 Oct 2023 11:43:14 -0400 Subject: [PATCH 4/7] fix tweak missed during merge --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f1fac83f8e..4dccdddd5b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -318,5 +318,5 @@ jobs: if: always() runs-on: ubuntu-latest steps: - - if: needs.dev-package.result != 'success' || needs.tests.result != 'success' || needs.np-tests.result != 'success' || needs.libtester-tests.result != 'success' + - if: needs.tests.result != 'success' || needs.np-tests.result != 'success' || needs.libtester-tests.result != 'success' run: false From a4e9df1e4c687d17919bf37694be35c5c1a0e595 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 24 Oct 2023 14:38:49 -0500 Subject: [PATCH 5/7] GH-1815 Avoid deadlock on app_thread --- tests/test_read_only_trx.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_read_only_trx.cpp b/tests/test_read_only_trx.cpp index 27a6c6be1e..e1eb419ff6 100644 --- a/tests/test_read_only_trx.cpp +++ b/tests/test_read_only_trx.cpp @@ -117,10 +117,6 @@ void test_trxs_common(std::vector& specific_args, bool test_disable } FC_LOG_AND_DROP() BOOST_CHECK(!"app threw exception see logged error"); } ); - fc::scoped_exit> on_except = [&](){ - if (app_thread.joinable()) - app_thread.join(); - }; auto[prod_plug, chain_plug] = plugin_fut.get(); @@ -168,6 +164,7 @@ void test_trxs_common(std::vector& specific_args, bool test_disable } app->quit(); + app_thread.join(); } BOOST_CHECK_EQUAL( trace_with_except, 0u ); // should not have any traces with except in it From 894648dc5239a7095c5e0a89ee074817578d2407 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 24 Oct 2023 14:46:44 -0500 Subject: [PATCH 6/7] GH-1815 Call app->quit() before app_thread.join() --- tests/test_read_only_trx.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_read_only_trx.cpp b/tests/test_read_only_trx.cpp index e1eb419ff6..1495476779 100644 --- a/tests/test_read_only_trx.cpp +++ b/tests/test_read_only_trx.cpp @@ -117,6 +117,11 @@ void test_trxs_common(std::vector& specific_args, bool test_disable } FC_LOG_AND_DROP() BOOST_CHECK(!"app threw exception see logged error"); } ); + fc::scoped_exit> on_except = [&](){ + app->quit(); + if (app_thread.joinable()) + app_thread.join(); + }; auto[prod_plug, chain_plug] = plugin_fut.get(); @@ -162,9 +167,6 @@ void test_trxs_common(std::vector& specific_args, bool test_disable while ( (next_calls < num_pushes || num_get_account_calls < num_pushes) && fc::time_point::now() < hard_deadline ){ std::this_thread::sleep_for( 100ms ); } - - app->quit(); - app_thread.join(); } BOOST_CHECK_EQUAL( trace_with_except, 0u ); // should not have any traces with except in it From 841d5476bcadd5dd7dbb3c9c731c417ce8d567a0 Mon Sep 17 00:00:00 2001 From: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> Date: Tue, 24 Oct 2023 20:08:03 -0400 Subject: [PATCH 7/7] refactor threading of snapshot_scheduler_test --- tests/test_snapshot_scheduler.cpp | 128 ++++++++++++++++-------------- 1 file changed, 68 insertions(+), 60 deletions(-) diff --git a/tests/test_snapshot_scheduler.cpp b/tests/test_snapshot_scheduler.cpp index ed1f5286d7..1a560ca079 100644 --- a/tests/test_snapshot_scheduler.cpp +++ b/tests/test_snapshot_scheduler.cpp @@ -60,6 +60,9 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) { std::promise> plugin_promise; std::future> plugin_fut = plugin_promise.get_future(); + std::promise at_block_20_promise; + std::future at_block_20_fut = at_block_20_promise.get_future(); + std::thread app_thread([&]() { try { fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug); @@ -68,8 +71,48 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) { "-p", "eosio", "-e"}; app->initialize(argv.size(), (char**) &argv[0]); app->startup(); - plugin_promise.set_value( - {app->find_plugin(), app->find_plugin()}); + + producer_plugin* prod_plug = app->find_plugin(); + chain_plugin* chain_plug = app->find_plugin(); + plugin_promise.set_value({prod_plug, chain_plug}); + + auto bs = chain_plug->chain().block_start.connect([&prod_plug, &at_block_20_promise](uint32_t bn) { + if(bn == 20u) + at_block_20_promise.set_value(); + // catching pending snapshot + if (!prod_plug->get_snapshot_requests().snapshot_requests.empty()) { + const auto& snapshot_requests = prod_plug->get_snapshot_requests().snapshot_requests; + + auto validate_snapshot_request = [&](uint32_t sid, uint32_t block_num, uint32_t spacing = 0, bool fuzzy_start = false) { + auto it = find_if(snapshot_requests.begin(), snapshot_requests.end(), [sid](const snapshot_scheduler::snapshot_schedule_information& obj) {return obj.snapshot_request_id == sid;}); + if (it != snapshot_requests.end()) { + auto& pending = it->pending_snapshots; + if (pending.size()==1u) { + // pending snapshot block number + auto pbn = pending.begin()->head_block_num; + + // first pending snapshot + auto ps_start = (spacing != 0) ? (spacing + (pbn%spacing)) : pbn; + + if (!fuzzy_start) { + BOOST_CHECK_EQUAL(block_num, ps_start); + } + else { + int diff = block_num - ps_start; + BOOST_CHECK(std::abs(diff) <= 5); // accept +/- 5 blocks if start block not specified + } + } + return true; + } + return false; + }; + + BOOST_REQUIRE(validate_snapshot_request(0, 9, 8)); // snapshot #0 should have pending snapshot at block #9 (8 + 1) and it never expires + BOOST_REQUIRE(validate_snapshot_request(4, 12, 10, true)); // snapshot #4 should have pending snapshot at block # at the moment of scheduling (2) plus 10 = 12 + BOOST_REQUIRE(validate_snapshot_request(5, 10, 10)); // snapshot #5 should have pending snapshot at block #10, #20 etc + } + }); + app->exec(); return; } FC_LOG_AND_DROP() @@ -77,45 +120,6 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) { }); auto [prod_plug, chain_plug] = plugin_fut.get(); - std::deque all_blocks; - std::promise empty_blocks_promise; - std::future empty_blocks_fut = empty_blocks_promise.get_future(); - auto pp = app->find_plugin(); - - auto bs = chain_plug->chain().block_start.connect([&pp](uint32_t bn) { - // catching pending snapshot - if (!pp->get_snapshot_requests().snapshot_requests.empty()) { - const auto& snapshot_requests = pp->get_snapshot_requests().snapshot_requests; - - auto validate_snapshot_request = [&](uint32_t sid, uint32_t block_num, uint32_t spacing = 0, bool fuzzy_start = false) { - auto it = find_if(snapshot_requests.begin(), snapshot_requests.end(), [sid](const snapshot_scheduler::snapshot_schedule_information& obj) {return obj.snapshot_request_id == sid;}); - if (it != snapshot_requests.end()) { - auto& pending = it->pending_snapshots; - if (pending.size()==1u) { - // pending snapshot block number - auto pbn = pending.begin()->head_block_num; - - // first pending snapshot - auto ps_start = (spacing != 0) ? (spacing + (pbn%spacing)) : pbn; - - if (!fuzzy_start) { - BOOST_CHECK_EQUAL(block_num, ps_start); - } - else { - int diff = block_num - ps_start; - BOOST_CHECK(std::abs(diff) <= 5); // accept +/- 5 blocks if start block not specified - } - } - return true; - } - return false; - }; - - BOOST_REQUIRE(validate_snapshot_request(0, 9, 8)); // snapshot #0 should have pending snapshot at block #9 (8 + 1) and it never expires - BOOST_REQUIRE(validate_snapshot_request(4, 12, 10, true)); // snapshot #4 should have pending snapshot at block # at the moment of scheduling (2) plus 10 = 12 - BOOST_REQUIRE(validate_snapshot_request(5, 10, 10)); // snapshot #5 should have pending snapshot at block #10, #20 etc - } - }); snapshot_request_params sri1 = {.block_spacing = 8, .start_block_num = 1, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 1"}; snapshot_request_params sri2 = {.block_spacing = 5000, .start_block_num = 100000, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2 that wont happen in test"}; @@ -124,31 +128,35 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) { snapshot_request_params sri5 = {.block_spacing = 10, .snapshot_description = "Recurring every 10 blocks snapshot starting now"}; snapshot_request_params sri6 = {.block_spacing = 10, .start_block_num = 0, .snapshot_description = "Recurring every 10 blocks snapshot starting from 0"}; - pp->schedule_snapshot(sri1); - pp->schedule_snapshot(sri2); - pp->schedule_snapshot(sri3); - pp->schedule_snapshot(sri4); - pp->schedule_snapshot(sri5); - pp->schedule_snapshot(sri6); + app->post(appbase::priority::medium_low, [&]() { + prod_plug->schedule_snapshot(sri1); + prod_plug->schedule_snapshot(sri2); + prod_plug->schedule_snapshot(sri3); + prod_plug->schedule_snapshot(sri4); + prod_plug->schedule_snapshot(sri5); + prod_plug->schedule_snapshot(sri6); - // all six snapshot requests should be present now - BOOST_CHECK_EQUAL(6u, pp->get_snapshot_requests().snapshot_requests.size()); + // all six snapshot requests should be present now + BOOST_CHECK_EQUAL(6u, prod_plug->get_snapshot_requests().snapshot_requests.size()); + }); - empty_blocks_fut.wait_for(std::chrono::seconds(10)); + at_block_20_fut.get(); - // two of the snapshots are done here and requests, corresponding to them should be deleted - BOOST_CHECK_EQUAL(4u, pp->get_snapshot_requests().snapshot_requests.size()); + app->post(appbase::priority::medium_low, [&]() { + // two of the snapshots are done here and requests, corresponding to them should be deleted + BOOST_CHECK_EQUAL(4u, prod_plug->get_snapshot_requests().snapshot_requests.size()); - // check whether no pending snapshots present for a snapshot with id 0 - const auto& snapshot_requests = pp->get_snapshot_requests().snapshot_requests; - auto it = find_if(snapshot_requests.begin(), snapshot_requests.end(),[](const snapshot_scheduler::snapshot_schedule_information& obj) {return obj.snapshot_request_id == 0;}); + // check whether no pending snapshots present for a snapshot with id 0 + const auto& snapshot_requests = prod_plug->get_snapshot_requests().snapshot_requests; + auto it = find_if(snapshot_requests.begin(), snapshot_requests.end(),[](const snapshot_scheduler::snapshot_schedule_information& obj) {return obj.snapshot_request_id == 0;}); - // snapshot request with id = 0 should be found and should not have any pending snapshots - BOOST_REQUIRE(it != snapshot_requests.end()); - BOOST_CHECK(!it->pending_snapshots.size()); + // snapshot request with id = 0 should be found and should not have any pending snapshots + BOOST_REQUIRE(it != snapshot_requests.end()); + BOOST_CHECK(!it->pending_snapshots.size()); - // quit app - app->quit(); + // quit app + app->quit(); + }); app_thread.join(); // lets check whether schedule can be read back after restart