From ce6becbd67e533f49004222be42919e13db19993 Mon Sep 17 00:00:00 2001 From: Alexey Alter-Pesotskiy Date: Tue, 30 Jul 2024 11:54:58 +0100 Subject: [PATCH] [CI] Comment SDK size on every commit (#560) --- .github/workflows/sdk-size-metrics.yml | 38 +++++++++ .github/workflows/smoke-checks.yml | 1 + .gitignore | 3 + fastlane/Fastfile | 105 ++++++++++++++++++++++++- fastlane/sdk_size_export_options.plist | 27 +++++++ 5 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/sdk-size-metrics.yml create mode 100644 fastlane/sdk_size_export_options.plist diff --git a/.github/workflows/sdk-size-metrics.yml b/.github/workflows/sdk-size-metrics.yml new file mode 100644 index 00000000..50745c71 --- /dev/null +++ b/.github/workflows/sdk-size-metrics.yml @@ -0,0 +1,38 @@ +name: SDK Size + +on: + pull_request: + + workflow_dispatch: + + push: + branches: + - develop + +env: + HOMEBREW_NO_INSTALL_CLEANUP: 1 # Disable cleanup for homebrew, we don't need it on CI + +jobs: + sdk_size: + name: Metrics + runs-on: macos-14 + env: + GITHUB_TOKEN: '${{ secrets.CI_BOT_GITHUB_TOKEN }}' + steps: + - name: Install Bot SSH Key + uses: webfactory/ssh-agent@v0.7.0 + with: + ssh-private-key: ${{ secrets.BOT_SSH_PRIVATE_KEY }} + + - uses: actions/checkout@v3.1.0 + + - uses: ./.github/actions/bootstrap + + - name: Run SDK Size Metrics + run: bundle exec fastlane show_frameworks_sizes + timeout-minutes: 30 + env: + GITHUB_PR_NUM: ${{ github.event.pull_request.number }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }} + APPSTORE_API_KEY: ${{ secrets.APPSTORE_API_KEY }} diff --git a/.github/workflows/smoke-checks.yml b/.github/workflows/smoke-checks.yml index 28dc268c..971ac78a 100644 --- a/.github/workflows/smoke-checks.yml +++ b/.github/workflows/smoke-checks.yml @@ -165,6 +165,7 @@ jobs: env: ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_PR_NUM: ${{ github.event.number }} GITHUB_EVENT: ${{ toJson(github.event) }} - id: get_launch_id run: echo "launch_id=${{env.LAUNCH_ID}}" >> $GITHUB_OUTPUT diff --git a/.gitignore b/.gitignore index bf116119..7e8e9234 100644 --- a/.gitignore +++ b/.gitignore @@ -71,6 +71,7 @@ fastlane/screenshots fastlane/test_output fastlane/allurectl fastlane/xcresults +fastlane/metrics fastlane/recordings StreamChatCore.framework.coverage.txt StreamChatCoreTests.xctest.coverage.txt @@ -87,4 +88,6 @@ derived_data/ spm_cache/ .buildcache buildcache +App Thinning Size Report.txt +app-thinning.plist *.dmg diff --git a/fastlane/Fastfile b/fastlane/Fastfile index f5cdc443..98190816 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -13,11 +13,18 @@ sdk_names = ['StreamChatSwiftUI'] github_repo = ENV['GITHUB_REPOSITORY'] || 'GetStream/stream-chat-swiftui' derived_data_path = 'derived_data' source_packages_path = 'spm_cache' +metrics_git = 'git@github.com:GetStream/apple-internal-metrics.git' +sdk_size_path = "metrics/#{github_repo.split('/').last}-size.json" buildcache_xcargs = 'CC=clang CPLUSPLUS=clang++ LD=clang LDPLUSPLUS=clang++' is_localhost = !is_ci project_package_resolved = "#{xcode_project}/project.xcworkspace/xcshareddata/swiftpm/Package.resolved" @force_check = false +warning_status = '🟡' # Warning if a branch is #{max_tolerance} less performant than the benchmark +fail_status = '🔴' # Failure if a branch is more than #{max_tolerance} less performant than the benchmark +success_status = '🟢' # Success if a branch is more performant or equals to the benchmark +outstanding_status = '🚀' # Outstanding performance + before_all do |lane| if is_ci setup_ci @@ -463,7 +470,8 @@ lane :sources_matrix do e2e: ['Sources', 'StreamChatSwiftUITestsAppTests', 'StreamChatSwiftUITestsApp'], ui: ['Sources', 'StreamChatSwiftUITests', xcode_project], sample_apps: ['Sources', 'DemoAppSwiftUI', xcode_project], - ruby: ['fastlane', 'Gemfile', 'Gemfile.lock'] + ruby: ['fastlane', 'Gemfile', 'Gemfile.lock'], + size: ['Sources', xcode_project] } end @@ -477,6 +485,93 @@ lane :copyright do ) end +desc 'Show current frameworks size' +lane :show_frameworks_sizes do |options| + next unless is_check_required(sources: sources_matrix[:size], force_check: @force_check) + + ['metrics/'].each { |dir| FileUtils.remove_dir(dir, force: true) } + + sh("git clone #{metrics_git} #{File.dirname(sdk_size_path)}") + develop_sizes = JSON.parse(File.read(sdk_size_path)) + branch_sizes = options[:sizes] || frameworks_sizes + + table_header = '## SDK Size' + markdown_table = "#{table_header}\n| `title` | `develop` | `branch` | `diff` | `status` |\n| - | - | - | - | - |\n" + sdk_names.each do |title| + benchmark_value = develop_sizes[title] + branch_value = branch_sizes[title.to_sym] + max_tolerance = 0.5 # Max Tolerance is 0.5MB + fine_tolerance = 0.25 # Fine Tolerance is 0.25MB + + diff = (branch_value - benchmark_value).round(2) + + status_emoji = + if diff < 0 + outstanding_status + elsif diff >= max_tolerance + fail_status + elsif diff >= fine_tolerance + warning_status + else + success_status + end + + markdown_table << "|#{title}|#{benchmark_value}MB|#{branch_value}MB|#{diff}MB|#{status_emoji}|\n" + end + + FastlaneCore::PrintTable.print_values(title: 'Benchmark', config: develop_sizes) + FastlaneCore::PrintTable.print_values(title: 'SDK Size', config: branch_sizes) + + if is_ci + if ENV['GITHUB_EVENT_NAME'].to_s == 'push' + File.write(sdk_size_path, JSON.pretty_generate(branch_sizes)) + Dir.chdir(File.dirname(sdk_size_path)) do + if sh('git status -s', log: false).to_s.empty? + UI.important('No changes in SDK sizes benchmarks.') + else + sh('git add -A') + sh("git commit -m 'Update #{sdk_size_path}'") + sh('git push') + end + end + end + + create_pr_comment(pr_num: ENV.fetch('GITHUB_PR_NUM'), text: markdown_table, edit_last_comment_with_text: table_header) + end + + UI.user_error!("#{table_header} benchmark failed.") if markdown_table.include?(fail_status) +end + +def frameworks_sizes + root_dir = 'Build/SDKSize' + archive_dir = "#{root_dir}/DemoApp.xcarchive" + + FileUtils.rm_rf("../#{root_dir}/") + + match_me + + gym( + scheme: 'DemoAppSwiftUI', + archive_path: archive_dir, + export_method: 'ad-hoc', + export_options: 'fastlane/sdk_size_export_options.plist' + ) + + # Parse the thinned size of Assets.car from Packaging.log + assets_size_regex = %r{\b(\d+)\sbytes\sfor\s./Payload/DemoAppSwiftUI.app/Frameworks/StreamChatSwiftUI.framework/Assets.car\b} + packaging_log_content = File.read("#{Gym.cache[:temporary_output_path]}/Packaging.log") + match = packaging_log_content.match(assets_size_regex) + assets_thinned_size = match[1].to_i + + frameworks_path = "../#{archive_dir}/Products/Applications/DemoAppSwiftUI.app/Frameworks" + stream_chat_swiftui_size = File.size("#{frameworks_path}/StreamChatSwiftUI.framework/StreamChatSwiftUI") + stream_chat_swiftui_size_mb = ((stream_chat_swiftui_size + assets_thinned_size).to_f / 1024 / 1024).round(2) + + { + StreamChatSwiftUI: stream_chat_swiftui_size_mb + } +end + private_lane :create_pr do |options| options[:base_branch] ||= 'develop' sh("git checkout -b #{options[:head_branch]}") @@ -494,6 +589,14 @@ private_lane :create_pr do |options| ) end +private_lane :create_pr_comment do |options| + if is_ci && !options[:pr_num].to_s.empty? + last_comment = sh("gh pr view #{options[:pr_num]} --json comments --jq '.comments | map(select(.author.login == \"Stream-iOS-Bot\")) | last'") + edit_last_comment = last_comment.include?(options[:edit_last_comment_with_text]) ? '--edit-last' : '' + sh("gh pr comment #{options[:pr_num]} #{edit_last_comment} -b '#{options[:text]}'") + end +end + private_lane :current_branch do ENV['BRANCH_NAME'].to_s.empty? ? git_branch : ENV.fetch('BRANCH_NAME') end diff --git a/fastlane/sdk_size_export_options.plist b/fastlane/sdk_size_export_options.plist new file mode 100644 index 00000000..a6c7e291 --- /dev/null +++ b/fastlane/sdk_size_export_options.plist @@ -0,0 +1,27 @@ + + + + + compileBitcode + + destination + export + method + release-testing + provisioningProfiles + + io.getstream.iOS.DemoAppSwiftUI + match AdHoc io.getstream.iOS.DemoAppSwiftUI + + signingCertificate + Apple Distribution + signingStyle + manual + stripSwiftSymbols + + teamID + EHV7XZLAHA + thinning + iPhone15,2 + +