Merge remote-tracking branch 'origin/develop' into anya/autofill-on-d…

…efault-prep-tasks

.github/workflows/pr.yml

@@ -99,6 +99,19 @@ jobs:
       with:
         report_paths: unittests.xml
 
+    - name: Update Asana with failed unit tests
+      if: always() # always run even if the previous step fails
+      env:
+        ASANA_ACCESS_TOKEN: ${{ secrets.ASANA_ACCESS_TOKEN }}
+        WORKFLOW_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+      run: |
+        # Extract failed tests from the junit report
+        # Only keep failures unique by classname and name (column 1 and 2 of the yq output)
+        yq < unittests.xml -p xml -o json -r \
+          $'[.testsuites.testsuite[].testcase] | flatten | map(select(.failure) | .+@classname + " " + .+@name + " \'" + .failure.+@message + "\' ${{ env.WORKFLOW_URL }}") | .[]' \
+          | sort -u -k 1,2 \
+          | xargs -L 1 ./scripts/report-failed-unit-test.sh -s ${{ vars.APPLE_CI_FAILING_TESTS_FAILED_TESTS_SECTION_ID }}
+
   release-build:
 
     name: Make Release Build

.github/workflows/release.yml

@@ -73,20 +73,21 @@ jobs:
         APPLE_API_KEY_ISSUER: ${{ secrets.APPLE_API_KEY_ISSUER }}
         MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
       run: |
-        bundle exec fastlane release_${{ steps.destination.outputs.destination }}
         app_version="$(cut -d ' ' -f 3 < Configuration/Version.xcconfig)"
         echo "dsyms_path=${{ github.workspace }}/DuckDuckGo.app.dSYM.zip" >> $GITHUB_ENV
         echo "app_version=${app_version}" >> $GITHUB_ENV
+        bundle exec fastlane release_${{ steps.destination.outputs.destination }}
 
     - name: Upload dSYMs artifact
+      if: always()
       uses: actions/upload-artifact@v3
       with:
         name: DuckDuckGo-${{ steps.destination.outputs.destination }}-dSYM-${{ env.app_version }}
         path: ${{ env.dsyms_path }}
 
     - name: Get Asana Task ID
       id: get-task-id
-      if: github.event.inputs.asana-task-url
+      if: ${{ always() && github.event.inputs.asana-task-url }}
       run: |
         task_url_regex='^https://app.asana.com/[0-9]/[0-9]*/([0-9]*)/f$'
         if [[ "${{ github.event.inputs.asana-task-url }}" =~ ${task_url_regex} ]]; then
@@ -96,16 +97,18 @@ jobs:
         fi
 
     - name: Upload debug symbols to Asana
-      if: github.event.inputs.asana-task-url
+      if: ${{ always() && github.event.inputs.asana-task-url }}
       env:
         ASANA_ACCESS_TOKEN: ${{ secrets.ASANA_ACCESS_TOKEN }}
       run: |
-        asana_dsyms_path="${{ github.workspace }}/DuckDuckGo-${{ env.app_version }}-dSYM.zip"
-        mv -f "${{ env.dsyms_path }}" "$asana_dsyms_path"
+        if [[ -f ${{ env.dsyms_path }} ]]; then
+          asana_dsyms_path="${{ github.workspace }}/DuckDuckGo-${{ env.app_version }}-dSYM.zip"
+          mv -f "${{ env.dsyms_path }}" "$asana_dsyms_path"
 
-        curl -s "https://app.asana.com/api/1.0/tasks/${{ steps.get-task-id.outputs.task_id }}/attachments" \
-          -H "Authorization: Bearer ${{ secrets.ASANA_ACCESS_TOKEN }}" \
-          --form "file=@${asana_dsyms_path};type=application/zip"
+          curl -s "https://app.asana.com/api/1.0/tasks/${{ steps.get-task-id.outputs.task_id }}/attachments" \
+            -H "Authorization: Bearer ${{ secrets.ASANA_ACCESS_TOKEN }}" \
+            --form "file=@${asana_dsyms_path};type=application/zip"
+        fi
 
     - name: Send Mattermost message
       env:

Configuration/Version.xcconfig

@@ -1 +1 @@
-MARKETING_VERSION = 7.88.0
+MARKETING_VERSION = 7.90.0

Core/AppPrivacyConfigurationDataProvider.swift

@@ -23,8 +23,8 @@ import BrowserServicesKit
 final public class AppPrivacyConfigurationDataProvider: EmbeddedDataProvider {
 
     public struct Constants {
-        public static let embeddedDataETag = "\"b3acf772994c82ed870835c27a94de36\""
-        public static let embeddedDataSHA = "955ebdcac92bf5589e34e174e9cb8bcbe5170c9bbb5f6d0b4fa32290368a22b2"
+        public static let embeddedDataETag = "\"18cf17e7f3383d2f9d1f0f6643c90c04\""
+        public static let embeddedDataSHA = "3f37f6c999e2d358a343c9150b6f1ac3120931c027a173aa99dae571c2398f37"
     }
 
     public var embeddedDataEtag: String {

Core/DefaultVariantManager.swift

@@ -46,21 +46,32 @@ public struct VariantIOS: Variant {
             return false
         }
     }
-
+
+    /// This variant is used for returning users to separate them from really new users.
+    static let returningUser = VariantIOS(name: "ru", weight: doNotAllocate, isIncluded: When.always, features: [])
+
     static let doNotAllocate = 0
-
-    // Note: Variants with `doNotAllocate` weight, should always be included so that previous installations are unaffected
+
+    /// The list of cohorts in active ATB experiments.
+    ///
+    /// Variants set to `doNotAllocate` are active, but not adding users to a new cohort, do not change them unless you're sure the experiment is finished.
     public static let defaultVariants: [Variant] = [
-        // SERP testing
         VariantIOS(name: "sc", weight: doNotAllocate, isIncluded: When.always, features: []),
         VariantIOS(name: "sd", weight: doNotAllocate, isIncluded: When.always, features: []),
-        VariantIOS(name: "se", weight: doNotAllocate, isIncluded: When.always, features: [])
-
+        VariantIOS(name: "se", weight: doNotAllocate, isIncluded: When.always, features: []),
+        returningUser
     ]
-
+
+    /// The name of the variant.  Shuld be a two character string like `ma` or `mb`
     public var name: String
+
+    /// The relative weight of this variant, e.g. if two variants have the same weight they will get 50% of the cohorts each.
     public var weight: Int
+
+    /// Function to determine inclusion, e.g. if you want to only run an experiment on English users use `When.inEnglish`
     public var isIncluded: () -> Bool
+
+    /// The experimental feature(s) being tested.
     public var features: [FeatureName]
 
 }
@@ -81,13 +92,26 @@ public class DefaultVariantManager: VariantManager {
     private let variants: [Variant]
     private let storage: StatisticsStore
     private let rng: VariantRNG
+    private let returningUserMeasurement: ReturnUserMeasurement
 
-    public init(variants: [Variant] = VariantIOS.defaultVariants,
-                storage: StatisticsStore = StatisticsUserDefaults(),
-                rng: VariantRNG = Arc4RandomUniformVariantRNG()) {
+    init(variants: [Variant],
+         storage: StatisticsStore,
+         rng: VariantRNG,
+         returningUserMeasurement: ReturnUserMeasurement) {
+
         self.variants = variants
         self.storage = storage
         self.rng = rng
+        self.returningUserMeasurement = returningUserMeasurement
+    }
+
+    public convenience init() {
+        self.init(
+            variants: VariantIOS.defaultVariants,
+            storage: StatisticsUserDefaults(),
+            rng: Arc4RandomUniformVariantRNG(),
+            returningUserMeasurement: KeychainReturnUserMeasurement()
+        )
     }
 
     public func isSupported(feature: FeatureName) -> Bool {
@@ -118,6 +142,10 @@ public class DefaultVariantManager: VariantManager {
     }
 
     private func selectVariant() -> Variant? {
+        if returningUserMeasurement.isReturningUser {
+            return VariantIOS.returningUser
+        }
+
         let totalWeight = variants.reduce(0, { $0 + $1.weight })
         let randomPercent = rng.nextInt(upperBound: totalWeight)
 

Core/FileLoader.swift

@@ -32,7 +32,12 @@ class FileLoader {
         let baseName = fileUrl.deletingPathExtension().path
         let ext = fileUrl.pathExtension
 
-        guard let path = bundle.path(forResource: baseName, ofType: ext) else { throw  FileError.unknownFile }
+        var path = bundle.path(forResource: baseName, ofType: ext)
+        if path == nil {
+            let fileName = fileName.dropping(suffix: ext).dropping(suffix: ".")
+            path = bundle.path(forResource: fileName, ofType: ext)
+        }
+        guard let path else { throw  FileError.unknownFile }
         let url = URL(fileURLWithPath: path)
         guard let data = try? Data(contentsOf: url, options: [.mappedIfSafe]) else { throw  FileError.invalidFileContents }
         return data

Core/Pixel.swift

@@ -110,7 +110,10 @@ public struct PixelParameters {
     public static let message = "message"
     public static let sheetResult = "success"
 
-    public static let defaultBrowser = "default_browser"
+    // Return user
+    public static let returnUserErrorCode = "error_code"
+    public static let returnUserOldATB = "old_atb"
+    public static let returnUserNewATB = "new_atb"
 }
 
 public struct PixelValues {

Core/PixelEvent.swift

@@ -302,16 +302,8 @@ extension Pixel {
         case remoteMessageSecondaryActionClicked
         case remoteMessageSheet
 
-        // MARK: Lock Screen widgets
-        case lockScreenWidgetNewSearch
-        case lockScreenWidgetFavorites
-        case lockScreenWidgetFireButton
-        case lockScreenWidgetVoiceSearch
-        case lockScreenWidgetNewEmail
-
-        // MARK: Set as Default
-        case onboardingSetDefaultOpened
-        case onboardingSetDefaultSkipped
+        // MARK: Return user measurement
+        case returnUser
 
         // MARK: debug pixels
         case dbCrashDetected
@@ -399,7 +391,12 @@ extension Pixel {
         case debugCantSaveBookmarkFix
 
         case debugCannotClearObservationsDatabase
-
+
+        // Return user measurement
+        case debugReturnUserReadATB
+        case debugReturnUserAddATB
+        case debugReturnUserUpdateATB
+
         // Errors from Bookmarks Module
         case bookmarkFolderExpected
         case bookmarksListIndexNotMatchingBookmark
@@ -442,8 +439,6 @@ extension Pixel {
 
         case invalidPayload(Configuration)
 
-        case dailyActiveUser
-
         case emailIncontextPromptDisplayed
         case emailIncontextPromptConfirmed
         case emailIncontextPromptDismissed
@@ -452,6 +447,8 @@ extension Pixel {
         case emailIncontextModalDismissed
         case emailIncontextModalExitEarly
         case emailIncontextModalExitEarlyContinue
+
+        case incrementalRolloutTest
     }
 
 }
@@ -741,17 +738,6 @@ extension Pixel.Event {
         case .remoteMessageSecondaryActionClicked: return "m_remote_message_secondary_action_clicked"
         case .remoteMessageSheet: return "m_remote_message_sheet"
 
-        // MARK: Lock Screen Widgets
-        case .lockScreenWidgetNewSearch: return "m_lockscreen_newsearch"
-        case .lockScreenWidgetFavorites: return "m_lockscreen_favorites"
-        case .lockScreenWidgetFireButton: return "m_lockscreen_fire"
-        case .lockScreenWidgetVoiceSearch: return "m_lockscreen_voicesearch"
-        case .lockScreenWidgetNewEmail: return "m_lockscreen_newemail"
-
-        // MARK: Set as default measuring
-        case .onboardingSetDefaultOpened: return "m_onboarding_set-default-opened"
-        case .onboardingSetDefaultSkipped: return "m_onboarding_set-default-skipped"
-
         // MARK: debug pixels
 
         case .dbCrashDetected: return "m_d_crash"
@@ -884,8 +870,6 @@ extension Pixel.Event {
 
         case .invalidPayload(let configuration): return "m_d_\(configuration.rawValue)_invalid_payload".lowercased()
 
-        case .dailyActiveUser: return "m_daily_active_user"
-
         // MARK: - InContext Email Protection
         case .emailIncontextPromptDisplayed: return "m_email_incontext_prompt_displayed"
         case .emailIncontextPromptConfirmed: return "m_email_incontext_prompt_confirmed"
@@ -895,6 +879,14 @@ extension Pixel.Event {
         case .emailIncontextModalDismissed: return "m_email_incontext_modal_dismissed"
         case .emailIncontextModalExitEarly: return "m_email_incontext_modal_exit_early"
         case .emailIncontextModalExitEarlyContinue: return "m_email_incontext_modal_exit_early_continue"
+
+        // MARK: - Return user measurement
+        case .returnUser: return "m_return_user"
+        case .debugReturnUserAddATB: return "m_debug_return_user_add_atb"
+        case .debugReturnUserReadATB: return "m_debug_return_user_read_atb"
+        case .debugReturnUserUpdateATB: return "m_debug_return_user_update_atb"
+
+        case .incrementalRolloutTest: return "m_autofill_incremental_rollout_test"
         }
 
     }