Merge pull request #144 from ably/schedule-integration-test-workflow

Skip 1 failing test, fix 1 failing test and schedule tests on main twice a day

.github/workflows/check.yaml

@@ -3,6 +3,10 @@ on:
   push:
     branches:
       - main
+  schedule:
+    # Runs on the 39th minute of the 4 & 15th hour of each day. i.e. 4:39 and 15:39.
+    # See more cron syntax documentation at https://docs.github.com/en/actions/reference/events-that-trigger-workflows#schedule
+    - cron: '39 4,15 * * *'
 
 jobs:
   check:

.github/workflows/flutter_integration.yaml

@@ -9,8 +9,7 @@ jobs:
     strategy:
       matrix:
         device:
-          - 'iPhone 8'  # we are not specifying the iOS version as it tends to change
-          - 'iPhone 11'
+          - 'iPhone 12'  # we are not specifying the iOS version as it tends to change
       fail-fast: false
     runs-on: 'macos-latest'
     steps:
@@ -41,7 +40,7 @@ jobs:
   android:
     strategy:
       matrix:
-        api-level: [19, 22, 24, 26, 28]
+        api-level: [21, 29]
       fail-fast: false
 
     # ubuntu-latest cannot be used as it is only a docker container, and unfortunately running

.run/Integration Test Driver.run.xml

@@ -0,0 +1,10 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Integration Test Driver" type="DartCommandLineRunConfigurationType" factoryName="Dart Command Line Application">
+    <option name="envs">
+      <entry key="VM_SERVICE_URL" value="http://127.0.0.1:8888/" />
+    </option>
+    <option name="filePath" value="$PROJECT_DIR$/test_integration/test_driver/runner.dart" />
+    <option name="workingDirectory" value="$PROJECT_DIR$/test_integration" />
+    <method v="2" />
+  </configuration>
+</component>

.run/Integration Test.run.xml

@@ -0,0 +1,7 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Integration Test" type="CompoundRunConfigurationType">
+    <toRun name="Integration Test App" type="FlutterRunConfigurationType" />
+    <toRun name="Integration Test Driver" type="DartCommandLineRunConfigurationType" />
+    <method v="2" />
+  </configuration>
+</component>

.run/integration_test.run.xml

@@ -1,5 +1,5 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="integration_test" type="FlutterRunConfigurationType" factoryName="Flutter">
+  <configuration default="false" name="Integration Test App" type="FlutterRunConfigurationType" factoryName="Flutter">
     <option name="additionalArgs" value="--observatory-port 8888 --disable-service-auth-codes" />
     <option name="filePath" value="$PROJECT_DIR$/test_integration/lib/main.dart" />
     <method v="2" />

test_integration/README.md

@@ -1,8 +1,38 @@
 # Ably Realtime integration tests
 
-This is for tests that depend on the Ably plugin to be available and
-functional and for tests to be executed on Android and iOS devices or 
-emulators.
+Integration tests allow for tests to execute on Android and iOS. This allows us to ensure
+platform specific behaviour (native ably-flutter code and dependencies, ably-cocoa and ably-java)
+are behaving as expected.
+
+## Test Structure 
+
+### Test Target
+
+This project has only one integration test target, `main.dart`. 
+To run all tests: 
+ - run `flutter drive --target lib/main.dart`, or 
+ - run `flutter drive` which runs them all (one and only, `main.dart`).
+
+### Test Module
+
+Test modules include `basicTests` and `realtime` (For the full list, `TestModules` in [tests_config.dart](./test_driver/tests_config.dart)). To run one module (a collection of test groups), either:
+- In Android Studio: 
+  - Modify the `Integration Test Driver` run/debug configuration by adding `-m module_name_1,module_name_2` in `Program Arguments`.
+    - Note: No spaces between module_names. Alternatively, specify the modules separately: `-m module_name_1 -m module_name_2` 
+- Launch each application manually:
+  - Launch the integration test app manually
+  - Launch the driver and pass the modules with the `-m` option: `dart test_integration/test_driver/runner.dart -m rest,realtime`
+
+### Test Group
+
+To run a specific test group:
+- In [tests_config.dart](test_driver/tests_config.dart), comment out the **test group** key: value pairs which you don't want to run.
+- Launch both integration test app and driver:
+  - Launch `Integration Test` compound run configuration in Android, or
+  - Launch `Integration Test App` and `Integration Test Driver` run configurations, or
+  - Launch each application manually:
+    - Launch the integration test app manually
+    - Launch the driver: `dart test_integration/test_driver/runner.dart`
 
 ## Execute the Flutter Driver Tests
 

test_integration/lib/driver_data_handler.dart

@@ -7,7 +7,7 @@ export 'config/test_names.dart';
 /// Send a message to run a widget test and receive a response.
 ///
 /// Helper to minimize repeatedly used code in driver tests.
-Future<TestControlResponseMessage> getTestResponse(
+Future<TestControlResponseMessage> requestDataForTest(
   FlutterDriver driver,
   TestControlMessage message,
 ) async {

test_integration/lib/test/realtime/realtime_events_test.dart

@@ -1,7 +1,7 @@
 import 'package:ably_flutter/ably_flutter.dart';
 import 'package:ably_flutter_example/provisioning.dart';
 
-import '../../config/test_config.dart';
+import '../../config/test_constants.dart';
 import '../../factory/reporter.dart';
 import '../../utils/encoders.dart';
 

test_integration/lib/test/realtime/realtime_history_test.dart

@@ -1,7 +1,7 @@
 import 'package:ably_flutter/ably_flutter.dart';
 import 'package:ably_flutter_example/provisioning.dart';
 
-import '../../config/test_config.dart';
+import '../../config/test_constants.dart';
 import '../../factory/reporter.dart';
 import '../../utils/encoders.dart';
 import '../../utils/realtime.dart';

test_integration/lib/test/realtime/realtime_presence_get.dart

@@ -1,7 +1,7 @@
 import 'package:ably_flutter/ably_flutter.dart';
 import 'package:ably_flutter_example/provisioning.dart';
 
-import '../../config/test_config.dart';
+import '../../config/test_constants.dart';
 import '../../factory/reporter.dart';
 import '../../utils/data.dart';
 import '../../utils/realtime.dart';

test_integration/lib/test/realtime/realtime_presence_history_test.dart

@@ -1,7 +1,7 @@
 import 'package:ably_flutter/ably_flutter.dart';
 import 'package:ably_flutter_example/provisioning.dart';
 
-import '../../config/test_config.dart';
+import '../../config/test_constants.dart';
 import '../../factory/reporter.dart';
 import '../../utils/data.dart';
 import '../../utils/realtime.dart';

test_integration/lib/test/realtime/realtime_presence_subscribe.dart

@@ -3,7 +3,7 @@ import 'dart:async';
 import 'package:ably_flutter/ably_flutter.dart';
 import 'package:ably_flutter_example/provisioning.dart';
 
-import '../../config/test_config.dart';
+import '../../config/test_constants.dart';
 import '../../factory/reporter.dart';
 import '../../utils/data.dart';
 import '../../utils/encoders.dart';

test_integration/lib/test/rest/rest_history_test.dart

@@ -1,7 +1,7 @@
 import 'package:ably_flutter/ably_flutter.dart';
 import 'package:ably_flutter_example/provisioning.dart';
 
-import '../../config/test_config.dart';
+import '../../config/test_constants.dart';
 import '../../factory/reporter.dart';
 import '../../utils/encoders.dart';
 import '../../utils/rest.dart';

test_integration/lib/test/rest/rest_presence_get_test.dart

@@ -1,7 +1,7 @@
 import 'package:ably_flutter/ably_flutter.dart';
 import 'package:ably_flutter_example/provisioning.dart';
 
-import '../../config/test_config.dart';
+import '../../config/test_constants.dart';
 import '../../factory/reporter.dart';
 import '../../utils/data.dart';
 import '../../utils/rest.dart';

test_integration/lib/test/rest/rest_presence_history_test.dart

@@ -1,7 +1,7 @@
 import 'package:ably_flutter/ably_flutter.dart';
 import 'package:ably_flutter_example/provisioning.dart';
 
-import '../../config/test_config.dart';
+import '../../config/test_constants.dart';
 import '../../factory/reporter.dart';
 import '../../utils/data.dart';
 import '../../utils/rest.dart';

test_integration/lib/test/rest/rest_publish_test.dart

@@ -1,7 +1,7 @@
 import 'package:ably_flutter/ably_flutter.dart';
 import 'package:ably_flutter_example/provisioning.dart';
 
-import '../../config/test_config.dart';
+import '../../config/test_constants.dart';
 import '../../factory/reporter.dart';
 import '../../utils/data.dart';
 import '../../utils/encoders.dart';

test_integration/lib/utils/encoders.dart

@@ -15,7 +15,7 @@ Map<String, dynamic>? encodeMessageExtras(MessageExtras? extras) =>
 
 Map<String, dynamic> encodeMessage(Message message) => {
       'id': message.id,
-      'timestamp': message.timestamp!.toIso8601String(),
+      'timestamp': message.timestamp?.toIso8601String(),
       'clientId': message.clientId,
       'connectionId': message.connectionId,
       'encoding': message.encoding,
@@ -32,7 +32,7 @@ Map<String, dynamic> encodePresenceMessage(PresenceMessage message) => {
       'data': message.data,
       'encoding': message.encoding,
       'extras': encodeMessageExtras(message.extras),
-      'timestamp': message.timestamp!.toIso8601String(),
+      'timestamp': message.timestamp?.toIso8601String(),
     };
 
 String enumValueToString(Object value) =>

test_integration/pubspec.lock

@@ -113,6 +113,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "3.0.1"
+  enum_to_string:
+    dependency: "direct main"
+    description:
+      name: enum_to_string
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "2.0.1"
   fake_async:
     dependency: transitive
     description:

test_integration/pubspec.yaml

@@ -14,6 +14,7 @@ dependencies:
   args: ^2.0.0
   flutter:
     sdk: flutter
+  enum_to_string: ^2.0.1
 
 dev_dependencies:
   flutter_driver:

test_integration/test_driver/runner.dart

@@ -1,17 +1,17 @@
 import 'package:args/args.dart';
+import 'package:enum_to_string/enum_to_string.dart';
 
 import 'tests_abstract.dart';
 import 'tests_config.dart';
 
 void main(List<String> args) {
   final parser = ArgParser();
-  final allowedModules =
-      TestGroup.values.map((group) => group.toString().split('.')[1]).toList();
+
   parser
     ..addMultiOption(
       'modules',
       abbr: 'm',
-      allowed: allowedModules,
+      allowed: TestModules.values.map(EnumToString.convertToString).toSet(),
     )
     ..addFlag(
       'help',
@@ -27,17 +27,21 @@ void main(List<String> args) {
     return;
   }
 
-  final modules = argv['modules'] as List;
-  if (modules.isEmpty) {
+  final selectedModules =
+      _parseModulesStringIntoTestModules(argv['modules'] as List<String>);
+
+  if (selectedModules.isEmpty) {
     runTests(all: true);
-  } else if (modules is List) {
-    runTests(
-        groups: modules
-            .map((module) =>
-                TestGroup.values[allowedModules.indexOf(module as String)])
-            .toList());
-  } else {
-    runTests(
-        groupName: TestGroup.values[allowedModules.indexOf(modules as String)]);
+  } else if (selectedModules is Iterable) {
+    runTests(testModules: selectedModules);
   }
 }
+
+Iterable<TestModules> _parseModulesStringIntoTestModules(
+    List<String> modulesStrings) {
+  final modules = modulesStrings
+      .map((module) => EnumToString.fromString(TestModules.values, module))
+      .whereType<TestModules>()
+      .toSet();
+  return TestModules.values.toSet().intersection(modules);
+}

test_integration/test_driver/test_implementation/basic_platform_tests.dart

@@ -5,7 +5,8 @@ import 'package:test/test.dart';
 void testPlatformAndAblyVersion(FlutterDriver Function() getDriver) {
   const message = TestControlMessage(TestName.platformAndAblyVersion);
   late TestControlResponseMessage response;
-  setUpAll(() async => response = await getTestResponse(getDriver(), message));
+  setUpAll(
+      () async => response = await requestDataForTest(getDriver(), message));
 
   test('platformVersion is a string', () {
     expect(response.payload['platformVersion'], isA<String>());
@@ -27,7 +28,8 @@ void testPlatformAndAblyVersion(FlutterDriver Function() getDriver) {
 void testDemoDependencies(FlutterDriver Function() getDriver) {
   const message = TestControlMessage(TestName.appKeyProvisioning);
   late TestControlResponseMessage response;
-  setUpAll(() async => response = await getTestResponse(getDriver(), message));
+  setUpAll(
+      () async => response = await requestDataForTest(getDriver(), message));
 
   test('appKey is a string', () {
     expect(response.payload['appKey'], isA<String>());

test_integration/test_driver/test_implementation/helper_tests.dart

@@ -5,7 +5,8 @@ import 'package:test/test.dart';
 void testShouldReportUnhandledException(FlutterDriver Function() getDriver) {
   const message = TestControlMessage(TestName.testHelperUnhandledExceptionTest);
   late TestControlResponseMessage response;
-  setUpAll(() async => response = await getTestResponse(getDriver(), message));
+  setUpAll(
+      () async => response = await requestDataForTest(getDriver(), message));
 
   test('returns appropriate error message', () {
     expect(response.payload['error']['exceptionType'], 'String');

test_integration/test_driver/test_implementation/realtime_tests.dart

@@ -10,8 +10,8 @@ void testRealtimePublish(FlutterDriver Function() getDriver) {
   late TestControlResponseMessage response;
   late TestControlResponseMessage response2;
   setUpAll(() async {
-    response = await getTestResponse(getDriver(), message);
-    response2 = await getTestResponse(getDriver(), message2);
+    response = await requestDataForTest(getDriver(), message);
+    response2 = await requestDataForTest(getDriver(), message2);
   });
 
   test('publishes message without any response', () {
@@ -40,7 +40,7 @@ void testRealtimeEvents(FlutterDriver Function() getDriver) {
       List.from(items as List).map((t) => t as Map<String, dynamic>).toList();
 
   setUpAll(() async {
-    response = await getTestResponse(getDriver(), message);
+    response = await requestDataForTest(getDriver(), message);
     connectionStates = transformState(response.payload['connectionStates']);
     connectionStateChanges = transformStateChange(
       response.payload['connectionStateChanges'],
@@ -206,7 +206,7 @@ void testRealtimeSubscribe(FlutterDriver Function() getDriver) {
           .toList();
 
   setUpAll(() async {
-    response = await getTestResponse(getDriver(), message);
+    response = await requestDataForTest(getDriver(), message);
     all = transformMessages(response.payload['all']);
     filteredWithName = transformMessages(response.payload['filteredWithName']);
     filteredWithNames = transformMessages(
@@ -281,7 +281,7 @@ void testRealtimeHistory(FlutterDriver Function() getDriver) {
       List.from(items as List).map((t) => t as Map<String, dynamic>).toList();
 
   setUpAll(() async {
-    response = await getTestResponse(getDriver(), message);
+    response = await requestDataForTest(getDriver(), message);
     paginatedResult =
         response.payload['paginatedResult'] as Map<String, dynamic>;
     historyDefault = transform(response.payload['historyDefault']);
@@ -350,7 +350,7 @@ void testRealtimePresenceGet(FlutterDriver Function() getDriver) {
       List.from(items as List).map((t) => t as Map<String, dynamic>).toList();
 
   setUpAll(() async {
-    response = await getTestResponse(getDriver(), message);
+    response = await requestDataForTest(getDriver(), message);
     membersInitial = transform(response.payload['membersInitial']);
     membersDefault = transform(response.payload['membersDefault']);
     membersClientId = transform(response.payload['membersClientId']);
@@ -396,7 +396,7 @@ void testRealtimePresenceHistory(FlutterDriver Function() getDriver) {
       List.from(items as List).map((t) => t as Map<String, dynamic>).toList();
 
   setUpAll(() async {
-    response = await getTestResponse(getDriver(), message);
+    response = await requestDataForTest(getDriver(), message);
     historyInitial = transform(response.payload['historyInitial']);
     historyDefault = transform(response.payload['historyDefault']);
     historyLimit4 = transform(response.payload['historyLimit4']);
@@ -456,7 +456,7 @@ void testRealtimeEnterUpdateLeave(FlutterDriver Function() getDriver) {
       List.from(items as List).map((t) => t as Map<String, dynamic>).toList();
 
   setUpAll(() async {
-    response = await getTestResponse(getDriver(), message);
+    response = await requestDataForTest(getDriver(), message);
     clientIDClashMatrix = transform(response.payload['clientIDClashMatrix']);
     actionMatrix = transform(response.payload['actionMatrix']);
   });
@@ -583,7 +583,7 @@ void testRealtimePresenceSubscription(FlutterDriver Function() getDriver) {
       List.from(items as List).map((t) => t as Map<String, dynamic>).toList();
 
   setUpAll(() async {
-    response = await getTestResponse(getDriver(), message);
+    response = await requestDataForTest(getDriver(), message);
     allMessages = transform(response.payload['allMessages']);
     enterMessages = transform(response.payload['enterMessages']);
     enterUpdateMessages = transform(response.payload['enterUpdateMessages']);