Retry starting device when we find potential crash in the logs (#30)

* Checking the output of the emulator periodically (when we poll for the serial)
   If output contains the fault indicator we kill the process and retry at most 5 times.

Co-authored-by: Laszlo Pusok <[email protected]>

main.go

@@ -32,6 +32,14 @@ type config struct {
 	EmulatorChannel   string `env:"emulator_channel,opt[0,1,2,3]"`
 }
 
+var (
+	faultIndicators = []string{" BUG: ", "Kernel panic"}
+)
+
+const (
+	maxAttempts = 5
+)
+
 func runningDeviceInfos(androidHome string) (map[string]string, error) {
 	cmd := command.New(filepath.Join(androidHome, "platform-tools", "adb"), "devices")
 	out, err := cmd.RunAndReturnTrimmedCombinedOutput()
@@ -193,8 +201,7 @@ func main() {
 		fmt.Println()
 	}
 
-	var output bytes.Buffer
-	deviceStartCmd := command.New(emulatorPath, append([]string{
+	args := append([]string{
 		"@" + cfg.ID,
 		"-verbose",
 		"-show-kernel",
@@ -204,8 +211,21 @@ func main() {
 		"-netdelay", "none",
 		"-no-snapshot",
 		"-wipe-data",
-		"-gpu", "swiftshader_indirect"}, startCustomFlags...)...,
-	).SetStdout(&output).SetStderr(&output)
+		"-gpu", "swiftshader_indirect"}, startCustomFlags...)
+
+	serial := startEmulator(emulatorPath, args, androidHome, runningDevices, 1)
+
+	if err := tools.ExportEnvironmentWithEnvman("BITRISE_EMULATOR_SERIAL", serial); err != nil {
+		log.Warnf("Failed to export environment (BITRISE_EMULATOR_SERIAL), error: %s", err)
+	}
+	log.Printf("- Device with serial: %s started", serial)
+
+	log.Donef("- Done")
+}
+
+func startEmulator(emulatorPath string, args []string, androidHome string, runningDevices map[string]string, attempt int) string {
+	var output bytes.Buffer
+	deviceStartCmd := command.New(emulatorPath, args...).SetStdout(&output).SetStderr(&output)
 
 	log.Infof("Starting device")
 	log.Donef("$ %s", deviceStartCmd.PrintableCommandArgs())
@@ -222,7 +242,7 @@ func main() {
 	const bootWaitTime = time.Duration(300)
 	timeout := time.NewTimer(bootWaitTime * time.Second)
 	deviceCheckTicker := time.NewTicker(5 * time.Second)
-
+	retry := false
 waitLoop:
 	for {
 		select {
@@ -236,21 +256,43 @@ waitLoop:
 			log.Warnf("Emulator log: %s", output)
 			failf("Failed to boot emulator device within %d seconds.", bootWaitTime)
 		case <-deviceCheckTicker.C:
+			var err error
 			serial, err = queryNewDeviceSerial(androidHome, runningDevices)
 			if err != nil {
 				failf("Error: %s", err)
 			} else if serial != "" {
 				break waitLoop
 			}
+			if containsAny(output.String(), faultIndicators) {
+				log.Warnf("Emulator log contains fault")
+				log.Warnf("Emulator log: %s", output)
+				if err := deviceStartCmd.GetCmd().Process.Kill(); err != nil {
+					failf("Couldn't finish emulator process: %v", err)
+				}
+				if attempt < maxAttempts {
+					log.Warnf("Trying to start emulator process again...")
+					retry = true
+					break waitLoop
+				} else {
+					failf("Failed to boot device due to faults after %d tries", maxAttempts)
+				}
+			}
 		}
 	}
 	timeout.Stop()
 	deviceCheckTicker.Stop()
+	if retry {
+		return startEmulator(emulatorPath, args, androidHome, runningDevices, attempt+1)
+	}
+	return serial
+}
 
-	if err := tools.ExportEnvironmentWithEnvman("BITRISE_EMULATOR_SERIAL", serial); err != nil {
-		log.Warnf("Failed to export environment (BITRISE_EMULATOR_SERIAL), error: %s", err)
+func containsAny(output string, any []string) bool {
+	for _, fault := range any {
+		if strings.Contains(output, fault) {
+			return true
+		}
 	}
-	log.Printf("- Device with serial: %s started", serial)
 
-	log.Donef("- Done")
+	return false
 }