PRT-975 - lavavisor linker verbosity checks (#902)

* linker verbosity checks

* lint fix

ecosystem/lavavisor/pkg/process/linker.go

@@ -18,7 +18,7 @@ type ProtocolBinaryLinker struct {
 }
 
 func (pbl *ProtocolBinaryLinker) CreateLink(binaryPath string) error {
-	dest, err := pbl.findLavaProtocolPath(binaryPath)
+	dest, err := pbl.FindLavaProtocolPath(binaryPath)
 	if err != nil {
 		return utils.LavaFormatError("Error in findLavaProtocolPath", err)
 	}
@@ -28,15 +28,17 @@ func (pbl *ProtocolBinaryLinker) CreateLink(binaryPath string) error {
 	return nil
 }
 
-func (pbl *ProtocolBinaryLinker) findLavaProtocolPath(binaryPath string) (string, error) {
+func (pbl *ProtocolBinaryLinker) FindLavaProtocolPath(binaryPath string) (string, error) {
 	out, err := exec.LookPath("lavap")
 	if err == nil {
 		return strings.TrimSpace(out), nil
 	}
-	return pbl.copyBinaryToSystemPath(binaryPath)
+	// if failed searching for lavap using "which lavap", try searching it in the goBin directory
+	return pbl.searchLavapInGoPath(binaryPath)
 }
 
-func (pbl *ProtocolBinaryLinker) copyBinaryToSystemPath(binaryPath string) (string, error) {
+// sometimes lavap is not in this context's path. we are looking for it where we assume go will be located in.
+func (pbl *ProtocolBinaryLinker) searchLavapInGoPath(binaryPath string) (string, error) {
 	goPath, err := pbl.Fetcher.VerifyGoInstallation()
 	if err != nil {
 		return "", utils.LavaFormatError("Couldn't get go binary path", err)
@@ -45,28 +47,37 @@ func (pbl *ProtocolBinaryLinker) copyBinaryToSystemPath(binaryPath string) (stri
 	if err != nil {
 		return "", utils.LavaFormatError("Couldn't determine Go binary path", err)
 	}
-
 	goBinPath := strings.TrimSpace(string(goBin)) + "/bin/"
-	pbl.validateBinaryExecutable(binaryPath)
+	err = pbl.validateBinaryExecutable(binaryPath)
+	if err != nil {
+		// failed to validate binary path is exeutable we need to remove it and re download next block.
+		if osErr := os.Remove(binaryPath); osErr != nil {
+			return "", utils.LavaFormatError("Couldn't remove existing link", osErr)
+		}
+		return "", utils.LavaFormatError("lavavisor removed the binary path and will attempt to download the lavap binary again", nil)
+	}
 	return goBinPath + "lavap", nil
 }
 
-func (pbl *ProtocolBinaryLinker) validateBinaryExecutable(path string) {
+func (pbl *ProtocolBinaryLinker) validateBinaryExecutable(path string) error {
 	version, err := exec.Command(path, "version").Output()
 	if err != nil {
-		utils.LavaFormatFatal("Binary is not a valid executable: ", err)
+		return utils.LavaFormatError("Binary is not a valid executable: ", err)
 	}
 	utils.LavaFormatInfo("Executable binary validated.", utils.Attribute{Key: "version", Value: strings.TrimSpace(string(version))})
+	return nil
 }
 
 func (pbl *ProtocolBinaryLinker) removeExistingLink(linkPath string) {
 	if _, err := os.Lstat(linkPath); err == nil {
 		utils.LavaFormatInfo("Discovered an existing link. Attempting to refresh.")
 		if err := os.Remove(linkPath); err != nil {
-			utils.LavaFormatFatal("Couldn't remove existing link", err)
+			utils.LavaFormatError("Couldn't remove existing link", err)
+			return
 		}
 	} else if !os.IsNotExist(err) {
-		utils.LavaFormatFatal("Unexpected error when checking for existing link", err)
+		utils.LavaFormatError("Unexpected error when checking for existing link", err)
+		return
 	}
 	utils.LavaFormatInfo("Removed Link Successfully")
 }

ecosystem/lavavisor/pkg/process/version_monitor.go

@@ -42,11 +42,9 @@ func NewVersionMonitor(initVersion string, lavavisorPath string, processes []str
 	}
 }
 
-func (vm *VersionMonitor) handleUpdateTrigger(incoming *protocoltypes.Version) error {
+func (vm *VersionMonitor) handleUpdateTrigger() error {
 	// set latest known version to incoming.
 	utils.LavaFormatInfo("Update detected. Lavavisor starting the auto-upgrade...")
-	vm.lastKnownVersion = incoming
-
 	// 1. check lavavisor directory first and attempt to fetch new binary from there
 	versionDir := filepath.Join(vm.LavavisorPath, "upgrades", "v"+vm.lastKnownVersion.ProviderTarget)
 	binaryPath := filepath.Join(versionDir, "lavap")
@@ -57,9 +55,13 @@ func (vm *VersionMonitor) handleUpdateTrigger(incoming *protocoltypes.Version) e
 	if err != nil {
 		return utils.LavaFormatError("Lavavisor was not able to fetch updated version. Skipping.", err, utils.Attribute{Key: "Version", Value: vm.lastKnownVersion.ProviderTarget})
 	}
+	return vm.createLinkAndRestartServices()
+}
 
+// create a link for lavap from the binary path and restart the services
+func (vm *VersionMonitor) createLinkAndRestartServices() error {
 	// linker
-	err = vm.protocolBinaryLinker.CreateLink(binaryPath)
+	err := vm.protocolBinaryLinker.CreateLink(vm.BinaryPath)
 	if err != nil {
 		return utils.LavaFormatError("Lavavisor was not able to create link to the binaries. Skipping.", err, utils.Attribute{Key: "Version", Value: vm.lastKnownVersion.ProviderTarget})
 	}
@@ -96,22 +98,59 @@ func (vm *VersionMonitor) handleUpdateTrigger(incoming *protocoltypes.Version) e
 	return nil
 }
 
+func (vm *VersionMonitor) validateLinkPointsToTheRightTarget() error {
+	lavapPath, err := vm.protocolBinaryLinker.FindLavaProtocolPath(vm.BinaryPath)
+	if err != nil {
+		return utils.LavaFormatError("Failed searching for lavap path, failed to validate link exists for lavap latest version", err)
+	}
+	var createLink bool
+	_, err = os.Stat(lavapPath)
+	if err != nil {
+		// failed to validate lavap path.
+		utils.LavaFormatDebug("lavap link path was not found attempting to create link", utils.Attribute{Key: "lavap_path", Value: lavapPath})
+		createLink = true
+	} else {
+		utils.LavaFormatDebug("lavap link path found, validating linked binary is the right one", utils.Attribute{Key: "lavap_path", Value: lavapPath})
+		// read link
+		targetPath, err := os.Readlink(lavapPath)
+		if err != nil {
+			utils.LavaFormatInfo("failed reading link from lavap path", utils.Attribute{Key: "error", Value: err})
+			createLink = true
+		} else if targetPath != vm.BinaryPath {
+			// utils.LavaFormatDebug("target validation", utils.Attribute{Key: "targetPath", Value: targetPath})
+			utils.LavaFormatInfo("lavap link was pointing to the wrong binary. removing and creating a new link")
+			createLink = true
+		}
+	}
+	if createLink {
+		utils.LavaFormatInfo("Attempting new link creation for lavap path", utils.Attribute{Key: "lavap", Value: lavapPath}, utils.Attribute{Key: "binary path", Value: vm.BinaryPath})
+		err = vm.createLinkAndRestartServices()
+	}
+	return err
+}
+
 func (vm *VersionMonitor) ValidateProtocolVersion(incoming *statetracker.ProtocolVersionResponse) error {
 	if !vm.lock.TryLock() { // if an upgrade is currently ongoing we don't need to check versions. just wait for the flow to end.
 		utils.LavaFormatDebug("ValidateProtocolVersion is locked, assuming upgrade is ongoing")
 		return nil
 	}
 	defer vm.lock.Unlock()
 	currentBinaryVersion, _ := GetBinaryVersion(vm.BinaryPath)
+	vm.lastKnownVersion = incoming.Version
 
 	if currentBinaryVersion == "" || ValidateMismatch(incoming.Version, currentBinaryVersion) {
 		utils.LavaFormatInfo("New version detected", utils.Attribute{Key: "incoming", Value: incoming})
 		utils.LavaFormatInfo("Started Version Upgrade flow")
-		err := vm.handleUpdateTrigger(incoming.Version)
+		err := vm.handleUpdateTrigger()
 		if err != nil {
 			utils.LavaFormatInfo("protocol update failed, lavavisor will continue trying to upgrade version every block until it succeeds")
 		}
 		return err
+	} else if currentBinaryVersion != "" { // in case we have the latest version already installed we need to validate a few things.
+		err := vm.validateLinkPointsToTheRightTarget()
+		if err != nil {
+			return utils.LavaFormatError("Failed to validateLinkPointsToTheRightTarget", err)
+		}
 	}
 
 	// version is ok.