diff --git a/tools/catch-catchpoint/catch-catchpoint.go b/tools/catch-catchpoint/catch-catchpoint.go index a1d59f2..25c6d8e 100644 --- a/tools/catch-catchpoint/catch-catchpoint.go +++ b/tools/catch-catchpoint/catch-catchpoint.go @@ -22,10 +22,10 @@ const ( httpRetryAttempts = 10 ) -var network string +var networkArgument string func init() { - flag.StringVar(&network, "network", "testnet", "Specify the network (testnet)") + flag.StringVar(&networkArgument, "network", "testnet", "Specify the network (testnet)") } func getLastNodeRound(pu utils.ProcessUtils) (int, error) { @@ -49,14 +49,16 @@ func main() { envNetwork := os.Getenv(envNetworkVar) if envNetwork != "" { log.Printf("Using network from environment variable: %s", envNetwork) - network = envNetwork + networkArgument = envNetwork } - log.Printf("Catchup on network: %s", network) + nu := utils.NetworkUtils{} + network, err := nu.NewNetwork(networkArgument) + + log.Printf("Catchup on network: %s", network.Name) pu := utils.ProcessUtils{} - nu := utils.NetworkUtils{} - statusURL, err := nu.GetStatusURL(network) + statusURL := network.StatusURL if err != nil { log.Fatalf("Error: %v", err) } @@ -126,13 +128,13 @@ func main() { return } - log.Printf("Last node round: %d, Last network round: %d\n", lastNodeRound, lastNetworkRound) + log.Printf("Last node round: %d, Last networkArgument round: %d\n", lastNodeRound, lastNetworkRound) if (lastNodeRound) > lastNetworkRound-1000 { log.Print("Current round is not that far behind (if at all), skipping catchup") return } else if catchpointRound < lastNodeRound-1000 { - log.Print("Catchpoint round is behind the network, skipping catchup") + log.Print("Catchpoint round is behind the networkArgument, skipping catchup") return } diff --git a/tools/start-node/start-node.go b/tools/start-node/start-node.go index 54b4703..c1ef8ae 100644 --- a/tools/start-node/start-node.go +++ b/tools/start-node/start-node.go @@ -13,6 +13,7 @@ const ( algodDataDir = "/algod/data" algodCmd = "/node/bin/algod" catchupCmd = "/node/bin/catch-catchpoint" + goalCmd = "/node/bin/goal" ) var network string @@ -21,7 +22,7 @@ var overwriteConfig bool func init() { flag.StringVar(&network, "network", "testnet", "Specify the network (testnet)") - flag.StringVar(&profile, "profile", "relay", "Specify the profile (archiver, relay)") + flag.StringVar(&profile, "profile", "relay", "Specify the profile (archiver, relay, developer)") flag.BoolVar(&overwriteConfig, "overwrite-config", true, "Specify whether to overwrite the configuration files (true, false)") } @@ -51,30 +52,58 @@ func main() { cu.HandleConfiguration(urlSet, genesisURL, network, profile, overwriteConfig, algodDataDir) pu := utils.ProcessUtils{} - - // Start algod - done := pu.StartProcess(algodCmd, "-d", algodDataDir) + var done <-chan error envVar := os.Getenv(envCatchupVar) - if envVar != "0" && !urlSet { - retryCount := 0 - maxRetries := 10 - for retryCount < maxRetries { - _, err := pu.ExecuteCommand(catchupCmd) - if err == nil { - break - } - retryCount++ - log.Printf("Retry %d/%d: Failed to execute catchup command, retrying in 5 seconds...", retryCount, maxRetries) - time.Sleep(5 * time.Second) + if profile == "archiver" && envVar != "0" { + predefinedNetwork, err := nu.NewNetwork(network) + if err != nil { + log.Fatalf("Error: %v", err) } - if retryCount == maxRetries { - log.Printf("Failed to execute catchup command after %d retries", maxRetries) - return + niou := utils.NetworkIOUtils{} + srvRecords, err := niou.LookupSRVRecords(predefinedNetwork) + if err != nil { + log.Fatalf("Error: %v", err) } + + log.Printf("Catching up using direct archiver connection to %s: ", srvRecords[0]) + done = pu.StartProcess(goalCmd, "node", "start", "-d", algodDataDir, "-p", srvRecords[0]) } else { - log.Printf("Skipping catchup execution as %s is set to 0", envCatchupVar) + + done = pu.StartProcess(algodCmd, "-d", algodDataDir) + + if envVar != "0" && !urlSet && profile != "archiver" { + retryCount := 0 + maxRetries := 10 + for retryCount < maxRetries { + _, err := pu.ExecuteCommand(catchupCmd) + if err == nil { + break + } + retryCount++ + log.Printf("Retry %d/%d: Failed to execute catchup command, retrying in 5 seconds...", retryCount, maxRetries) + time.Sleep(5 * time.Second) + } + if retryCount == maxRetries { + log.Printf("Failed to execute catchup command after %d retries", maxRetries) + return + } + } else { + log.Printf("Skipping catchup execution as %s is set to 0", envCatchupVar) + } } - <-done + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + select { + case err := <-done: + if err != nil { + log.Fatalf("Process finished with error: %v", err) + } + case <-ticker.C: + // do nothing + } + } } diff --git a/tools/utils/config_utils.go b/tools/utils/config_utils.go index 46cb017..ad33de3 100644 --- a/tools/utils/config_utils.go +++ b/tools/utils/config_utils.go @@ -19,11 +19,11 @@ type ConfigUtils struct{} func (cu ConfigUtils) HandleConfiguration(urlSet bool, genesisURL string, network string, profile string, overwriteConfig bool, algodDataDir string) { fu := FileUtils{} - nu := NetworkUtils{} + niou := NetworkIOUtils{} if urlSet { log.Printf("Using genesis and configuration URLs from environment variables: %s", genesisURL) - if err := nu.DownloadNetworkConfiguration(genesisURL, algodDataDir); err != nil { + if err := niou.DownloadNetworkConfiguration(genesisURL, algodDataDir); err != nil { fmt.Printf("Failed to download network configuration: %v", err) os.Exit(1) } diff --git a/tools/utils/network_utils.go b/tools/utils/network_utils.go index 13fa2bf..d7c09f0 100644 --- a/tools/utils/network_utils.go +++ b/tools/utils/network_utils.go @@ -2,42 +2,37 @@ package utils import ( "fmt" - "io" "log" - "net" - "net/http" "os" - "path/filepath" - "time" ) // TODO: Separate network io from blockchain network const ( - // TODO: Create an enum to hold predefined network values testNet = "testnet" envNetworkVar = "VOINETWORK_NETWORK" envGenesisURLVar = "VOINETWORK_GENESIS" envProfileVar = "VOINETWORK_PROFILE" ) -type NetworkUtils struct{} - -func (nu NetworkUtils) IsPortOpen(address string) bool { - conn, err := net.DialTimeout("tcp", address, 5*time.Second) - if err != nil { - return false - } - conn.Close() - return true +type Network struct { + Name string + StatusURL string + ArchivalDNS string } -func (nu NetworkUtils) GetStatusURL(network string) (string, error) { - switch network { +type NetworkUtils struct{} + +func (nu NetworkUtils) NewNetwork(name string) (Network, error) { + switch name { case testNet: - return "https://testnet-api.voi.nodly.io/v2/status", nil + return Network{ + Name: testNet, + StatusURL: "https://testnet-api.voi.nodly.io/v2/status", + ArchivalDNS: "voitest.voi.network", + }, nil default: - return "", fmt.Errorf("unsupported network: %s", network) + return Network{}, fmt.Errorf("unsupported network: %s", name) } } @@ -84,43 +79,3 @@ func (nu NetworkUtils) GetGenesisFromEnv() (string, bool) { } return "", false } - -func (nu NetworkUtils) DownloadNetworkConfiguration(genesisURL, algodDataDir string) error { - if err := downloadFile(genesisURL, filepath.Join(algodDataDir, "genesis.json")); err != nil { - return fmt.Errorf("failed to download genesis.json: %w", err) - } - - return nil -} - -func downloadFile(url, destFile string) error { - resp, err := http.Get(url) - if err != nil { - return fmt.Errorf("error making GET request to %s: %w", url, err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("received non-200 response code: %d", resp.StatusCode) - } - - fu := FileUtils{} - err = fu.EnsureDirExists(destFile) - if err != nil { - return err - } - - out, err := os.Create(destFile) - if err != nil { - return fmt.Errorf("error creating file %s: %w", destFile, err) - } - defer out.Close() - - _, err = io.Copy(out, resp.Body) - if err != nil { - return fmt.Errorf("error writing data to %s: %w", destFile, err) - } - - log.Printf("Successfully downloaded %s to %s", url, destFile) - return nil -} diff --git a/tools/utils/network_utils_test.go b/tools/utils/network_utils_test.go index fe96303..6569e30 100644 --- a/tools/utils/network_utils_test.go +++ b/tools/utils/network_utils_test.go @@ -5,58 +5,94 @@ import ( "testing" ) -func TestGetStatusURL(t *testing.T) { - tests := []struct { - name string - network string - wantURL string - wantErr bool - }{ - {"testNet", "testnet", "https://testnet-api.voi.nodly.io/v2/status", false}, - {"Unknown", "unknown", "", true}, +func TestNewNetwork(t *testing.T) { + nu := NetworkUtils{} + + network, err := nu.NewNetwork(testNet) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if network.Name != testNet { + t.Errorf("Expected network name %s, got %s", testNet, network.Name) } + _, err = nu.NewNetwork("invalid") + if err == nil { + t.Fatalf("Expected error for unsupported network, got none") + } +} + +func TestCheckIfPredefinedNetwork(t *testing.T) { nu := NetworkUtils{} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotURL, err := nu.GetStatusURL(tt.network) - if (err != nil) != tt.wantErr { - t.Errorf("GetStatusURL() error = %v, wantErr %v", err, tt.wantErr) - return - } - if gotURL != tt.wantURL { - t.Errorf("GetStatusURL() = %v, want %v", gotURL, tt.wantURL) - } - }) + if !nu.CheckIfPredefinedNetwork(testNet) { + t.Errorf("Expected true for predefined network %s", testNet) + } + + if nu.CheckIfPredefinedNetwork("invalid") { + t.Errorf("Expected false for unsupported network") } } func TestGetEnvNetworkVar(t *testing.T) { - expected := envNetworkVar nu := NetworkUtils{} - if got := nu.GetEnvNetworkVar(); got != expected { - t.Errorf("GetEnvNetworkVar() = %v, want %v", got, expected) + expected := envNetworkVar + + if nu.GetEnvNetworkVar() != expected { + t.Errorf("Expected %s, got %s", expected, nu.GetEnvNetworkVar()) } } func TestGetNetworkFromEnv(t *testing.T) { nu := NetworkUtils{} - - // Test with environment variable set - expectedNetwork := "testnet" - os.Setenv(envNetworkVar, expectedNetwork) + expected := "test_network" + os.Setenv(envNetworkVar, expected) defer os.Unsetenv(envNetworkVar) - got, ok := nu.GetNetworkFromEnv() - if !ok || got != expectedNetwork { - t.Errorf("GetNetworkFromEnv() = %v, %v, want %v, true", got, ok, expectedNetwork) + network, found := nu.GetNetworkFromEnv() + if !found { + t.Fatalf("Expected to find network from environment variable") + } + if network != expected { + t.Errorf("Expected %s, got %s", expected, network) + } +} + +func TestGetEnvProfileVar(t *testing.T) { + nu := NetworkUtils{} + expected := envProfileVar + + if nu.GetEnvProfileVar() != expected { + t.Errorf("Expected %s, got %s", expected, nu.GetEnvProfileVar()) + } +} + +func TestGetProfileFromEnv(t *testing.T) { + nu := NetworkUtils{} + expected := "test_profile" + os.Setenv(envProfileVar, expected) + defer os.Unsetenv(envProfileVar) + + profile, found := nu.GetProfileFromEnv() + if !found { + t.Fatalf("Expected to find profile from environment variable") } + if profile != expected { + t.Errorf("Expected %s, got %s", expected, profile) + } +} - // Test without environment variable set - os.Unsetenv(envNetworkVar) - _, ok = nu.GetNetworkFromEnv() - if ok { - t.Error("GetNetworkFromEnv() expected to return false when environment variable is not set") +func TestGetGenesisFromEnv(t *testing.T) { + nu := NetworkUtils{} + expected := "test_genesis_url" + os.Setenv(envGenesisURLVar, expected) + defer os.Unsetenv(envGenesisURLVar) + + genesisURL, found := nu.GetGenesisFromEnv() + if !found { + t.Fatalf("Expected to find genesis URL from environment variable") + } + if genesisURL != expected { + t.Errorf("Expected %s, got %s", expected, genesisURL) } } diff --git a/tools/utils/networkio_utils.go b/tools/utils/networkio_utils.go new file mode 100644 index 0000000..67123eb --- /dev/null +++ b/tools/utils/networkio_utils.go @@ -0,0 +1,81 @@ +package utils + +import ( + "fmt" + "io" + "log" + "net" + "net/http" + "os" + "path/filepath" + "time" +) + +type NetworkIOUtils struct{} + +func (nu NetworkIOUtils) IsPortOpen(address string) bool { + conn, err := net.DialTimeout("tcp", address, 5*time.Second) + if err != nil { + return false + } + conn.Close() + return true +} + +func (nu NetworkIOUtils) LookupSRVRecords(network Network) ([]string, error) { + _, srvs, err := net.LookupSRV("archive", "tcp", network.ArchivalDNS) + if err != nil { + return nil, fmt.Errorf("failed to lookup SRV records: %v", err) + } + + if len(srvs) == 0 { + return nil, fmt.Errorf("no SRV records found") + } + + var records []string + for _, srv := range srvs { + records = append(records, fmt.Sprintf("%s:%d", srv.Target, srv.Port)) + } + + return records, nil +} + +func (nu NetworkIOUtils) DownloadNetworkConfiguration(genesisURL, algodDataDir string) error { + if err := downloadFile(genesisURL, filepath.Join(algodDataDir, "genesis.json")); err != nil { + return fmt.Errorf("failed to download genesis.json: %w", err) + } + + return nil +} + +func downloadFile(url, destFile string) error { + resp, err := http.Get(url) + if err != nil { + return fmt.Errorf("error making GET request to %s: %w", url, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("received non-200 response code: %d", resp.StatusCode) + } + + fu := FileUtils{} + err = fu.EnsureDirExists(destFile) + if err != nil { + return err + } + + out, err := os.Create(destFile) + if err != nil { + return fmt.Errorf("error creating file %s: %w", destFile, err) + } + defer out.Close() + + _, err = io.Copy(out, resp.Body) + if err != nil { + return fmt.Errorf("error writing data to %s: %w", destFile, err) + } + + log.Printf("Successfully downloaded %s to %s", url, destFile) + return nil +} diff --git a/tools/utils/networkio_utils_test.go b/tools/utils/networkio_utils_test.go new file mode 100644 index 0000000..b8d4f2c --- /dev/null +++ b/tools/utils/networkio_utils_test.go @@ -0,0 +1,65 @@ +package utils + +import ( + "net/http" + "net/http/httptest" + "os" + "testing" +) + +func TestIsPortOpen(t *testing.T) { + nu := NetworkIOUtils{} + + t.Run("Port is open", func(t *testing.T) { + // Start a test server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + defer server.Close() + + address := server.Listener.Addr().String() + if !nu.IsPortOpen(address) { + t.Errorf("Expected port %s to be open", address) + } + }) + + t.Run("Port is closed", func(t *testing.T) { + address := "localhost:12345" + if nu.IsPortOpen(address) { + t.Errorf("Expected port %s to be closed", address) + } + }) +} + +func TestDownloadNetworkConfiguration(t *testing.T) { + nu := NetworkIOUtils{} + algodDataDir := os.TempDir() + + t.Run("Successful download", func(t *testing.T) { + // Mock http.Get + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte(`{"genesis": "data"}`)) + if err != nil { + return + } + })) + defer server.Close() + + err := nu.DownloadNetworkConfiguration(server.URL, algodDataDir) + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + }) + + t.Run("Failed download", func(t *testing.T) { + // Mock http.Get to return a non-200 status code + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + err := nu.DownloadNetworkConfiguration(server.URL, algodDataDir) + if err == nil { + t.Errorf("Expected error, got none") + } + }) +}