diff --git a/HackMdApi.cs b/HackMdApi.cs index 3c5f381..de497c2 100644 --- a/HackMdApi.cs +++ b/HackMdApi.cs @@ -189,6 +189,7 @@ public async Task GetAllNotes() bool returnedNotes = true; int page = 1; List allNotes = new List(); + Random random = new Random(); while (returnedNotes) { @@ -196,12 +197,18 @@ public async Task GetAllNotes() var response = await _client.SendAsync(request); response.EnsureSuccessStatusCode(); + if (response.Headers.Contains("aws-waf-token")) + { + Console.WriteLine("New AWS WAF token received."); + } + string responseBody = string.Empty; NotesResponse notes = null; try { responseBody = await response.Content.ReadAsStringAsync(); notes = JsonSerializer.Deserialize(responseBody)!; + Console.WriteLine($"Notes grabbed: {notes.page} of {notes.total} | Limit at {notes.limit}"); } catch (Exception ex) { @@ -222,9 +229,12 @@ public async Task GetAllNotes() } else { - Console.WriteLine("Skipping next page."); - page++; + Console.WriteLine("Got caught be the AWS WAF. Waiting for 15 minutes before trying again..."); + Thread.Sleep(TimeSpan.FromMinutes(15)); } + + + Thread.Sleep(random.Next(1000, 10000)); } Console.WriteLine("Start reading notes..."); @@ -281,20 +291,24 @@ private HttpRequestMessage CreateRequest(string path) return new HttpRequestMessage { Method = HttpMethod.Get, + Version = HttpVersion.Version20, RequestUri = new Uri($"https://{HackMdHost}{path}"), Headers = { { "accept", "application/json, text/plain, */*" }, + { "accept-language", "en-GB,en;q=0.5"}, { "cookie", - $"locale=en-GB; connect.sid={_sessionCookie}; _csrf={_crsfCookie}; loginstate=true; userid={_useridCookie}" + $"locale=en-GB; indent_type=space; space_units=4; keymap=sublime; connect.sid={_sessionCookie}; _csrf={_crsfCookie}; loginstate=true; userid={_useridCookie}; aws-waf-token=bdbb1976-bc58-4db7-a6a8-430e10d3c9c6:AQoAlYZUH3wHAQAA:ptGNgPAiUpz5V1UyRnJAUP92xWgXrhvNPkNgZy2vD2k6g0oBUL5IxnHJlwxh3tvbUfEijkKsKqT1HBzGWk4LndJ5+r9EFo+Sh/XA42S0bYz9cH3D7chtwhtpXgecbQynGi3MHmVO2pR5P0X/QFvnyK2479gyCgfISM+MyyeDZMxsrnyY8wBOKBfApJwelijsqvRehbQ9/Bz8ZkQjuUa5lsIx8a76XQIpZCJz97gBVoa23LrNluPjS9Diog==" }, - { "referer", $"https://{HackMdHost}/dashboard/note" }, + { "referer", $"https://{HackMdHost}/dashboard/note/11" }, { "user-agent", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" }, - { "x-xsrf-token", "FTxa1K8n-Bb877mY1mqBvZJNiWRqUf4sIqh0" }, + {"if-none-match", "W/\"69b7-zbbviSdQ+/166WDeKO4GtTKrp24\""}, + { "x-xsrf-token", "Hw2iROpR-yJZeYLSl2UlY_Kw1EQ2FyWDsAFI" } + }, }; } diff --git a/Program.cs b/Program.cs index ecfb491..9931adc 100644 --- a/Program.cs +++ b/Program.cs @@ -2,11 +2,11 @@ using HackMdBackup; -string GetConfig(string envName, string defaultValue = "") +string GetConfig(string envName, string defaultValue = "",bool isOptional = false) { string? envVar = Environment.GetEnvironmentVariable(envName); - if(string.IsNullOrEmpty(envVar) && string.IsNullOrEmpty(envVar)) + if(!isOptional && string.IsNullOrEmpty(envVar) && string.IsNullOrEmpty(defaultValue)) { throw new Exception($"Missing configuration env: {envName}"); } @@ -16,10 +16,13 @@ string GetConfig(string envName, string defaultValue = "") var pingHttp = new HttpClient(); -string hcUrl = GetConfig("HEALTHCHECK_URL"); -await pingHttp.GetAsync($"{hcUrl}/start"); +string hcUrl = GetConfig("HEALTHCHECK_URL", isOptional: true); +if (!String.IsNullOrEmpty(hcUrl)) +{ + await pingHttp.GetAsync($"{hcUrl}/start"); +} -string gpgPublicKey = GetConfig("GPG_PUBKEY_FILE"); +string gpgPublicKey = GetConfig("GPG_PUBKEY_FILE","foo"); string tarFile = GetConfig("BACKUP_TAR_PATH","/tmp/backup.tar.gz"); string webDriverEndpoint = GetConfig("WEBDRIVER_URL", "http://localhost:4444"); @@ -37,8 +40,8 @@ string GetConfig(string envName, string defaultValue = "") GitHubUsername = GetConfig("GITHUB_USERNAME"), GitHubPassword = GetConfig("GITHUB_PASSWORD"), GitHub2FaSeed = GetConfig("GITHUB_OTP_SEED"), - BackupPath = GetConfig("BACKUP_PATH","/tmp/backup"), - CredetialCachePath = GetConfig("CREDENTIAL_CACHE_PATH","/tmp/creds"), + BackupPath = GetConfig("BACKUP_PATH","."), + CredetialCachePath = GetConfig("CREDENTIAL_CACHE_PATH","."), HackMdHost = GetConfig("HACKMD_HOST"), };