possible fix for AWS WAF

EthDevOps · Aug 26, 2024 · 7b87037 · 7b87037
1 parent 21e80ce
commit 7b87037
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 13 deletions.
diff --git a/HackMdApi.cs b/HackMdApi.cs
@@ -189,19 +189,26 @@ public async Task GetAllNotes()
         bool returnedNotes = true;
         int page = 1;
         List<NoteMeta> allNotes = new List<NoteMeta>();
+        Random random = new Random();
 
         while (returnedNotes)
         {
             var request = CreateRequest($"/api/notes?page={page}");
             var response = await _client.SendAsync(request);
             response.EnsureSuccessStatusCode();
 
+            if (response.Headers.Contains("aws-waf-token"))
+            {
+                Console.WriteLine("New AWS WAF token received.");
+            }
+
             string responseBody = string.Empty;
             NotesResponse notes = null;
             try
             {
                 responseBody = await response.Content.ReadAsStringAsync();
                 notes = JsonSerializer.Deserialize<NotesResponse>(responseBody)!;
+                Console.WriteLine($"Notes grabbed: {notes.page} of {notes.total} | Limit at {notes.limit}");
             }
             catch (Exception ex)
             {
@@ -222,9 +229,12 @@ public async Task GetAllNotes()
             }
             else
             {
-                Console.WriteLine("Skipping next page.");
-                page++;
+                Console.WriteLine("Got caught be the AWS WAF. Waiting for 15 minutes before trying again...");
+                Thread.Sleep(TimeSpan.FromMinutes(15));
             }
+
+
+            Thread.Sleep(random.Next(1000, 10000));
         }
 
         Console.WriteLine("Start reading notes...");
@@ -281,20 +291,24 @@ private HttpRequestMessage CreateRequest(string path)
        return new HttpRequestMessage
         {
             Method = HttpMethod.Get,
+            Version = HttpVersion.Version20,
             RequestUri = new Uri($"https://{HackMdHost}{path}"),
             Headers =
             {
                 { "accept", "application/json, text/plain, */*" },
+                { "accept-language", "en-GB,en;q=0.5"},
                 {
                     "cookie",
-                    $"locale=en-GB; connect.sid={_sessionCookie}; _csrf={_crsfCookie}; loginstate=true; userid={_useridCookie}"
+                    $"locale=en-GB; indent_type=space; space_units=4; keymap=sublime; connect.sid={_sessionCookie}; _csrf={_crsfCookie}; loginstate=true; userid={_useridCookie}; aws-waf-token=bdbb1976-bc58-4db7-a6a8-430e10d3c9c6:AQoAlYZUH3wHAQAA:ptGNgPAiUpz5V1UyRnJAUP92xWgXrhvNPkNgZy2vD2k6g0oBUL5IxnHJlwxh3tvbUfEijkKsKqT1HBzGWk4LndJ5+r9EFo+Sh/XA42S0bYz9cH3D7chtwhtpXgecbQynGi3MHmVO2pR5P0X/QFvnyK2479gyCgfISM+MyyeDZMxsrnyY8wBOKBfApJwelijsqvRehbQ9/Bz8ZkQjuUa5lsIx8a76XQIpZCJz97gBVoa23LrNluPjS9Diog=="
                 },
-                { "referer", $"https://{HackMdHost}/dashboard/note" },
+                { "referer", $"https://{HackMdHost}/dashboard/note/11" },
                 {
                     "user-agent",
-                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
+                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
                 },
-                { "x-xsrf-token", "FTxa1K8n-Bb877mY1mqBvZJNiWRqUf4sIqh0" },
+                {"if-none-match", "W/\"69b7-zbbviSdQ+/166WDeKO4GtTKrp24\""},
+                { "x-xsrf-token", "Hw2iROpR-yJZeYLSl2UlY_Kw1EQ2FyWDsAFI" }
+
             },
         };
     }

diff --git a/Program.cs b/Program.cs
@@ -2,11 +2,11 @@
 
 using HackMdBackup;
 
-string GetConfig(string envName, string defaultValue = "")
+string GetConfig(string envName, string defaultValue = "",bool isOptional = false)
 {
     string? envVar = Environment.GetEnvironmentVariable(envName);
 
-    if(string.IsNullOrEmpty(envVar) && string.IsNullOrEmpty(envVar))
+    if(!isOptional && string.IsNullOrEmpty(envVar) && string.IsNullOrEmpty(defaultValue))
     {
         throw new Exception($"Missing configuration env: {envName}");
     }
@@ -16,10 +16,13 @@ string GetConfig(string envName, string defaultValue = "")
 
 var pingHttp = new HttpClient();
 
-string hcUrl = GetConfig("HEALTHCHECK_URL");
-await pingHttp.GetAsync($"{hcUrl}/start");
+string hcUrl = GetConfig("HEALTHCHECK_URL", isOptional: true);
+if (!String.IsNullOrEmpty(hcUrl))
+{
+    await pingHttp.GetAsync($"{hcUrl}/start");
+}
 
-string gpgPublicKey = GetConfig("GPG_PUBKEY_FILE");
+string gpgPublicKey = GetConfig("GPG_PUBKEY_FILE","foo");
 string tarFile = GetConfig("BACKUP_TAR_PATH","/tmp/backup.tar.gz");
 string webDriverEndpoint = GetConfig("WEBDRIVER_URL", "http://localhost:4444");
 
@@ -37,8 +40,8 @@ string GetConfig(string envName, string defaultValue = "")
     GitHubUsername = GetConfig("GITHUB_USERNAME"),
     GitHubPassword = GetConfig("GITHUB_PASSWORD"),
     GitHub2FaSeed = GetConfig("GITHUB_OTP_SEED"),
-    BackupPath = GetConfig("BACKUP_PATH","/tmp/backup"),
-    CredetialCachePath = GetConfig("CREDENTIAL_CACHE_PATH","/tmp/creds"),
+    BackupPath = GetConfig("BACKUP_PATH","."),
+    CredetialCachePath = GetConfig("CREDENTIAL_CACHE_PATH","."),
     HackMdHost = GetConfig("HACKMD_HOST"),
 };