Fixes #1388 by correctly checking whether we are in ec2

- The existing tcp_ping check didn't work.
- Uses the suggested by aws docs to query aws imds using libcurl.
- Tested on an ec2 machine manually to verify it works.

cpp/arcticdb/storage/s3/ec2_utils.hpp

@@ -0,0 +1,62 @@
+#include <arcticdb/log/log.hpp>
+#include <curl/curl.h>
+
+namespace arcticdb::storage::s3 {
+
+// We only care about the response codes, so we just pass a dummy write function to libcurl to not print the responses.
+size_t write_callback([[maybe_unused]] void *buffer, size_t size, size_t nmemb, [[maybe_unused]] void *userp) {
+    return size * nmemb;
+}
+
+// A fast check to identify whether we're running on AWS EC2.
+// According to AWS docs a reliable way to identify if we're in EC2 is to query the instance metadata service:
+// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/identify_ec2_instances.html
+//
+// Since there are two versions IMDSv1 and IMDSv2 we first try to connect to v2 and if we fail then attempt the legacy
+// v1 connection. If both fail we're most likely running outside of EC2 (unless IMDS is under heavy load and takes more
+// than 100ms to respond)
+bool is_running_inside_ec2(){
+    CURL *curl = curl_easy_init();
+    if(curl) {
+        CURLcode res;
+
+        // Suggested approach by aws docs for IMDSv2 (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-identity-documents.html):
+        // curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"
+        // The below libcurl options should mimic the command above.
+        curl_slist *headers = nullptr;
+        headers = curl_slist_append(headers, "X-aws-ec2-metadata-token-ttl-seconds: 21600");
+        curl_easy_setopt(curl, CURLOPT_URL, "http://169.254.169.254/latest/api/token");
+        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+        curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
+        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
+
+        // We use a timeout of 100ms because it should be enough for IMDS to respond. Be wary of increasing the timeout
+        // since we need to wait the whole 100ms every time we init the aws sdk outside of aws ec2.
+        curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, 100);
+
+        res = curl_easy_perform(curl);
+        curl_slist_free_all(headers);
+
+        if (res == CURLE_OK){
+            curl_easy_cleanup(curl);
+            return true;
+        }
+
+        // If attempting to connect via IMDSv2 fails we want to attempt a connection to IMDSv1:
+        // curl http://169.254.169.254/latest/dynamic/instance-identity/document
+        curl_easy_reset(curl);
+        curl_easy_setopt(curl, CURLOPT_URL, "http://169.254.169.254/latest/dynamic/instance-identity/document");
+        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
+        curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, 100);
+
+        res = curl_easy_perform(curl);
+        curl_easy_cleanup(curl);
+
+        if (res == CURLE_OK){
+            return true;
+        }
+        return false;
+    }
+    return false;
+}
+}

cpp/arcticdb/storage/s3/s3_api.cpp

@@ -12,7 +12,7 @@
 #include <aws/core/platform/Environment.h>
 #include <arcticdb/util/configs_map.hpp>
 #include <arcticdb/log/log.hpp>
-#include <arcticdb/storage/s3/tcp_ping_ec2.hpp>
+#include <arcticdb/storage/s3/ec2_utils.hpp>
 #include <cstdlib>
 
 namespace arcticdb::storage::s3 {
@@ -37,7 +37,7 @@ S3ApiInstance::S3ApiInstance(Aws::Utils::Logging::LogLevel log_level) :
         if (!Aws::Environment::GetEnv(name).empty())
             return;
     }
-    if (ec2_metadata_endpoint_reachable())
+    if (is_running_inside_ec2())
         return;
     ARCTICDB_RUNTIME_DEBUG(log::storage(),
         "Does not appear to be using AWS. Will set AWS_EC2_METADATA_DISABLED");