Merge branch 'master' into datadog-api-spec/generated/2733

.generator/src/generator/templates/api.j2

@@ -2,6 +2,7 @@
 use reqwest;
 use serde::{Serialize, Deserialize};
 use crate::datadog::*;
+use reqwest::header::{HeaderMap, HeaderValue};
 {%- for _, _, operation in operations if "x-pagination" in operation %}
 {%- if loop.first %}
 use async_stream::try_stream;
@@ -13,6 +14,12 @@ use futures_core::stream::Stream;
 use log::warn;
 {%- endif %}
 {%- endfor %}
+{%- for _, _, operation in operations if operation.requestBody is defined and not operation|form_parameter %}
+{%- if loop.first %}
+use std::io::Write;
+use flate2::{write::{GzEncoder, ZlibEncoder}, Compression};
+{%- endif %}
+{%- endfor %}
 
 {%- set structName = name.replace(" ", "")+"API" %}
 {% for path, method, operation in operations|sort(attribute="2.operationId", case_sensitive=true) %}
@@ -83,10 +90,7 @@ pub struct {{ structName }} {
 
 impl Default for {{ structName }} {
     fn default() -> Self {
-        Self {
-            config: configuration::Configuration::new(),
-            client: reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(),
-        }
+        Self::with_config(configuration::Configuration::default())
     }
 }
 
@@ -103,9 +107,37 @@ impl {{ structName }} {
             reqwest_client_builder = reqwest_client_builder.proxy(proxy);
         }
 
-        let middleware_client_builder = reqwest_middleware::ClientBuilder::new(reqwest_client_builder.build().unwrap());
+        let mut middleware_client_builder =
+            reqwest_middleware::ClientBuilder::new(reqwest_client_builder.build().unwrap());
+
+        if config.enable_retry {
+            struct RetryableStatus;
+            impl reqwest_retry::RetryableStrategy for RetryableStatus {
+                fn handle(
+                    &self,
+                    res: &Result<reqwest::Response, reqwest_middleware::Error>,
+                ) -> Option<reqwest_retry::Retryable> {
+                    match res {
+                        Ok(success) => reqwest_retry::default_on_request_success(success),
+                        Err(_) => None,
+                    }
+                }
+            }
+            let backoff_policy = reqwest_retry::policies::ExponentialBackoff::builder()
+                .build_with_max_retries(config.max_retries);
+
+            let retry_middleware =
+                reqwest_retry::RetryTransientMiddleware::new_with_policy_and_strategy(
+                    backoff_policy,
+                    RetryableStatus,
+                );
+
+            middleware_client_builder = middleware_client_builder.with(retry_middleware);
+        }
+
         let client = middleware_client_builder.build();
-        Self {config, client}
+
+        Self { config, client }
     }
 
     pub fn with_client_and_config(config: configuration::Configuration, client: reqwest_middleware::ClientWithMiddleware) -> Self {
@@ -300,18 +332,50 @@ impl {{ structName }} {
         {%- endif %}
         {%- endfor %}
 
+        // build headers
+        let mut headers = HeaderMap::new();
+        {%- if operation.requestBody %}
+        {%- set contentTypeHeaders = operation.requestBody.content.keys()|list %}
+        {%- if contentTypeHeaders %}
+        {%- if "application/json" in contentTypeHeaders %}
+        headers.insert("Content-Type", HeaderValue::from_static("application/json"));
+        {%- else %}
+        headers.insert("Content-Type", HeaderValue::from_static("{{ contentTypeHeaders[0] }}"));
+        {%- endif %}
+        {%- endif %}
+        {%- endif %}
+        {%- if operation.responses %}
+        {%- set acceptHeaders = operation|accept_headers %}
+        {%- if acceptHeaders %}
+        {%- if "application/json" in acceptHeaders %}
+        headers.insert("Accept", HeaderValue::from_static("application/json"));
+        {%- else  %}
+        headers.insert("Accept", HeaderValue::from_static("{{ acceptHeaders|join(",") }}"));
+        {%- endif %}
+        {%- endif %}
+        {%- endif %}
+
         {% for name, parameter in operation|parameters if parameter.in == "header" %}
         {%- if not parameter.required %}
         if let Some(ref local) = {{name|variable_name}} {
-            local_req_builder = local_req_builder.header("{{name}}", &local.to_string());
-        };
+            headers.insert("{{name}}", local.to_string().parse().expect("failed to parse {{name}} header"));
+        }
         {%- else %}
-        local_req_builder = local_req_builder.header("{{name}}", &{{name|variable_name}}.to_string());
+        headers.insert("{{name}}", {{name|variable_name}}.to_string().parse().expect("failed to parse {{name}} header"));
         {%- endif %}
         {%- endfor %}
 
         // build user agent
-        local_req_builder = local_req_builder.header(reqwest::header::USER_AGENT, local_configuration.user_agent.clone());
+        match HeaderValue::from_str(local_configuration.user_agent.as_str()) {
+            Ok(user_agent) => headers.insert(reqwest::header::USER_AGENT, user_agent),
+            Err(e) => {
+                log::warn!("Failed to parse user agent header: {e}, falling back to default");
+                headers.insert(
+                    reqwest::header::USER_AGENT,
+                    HeaderValue::from_static(configuration::DEFAULT_USER_AGENT.as_str()),
+                )
+            }
+        };
 
         // build auth
         {%- set authMethods = operation.security if "security" in operation else openapi.security %}
@@ -321,7 +385,7 @@ impl {{ structName }} {
         {%- set schema = openapi.components.securitySchemes[name] %}
         {%- if schema.type == "apiKey" and schema.in != "cookie" %}
         if let Some(local_key) = local_configuration.auth_keys.get("{{ name }}") {
-            local_req_builder = local_req_builder.header("{{schema.name}}", &local_key.key);
+            headers.insert("{{schema.name}}", HeaderValue::from_str(local_key.key.as_str()).expect("failed to parse {{schema.name}} header"));
         };
         {%- endif %}
         {%- endfor %}
@@ -333,11 +397,13 @@ impl {{ structName }} {
         {%- if formParameter.required %}
         let mut local_form = reqwest::multipart::Form::new();
         local_form = local_form.part("{{formParameter.name}}", reqwest::multipart::Part::bytes({{formParameter.name}}).file_name("{{formParameter.name}}"));
+        headers.insert("Content-Type", format!("multipart/form-data; boundary={}", local_form.boundary()).parse().unwrap());
         local_req_builder = local_req_builder.multipart(local_form);
         {%- else %}
         if let Some({{formParameter.name}}) = {{formParameter.name}} {
             let mut local_form = reqwest::multipart::Form::new();
             local_form = local_form.part("{{formParameter.name}}", reqwest::multipart::Part::bytes({{formParameter.name}}).file_name("{{formParameter.name}}"));
+            headers.insert("Content-Type", format!("multipart/form-data; boundary={}", local_form.boundary()).parse().unwrap());
             local_req_builder = local_req_builder.multipart(local_form);
         };
         {%- endif %}
@@ -348,10 +414,55 @@ impl {{ structName }} {
         let output = Vec::new();
         let mut ser = serde_json::Serializer::with_formatter(output, DDFormatter);
         if {{operation.get("x-codegen-request-body-name", "body")|variable_name}}.serialize(&mut ser).is_ok() {
-            local_req_builder = local_req_builder.body(ser.into_inner());
+            if let Some(content_encoding) = headers.get("Content-Encoding") {
+                match content_encoding.to_str().unwrap_or_default() {
+                    "gzip" => {
+                        let mut enc = GzEncoder::new(
+                            Vec::new(),
+                            Compression::default(),
+                        );
+                        let _ = enc.write_all(ser.into_inner().as_slice());
+                        match enc.finish() {
+                            Ok(buf) => {
+                                local_req_builder = local_req_builder.body(buf);
+                            }
+                            Err(e) => return Err(Error::Io(e)),
+                        }
+                    }
+                    "deflate" => {
+                        let mut enc = ZlibEncoder::new(
+                            Vec::new(),
+                            Compression::default(),
+                        );
+                        let _ = enc.write_all(ser.into_inner().as_slice());
+                        match enc.finish() {
+                            Ok(buf) => {
+                                local_req_builder = local_req_builder.body(buf);
+                            }
+                            Err(e) => return Err(Error::Io(e)),
+                        }
+                    }
+                    "zstd1" => {
+                        let mut enc = zstd::stream::Encoder::new(Vec::new(), 0).unwrap();
+                        let _ = enc.write_all(ser.into_inner().as_slice());
+                        match enc.finish() {
+                            Ok(buf) => {
+                                local_req_builder = local_req_builder.body(buf);
+                            }
+                            Err(e) => return Err(Error::Io(e)),
+                        }
+                    }
+                    _ => {
+                        local_req_builder = local_req_builder.body(ser.into_inner());
+                    },
+                }
+            } else {
+                local_req_builder = local_req_builder.body(ser.into_inner());
+            }
         }
         {%- endif %}
 
+        local_req_builder = local_req_builder.headers(headers);
         let local_req = local_req_builder.build()?;
         let local_resp = local_client.execute(local_req).await?;
 

.generator/src/generator/templates/configuration.j2

@@ -49,6 +49,8 @@ pub struct Configuration {
     pub server_operation_index: HashMap<String, usize>,
     pub server_operation_variables: HashMap<String, HashMap<String, String>>,
     pub proxy_url: Option<String>,
+    pub enable_retry: bool,
+    pub max_retries: u32,
 }   
 
 impl Configuration {
@@ -114,17 +116,16 @@ impl Configuration {
     pub fn set_proxy_url(&mut self, proxy_url: Option<String>) {
         self.proxy_url = proxy_url;
     }
+
+    pub fn set_retry(&mut self, enable_retry: bool, max_retries: u32) {
+        self.enable_retry = enable_retry;
+        self.max_retries = max_retries;
+    }
+
 }
 
 impl Default for Configuration {
     fn default() -> Self {
-        let user_agent = format!(
-            "datadog-api-client-rust/{} (rust {}; os {}; arch {})",
-            option_env!("CARGO_PKG_VERSION").unwrap_or("?"),
-            option_env!("DD_RUSTC_VERSION").unwrap_or("?"),
-            env::consts::OS,
-            env::consts::ARCH,
-        );
         let unstable_operations = HashMap::from([
             {%- for version, api in apis.items() %}
             {%- for operations in api.values() %}
@@ -156,14 +157,16 @@ impl Default for Configuration {
         {%- endif %}
 
         Self {
-            user_agent,
+            user_agent: DEFAULT_USER_AGENT.clone(),
             unstable_operations,
             auth_keys,
             server_index: 0,
             server_variables: HashMap::new(),
             server_operation_index: HashMap::new(),
             server_operation_variables: HashMap::new(),
             proxy_url: None,
+            enable_retry: false,
+            max_retries: 3,
         }
     }
 }
@@ -192,6 +195,13 @@ ServerConfiguration {
 {%- endmacro %}
 
 lazy_static! {
+    pub static ref DEFAULT_USER_AGENT: String = format!(
+        "datadog-api-client-rust/{} (rust {}; os {}; arch {})",
+        option_env!("CARGO_PKG_VERSION").unwrap_or("?"),
+        option_env!("DD_RUSTC_VERSION").unwrap_or("?"),
+        env::consts::OS,
+        env::consts::ARCH,
+    );
     static ref SERVERS: Vec<ServerConfiguration> = {
         vec![
             {%- for server in openapi.servers %}

.generator/src/generator/templates/function_mappings.j2

@@ -64,14 +64,14 @@ fn test_{{version}}_{{ operation['operationId'] | snake_case }}(world: &mut Data
     {%- set schema = parameter[1] | parameter_schema %}
     {%- if parameter[1].required %}
     {%- if schema | is_primitive and schema.get("format") == "binary" -%}
-    let {{ parameter[0] | variable_name }} = _parameters.get("{{ parameter[0] }}").unwrap().as_str().unwrap().as_bytes().to_vec();
+    let {{ parameter[0] | variable_name }} = std::fs::read(format!("tests/scenarios/features/v{}/{}", world.api_version, _parameters.get("{{ parameter[0] }}").unwrap().as_str().unwrap())).unwrap();
     {%- else -%}
     let {{ parameter[0] | variable_name }} = serde_json::from_value(_parameters.get("{{ parameter[0] }}").unwrap().clone()).unwrap();
     {%- endif -%}
     {%- else %}
     let {{ parameter[0] | variable_name }} = _parameters.get("{{ parameter[0] }}").and_then(|param| 
         {%- if schema | is_primitive and schema.get("format") == "binary" -%}
-        Some(param.as_str().unwrap().as_bytes().to_vec())
+        std::fs::read(format!("tests/scenarios/features/v{}/{}", world.api_version, param.as_str().unwrap())).ok()
         {%- else -%}
         Some(serde_json::from_value(param.clone()).unwrap())
         {%- endif -%}
@@ -113,14 +113,14 @@ fn test_{{version}}_{{ operation['operationId'] | snake_case }}_with_pagination(
     {%- set schema = parameter[1] | parameter_schema %}
     {%- if parameter[1].required %}
     {%- if schema | is_primitive and schema.get("format") == "binary" -%}
-    let {{ parameter[0] | variable_name }} = _parameters.get("{{ parameter[0] }}").unwrap().as_str().unwrap().as_bytes().to_vec();
+    let {{ parameter[0] | variable_name }} = std::fs::read(_parameters.get("{{ parameter[0] }}").unwrap().as_str().unwrap()).unwrap();
     {%- else -%}
     let {{ parameter[0] | variable_name }} = serde_json::from_value(_parameters.get("{{ parameter[0] }}").unwrap().clone()).unwrap();
     {%- endif -%}
     {%- else %}
     let {{ parameter[0] | variable_name }} = _parameters.get("{{ parameter[0] }}").and_then(|param| 
         {%- if schema | is_primitive and schema.get("format") == "binary" -%}
-        Some(param.as_str().unwrap().as_bytes().to_vec())
+        std::fs::read(param.as_str().unwrap()).ok()
         {%- else -%}
         Some(serde_json::from_value(param.clone()).unwrap())
         {%- endif -%}

.github/workflows/test.yml

@@ -8,8 +8,8 @@ on:
   pull_request:
     branches:
       - master
-  # schedule:
-  #   - cron: "0 1 * * *"
+  schedule:
+    - cron: "0 6 * * *"
 
 concurrency:
   group: unit-${{ github.head_ref }}
@@ -25,11 +25,14 @@ jobs:
       github.event_name == 'schedule'
     steps:
       - name: Get GitHub App token
+        if: github.event_name == 'pull_request'
         id: get_token
-        uses: tibdex/github-app-token@v1
+        uses: tibdex/github-app-token@v2.1.0
         with:
           app_id: ${{ secrets.PIPELINE_GITHUB_APP_ID }}
           private_key: ${{ secrets.PIPELINE_GITHUB_APP_PRIVATE_KEY }}
+          installation_retrieval_mode: repository
+          installation_retrieval_payload: DataDog/datadog-api-spec
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
@@ -84,7 +87,7 @@ jobs:
           toolchain: ${{ matrix.rust-version }}
       - uses: Swatinem/rust-cache@v2
       - name: Test
-        run: cargo build
+        run: ./run-tests.sh
 
   examples:
     runs-on: ubuntu-latest