diff --git a/.gitignore b/.gitignore index 6c70190..a26489d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ cloud-config-vultr.ini output resources vultr-cloud-*.tar.gz +.vscode/ \ No newline at end of file diff --git a/plugins/doc_fragments/vultr_v2.py b/plugins/doc_fragments/vultr_v2.py index 76eaa8c..b55897a 100644 --- a/plugins/doc_fragments/vultr_v2.py +++ b/plugins/doc_fragments/vultr_v2.py @@ -38,6 +38,15 @@ class ModuleDocFragment(object): - Fallback environment variable C(VULTR_API_RETRY_MAX_DELAY). type: int default: 12 + api_results_per_page: + description: + - When receiving large numbers of resources, specify how many results should be returned per call to API. + - This does not determine how many results are returned; all resources are returned according to other filters. + - Vultr API maximum is 500. + - Fallback environment variable C(VULTR_API_RESULTS_PER_PAGE). + type: int + default: 100 + version_added: 1.14.0 api_endpoint: description: - URL to API endpint (without trailing slash). diff --git a/plugins/module_utils/vultr_v2.py b/plugins/module_utils/vultr_v2.py index db8eded..cc284dc 100644 --- a/plugins/module_utils/vultr_v2.py +++ b/plugins/module_utils/vultr_v2.py @@ -11,6 +11,7 @@ from ansible.module_utils._text import to_text from ansible.module_utils.basic import env_fallback +from ansible.module_utils.common.dict_transformations import dict_merge from ansible.module_utils.six.moves.urllib.parse import quote from ansible.module_utils.urls import fetch_url @@ -45,6 +46,11 @@ def vultr_argument_spec(): fallback=(env_fallback, ["VULTR_API_RETRY_MAX_DELAY"]), default=12, ), + api_results_per_page=dict( + type="int", + fallback=(env_fallback, ["VULTR_API_RESULTS_PER_PAGE"]), + default=100, + ), validate_certs=dict( type="bool", default=True, @@ -113,6 +119,7 @@ def __init__( "api_timeout": module.params["api_timeout"], "api_retries": module.params["api_retries"], "api_retry_max_delay": module.params["api_retry_max_delay"], + "api_results_per_page": module.params["api_results_per_page"], "api_endpoint": module.params["api_endpoint"], }, } @@ -135,6 +142,35 @@ def transform_resource(self, resource): """ return resource + def paginate_api_query(self, path, method="GET", data=None, query_params=None, result_key=None): + result_key = result_key or self.ressource_result_key_plural + pager_param = dict(per_page=self.module.params["api_results_per_page"]) + if query_params is not None: + query_params = dict_merge(query_params, pager_param) + else: + query_params = pager_param + + result = dict() + cursor = dict() + while True: + resp = self.api_query( + path=path, + method=method, + data=data, + query_params=dict_merge(query_params, cursor), + ) + + resp_body = resp.get(result_key, {}) + if isinstance(resp_body, list): + if isinstance(result.get(result_key), list): + result[result_key].extend(resp_body) + else: + result[result_key] = resp_body + cursor["cursor"] = resp.get("meta", {}).get("links", {}).get("next", "") + + if cursor["cursor"] == "": + return result + def api_query(self, path, method="GET", data=None, query_params=None): if query_params: query = "?" @@ -151,12 +187,12 @@ def api_query(self, path, method="GET", data=None, query_params=None): resp_body = None for retry in range(0, self.module.params["api_retries"]): resp, info = fetch_url( - self.module, - self.module.params["api_endpoint"] + path, + module=self.module, + url=self.module.params["api_endpoint"] + path, method=method, data=data, headers=self.headers, - timeout=self.module.params["api_timeout"], + timeout=int(self.module.params["api_timeout"]), ) resp_body = resp.read() if resp is not None else "" @@ -254,7 +290,10 @@ def query_by_id(self, resource_id=None, path=None, result_key=None, skip_transfo path = path or self.resource_path result_key = result_key or self.ressource_result_key_singular - resource = self.api_query(path="%s%s" % (path, "/" + resource_id if resource_id else resource_id)) + resource = self.api_query( + path="%s%s" % (path, "/" + resource_id if resource_id else resource_id) + ) + if resource: if skip_transform: return resource[result_key] @@ -272,7 +311,11 @@ def query_list(self, path=None, result_key=None, query_params=None): path = path or self.resource_path result_key = result_key or self.ressource_result_key_plural - resources = self.api_query(path=path, query_params=query_params) + resources = self.paginate_api_query( + path=path, + query_params=query_params, + result_key=result_key, + ) return resources[result_key] if resources else [] def wait_for_state(self, resource, key, states, cmp="=", retries=60, skip_wait=False): diff --git a/plugins/modules/account_info.py b/plugins/modules/account_info.py index ecf55e2..1115f9d 100644 --- a/plugins/modules/account_info.py +++ b/plugins/modules/account_info.py @@ -58,6 +58,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/bare_metal.py b/plugins/modules/bare_metal.py index f700fb9..f52f937 100644 --- a/plugins/modules/bare_metal.py +++ b/plugins/modules/bare_metal.py @@ -177,6 +177,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/block_storage.py b/plugins/modules/block_storage.py index abcb49d..10c6c7a 100644 --- a/plugins/modules/block_storage.py +++ b/plugins/modules/block_storage.py @@ -120,6 +120,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/block_storage_info.py b/plugins/modules/block_storage_info.py index 1d43a02..52dec78 100644 --- a/plugins/modules/block_storage_info.py +++ b/plugins/modules/block_storage_info.py @@ -55,6 +55,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/dns_domain.py b/plugins/modules/dns_domain.py index 3f0132b..271e339 100644 --- a/plugins/modules/dns_domain.py +++ b/plugins/modules/dns_domain.py @@ -82,6 +82,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/dns_domain_info.py b/plugins/modules/dns_domain_info.py index aa198b1..55bc8d0 100644 --- a/plugins/modules/dns_domain_info.py +++ b/plugins/modules/dns_domain_info.py @@ -55,6 +55,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/dns_record.py b/plugins/modules/dns_record.py index 2e4352e..2d50311 100644 --- a/plugins/modules/dns_record.py +++ b/plugins/modules/dns_record.py @@ -148,6 +148,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/firewall_group.py b/plugins/modules/firewall_group.py index 962936a..7f2d8b2 100644 --- a/plugins/modules/firewall_group.py +++ b/plugins/modules/firewall_group.py @@ -67,6 +67,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/firewall_group_info.py b/plugins/modules/firewall_group_info.py index 866ac48..2090002 100644 --- a/plugins/modules/firewall_group_info.py +++ b/plugins/modules/firewall_group_info.py @@ -55,6 +55,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/firewall_rule.py b/plugins/modules/firewall_rule.py index 67dd85f..335511c 100644 --- a/plugins/modules/firewall_rule.py +++ b/plugins/modules/firewall_rule.py @@ -122,6 +122,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/firewall_rule_info.py b/plugins/modules/firewall_rule_info.py index 6d51d6f..153413a 100644 --- a/plugins/modules/firewall_rule_info.py +++ b/plugins/modules/firewall_rule_info.py @@ -59,6 +59,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/instance.py b/plugins/modules/instance.py index c116512..8ac5fea 100644 --- a/plugins/modules/instance.py +++ b/plugins/modules/instance.py @@ -221,6 +221,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/instance_info.py b/plugins/modules/instance_info.py index d473eae..30d59a2 100644 --- a/plugins/modules/instance_info.py +++ b/plugins/modules/instance_info.py @@ -72,6 +72,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/object_storage.py b/plugins/modules/object_storage.py index 937e6f7..1776f24 100644 --- a/plugins/modules/object_storage.py +++ b/plugins/modules/object_storage.py @@ -81,6 +81,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/object_storage_cluster_info.py b/plugins/modules/object_storage_cluster_info.py index ccfdf00..6d9b3ca 100644 --- a/plugins/modules/object_storage_cluster_info.py +++ b/plugins/modules/object_storage_cluster_info.py @@ -56,6 +56,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/object_storage_info.py b/plugins/modules/object_storage_info.py index 52313a5..21c22b4 100644 --- a/plugins/modules/object_storage_info.py +++ b/plugins/modules/object_storage_info.py @@ -56,6 +56,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/os_info.py b/plugins/modules/os_info.py index 8edb7a7..d2d3212 100644 --- a/plugins/modules/os_info.py +++ b/plugins/modules/os_info.py @@ -56,6 +56,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/plan_info.py b/plugins/modules/plan_info.py index 639a107..3fa76c4 100644 --- a/plugins/modules/plan_info.py +++ b/plugins/modules/plan_info.py @@ -56,6 +56,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/plan_metal_info.py b/plugins/modules/plan_metal_info.py index c7ff478..5b8e6b9 100644 --- a/plugins/modules/plan_metal_info.py +++ b/plugins/modules/plan_metal_info.py @@ -58,6 +58,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/region_info.py b/plugins/modules/region_info.py index 1e222b9..7d6fa60 100644 --- a/plugins/modules/region_info.py +++ b/plugins/modules/region_info.py @@ -56,6 +56,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/reserved_ip.py b/plugins/modules/reserved_ip.py index 4bf1c66..175dde3 100644 --- a/plugins/modules/reserved_ip.py +++ b/plugins/modules/reserved_ip.py @@ -101,6 +101,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/snapshot.py b/plugins/modules/snapshot.py index 9dba03c..463cd35 100644 --- a/plugins/modules/snapshot.py +++ b/plugins/modules/snapshot.py @@ -85,6 +85,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/snapshot_info.py b/plugins/modules/snapshot_info.py index 38fa4b8..aa75247 100644 --- a/plugins/modules/snapshot_info.py +++ b/plugins/modules/snapshot_info.py @@ -53,6 +53,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/ssh_key.py b/plugins/modules/ssh_key.py index 2c85eeb..8f769a7 100644 --- a/plugins/modules/ssh_key.py +++ b/plugins/modules/ssh_key.py @@ -73,6 +73,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/ssh_key_info.py b/plugins/modules/ssh_key_info.py index 2444188..d0a88a3 100644 --- a/plugins/modules/ssh_key_info.py +++ b/plugins/modules/ssh_key_info.py @@ -58,6 +58,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/startup_script.py b/plugins/modules/startup_script.py index 4211b1a..0ce3bfb 100644 --- a/plugins/modules/startup_script.py +++ b/plugins/modules/startup_script.py @@ -85,6 +85,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/startup_script_info.py b/plugins/modules/startup_script_info.py index dd91166..5d183f4 100644 --- a/plugins/modules/startup_script_info.py +++ b/plugins/modules/startup_script_info.py @@ -56,6 +56,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/user.py b/plugins/modules/user.py index dd51243..30e210b 100644 --- a/plugins/modules/user.py +++ b/plugins/modules/user.py @@ -111,6 +111,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/user_info.py b/plugins/modules/user_info.py index 5d0a61e..6d20838 100644 --- a/plugins/modules/user_info.py +++ b/plugins/modules/user_info.py @@ -56,6 +56,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/vpc.py b/plugins/modules/vpc.py index 83a183c..da0acf7 100644 --- a/plugins/modules/vpc.py +++ b/plugins/modules/vpc.py @@ -85,6 +85,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/vpc2.py b/plugins/modules/vpc2.py index c2caa17..9adb9c0 100644 --- a/plugins/modules/vpc2.py +++ b/plugins/modules/vpc2.py @@ -92,6 +92,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/vpc2_info.py b/plugins/modules/vpc2_info.py index c71c25d..eac6c17 100644 --- a/plugins/modules/vpc2_info.py +++ b/plugins/modules/vpc2_info.py @@ -53,6 +53,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success diff --git a/plugins/modules/vpc_info.py b/plugins/modules/vpc_info.py index 5dc67a5..dc0921b 100644 --- a/plugins/modules/vpc_info.py +++ b/plugins/modules/vpc_info.py @@ -53,6 +53,11 @@ returned: success type: int sample: 12 + api_results_per_page: + description: Number of results returned per call to API. + returned: success + type: int + sample: 100 api_endpoint: description: Endpoint used for the API requests. returned: success