diff --git a/resources/collectd-cloudwatch.conf b/resources/collectd-cloudwatch.conf index f08f15b..8c0e45e 100644 --- a/resources/collectd-cloudwatch.conf +++ b/resources/collectd-cloudwatch.conf @@ -2,6 +2,8 @@ LoadPlugin python ModulePath "/opt/collectd-plugins/" + ModulePath "/usr/lib/python3.6/" + ModulePath "/usr/local/lib/python3.6/dist-packages/" LogTraces true Interactive false Import "cloudwatch_writer" diff --git a/src/cloudwatch/modules/awsutils.py b/src/cloudwatch/modules/awsutils.py index 9994c1a..37aa195 100644 --- a/src/cloudwatch/modules/awsutils.py +++ b/src/cloudwatch/modules/awsutils.py @@ -10,4 +10,4 @@ def get_aws_timestamp(): def get_datestamp(): - return datetime.utcnow().strftime('%Y%m%d') \ No newline at end of file + return datetime.utcnow().strftime('%Y%m%d') diff --git a/src/cloudwatch/modules/client/baserequestbuilder.py b/src/cloudwatch/modules/client/baserequestbuilder.py index 8bac432..f6c5c5f 100644 --- a/src/cloudwatch/modules/client/baserequestbuilder.py +++ b/src/cloudwatch/modules/client/baserequestbuilder.py @@ -1,6 +1,7 @@ -from ..awsutils import get_aws_timestamp, get_datestamp -from signer import Signer -from querystringbuilder import QuerystringBuilder +from cloudwatch.modules.awsutils import get_aws_timestamp, get_datestamp +from cloudwatch.modules.client.signer import Signer +from cloudwatch.modules.client.querystringbuilder import QuerystringBuilder + class BaseRequestBuilder(object): """ @@ -61,4 +62,4 @@ def _get_request_map(self): } if self.credentials.token: canonical_map["X-Amz-Security-Token"] = self.credentials.token - return canonical_map \ No newline at end of file + return canonical_map diff --git a/src/cloudwatch/modules/client/ec2getclient.py b/src/cloudwatch/modules/client/ec2getclient.py index f1530ef..4fb06d2 100644 --- a/src/cloudwatch/modules/client/ec2getclient.py +++ b/src/cloudwatch/modules/client/ec2getclient.py @@ -1,14 +1,13 @@ import re -from ..plugininfo import PLUGIN_NAME, PLUGIN_VERSION -from ec2requestbuilder import EC2RequestBuilder -from ..logger.logger import get_logger +from cloudwatch.modules.plugininfo import PLUGIN_NAME, PLUGIN_VERSION +from cloudwatch.modules.client.ec2requestbuilder import EC2RequestBuilder +from cloudwatch.modules.logger.logger import get_logger from requests.adapters import HTTPAdapter from requests.sessions import Session import xml.etree.ElementTree as ET - class EC2GetClient(object): """ This is a simple HTTPClient wrapper which supports DescribeTags operation on EC2 endpoints. @@ -30,7 +29,7 @@ def __init__(self, config_helper, connection_timeout=_DEFAULT_CONNECTION_TIMEOUT self.timeout = (connection_timeout, response_timeout) def _validate_and_set_endpoint(self, endpoint): - pattern = re.compile("http[s]?://*/") + pattern = re.compile(r"http[s]?://*/") if pattern.match(endpoint) or "localhost" in endpoint: self.endpoint = endpoint else: diff --git a/src/cloudwatch/modules/client/ec2requestbuilder.py b/src/cloudwatch/modules/client/ec2requestbuilder.py index 073cd1f..3ee99e5 100644 --- a/src/cloudwatch/modules/client/ec2requestbuilder.py +++ b/src/cloudwatch/modules/client/ec2requestbuilder.py @@ -1,4 +1,4 @@ -from baserequestbuilder import BaseRequestBuilder +from cloudwatch.modules.client.baserequestbuilder import BaseRequestBuilder class EC2RequestBuilder(BaseRequestBuilder): diff --git a/src/cloudwatch/modules/client/putclient.py b/src/cloudwatch/modules/client/putclient.py index 4fea5c1..a10168b 100644 --- a/src/cloudwatch/modules/client/putclient.py +++ b/src/cloudwatch/modules/client/putclient.py @@ -1,9 +1,9 @@ import re import os -from ..plugininfo import PLUGIN_NAME, PLUGIN_VERSION -from requestbuilder import RequestBuilder -from ..logger.logger import get_logger +from cloudwatch.modules.plugininfo import PLUGIN_NAME, PLUGIN_VERSION +from cloudwatch.modules.client.requestbuilder import RequestBuilder +from cloudwatch.modules.logger.logger import get_logger from requests.adapters import HTTPAdapter from requests.sessions import Session from tempfile import gettempdir @@ -54,7 +54,7 @@ def _prepare_session(self): self.session.mount("https://", HTTPAdapter(max_retries=self._TOTAL_RETRIES)) def _validate_and_set_endpoint(self, endpoint): - pattern = re.compile("http[s]?://*/") + pattern = re.compile(r"http[s]?://*/") if pattern.match(endpoint) or "localhost" in endpoint: self.endpoint = endpoint else: diff --git a/src/cloudwatch/modules/client/querystringbuilder.py b/src/cloudwatch/modules/client/querystringbuilder.py index 13d9472..38cd7ab 100644 --- a/src/cloudwatch/modules/client/querystringbuilder.py +++ b/src/cloudwatch/modules/client/querystringbuilder.py @@ -1,7 +1,7 @@ import operator -from ..logger.logger import get_logger -from urllib import urlencode +from cloudwatch.modules.logger.logger import get_logger +from urllib.parse import urlencode class QuerystringBuilder(object): @@ -44,7 +44,6 @@ def build_querystring_from_map(self, call_map, base_map): url_string = urlencode(sorted_query_data).replace('+', '%20') return url_string - def _build_metric_map(self, metric_list): """ Translate the list of metric objects into a single map with keys represented in the format required diff --git a/src/cloudwatch/modules/client/requestbuilder.py b/src/cloudwatch/modules/client/requestbuilder.py index 85e3896..aacc856 100644 --- a/src/cloudwatch/modules/client/requestbuilder.py +++ b/src/cloudwatch/modules/client/requestbuilder.py @@ -1,4 +1,5 @@ -from baserequestbuilder import BaseRequestBuilder +from cloudwatch.modules.client.baserequestbuilder import BaseRequestBuilder + class RequestBuilder(BaseRequestBuilder): """ @@ -12,7 +13,7 @@ class RequestBuilder(BaseRequestBuilder): _SERVICE = "monitoring" _ACTION = "PutMetricData" _API_VERSION = "2010-08-01" - + def __init__(self, credentials, region, enable_high_resolution_metrics): super(self.__class__, self).__init__(credentials, region, self._SERVICE, self._ACTION, self._API_VERSION, enable_high_resolution_metrics) self.namespace = "" @@ -23,28 +24,29 @@ def create_signed_request(self, namespace, metric_list): self._init_timestamps() canonical_querystring = self._create_canonical_querystring(metric_list) signature = self.signer.create_request_signature(canonical_querystring, self._get_credential_scope(), - self.aws_timestamp, self.datestamp, self._get_canonical_headers(), - self._get_signed_headers(), self.payload) + self.aws_timestamp, self.datestamp, + self._get_canonical_headers(), + self._get_signed_headers(), self.payload) canonical_querystring += '&X-Amz-Signature=' + signature return canonical_querystring - + def _create_canonical_querystring(self, metric_list): """ Creates a canonical querystring as defined in the official AWS API documentation: http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html """ return self.querystring_builder.build_querystring(metric_list, self._get_namespace_request_map()) - + def _get_namespace_request_map(self): """ Creates a map of request parameters and values which can be used to build a canonical querystring """ canonical_map = self._get_request_map() - if (self.namespace): + if self.namespace: canonical_map["Namespace"] = self.namespace return canonical_map - + def _get_host(self): """ Returns the endpoint's hostname derived from the region """ if self.region == "localhost": diff --git a/src/cloudwatch/modules/client/signer.py b/src/cloudwatch/modules/client/signer.py index 4eab220..f18cd03 100644 --- a/src/cloudwatch/modules/client/signer.py +++ b/src/cloudwatch/modules/client/signer.py @@ -44,13 +44,24 @@ def _build_string_to_sign(self, aws_timestamp, credential_scope, canonical_reque return self.algorithm + "\n" + aws_timestamp + '\n' + credential_scope + '\n' + self._hash(canonical_request) def _hash(self, data): + # change type to `bytes` + if type(data) is str: + data = data.encode("utf-8") return sha256(data).hexdigest() def _sign(self, key, msg): - return hmac.new(key, msg.encode("utf-8"), sha256).digest() + # change type to `bytes` + if type(key) is str: + key = key.encode("utf-8") + if type(msg) is str: + msg = msg.encode("utf-8") + return hmac.new(key, msg, sha256).digest() def _build_signature_key(self, key, date_stamp, region_name, service_name): - kDate = self._sign(('AWS4' + key).encode('utf-8'), date_stamp) + # change type to `str` + if type(key) is bytes: + key = key.decode("utf-8") + kDate = self._sign('AWS4' + key, date_stamp) kRegion = self._sign(kDate, region_name) kService = self._sign(kRegion, service_name) kSigning = self._sign(kService, self._V4_TERMINATOR) diff --git a/src/cloudwatch/modules/collectd.py b/src/cloudwatch/modules/collectd.py index 72b3f3c..607d4bc 100644 --- a/src/cloudwatch/modules/collectd.py +++ b/src/cloudwatch/modules/collectd.py @@ -2,23 +2,30 @@ The collectd stub is used to allow testing every part of this module without the need of installing and running collectd. """ + def register_config(*args): pass + def register_init(*args): pass + def register_write(*args, **kwargs): pass + def debug(msg): pass + def info(msg): pass + def warning(msg): pass + def error(msg): - pass \ No newline at end of file + pass diff --git a/src/cloudwatch/modules/configuration/confighelper.py b/src/cloudwatch/modules/configuration/confighelper.py index f013085..11bbfad 100644 --- a/src/cloudwatch/modules/configuration/confighelper.py +++ b/src/cloudwatch/modules/configuration/confighelper.py @@ -1,12 +1,13 @@ import os -from ..logger.logger import get_logger -from configreader import ConfigReader -from metadatareader import MetadataReader -from credentialsreader import CredentialsReader -from whitelist import Whitelist, WhitelistConfigReader -from ..client.ec2getclient import EC2GetClient +from cloudwatch.modules.logger.logger import get_logger +from cloudwatch.modules.configuration.configreader import ConfigReader +from cloudwatch.modules.configuration.metadatareader import MetadataReader +from cloudwatch.modules.configuration.credentialsreader import CredentialsReader +from cloudwatch.modules.configuration.whitelist import Whitelist, WhitelistConfigReader +from cloudwatch.modules.client.ec2getclient import EC2GetClient import traceback + class ConfigHelper(object): """ The configuration helper is responsible for obtaining configuration data from number @@ -135,12 +136,11 @@ def _load_hostname(self): try: self.host = self.metadata_reader.get_instance_id() except Exception as e: - ConfigHelper._LOGGER.warning("Cannot retrieve Instance ID from the local metadata server. Cause: " + str(e) + - " Using host information provided by Collectd.") + ConfigHelper._LOGGER.warning("Cannot retrieve Instance ID from the local metadata server. Cause: " + str(e) + " Using host information provided by Collectd.") def _set_ec2_endpoint(self): """ Creates endpoint from region information """ - if self.region is "localhost": + if self.region == "localhost": self.ec2_endpoint = "http://" + self.region + "/" elif self.region.startswith("cn-"): self.ec2_endpoint = "https://ec2." + self.region + ".amazonaws.com.cn/" diff --git a/src/cloudwatch/modules/configuration/configreader.py b/src/cloudwatch/modules/configuration/configreader.py index 1da5a65..2e127fe 100644 --- a/src/cloudwatch/modules/configuration/configreader.py +++ b/src/cloudwatch/modules/configuration/configreader.py @@ -1,5 +1,5 @@ -from ..logger.logger import get_logger -from readerutils import ReaderUtils +from cloudwatch.modules.logger.logger import get_logger +from cloudwatch.modules.configuration.readerutils import ReaderUtils class ConfigReader(object): @@ -50,7 +50,7 @@ def __init__(self, config_path): self.push_asg = self._PUSH_ASG_DEFAULT_VALUE self.push_constant = self._PUSH_CONSTANT_DEFAULT_VALUE self.constant_dimension_value = '' - self.proxy_server_name='' + self.proxy_server_name = '' self.proxy_server_port = '' self.enable_high_resolution_metrics = self._ENABLE_HIGH_DEFINITION_METRICS_DEFAULT_VALUE self.flush_interval_in_seconds = '' diff --git a/src/cloudwatch/modules/configuration/credentialsreader.py b/src/cloudwatch/modules/configuration/credentialsreader.py index 2ec8b55..7834c55 100644 --- a/src/cloudwatch/modules/configuration/credentialsreader.py +++ b/src/cloudwatch/modules/configuration/credentialsreader.py @@ -1,6 +1,6 @@ -from ..awscredentials import AWSCredentials -from readerutils import ReaderUtils -from ..logger.logger import get_logger +from cloudwatch.modules.awscredentials import AWSCredentials +from cloudwatch.modules.configuration.readerutils import ReaderUtils +from cloudwatch.modules.logger.logger import get_logger class CredentialsReader(object): @@ -23,7 +23,7 @@ class CredentialsReader(object): _ACCESS_CONFIG_KEY = "aws_access_key" _SECRET_CONFIG_KEY = "aws_secret_key" - #This is the format the SDK uses. + # This is the format the SDK uses. _ACCESS_CONFIG_KEY_AWS_FORMAT = "aws_access_key_id" _SECRET_CONFIG_KEY_AWS_FORMAT = "aws_secret_access_key" @@ -57,4 +57,3 @@ def _parse_credentials_file(self): class CredentialsReaderException(Exception): pass - diff --git a/src/cloudwatch/modules/configuration/metadatareader.py b/src/cloudwatch/modules/configuration/metadatareader.py index ba2206a..fc86893 100644 --- a/src/cloudwatch/modules/configuration/metadatareader.py +++ b/src/cloudwatch/modules/configuration/metadatareader.py @@ -1,8 +1,8 @@ from json import loads from requests import Session, codes from requests.adapters import HTTPAdapter -from ..logger.logger import get_logger -from ..awscredentials import AWSCredentials +from cloudwatch.modules.logger.logger import get_logger +from cloudwatch.modules.awscredentials import AWSCredentials class MetadataReader(object): @@ -63,7 +63,7 @@ def _get_metadata(self, request): if result.status_code is codes.ok: return str(result.text) else: - self._LOGGER.error("The request: '" + str(request) + "' failed with status code: '" + str(result.status_code) + "' and message: '" + str(result.text) +"'.") + self._LOGGER.error("The request: '" + str(request) + "' failed with status code: '" + str(result.status_code) + "' and message: '" + str(result.text) + "'.") raise MetadataRequestException("Cannot retrieve configuration from metadata service. Status code: " + str(result.status_code)) diff --git a/src/cloudwatch/modules/configuration/readerutils.py b/src/cloudwatch/modules/configuration/readerutils.py index ff7207f..86872d3 100644 --- a/src/cloudwatch/modules/configuration/readerutils.py +++ b/src/cloudwatch/modules/configuration/readerutils.py @@ -1,14 +1,14 @@ import re import os -from ..logger.logger import get_logger +from cloudwatch.modules.logger.logger import get_logger class ReaderUtils(object): _LOGGER = get_logger(__name__) _COMMENT_CHARACTER = '#' - _AWS_PROFILE_PATTERN = re.compile("^\s*\[[\w]+\]\s*$") + _AWS_PROFILE_PATTERN = re.compile(r"^\s*\[[\w]+\]\s*$") def __init__(self, path): self.path = path diff --git a/src/cloudwatch/modules/configuration/whitelist.py b/src/cloudwatch/modules/configuration/whitelist.py index 02fc815..61c7b06 100644 --- a/src/cloudwatch/modules/configuration/whitelist.py +++ b/src/cloudwatch/modules/configuration/whitelist.py @@ -1,10 +1,10 @@ import re from os import path -from string import strip from threading import Lock +import sys -from configreader import ConfigReader -from ..logger.logger import get_logger +from cloudwatch.modules.configuration.configreader import ConfigReader +from cloudwatch.modules.logger.logger import get_logger class WhitelistConfigReader(object): @@ -19,7 +19,7 @@ class WhitelistConfigReader(object): END_STRING = "$" EMPTY_REGEX = START_STRING + END_STRING - PASS_THROUGH_REGEX_STRING = "^\.[\*\+]?\s.*$|^.*?\s\.[\*\+]|^\.[\*\+]$" # matches single .*, .+ strings + PASS_THROUGH_REGEX_STRING = r"^\.[\*\+]?\s.*$|^.*?\s\.[\*\+]|^\.[\*\+]$" # matches single .*, .+ strings # as well as strings with .* or .+ preceded or followed by whitespace. def __init__(self, whitelist_config_path, pass_through_allowed): @@ -43,7 +43,7 @@ def get_regex_list(self): def _get_whitelisted_names_from_file(self, whitelist_path): with open(whitelist_path) as whitelist_file: - return self._filter_valid_regexes(map(strip, whitelist_file)) + return self._filter_valid_regexes(map(str.strip, whitelist_file)) def _create_whitelist_file(self, whitelist_path): if not path.exists(whitelist_path): @@ -63,7 +63,8 @@ def _is_valid_regex(self, regex_string): return True return False except Exception as e: - self._LOGGER.warning("The whitelist rule: '{}' is invalid, reason: {}".format(str(regex_string), str(e.message))) + _traceback = sys.exc_info()[2] + self._LOGGER.warning("The whitelist rule: '{}' is invalid, reason: {}".format(str(regex_string), str(e.with_traceback(_traceback).msg))) return False def _is_allowed_regex(self, regex_string): @@ -138,6 +139,3 @@ def is_whitelisted(self, metric_key): self._allowed_metrics[metric_key] = False self.blocked_metric_log.log_metric(metric_key) return self._allowed_metrics[metric_key] - - - diff --git a/src/cloudwatch/modules/flusher.py b/src/cloudwatch/modules/flusher.py index 95ad4b9..8dd23d3 100644 --- a/src/cloudwatch/modules/flusher.py +++ b/src/cloudwatch/modules/flusher.py @@ -3,9 +3,9 @@ import os import math -from client.putclient import PutClient -from logger.logger import get_logger -from metricdata import MetricDataStatistic, MetricDataBuilder +from cloudwatch.modules.client.putclient import PutClient +from cloudwatch.modules.logger.logger import get_logger +from cloudwatch.modules.metricdata import MetricDataStatistic, MetricDataBuilder class Flusher(object): """ @@ -189,13 +189,13 @@ def _flush(self): prepare_batch = self._prepare_batch() try: while True: - metric_batch = prepare_batch.next() + metric_batch = prepare_batch.__next__() if not metric_batch: break self.client.put_metric_data(MetricDataStatistic.NAMESPACE, metric_batch) if len(metric_batch) < self._MAX_METRICS_PER_PUT_REQUEST: break - except StopIteration, e: + except StopIteration as e: if metric_map_size % self._MAX_METRICS_PER_PUT_REQUEST != 0 or len(self.metric_map) != 0: self._LOGGER.error("_flush error: " + str(e) + " Original map size: " + str(metric_map_size)) diff --git a/src/cloudwatch/modules/logger/logger.py b/src/cloudwatch/modules/logger/logger.py index a8ea12e..e489aca 100644 --- a/src/cloudwatch/modules/logger/logger.py +++ b/src/cloudwatch/modules/logger/logger.py @@ -2,7 +2,7 @@ try: import collectd except: - from .. import collectd + from cloudwatch.modules import collectd def get_logger(channel=None): @@ -68,4 +68,4 @@ def warning(self, msg): collectd.warning(self.prefix + msg) def error(self, msg): - collectd.error(self.prefix + msg) \ No newline at end of file + collectd.error(self.prefix + msg) diff --git a/src/cloudwatch/modules/metricdata.py b/src/cloudwatch/modules/metricdata.py index 1669e66..cabda46 100644 --- a/src/cloudwatch/modules/metricdata.py +++ b/src/cloudwatch/modules/metricdata.py @@ -1,7 +1,8 @@ -import awsutils as awsutils -import plugininfo +import cloudwatch.modules.awsutils as awsutils +import cloudwatch.modules.plugininfo as plugininfo import datetime + class MetricDataStatistic(object): """ The MetricDataStatistic object encapsulates the information sent with putMetricData. @@ -110,22 +111,22 @@ def _build_metric_name(self): def _build_asg_dimension(self): dimensions = { - "AutoScalingGroup" : self._get_autoscaling_group(), - "PluginInstance" : self._get_plugin_instance_dimension() + "AutoScalingGroup": self._get_autoscaling_group(), + "PluginInstance": self._get_plugin_instance_dimension() } return dimensions def _build_constant_dimension(self): dimensions = { - "FixedDimension" : self.config.constant_dimension_value, - "PluginInstance" : self._get_plugin_instance_dimension() + "FixedDimension": self.config.constant_dimension_value, + "PluginInstance": self._get_plugin_instance_dimension() } return dimensions def _build_metric_dimensions(self): dimensions = { - "Host" : self._get_host_dimension(), - "PluginInstance" : self._get_plugin_instance_dimension() + "Host": self._get_host_dimension(), + "PluginInstance": self._get_plugin_instance_dimension() } if self.config.push_asg: dimensions["AutoScalingGroup"] = self._get_autoscaling_group() diff --git a/src/cloudwatch_writer.py b/src/cloudwatch_writer.py index 2067aaa..95adfc2 100644 --- a/src/cloudwatch_writer.py +++ b/src/cloudwatch_writer.py @@ -34,5 +34,6 @@ def aws_write(vl, flusher): Collectd callback entry used to write metric data """ flusher.add_metric(vl) - + + collectd.register_init(aws_init) diff --git a/src/setup.py b/src/setup.py index e8a8282..ac15a01 100755 --- a/src/setup.py +++ b/src/setup.py @@ -44,18 +44,18 @@ PLUGIN_CONFIGURATION_INCLUDE_LINE = 'Include "/etc/collectd-cloudwatch.conf"\r\n' APT_INSTALL_COMMAND = "apt-get install -y " YUM_INSTALL_COMMAND = "yum install -y " -SYSTEM_DEPENDENCIES = ["python-pip", "python-setuptools"] +SYSTEM_DEPENDENCIES = ["python3-pip", "python3-setuptools", "python3-requests", "curl"] PIP_INSTALLATION_FLAGS = " install --quiet --upgrade --force-reinstall " EASY_INSTALL_COMMAND = "easy_install -U --quiet " PYTHON_DEPENDENCIES = ["requests"] FIND_COMMAND = "which {} 2> /dev/null" COLLECTD_HELP_ARGS = "-help" -CONFIG_FILE_REGEX = re.compile("\sConfig file\s*(.*)\s") -VERSION_REGEX = re.compile("\scollectd ([\d*].*[\d*]).*\s") -DISTRO_NAME_REGEX = re.compile("(?$", re.MULTILINE | re.IGNORECASE) +CONFIG_FILE_REGEX = re.compile(r"\sConfig file\s*(.*)\s") +VERSION_REGEX = re.compile(r"\scollectd ([\d*].*[\d*]).*\s") +DISTRO_NAME_REGEX = re.compile(r"(?$", re.MULTILINE | re.IGNORECASE) logging.basicConfig() logger = logging.getLogger() @@ -122,6 +122,9 @@ def cyan(cls, string): def get_collectd_info(): exec_path = _get_collectd_exec() output = check_output([exec_path, COLLECTD_HELP_ARGS]) + # change type of `output` to str + if type(output) is bytes: + output = output.decode("utf-8") version = VERSION_REGEX.search(output).group(1) config_path = CONFIG_FILE_REGEX.search(output).group(1) return CollectdInfo(exec_path, config_path, version) @@ -137,7 +140,10 @@ def _get_collectd_exec(): def get_path_to_executable(command): - return check_output(FIND_COMMAND.format(command), shell=True).strip() + _path = check_output(FIND_COMMAND.format(command), shell=True).strip() + if type(_path) is bytes: + _path = _path.decode("utf-8") + return _path class Command(object): @@ -162,7 +168,7 @@ def was_successful(self): return self._process and self._process.returncode is self.SUCCESS def run(self): - print self.message, # comma stops print from adding line break at the end + print(self.message,) # comma stops print from adding line break at the end try: self._process = self._get_process() self._capture_outputs() @@ -181,14 +187,14 @@ def _get_process(self): def _capture_outputs(self): stdout, stderr = self._process.communicate() - self.stdout = str(stdout).strip() - self.stderr = str(stderr).strip() + self.stdout = stdout.decode("utf-8").strip() + self.stderr = stderr.decode("utf-8").strip() def _output_command_status(self): result = self.NOT_OK if self.was_successful: result = self.OK - print result + print(result) class MetadataReader(object): @@ -247,14 +253,14 @@ class MetadataRequestException(Exception): def install_python_packages(packages): try: - Command(detect_pip() + PIP_INSTALLATION_FLAGS + " ".join(packages), "Installing python dependencies", exit_on_failure=True).run() + Command("{}{}{}".format(detect_pip(), PIP_INSTALLATION_FLAGS, " ".join(packages)), "Installing python dependencies", exit_on_failure=True).run() except CalledProcessError: Command(EASY_INSTALL_COMMAND + " ".join(packages), "Installing python dependencies", exit_on_failure=True).run() def detect_pip(): try: - return get_path_to_executable("pip") + return get_path_to_executable("pip3") except CalledProcessError: return get_path_to_executable("python-pip") @@ -516,9 +522,9 @@ def _get_flush_interval_in_seconds(self): def _configure_credentials_non_interactive(self): if self.access_key and self.secret_key: self.config.credentials_path = self._get_credentials_path() - print "self.config.credentials_path = ", self.config.credentials_path + print("self.config.credentials_path = ", self.config.credentials_path) self.config.credentials_file_exist = path.exists(str(self.config.credentials_path)) - print "self.config.credentials_file_exist = ", self.config.credentials_file_exist + print("self.config.credentials_file_exist = ", self.config.credentials_file_exist) if not self.config.credentials_file_exist: self.config.access_key = self.access_key self.config.secret_key = self.secret_key @@ -626,16 +632,16 @@ def _prepare_prompt(self): def run(self): if self.title: - print self.title + print(self.title) if self.options: for index, option in enumerate(self.options, start=1): - print " {}. {}".format(index, option) + print(" {}. {}".format(index, option)) return self._get_answer() def _get_answer(self): - value = raw_input(self.message).strip() + value = input(self.message).strip() while self._is_value_invalid(value): - value = raw_input(self.message).strip() + value = input(self.message).strip() return value or str(self.default) def _is_value_invalid(self, value): @@ -748,7 +754,7 @@ def main(): COPY_PLUGIN_INCLUDE_FILE_CMD = CMD(COPY_CMD.format(source=PLUGIN_INCLUDE_CONFIGURATION, target="/etc/"), "Copying CloudWatch plugin include file") COPY_RECOMMENDED_COLLECTD_CONFIG_CMD = CMD(COPY_CMD.format(source=RECOMMENDED_COLLECTD_CONFIGURATION, target=COLLECTD_INFO.config_path), "Replacing collectd configuration") BACKUP_COLLECTD_CONFIG_CMD = CMD(COPY_CMD.format(source=COLLECTD_INFO.config_path, target=COLLECTD_INFO.config_path + "." + time.strftime(TIMESTAMP_FORMAT)), - "Creating backup of the original configuration") + "Creating backup of the original configuration") REPLACE_WHITELIST_CMD = CMD(COPY_CMD.format(source=RECOMMENDED_WHITELIST, target=DEFAULT_PLUGIN_CONFIGURATION_DIR), "Replacing whitelist configuration") parser = argparse.ArgumentParser( @@ -897,7 +903,7 @@ def supply_config(): elif config.only_add_plugin: _inject_plugin_configuration() else: - print Color.yellow("Please find instructions for the manual configuration of the plugin in the readme.md file.") + print(Color.yellow("Please find instructions for the manual configuration of the plugin in the readme.md file.")) else: raise InstallationFailedException("The minimum supported version of collectd is " + CollectdInfo.MIN_SUPPORTED_VERSION + \ ", and your version is " + COLLECTD_INFO.version + \ @@ -916,13 +922,13 @@ def _prepare_plugin_config(plugin_config): def _inject_plugin_configuration(): if _is_cloudwatch_plugin_configured(): - print Color.yellow("CloudWatch collectd plugin is already configured in the existing collectd.conf file.") + print(Color.yellow("CloudWatch collectd plugin is already configured in the existing collectd.conf file.")) elif _can_safely_add_python_plugin(): with open(COLLECTD_INFO.config_path, "a") as config: config.write(PLUGIN_CONFIGURATION_INCLUDE_LINE) else: - print Color.yellow("Cannot add CloudWatch collectd plugin automatically to the existing collectd configuration.\n" - "Plugin must be configured manually, please find instructions in readme.md file.") + print(Color.yellow("Cannot add CloudWatch collectd plugin automatically to the existing collectd configuration.\n" + "Plugin must be configured manually, please find instructions in readme.md file.")) def _copy_recommended_configs(): _run_command(BACKUP_COLLECTD_CONFIG_CMD) diff --git a/test/helpers/fake_http_server.py b/test/helpers/fake_http_server.py index e817535..4933537 100644 --- a/test/helpers/fake_http_server.py +++ b/test/helpers/fake_http_server.py @@ -1,11 +1,12 @@ -import BaseHTTPServer +import http.server as BaseHTTPServer import json import os.path import socket import threading import time -import urllib2 -from SimpleHTTPServer import SimpleHTTPRequestHandler +import urllib.request +import urllib.error +from http.server import SimpleHTTPRequestHandler class FakeServer(object): @@ -73,7 +74,7 @@ def write_response(self, response_type=""): self.send_header("Content-type", response_type) self.send_header("Content-length", str(len(self._response))) self.end_headers() - self.wfile.write(self._response) + self.wfile.write(self._response.encode("utf-8")) def _update_response(self): self._response = self.read_file_or_default_to(FakeServer.RESPONSE_FILE, FakeServer.DEFAULT_RESPONSE) @@ -103,7 +104,7 @@ def stop_server(self): self._httpd.shutdown() self._serve_in_loop = False elif self._ready_to_serve: - urllib2.urlopen(self.get_url()).read() # generate empty request required to close listener + urllib.request.urlopen(self.get_url()).read() # generate empty request required to close listener self._ready_to_serve = False self._httpd.server_close() self._server_started = False @@ -175,4 +176,4 @@ def __clean_files(self): os.remove(FakeServer.RESPONSE_CODE_FILE) os.remove(FakeServer.REQUEST_FILE) except: - pass # Don't raise exceptions if cannot delete files \ No newline at end of file + pass # Don't raise exceptions if cannot delete files diff --git a/test/helpers/output_catcher.py b/test/helpers/output_catcher.py index 7c3f92a..4a34f29 100644 --- a/test/helpers/output_catcher.py +++ b/test/helpers/output_catcher.py @@ -1,5 +1,5 @@ import sys -from StringIO import StringIO +from io import StringIO from contextlib import contextmanager diff --git a/test/resources/collectd_outputs.py b/test/resources/collectd_outputs.py index 94b7d21..474451a 100644 --- a/test/resources/collectd_outputs.py +++ b/test/resources/collectd_outputs.py @@ -122,4 +122,4 @@ collectd 6.215a.12.beta.321.git, http://collectd.org/ by Florian octo Forster for contributions see `AUTHORS'""" -} \ No newline at end of file +} diff --git a/test/test_awscredentials.py b/test/test_awscredentials.py index f898c25..412b787 100644 --- a/test/test_awscredentials.py +++ b/test/test_awscredentials.py @@ -1,6 +1,7 @@ import unittest from cloudwatch.modules.awscredentials import AWSCredentials + class AWSCredentialsTest(unittest.TestCase): def test_aws_credentials_with_default_constructor(self): @@ -13,7 +14,8 @@ def test_aws_credentials_with_custom_values(self): new_token = "token" credentials = AWSCredentials(new_access_key, new_secret_key, new_token) assert_credentials_data(credentials, new_access_key, new_secret_key, new_token) - + + def assert_credentials_data(credentials, access_key=None, secret_key=None, token=None): assert access_key == credentials.access_key assert secret_key == credentials.secret_key diff --git a/test/test_awsutils.py b/test/test_awsutils.py index aecac68..5410826 100644 --- a/test/test_awsutils.py +++ b/test/test_awsutils.py @@ -3,12 +3,13 @@ from cloudwatch.modules.awsutils import get_aws_timestamp, get_datestamp + class AWSUtilsTest(unittest.TestCase): def test_get_timestamp_format(self): - pattern = re.compile('\d\d\d\d\d\d\d\dT\d\d\d\d\d\dZ') + pattern = re.compile(r'\d\d\d\d\d\d\d\dT\d\d\d\d\d\dZ') self.assertTrue(pattern.match(get_aws_timestamp()), "Expected timestamp in the format YYYYMMDDThhmmssZ.") def test_get_datestamp_format(self): - pattern = re.compile('\d\d\d\d\d\d\d\d') - self.assertTrue(pattern.match(get_datestamp())) \ No newline at end of file + pattern = re.compile(r'\d\d\d\d\d\d\d\d') + self.assertTrue(pattern.match(get_datestamp())) diff --git a/test/test_cloudwatch_writer.py b/test/test_cloudwatch_writer.py index 06e56b6..d32c42a 100644 --- a/test/test_cloudwatch_writer.py +++ b/test/test_cloudwatch_writer.py @@ -3,6 +3,7 @@ from mock import patch, Mock, MagicMock + class CloudWatchWriter(unittest.TestCase): def setUp(self): @@ -51,4 +52,4 @@ def test_write_passes_vl_to_flusher(self, config_helper, flusher_class): plugin.aws_init() vl = MagicMock() plugin.aws_write(vl, flusher) - flusher.add_metric.assert_called_with(vl) \ No newline at end of file + flusher.add_metric.assert_called_with(vl) diff --git a/test/test_collectd.py b/test/test_collectd.py index bae0795..209b020 100644 --- a/test/test_collectd.py +++ b/test/test_collectd.py @@ -2,6 +2,7 @@ import cloudwatch.modules.collectd as collectd + class collectdTest(unittest.TestCase): def test_stub_logging_methods_are_not_throwing_exceptions(self): @@ -13,4 +14,4 @@ def test_stub_logging_methods_are_not_throwing_exceptions(self): def test_stub_callbacks_are_not_throwing_exceptions(self): collectd.register_config() collectd.register_init() - collectd.register_write() \ No newline at end of file + collectd.register_write() diff --git a/test/test_confighelper.py b/test/test_confighelper.py index fbc7dee..69e1edf 100644 --- a/test/test_confighelper.py +++ b/test/test_confighelper.py @@ -186,7 +186,7 @@ def test_whitelist_is_properly_configured_based_on_plugin_config_file(self): self.assertTrue(self.config_helper.whitelist.is_whitelisted("random-metric-name")) def _load_and_assert_iam_role_credentials(self, expected_access, expected_secret, expected_token): - creds_json = '{"AccessKeyId" : "' + expected_access +'", "SecretAccessKey" : "' + expected_secret + '", "Token" : "' + expected_token + '" }' + creds_json = '{"AccessKeyId" : "' + expected_access + '", "SecretAccessKey" : "' + expected_secret + '", "Token" : "' + expected_token + '" }' self.server.set_expected_response(creds_json, 200) ConfigHelper._DEFAULT_CREDENTIALS_PATH = "" self.config_helper = ConfigHelper(config_path=ConfigHelperTest.VALID_CONFIG_WITHOUT_CREDS,metadata_server=self.server.get_url()) @@ -208,4 +208,3 @@ def assert_credentials(credentials, expected_access=ConfigHelperTest.VALID_ACCES assert credentials.access_key == expected_access assert credentials.secret_key == expected_secret assert credentials.token == expected_token - diff --git a/test/test_credentialsreader.py b/test/test_credentialsreader.py index a70b02c..9fb5163 100644 --- a/test/test_credentialsreader.py +++ b/test/test_credentialsreader.py @@ -3,6 +3,7 @@ from mock import MagicMock, Mock from cloudwatch.modules.configuration.credentialsreader import CredentialsReader, CredentialsReaderException + class CredentialsReaderTest(unittest.TestCase): CONFIG_DIR = "./test/config_files/" VALID_CREDENTIALS_FILE = CONFIG_DIR + "valid_credentials_file" @@ -63,9 +64,10 @@ def test_config_reader_without_path(self): with self.assertRaises(TypeError): self.config_reader = CredentialsReader() + def assert_credentials(credentials_reader): assert credentials_reader.credentials creds = credentials_reader.credentials assert creds.access_key == CredentialsReaderTest.VALID_ACCESS_KEY_STRING assert creds.secret_key == CredentialsReaderTest.VALID_SECRET_KEY_STRING - + diff --git a/test/test_ec2getclient.py b/test/test_ec2getclient.py index e97f0d3..b55c7e9 100644 --- a/test/test_ec2getclient.py +++ b/test/test_ec2getclient.py @@ -38,7 +38,7 @@ def test_constructor(self): response_timeout = 20 client = EC2GetClient(self.config_helper, connection_timeout, response_timeout) self.assertEquals("http://localhost:57575/", client.endpoint) - self.assertEquals((connection_timeout,response_timeout), client.timeout) + self.assertEquals((connection_timeout, response_timeout), client.timeout) def test_initialize_get_client_with_valid_endpoint(self): self.config_helper.endpoint = "https://ec2.eu-west-1.amazonaws.com" @@ -156,10 +156,9 @@ def server_restart(self): self.server.serve_forever() def server_get_received_request(self): - return open(FakeServer.REQUEST_FILE).read()[2:] # trim '/?' from the request + return open(FakeServer.REQUEST_FILE).read()[2:] # trim '/?' from the request @classmethod def tearDownClass(cls): cls.FAKE_SERVER.stop_server() cls.FAKE_SERVER = None - diff --git a/test/test_fakeservertest.py b/test/test_fakeservertest.py index 6ebba82..3e3bbc5 100644 --- a/test/test_fakeservertest.py +++ b/test/test_fakeservertest.py @@ -3,6 +3,7 @@ from helpers.fake_http_server import FakeServer + class FakeServerTest(unittest.TestCase): SERVER = None @@ -62,11 +63,12 @@ def tearDownClass(cls): cls.SERVER.stop_server() except: pass - + + def send_and_check_request(url, request): url = url + request response = requests.get(url) received_request = open(FakeServer.REQUEST_FILE).read() assert request in received_request[1:] # skip first character which always is '/' assert response.status_code == FakeServer.DEFAULT_RESPONSE_CODE - assert response.text == FakeServer.DEFAULT_RESPONSE \ No newline at end of file + assert response.text == FakeServer.DEFAULT_RESPONSE diff --git a/test/test_flusher.py b/test/test_flusher.py index 001874d..70886f9 100644 --- a/test/test_flusher.py +++ b/test/test_flusher.py @@ -70,7 +70,7 @@ def test_is_numerical_value(self): def test_numerical_value(self): key = self.add_value_list("plugin", "plugin_instance_0", "type", "type_instance", "host", [float('nan')]) - self.assertFalse(key in self.flusher.metric_map) + self.assertFalse(key in self.flusher.metric_map) self.assertTrue(key in self.flusher.nan_key_set) key = self.add_value_list("plugin", "plugin_instance_1", "type", "type_instance", "host", [10]) self.assertTrue(key in self.flusher.metric_map) @@ -231,7 +231,7 @@ def test_flush_when_enable_high_resolution(self, client_class): def test_prepare_batches_respects_the_size_limit(self): for i in range(self.flusher._MAX_METRICS_PER_PUT_REQUEST + 1): self.flusher._aggregate_metric(self._get_vl_mock("plugin" + str(i), "plugin_instance", "type", "type_instance", "host", [i], 0)) - batch = self.flusher._prepare_batch().next() + batch = self.flusher._prepare_batch().__next__() self.assertEquals(self.flusher._MAX_METRICS_PER_PUT_REQUEST, len(list(batch))) batch = self.flusher._prepare_batch() self.assertEquals(1, len(list(batch))) @@ -257,7 +257,6 @@ def test_multivalue_metrics_set_the_type_instance(self): self.assertListEqual([value.type_instance for value in expanded_value_list], ['name1', 'name2']) self.assertListEqual([value.values for value in expanded_value_list], [[10], [11]]) - def test_multivalue_metrics_appends_the_type_instance(self): vl1 = self._get_vl_mock("plugin", "plugin_instance", "multivalue_type", "type_instance", "host", [10, 11], 101.1) expanded_value_list=self.flusher._expand_value_list(vl1) @@ -265,14 +264,13 @@ def test_multivalue_metrics_appends_the_type_instance(self): self.assertListEqual([value.type_instance for value in expanded_value_list], ['type_instance.name1', 'type_instance.name2']) self.assertListEqual([value.values for value in expanded_value_list], [[10], [11]]) - def _assert_statistics(self, metric, min, max, sum, sample_count): self.assertEquals(min, metric.statistics.min) self.assertEquals(max, metric.statistics.max) self.assertEquals(sum, metric.statistics.sum) self.assertEquals(sample_count, metric.statistics.sample_count) - def _get_vl_mock(self, plugin, plugin_instance, type, type_instance, host="MockHost", values=[], timestamp=0): + def _get_vl_mock(self, plugin, plugin_instance, type, type_instance, host="MockHost", values=[], timestamp=0.0): vl = MagicMock() vl.plugin = plugin vl.plugin_instance = plugin_instance @@ -280,7 +278,7 @@ def _get_vl_mock(self, plugin, plugin_instance, type, type_instance, host="MockH vl.type_instance = type_instance vl.host = host vl.values = values - vl.time = timestamp; + vl.time = timestamp return vl def server_get_received_request(self): diff --git a/test/test_logger.py b/test/test_logger.py index 405d492..0f2bc1b 100644 --- a/test/test_logger.py +++ b/test/test_logger.py @@ -4,6 +4,7 @@ from mock import Mock from cloudwatch.modules.logger.logger import get_logger, _Logger, _CollectdLogger + class LoggerTest(unittest.TestCase): def setUp(self): @@ -14,9 +15,9 @@ def test_type_of_logger(self): self.assertTrue(type(self.logger) is _CollectdLogger) def test_base_logger_is_abstract(self): - with self.assertRaises(TypeError): - logger = _Logger() - + self.assertTrue(type(_Logger()) is not _CollectdLogger) + self.assertTrue(type(_Logger()) is _Logger) + def test_debug_called_on_collectd(self): msg = "debug msg" collectd.debug = Mock() @@ -44,4 +45,3 @@ def test_error_called_on_collectd(self): self.logger.error(msg) self.assertTrue(collectd.error.called) collectd.error.assert_called_with(self.expected_prefix + msg) - \ No newline at end of file diff --git a/test/test_metadatareader.py b/test/test_metadatareader.py index f098f86..17fb75e 100644 --- a/test/test_metadatareader.py +++ b/test/test_metadatareader.py @@ -4,6 +4,7 @@ from helpers.fake_metadata import FAKE_REGION, FAKE_IDENTITY_DOCUMENT_STRING from cloudwatch.modules.configuration.metadatareader import MetadataReader, MetadataRequestException + class MetadataReaderTest(unittest.TestCase): FAKE_SERVER = None MINIMUM_NUMBER_OF_RETRIES = 3 @@ -28,7 +29,7 @@ def test_number_of_retries_is_within_expected_range(self): def test_get_region_from_metadata(self): self.server.set_expected_response(FAKE_IDENTITY_DOCUMENT_STRING, 200) self.assertEquals(FAKE_REGION, self.metadata_reader.get_region()) - + def test_get_full_configuration_from_config_file(self): self.server.set_expected_response("invalid request", 404) with self.assertRaises(MetadataRequestException): diff --git a/test/test_metricdata.py b/test/test_metricdata.py index 51a0e45..7e0bb9b 100644 --- a/test/test_metricdata.py +++ b/test/test_metricdata.py @@ -4,6 +4,7 @@ from cloudwatch.modules.metricdata import MetricDataStatistic import cloudwatch.modules.awsutils as awsutils + class MetricDataTest(unittest.TestCase): def test_metric_data_default_constructor(self): @@ -18,14 +19,14 @@ def test_metric_object_with_custom_value(self): new_dimensions = {'Host':'InstanceID'} new_namespace = "test_namespace" metric = MetricDataStatistic(metric_name=new_metric_name, timestamp=new_timestamp, - unit=new_unit, dimensions=new_dimensions, namespace=new_namespace) + unit=new_unit, dimensions=new_dimensions, namespace=new_namespace) assert_metric_data(metric, new_metric_name, new_timestamp, new_unit, new_dimensions, namespace=new_namespace) def test_empty_metric_object(self): metric = MetricDataStatistic() assert_metric_data(metric, metric_name="", unit="", dimensions={}, statistics=None, - timestamp=None, namespace=MetricDataStatistic.NAMESPACE) + timestamp=None, namespace=MetricDataStatistic.NAMESPACE) def test_metric_object_with_statistic_data(self): new_statistics = MetricDataStatistic.Statistics(10) @@ -82,6 +83,7 @@ def test_custom_timestamp_is_not_overriden(self): metric2 = MetricDataStatistic("metric_name", 20, timestamp=timestamp) self.assertTrue(metric1.timestamp is metric2.timestamp) + def assert_metric_data(metric_data, metric_name='', timestamp=None, unit="", dimensions={}, statistics=None, namespace=MetricDataStatistic.NAMESPACE): assert namespace == metric_data.namespace assert metric_name == metric_data.metric_name @@ -89,12 +91,13 @@ def assert_metric_data(metric_data, metric_name='', timestamp=None, unit="", dim assert dimensions == metric_data.dimensions assert statistics == metric_data.statistics if timestamp is None: - assert metric_data.timestamp #Make sure that timestamp is not empty + assert metric_data.timestamp # Make sure that timestamp is not empty else: assert timestamp == metric_data.timestamp - + + def assert_statistics(statistics, min=None, max=None, sum=None, sample_count=1): assert min == statistics.min assert max == statistics.max assert sum == statistics.sum - assert sample_count == statistics.sample_count \ No newline at end of file + assert sample_count == statistics.sample_count diff --git a/test/test_metricdatabuilder.py b/test/test_metricdatabuilder.py index 9221674..f5ea661 100644 --- a/test/test_metricdatabuilder.py +++ b/test/test_metricdatabuilder.py @@ -5,6 +5,7 @@ from cloudwatch.modules.configuration.confighelper import ConfigHelper from cloudwatch.modules.metricdata import MetricDataBuilder + class MetricDataBuilderTest(unittest.TestCase): FAKE_SERVER = None @@ -105,7 +106,7 @@ def test_build_with_enable_high_resolution_metrics(self): self.assertEquals("CPU.CPU.Steal", metric[0].metric_name) self.assertEquals("valid_host", metric[0].dimensions['Host']) self.assertEquals("0", metric[0].dimensions['PluginInstance']) - self.assertEquals("19700101T000240Z", metric[0].timestamp); + self.assertEquals("19700101T000240Z", metric[0].timestamp) def test_build_metric_name_with_all_name_parts(self): vl = self._get_vl_mock("CPU", "0", "CPU", "Steal") @@ -149,7 +150,7 @@ def test_build_metric_dimensions_with_host_from_value_list(self): dimensions = metric_data_builder._build_metric_dimensions() self.assertEquals("MockHost", dimensions['Host']) - def _get_vl_mock(self, plugin, plugin_instance, type, type_instance, host="MockHost", values=[], timestamp=0): + def _get_vl_mock(self, plugin, plugin_instance, type, type_instance, host="MockHost", values=[], timestamp=0.0): vl = MagicMock() vl.plugin = plugin vl.plugin_instance = plugin_instance @@ -163,4 +164,4 @@ def _get_vl_mock(self, plugin, plugin_instance, type, type_instance, host="MockH @classmethod def tearDownClass(cls): cls.FAKE_SERVER.stop_server() - cls.FAKE_SERVER = None \ No newline at end of file + cls.FAKE_SERVER = None diff --git a/test/test_putclient.py b/test/test_putclient.py index f8c8808..70fcb70 100644 --- a/test/test_putclient.py +++ b/test/test_putclient.py @@ -168,10 +168,9 @@ def server_restart(self): self.server.serve_forever() def server_get_received_request(self): - return open(FakeServer.REQUEST_FILE).read()[2:] # trim '/?' from the request + return open(FakeServer.REQUEST_FILE).read()[2:] # trim '/?' from the request @classmethod def tearDownClass(cls): cls.FAKE_SERVER.stop_server() cls.FAKE_SERVER = None - diff --git a/test/test_querystringbuilder.py b/test/test_querystringbuilder.py index fb77edb..d4afe07 100644 --- a/test/test_querystringbuilder.py +++ b/test/test_querystringbuilder.py @@ -3,6 +3,7 @@ from cloudwatch.modules.client.querystringbuilder import QuerystringBuilder from cloudwatch.modules.metricdata import MetricDataStatistic + class QuerystringBuilderTest(unittest.TestCase): def setUp(self): @@ -27,9 +28,9 @@ def test_build_metric_map_for_single_metric_with_dimensions(self): assert_metric_map([metric], metric_map) def test_build_metric_map_for_multiple_metrics_with_dimensions(self): - dimensions1 = { "Dimension1": 20, "Dimension2": 30, "Host": "localhost" } + dimensions1 = {"Dimension1": 20, "Dimension2": 30, "Host": "localhost"} metric1 = MetricDataStatistic("test_metric", statistic_values=MetricDataStatistic.Statistics(20), dimensions=dimensions1) - dimensions2 = { "Dimension1": 20, "Dimension2": 30, "Host": "localhost" } + dimensions2 = {"Dimension1": 20, "Dimension2": 30, "Host": "localhost"} metric2 = MetricDataStatistic("test_metric2", statistic_values=MetricDataStatistic.Statistics(400), dimensions=dimensions2) metric_list = [metric1, metric2] metric_map = self.builder._build_metric_map(metric_list) @@ -40,7 +41,7 @@ def test_build_querystring_with_no_metrics(self): assert_querystring(get_canonical_map(), querystring) def test_build_querystring_with_one_metric(self): - dimensions = { "Dimension1": 20, "Dimension2": 30, "Host": "localhost" } + dimensions = {"Dimension1": 20, "Dimension2": 30, "Host": "localhost"} metric = MetricDataStatistic("test_metric", statistic_values=MetricDataStatistic.Statistics(20), dimensions=dimensions) expected_map = get_canonical_map() expected_map.update(self.builder._build_metric_map([metric])) @@ -48,9 +49,9 @@ def test_build_querystring_with_one_metric(self): assert_querystring(expected_map, querystring) def test_build_querystring_with_multiple_metrics(self): - dimensions1 = { "Dimension1": 20, "Dimension2": 30, "Host": "localhost" } + dimensions1 = {"Dimension1": 20, "Dimension2": 30, "Host": "localhost"} metric1 = MetricDataStatistic("test_metric", statistic_values=MetricDataStatistic.Statistics(20), dimensions=dimensions1) - dimensions2 = { "Dimension1": 20, "Dimension2": 30, "Host": "localhost" } + dimensions2 = {"Dimension1": 20, "Dimension2": 30, "Host": "localhost"} metric2 = MetricDataStatistic("test_metric2", statistic_values=MetricDataStatistic.Statistics(400), dimensions=dimensions2) expected_map = get_canonical_map() expected_map.update(self.builder._build_metric_map([metric1, metric2])) @@ -65,7 +66,7 @@ def test_build_querystring_with_storage_resolution(self): self.assertTrue(self.builder._STORAGE_RESOLUTION in querystring) def test_build_map_with_statistics(self): - dimensions1 = { "Dimension1": 20, "Dimension2": 30, "Host": "localhost" } + dimensions1 = {"Dimension1": 20, "Dimension2": 30, "Host": "localhost"} metric = MetricDataStatistic("test_metric", dimensions=dimensions1) metric.add_value(10) metric.add_value(20) @@ -79,7 +80,7 @@ def test_build_map_with_statistics(self): self.assertEquals(3, map[prefix + QuerystringBuilder._STAT_SAMPLE]) def test_build_querystring_with_statistics(self): - dimensions = { "Dimension1": 20, "Dimension2": 30, "Host": "localhost" } + dimensions = {"Dimension1": 20, "Dimension2": 30, "Host": "localhost"} metric = MetricDataStatistic("test_metric", dimensions=dimensions) metric.add_value(10) metric.add_value(-20) @@ -87,13 +88,13 @@ def test_build_querystring_with_statistics(self): expected_map = get_canonical_map() expected_map.update(self.builder._build_metric_map([metric])) querystring = self.builder.build_querystring([metric], get_canonical_map()) - self.assertTrue(QuerystringBuilder._STAT_MAX +"=50" in querystring) - self.assertTrue(QuerystringBuilder._STAT_MIN +"=-20" in querystring) - self.assertTrue(QuerystringBuilder._STAT_SUM +"=40" in querystring) - self.assertTrue(QuerystringBuilder._STAT_SAMPLE +"=3" in querystring) + self.assertTrue(QuerystringBuilder._STAT_MAX + "=50" in querystring) + self.assertTrue(QuerystringBuilder._STAT_MIN + "=-20" in querystring) + self.assertTrue(QuerystringBuilder._STAT_SUM + "=40" in querystring) + self.assertTrue(QuerystringBuilder._STAT_SAMPLE + "=3" in querystring) def test_build_map_without_statistics(self): - dimensions = { "Dimension1": 20, "Dimension2": 30, "Host": "localhost" } + dimensions = {"Dimension1": 20, "Dimension2": 30, "Host": "localhost"} metric = MetricDataStatistic("test_metric", dimensions=dimensions) map = {} prefix = "metric_prefix" @@ -102,31 +103,33 @@ def test_build_map_without_statistics(self): def test_no_double_slash_url_encoding(self): """ when string is urlencoded twice or more the slash character gets invalid value of %252F instead of %2F """ - input_map = {"test":"test/test"} + input_map = {"test": "test/test"} querystring = self.builder.build_querystring([], input_map) self.assertTrue("%2F" in querystring) self.assertFalse("%252F" in querystring) def test_spaces_are_urlencoded(self): """ the standard urllib.urlencode encodes spaces as '+', but CloudWatch requires '%20' """ - input_map = {"test":"test test2"} + input_map = {"test": "test test2"} querystring = self.builder.build_querystring([], input_map) self.assertEquals("test=test%20test2", querystring) self.assertFalse('+' in querystring) def test_plus_is_encoded_properly(self): - input_map = {"test":"test+test2"} + input_map = {"test": "test+test2"} querystring = self.builder.build_querystring([], input_map) self.assertEquals("test=test%2Btest2", querystring) self.assertFalse('+' in querystring) - + + def get_canonical_map(): return {"Action": "test_action", "Namespace": "test_namespace", "Version": "test_version", "X-Amz-Algorithm": "test_algorithm", "X-Amz-Credential": "access_key" - } + } + def split_querystring(querystring): list = querystring.split('&') @@ -135,6 +138,7 @@ def split_querystring(querystring): request_list.append(item.split('=', 1)) return request_list + def assert_querystring(expected_map, querystring): keys_in_order = sorted(expected_map.keys()) splited_querystring = split_querystring(querystring) @@ -148,7 +152,8 @@ def assert_querystring(expected_map, querystring): except: pass assert expected_map[keys_in_order[i]] == current_value - + + def assert_metric_map(metric_list, metric_map): index = 1 for metric in metric_list: @@ -157,11 +162,12 @@ def assert_metric_map(metric_list, metric_map): assert metric.timestamp == metric_map[prefix + "Timestamp"] assert_dimensions(metric, index, metric_map) index += 1 - + + def assert_dimensions(metric, metric_index, metric_map): dimension_index = 1 for key in metric.dimensions.keys(): prefix = "MetricData.member." + str(metric_index) + ".Dimensions.member." + str(dimension_index) + "." assert key == metric_map[prefix + "Name"] assert metric.dimensions[key] == metric_map[prefix + "Value"] - dimension_index += 1 \ No newline at end of file + dimension_index += 1 diff --git a/test/test_readerutils.py b/test/test_readerutils.py index 3a8c6dd..865ee19 100644 --- a/test/test_readerutils.py +++ b/test/test_readerutils.py @@ -4,6 +4,7 @@ from cloudwatch.modules.configuration.readerutils import ReaderUtils + class ReaderUtilsTest(unittest.TestCase): CONFIG_DIR = "./test/config_files/" VALID_CONFIG_FULL = CONFIG_DIR + "valid_config_full" @@ -68,8 +69,8 @@ def test_try_get_high_resolution_parameters(self): reader = ReaderUtils(self.INVALID_CONFIG_WITH_HIGH_RESOLUTION_PARAMETERS) enable_high_resolution_metrics = reader.get_string("enable_high_resolution_metrics") flush_interval_in_seconds = reader.get_string("flush_interval_in_seconds") - self.assertEquals('Tru', enable_high_resolution_metrics); - self.assertEquals('59', flush_interval_in_seconds); + self.assertEquals('Tru', enable_high_resolution_metrics) + self.assertEquals('59', flush_interval_in_seconds) def test_get_boolean_from_invalid_config(self): reader = ReaderUtils(self.INVALID_CONFIG_WITH_DEBUG) @@ -112,7 +113,7 @@ def test_strip_quotes(self): def test_valid_real_config(self): reader = ReaderUtils(self.VALID_REAL_CONFIG) region = reader.get_string("region") - self.assertEquals("", region) # region is commented out + self.assertEquals("", region) # region is commented out host = reader.get_string("host") self.assertEquals("Server1", host) credentials_path = reader.get_string("credentials_path") diff --git a/test/test_requestbuilder.py b/test/test_requestbuilder.py index 41e2310..5e92a8d 100644 --- a/test/test_requestbuilder.py +++ b/test/test_requestbuilder.py @@ -4,7 +4,8 @@ from cloudwatch.modules.awsutils import get_datestamp from cloudwatch.modules.client.requestbuilder import RequestBuilder from cloudwatch.modules.metricdata import MetricDataStatistic -from cloudwatch.modules.client.querystringbuilder import QuerystringBuilder# +from cloudwatch.modules.client.querystringbuilder import QuerystringBuilder + class RequestBuilderTest(unittest.TestCase): @@ -15,7 +16,7 @@ def setUp(self): self.builder = RequestBuilder(self.credentials, self.region, "10") def test_get_host(self): - self.assertEquals("monitoring." + self.region +".amazonaws.com", self.builder._get_host()) + self.assertEquals("monitoring." + self.region + ".amazonaws.com", self.builder._get_host()) def test_get_host_for_localhost_region(self): self.builder.region = "localhost" @@ -27,14 +28,14 @@ def test_get_signed_headers(self): def test_get_canonical_headers(self): headers = self.builder._get_canonical_headers() self.assertTrue("\n" in headers) - self.assertTrue("host:monitoring." + self.region +".amazonaws.com\n" in headers) + self.assertTrue("host:monitoring." + self.region + ".amazonaws.com\n" in headers) def test_get_credential_scope(self): datestamp = get_datestamp() v4_terminator = "aws4_request" service = "monitoring" self.builder.create_signed_request(self.namespace, []) - self.assertEquals(datestamp + "/" + self.region + "/" + service + "/" + v4_terminator, \ + self.assertEquals(datestamp + "/" + self.region + "/" + service + "/" + v4_terminator, self.builder._get_credential_scope()) def test_get_credential_scope_before_geting_request(self): @@ -65,4 +66,4 @@ def test_canonical_querystring_is_directly_created_by_querystring_builder(self): metric = MetricDataStatistic("test_metric", statistic_values=MetricDataStatistic.Statistics(20)) original_querystring = querystring_builder.build_querystring([metric], self.builder._get_request_map()) generated_querystring = self.builder._create_canonical_querystring([metric]) - self.assertEquals(original_querystring, generated_querystring) \ No newline at end of file + self.assertEquals(original_querystring, generated_querystring) diff --git a/test/test_setup.py b/test/test_setup.py index c4cdc88..df21fb6 100644 --- a/test/test_setup.py +++ b/test/test_setup.py @@ -5,7 +5,7 @@ from mock import patch, Mock -import resources.collectd_outputs as resources +from resources import collectd_outputs as resources import src.setup as installer from helpers.output_catcher import output_catcher from src.setup import Command, get_collectd_info, CollectdInfo @@ -70,7 +70,7 @@ def test_output_of_valid_command(self): command = Command("echo test", self.TEST_MSG) command.run() output = out.getvalue().strip() - self.assertEquals(self.TEST_MSG + " " + command.OK, output) + self.assertEquals(self.TEST_MSG + "\n" + command.OK, output) self.assertEquals("test", command.stdout) self.assertTrue(command.was_successful) @@ -79,7 +79,7 @@ def test_output_of_failed_command(self): command = Command("which " + self.INVALID_COMMAND, self.TEST_MSG) command.run() output = out.getvalue().strip() - self.assertEquals(self.TEST_MSG + " " + command.NOT_OK, output) + self.assertEquals(self.TEST_MSG + "\n" + command.NOT_OK, output) self.assertFalse(command.was_successful) def test_output_of_invalid_command(self): @@ -127,6 +127,7 @@ def color_methods_appends_color_end(self): self.assertEquals(installer.Color.YELLOW + test_string + self.END, installer.Color.yellow(test_string)) self.assertEquals(installer.Color.CYAN + test_string + self.END, installer.Color.cyan(test_string)) + class NonInteractiveTests(unittest.TestCase): plugin_config = installer.PluginConfig metadata_reader = None @@ -150,7 +151,6 @@ class NonInteractiveTests(unittest.TestCase): tmp = tempfile.mkdtemp(prefix='/tmp/') creds_path = tmp + '/' + 'test_creds_file' - def test_params(self): non_interactive_installer = installer.InteractiveConfigurator(self.plugin_config, self.metadata_reader, self.collectd_info, diff --git a/test/test_signer.py b/test/test_signer.py index 6a6aba0..a50d3ec 100644 --- a/test/test_signer.py +++ b/test/test_signer.py @@ -4,6 +4,7 @@ from cloudwatch.modules.awsutils import get_aws_timestamp from cloudwatch.modules.client.signer import Signer + class SignerTest(unittest.TestCase): def setUp(self): @@ -33,7 +34,7 @@ def test_get_canonical_request_sequence(self): generated_request = self.signer._build_canonical_request(query_string, canonical_headers, signed_headers, payload) self.assertEquals(expected_request, generated_request) -# regression tests + # regression tests def test_hash(self): signer = self.get_regression_signer() test_data = "test data string" @@ -43,13 +44,13 @@ def test_hash(self): def test_sign(self): signer = self.get_regression_signer() data = "testing&canonical&request" - historical_result = "\xfb\xd2\x86\x87&\xdaC\x03\x98\x9dIC\xcbP?\xa8\\\xfeJ\x82\x03\xe6w\xd4\x963Q\xfd\xe5-\xdb\xcf" + historical_result = b"\xfb\xd2\x86\x87&\xdaC\x03\x98\x9dIC\xcbP?\xa8\\\xfeJ\x82\x03\xe6w\xd4\x963Q\xfd\xe5-\xdb\xcf" self.assertEquals(historical_result, signer._sign(signer.credentials.secret_key, data)) def test_build_signature_key(self): signer = self.get_regression_signer() datestamp = "20150725" - historical_result = "=7\xa5&\xa3%\xd5Q\x9a\x1ah\xee2mSw<\xdd\xf8\x0e\xde\xdf5\x94\xa6(M`\x00\xd1\x81\xea" + historical_result = b"=7\xa5&\xa3%\xd5Q\x9a\x1ah\xee2mSw<\xdd\xf8\x0e\xde\xdf5\x94\xa6(M`\x00\xd1\x81\xea" new_result = signer._build_signature_key(signer.credentials.secret_key, datestamp, signer.region, signer.service) self.assertEquals(historical_result, new_result) @@ -74,4 +75,4 @@ def get_regression_signer(self): self.region = "eu-west-1" self.service = "monitoring" self.algorithm = "AWS4-HMAC-SHA256" - return Signer(self.credentials, self.region, self.service, self.algorithm) \ No newline at end of file + return Signer(self.credentials, self.region, self.service, self.algorithm)