diff --git a/mtools/mlaunch/mlaunch.py b/mtools/mlaunch/mlaunch.py index 2c181aa9..60546dec 100755 --- a/mtools/mlaunch/mlaunch.py +++ b/mtools/mlaunch/mlaunch.py @@ -185,8 +185,8 @@ def run(self, arguments=None): # to run can call different sub-commands self.argparser = argparse.ArgumentParser() self.argparser.add_argument('--version', action='version', - version="mtools version {0} || Python {1}".format( - __version__, sys.version)) + version="mtools version {0} || Python {1}". + format(__version__, sys.version)) self.argparser.add_argument('--no-progressbar', action='store_true', default=False, help='disables progress bar') @@ -323,7 +323,9 @@ def run(self, arguments=None): 'required to run the stop command ' '(requires --auth, default="%s")' % ' '.join(self._default_auth_roles))) - init_parser.add_argument('--auth-role-docs', action='store_true', default=False, help='auth-roles are json documents') + init_parser.add_argument('--auth-role-docs', action='store_true', + default=False, + help='auth-roles are json documents') init_parser.add_argument('--no-initial-user', action='store_false', default=True, dest='initial-user', help=('create an initial user if auth is ' @@ -579,7 +581,7 @@ def init(self): # add the 'csrs' parameter as default for MongoDB >= 3.3.0 if (LooseVersion(current_version) >= LooseVersion("3.3.0") or - LooseVersion(current_version) == LooseVersion("0.0.0")): + LooseVersion(current_version) == LooseVersion("0.0.0")): self.args['csrs'] = True # construct startup strings @@ -664,7 +666,7 @@ def init(self): print("adding shards.") shard_conns_and_names = list(zip(self.shard_connection_str, - shard_names)) + shard_names)) while True: try: nshards = con['config']['shards'].count() @@ -733,7 +735,6 @@ def init(self): raise RuntimeError("can't connect to server, so adding admin " "user isn't possible") - roles = [] found_cluster_admin = False if self.args['auth_role_docs']: @@ -762,8 +763,11 @@ def init(self): members = sorted(self.get_tagged([shard])) if self.args['verbose']: print("adding users to %s" % shard) - self._add_user(members[0], name=self.args['username'], password=self.args['password'], - database=self.args['auth_db'], roles=self.args['auth_roles']) + self._add_user(members[0], + name=self.args['username'], + password=self.args['password'], + database=self.args['auth_db'], + roles=self.args['auth_roles']) if self.args['verbose']: print("added user %s on %s database" % (self.args['username'], @@ -1172,7 +1176,7 @@ def discover(self): # tag all nodes with 'all' self.cluster_tags['all'].extend(list(range(current_port, - current_port + num_nodes))) + current_port + num_nodes))) # tag all nodes with their port number (as string) and whether # they are running @@ -1199,8 +1203,8 @@ def discover(self): shard_names = [None] for shard in shard_names: - port_range = list(range(current_port, current_port + - num_nodes_per_shard)) + port_range = list(range(current_port, + current_port + num_nodes_per_shard)) # all of these are mongod nodes self.cluster_tags['mongod'].extend(port_range) @@ -1228,7 +1232,7 @@ def discover(self): (itemgetter(1), mrsc.secondaries))) self.cluster_tags['arbiter'].extend(list(map(itemgetter(1), - mrsc.arbiters))) + mrsc.arbiters))) # secondaries in cluster_tree (order is now important) self.cluster_tree.setdefault('secondary', []) @@ -1507,21 +1511,23 @@ def _filter_valid_arguments(self, arguments, binary="mongod", result = [] for i, arg in enumerate(arguments): if arg.startswith('-'): - # check if the binary accepts this argument or special case -vvv for any number of v + # check if the binary accepts this argument + # or special case -vvv for any number of v if arg in accepted_arguments or re.match(r'-v+', arg): result.append(arg) - elif binary.endswith('mongod') and arg in self.UNDOCUMENTED_MONGOD_ARGS: + elif (binary.endswith('mongod') and + arg in self.UNDOCUMENTED_MONGOD_ARGS): result.append(arg) elif self.ignored_arguments.get(binary + arg) is None: # warn once for each combination of binary and unknown arg self.ignored_arguments[binary + arg] = True - print("warning: ignoring unknown argument %s for %s" % (arg, binary)) + print("warning: ignoring unknown argument %s for %s" % + (arg, binary)) elif i > 0 and arguments[i - 1] in result: # if it doesn't start with a '-', it could be the value of # the last argument, e.g. `--slowms 1000` result.append(arg) - # return valid arguments as joined string return ' '.join(result) @@ -1618,16 +1624,13 @@ def _start_on_ports(self, ports, wait=False, override_auth=False): try: if os.name == 'nt': - ret = subprocess.check_call(command_str, - shell=True) + subprocess.check_call(command_str, shell=True) # create sub process on windows doesn't wait for output, # wait a few seconds for mongod instance up time.sleep(5) else: - ret = subprocess.check_output([command_str], - stderr=subprocess.STDOUT, - shell=True) - + subprocess.check_output([command_str], shell=True, + stderr=subprocess.STDOUT) binary = command_str.split()[0] if '--configsvr' in command_str: @@ -1648,7 +1651,6 @@ def _start_on_ports(self, ports, wait=False, override_auth=False): if wait: self.wait_for(ports) - def _initiate_replset(self, port, name, maxwait=30): """Initiate replica set.""" if not self.args['replicaset'] and name != 'configRepl': @@ -1677,7 +1679,7 @@ def _initiate_replset(self, port, name, maxwait=30): print("replica set '%s' initialized." % name) def _add_user(self, port, name, password, database, roles): - con = self.client('localhost:%i'%port) + con = self.client('localhost:%i' % port) v = con['admin'].command('isMaster').get('maxWireVersion', 0) if v >= 7: # Until drivers have implemented SCRAM-SHA-256, use old mechanism. @@ -1689,7 +1691,8 @@ def _add_user(self, port, name, password, database, roles): password = None try: - con[database].add_user(name, password=password, roles=roles, **opts) + con[database].add_user(name, password=password, roles=roles, + **opts) except OperationFailure as e: raise e except TypeError as e: @@ -1754,9 +1757,9 @@ def _wait_for_primary(self): # update cluster tags now that we have a primary self.cluster_tags['primary'].append(mrsc.primary[1]) self.cluster_tags['secondary'].extend(list(map(itemgetter(1), - mrsc.secondaries))) + mrsc.secondaries))) self.cluster_tags['arbiter'].extend(list(map(itemgetter(1), - mrsc.arbiters))) + mrsc.arbiters))) # secondaries in cluster_tree (order is now important) self.cluster_tree.setdefault('secondary', []) @@ -1812,20 +1815,25 @@ def _construct_sharded(self): # create shards as stand-alones or replica sets nextport = self.args['port'] + num_mongos for shard in shard_names: - if (self.args['single'] - and LooseVersion(current_version) >= LooseVersion("3.6.0")): - errmsg = " \n * In MongoDB 3.6 and above a Shard must be made up of a replica set. Please use --replicaset option when starting a sharded cluster.*" + if (self.args['single'] and + LooseVersion(current_version) >= LooseVersion("3.6.0")): + errmsg = " \n * In MongoDB 3.6 and above a Shard must be " \ + "made up of a replica set. Please use --replicaset " \ + "option when starting a sharded cluster.*" raise SystemExit(errmsg) - elif (self.args['single'] - and LooseVersion(current_version) < LooseVersion("3.6.0")): + elif (self.args['single'] and + LooseVersion(current_version) < LooseVersion("3.6.0")): self.shard_connection_str.append( - self._construct_single(self.dir, nextport, name=shard, extra='--shardsvr')) + self._construct_single( + self.dir, nextport, name=shard, extra='--shardsvr')) nextport += 1 elif self.args['replicaset']: self.shard_connection_str.append( - self._construct_replset(self.dir, nextport, shard, num_nodes=list(range(self.args['nodes'])), - arbiter=self.args['arbiter'], extra='--shardsvr')) + self._construct_replset( + self.dir, nextport, shard, + num_nodes=list(range(self.args['nodes'])), + arbiter=self.args['arbiter'], extra='--shardsvr')) nextport += self.args['nodes'] if self.args['arbiter']: nextport += 1 @@ -1926,8 +1934,8 @@ def _construct_config(self, basedir, port, name=None, isreplset=False): if isreplset: return self._construct_replset(basedir=basedir, portstart=port, name=name, - num_nodes=list(range(self - .args['config'])), + num_nodes=list(range( + self.args['config'])), arbiter=False, extra='--configsvr') else: datapath = self._create_paths(basedir, name) @@ -1980,7 +1988,10 @@ def _construct_mongod(self, dbpath, logpath, port, replset=None, extra=''): and (self.args['sharded'] or self.args['replicaset']) and '--bind_ip' not in extra): os.removedirs(dbpath) - errmsg = " \n * If hostname is specified, please include '--bind_ip_all' or '--bind_ip' options when deploying replica sets or sharded cluster with MongoDB version 3.6.0 or greater" + errmsg = " \n * If hostname is specified, please include "\ + "'--bind_ip_all' or '--bind_ip' options when deploying "\ + "replica sets or sharded cluster with MongoDB version 3.6.0 "\ + "or greater" raise SystemExit(errmsg) extra += self._get_ssl_server_args() @@ -1989,12 +2000,14 @@ def _construct_mongod(self, dbpath, logpath, port, replset=None, extra=''): if os.name == 'nt': newdbpath = dbpath.replace('\\', '\\\\') newlogpath = logpath.replace('\\', '\\\\') - command_str = ("start /b \"\" \"%s\" %s --dbpath \"%s\" --logpath \"%s\" --port %i " + command_str = ("start /b \"\" \"%s\" %s --dbpath \"%s\" " + " --logpath \"%s\" --port %i " "%s %s" % (os.path.join(path, 'mongod.exe'), rs_param, newdbpath, newlogpath, port, auth_param, extra)) else: - command_str = ("\"%s\" %s --dbpath \"%s\" --logpath \"%s\" --port %i --fork " + command_str = ("\"%s\" %s --dbpath \"%s\" --logpath \"%s\" " + "--port %i --fork " "%s %s" % (os.path.join(path, 'mongod'), rs_param, dbpath, logpath, port, auth_param, extra)) diff --git a/mtools/mlogfilter/mlogfilter.py b/mtools/mlogfilter/mlogfilter.py index aa9aa799..ad3d18cf 100644 --- a/mtools/mlogfilter/mlogfilter.py +++ b/mtools/mlogfilter/mlogfilter.py @@ -103,7 +103,8 @@ def _outputLine(self, logevent, length=None, human=False): if length: if len(line) > length: - line = line[:int(length / 2 - 2)] + '...' + line[int(-length/2 + 1):] + line = (line[:int(length / 2 - 2)] + '...' + + line[int(-length / 2 + 1):]) if human: line = self._changeMs(line) line = self._formatNumbers(line) @@ -196,20 +197,19 @@ def _merge_logfiles(self): while any(lines): min_line = min(lines, key=self._datetime_key_for_merge) - min_index = lines.index(min_line) + min_idx = lines.index(min_line) - if self.args['markers'][min_index]: - min_line.merge_marker_str = self.args['markers'][min_index] + if self.args['markers'][min_idx]: + min_line.merge_marker_str = self.args['markers'][min_idx] yield min_line - # update lines array with a new line from the min_index'th logfile - lines[min_index] = next(iter(self.args['logfile'][min_index]), None) - if lines[min_index] and lines[min_index].datetime: - lines[min_index]._datetime = (lines[min_index].datetime + - timedelta(hours=self - .args['timezone'] - [min_index])) + # update lines array with a new line from the min_idx'th logfile + lines[min_idx] = next(iter(self.args['logfile'][min_idx]), None) + if lines[min_idx] and lines[min_idx].datetime: + lines[min_idx]._datetime = ( + lines[min_idx].datetime + + timedelta(hours=self.args['timezone'][min_idx])) def logfile_generator(self): """Yield each line of the file, or the next line if several files.""" diff --git a/mtools/mloginfo/sections/connection_section.py b/mtools/mloginfo/sections/connection_section.py index c664fcae..44788088 100644 --- a/mtools/mloginfo/sections/connection_section.py +++ b/mtools/mloginfo/sections/connection_section.py @@ -102,7 +102,6 @@ def run(self): connections_start[connid] = dt - pos = line.find('end connection') if pos != -1: # connection was closed, increase counter @@ -193,11 +192,22 @@ def run(self): closed = ip_closed[ip] if ip in ip_closed else 0 if genstats: - covered_count = ipwise_count[ip] if ip in ipwise_count else 1 - connection_duration_ip = (ipwise_sum_durations[ip] - if ip in ipwise_sum_durations else 0) - ipwise_min_connection_duration_final = ipwise_min_connection_duration[ip] if ipwise_min_connection_duration[ip] != MIN_DURATION_EMPTY else 0 - ipwise_max_connection_duration_final = ipwise_max_connection_duration[ip] if ipwise_max_connection_duration[ip] != MAX_DURATION_EMPTY else 0 + covered_count = ( + ipwise_count[ip] + if ip in ipwise_count + else 1) + connection_duration_ip = ( + ipwise_sum_durations[ip] + if ip in ipwise_sum_durations + else 0) + ipwise_min_connection_duration_final = ( + ipwise_min_connection_duration[ip] + if ipwise_min_connection_duration[ip] != MIN_DURATION_EMPTY + else 0) + ipwise_max_connection_duration_final = ( + ipwise_max_connection_duration[ip] + if ipwise_max_connection_duration[ip] != MAX_DURATION_EMPTY + else 0) print("%-15s opened: %-8i closed: %-8i dur-avg(s): %-8i " "dur-min(s): %-8i dur-max(s): %-8i" diff --git a/mtools/mloginfo/sections/rs_info_section.py b/mtools/mloginfo/sections/rs_info_section.py index bcc61b21..ef195490 100644 --- a/mtools/mloginfo/sections/rs_info_section.py +++ b/mtools/mloginfo/sections/rs_info_section.py @@ -31,12 +31,15 @@ def run(self): print(" rs name: %s" % self.mloginfo.logfile.repl_set) print(" rs members: %s" % (self.mloginfo.logfile.repl_set_members - if self.mloginfo.logfile.repl_set_members else "unknown")) + if self.mloginfo.logfile.repl_set_members + else "unknown")) print(" rs version: %s" % (self.mloginfo.logfile.repl_set_version - if self.mloginfo.logfile.repl_set_version else "unknown")) + if self.mloginfo.logfile.repl_set_version + else "unknown")) print("rs protocol: %s" % (self.mloginfo.logfile.repl_set_protocol - if self.mloginfo.logfile.repl_set_protocol else "unknown")) + if self.mloginfo.logfile.repl_set_protocol + else "unknown")) else: print(" no rs info changes found") diff --git a/mtools/mplotqueries/mplotqueries.py b/mtools/mplotqueries/mplotqueries.py index 7256eca3..7ac1c70f 100644 --- a/mtools/mplotqueries/mplotqueries.py +++ b/mtools/mplotqueries/mplotqueries.py @@ -175,7 +175,10 @@ def parse_logevents(self): multiple_files = True self.args['group'] = 'filename' - self.plot_instance = self.plot_types[self.args['type']](args=self.args, unknown_args=self.unknown_args) + self.plot_instance = self.plot_types[ + self.args['type']]( + args=self.args, + unknown_args=self.unknown_args) for logfile in self.logfiles: @@ -586,7 +589,7 @@ def plot(self): # use timezone of first log file (may not always be what user wants # but must make a choice) tz = self.logfiles[0].timezone - tzformat = '%b %d\n%H:%M:%S' if tz == tzutc() else '%b %d\n%H:%M:%S%z' + # tzformat='%b %d\n%H:%M:%S' if tz == tzutc() else '%b %d\n%H:%M:%S%z' locator = AutoDateLocator(tz=tz, minticks=5, maxticks=10) formatter = AutoDateFormatter(locator, tz=tz) diff --git a/mtools/mplotqueries/plottypes/range_type.py b/mtools/mplotqueries/plottypes/range_type.py index cf8fcef3..9c678ec6 100644 --- a/mtools/mplotqueries/plottypes/range_type.py +++ b/mtools/mplotqueries/plottypes/range_type.py @@ -97,7 +97,7 @@ def plot_group(self, group, idx, axis): return artists def clicked(self, event): - group = event.artist._mt_group + # group = event.artist._mt_group print(num2date(event.artist._mt_left).strftime("%a %b %d %H:%M:%S") + ' - ' + num2date(event.artist._mt_right).strftime("%a %b %d %H:%M:%S")) diff --git a/mtools/mplotqueries/plottypes/scatter_type.py b/mtools/mplotqueries/plottypes/scatter_type.py index 16a45596..b5bda8a1 100644 --- a/mtools/mplotqueries/plottypes/scatter_type.py +++ b/mtools/mplotqueries/plottypes/scatter_type.py @@ -7,6 +7,7 @@ try: from matplotlib import __version__ as mpl_version + import matplotlib.pyplot as plt from matplotlib.dates import date2num from matplotlib.patches import Polygon diff --git a/mtools/test/test_mlaunch.py b/mtools/test/test_mlaunch.py index 1d4e764a..192a96d4 100644 --- a/mtools/test/test_mlaunch.py +++ b/mtools/test/test_mlaunch.py @@ -37,7 +37,6 @@ def __init__(self): self.use_auth = False self.data_dir = '' - def setup(self): """Start up method to create mlaunch tool and find free port.""" self.tool = MLaunchTool(test=True) @@ -705,7 +704,6 @@ def test_adding_default_user_no_mongos(self): for x in user['roles']]) == set(self.tool. _default_auth_roles)) - @attr('auth') def test_adding_custom_user(self): """mlaunch: test custom username and password and custom roles.""" diff --git a/mtools/test/test_mlaunch_commandlines.py b/mtools/test/test_mlaunch_commandlines.py index 1ca2b334..e6b44ea7 100644 --- a/mtools/test/test_mlaunch_commandlines.py +++ b/mtools/test/test_mlaunch_commandlines.py @@ -48,7 +48,7 @@ def read_config(self): def cmdlist_filter(self, cmdlist): """Filter command lines to contain only [mongod|mongos] --parameter.""" - # NOTE: The command "mongo was intentionally written with a leading quote + # NOTE: Command "mongo was intentionally written with a leading quote res = map(lambda cmd: set([param for param in cmd.split() if param.startswith('"mongo') or param.startswith('--')]), @@ -375,12 +375,12 @@ def test_sharded_two_mongos_csrs(self): self.raises_ioerror() else: cmdlist = ( - [set(['"mongod"', '--port', '--logpath', '--dbpath', '--configsvr', - '--fork', '--replSet'])] + - [set(['"mongod"', '--port', '--shardsvr', '--logpath', '--dbpath', - '--fork'])] * 2 + + [set(['"mongod"', '--port', '--logpath', '--dbpath', + '--configsvr', '--fork', '--replSet'])] + + [set(['"mongod"', '--port', '--shardsvr', '--logpath', + '--dbpath', '--fork'])] * 2 + [set(['"mongos"', '--port', '--logpath', '--configdb', - '--fork'])] * 2 + '--fork'])] * 2 ) self.cmdlist_assert(cmdlist) @@ -425,11 +425,12 @@ def test_default_single_3_4(self): self.raises_ioerror() else: cmdlist = ( - [set(['"mongod"', '--port', '--logpath', '--dbpath', '--configsvr', - '--fork', '--replSet'])] + - [set(['"mongod"', '--port', '--logpath', '--dbpath', '--shardsvr', - '--fork'])] * 2 + - [set(['"mongos"', '--port', '--logpath', '--configdb', '--fork'])] + [set(['"mongod"', '--port', '--logpath', '--dbpath', + '--configsvr', '--fork', '--replSet'])] + + [set(['"mongod"', '--port', '--logpath', '--dbpath', + '--shardsvr', '--fork'])] * 2 + + [set(['"mongos"', '--port', '--logpath', '--configdb', + '--fork'])] ) self.cmdlist_assert(cmdlist) @@ -506,9 +507,9 @@ def test_storageengine_3_4(self): self.run_tool('init --sharded 2 --replicaset --storageEngine mmapv1') cmdlist = ( [set(['"mongod"', '--port', '--logpath', '--dbpath', '--configsvr', - '--fork', '--replSet'])] + + '--fork', '--replSet'])] + [set(['"mongod"', '--port', '--logpath', '--dbpath', '--shardsvr', - '--fork', '--storageEngine'])] * 6 + + '--fork', '--storageEngine'])] * 6 + [set(['"mongos"', '--port', '--logpath', '--configdb', '--fork'])] ) self.cmdlist_assert(cmdlist) diff --git a/mtools/test/test_mloginfo.py b/mtools/test/test_mloginfo.py index 4db594e9..4c1b56e4 100644 --- a/mtools/test/test_mloginfo.py +++ b/mtools/test/test_mloginfo.py @@ -364,7 +364,6 @@ def test_rsstate_mongos_2(self): self._test_rsinfo(logfile_path, **{'rs name': None, 'rs version': None, 'rs members': None}) - def _test_rsinfo(self, logfile_path, **expected): """ utility test runner for rsstate """ diff --git a/mtools/test/test_util_presplit.py b/mtools/test/test_util_presplit.py index 84261f36..d1bd64ed 100644 --- a/mtools/test/test_util_presplit.py +++ b/mtools/test/test_util_presplit.py @@ -1,4 +1,4 @@ -import mtools.util.presplit +#import mtools.util.presplit def test_presplit(): diff --git a/mtools/util/cmdlinetool.py b/mtools/util/cmdlinetool.py index 44ac4a9c..d4214a76 100644 --- a/mtools/util/cmdlinetool.py +++ b/mtools/util/cmdlinetool.py @@ -203,7 +203,6 @@ def __init__(self, multiple_logfiles=False, stdin_allowed=True): self.argparser.add_argument('logfile', **arg_opts) - if __name__ == '__main__': tool = LogFileTool(multiple_logfiles=True, stdin_allowed=True) tool.run() diff --git a/tox.ini b/tox.ini index 3877e0de..2d61e4de 100644 --- a/tox.ini +++ b/tox.ini @@ -52,9 +52,12 @@ show-source = True # E123, E125 skipped as they are invalid PEP-8. # N802 skipped (function name should be lowercase) # N806 skipped (variable in function should be lowercase) -ignore = E123,E125,N802,N806 +# F401 skipped (imported but unused) after verifying current usage is valid +# W503 skipped line break before binary operator +# C901 skipped: 'MLaunchTool.init' is too complex +ignore = E123,E125,N802,N806,F401,W503,C901 builtins = _ -exclude=.venv,.git,.tox,dist,*lib/python*,*egg,*figures/*,__init__.py +exclude=.venv,.git,.tox,dist,*lib/python*,*egg,*figures/*,__init__.py,build/*,setup.py,mtools/util/*,mtools/test/test_* count = true statistics = true max-complexity = 49