Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix: #652: Resolve flake8 complaints #653

Merged
merged 1 commit into from
Jul 10, 2018
Merged

Fix: #652: Resolve flake8 complaints #653

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 53 additions & 40 deletions mtools/mlaunch/mlaunch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,8 +185,8 @@ def run(self, arguments=None):
# to run can call different sub-commands
self.argparser = argparse.ArgumentParser()
self.argparser.add_argument('--version', action='version',
version="mtools version {0} || Python {1}".format(
__version__, sys.version))
version="mtools version {0} || Python {1}".
format(__version__, sys.version))
self.argparser.add_argument('--no-progressbar', action='store_true',
default=False,
help='disables progress bar')
Expand Down Expand Up @@ -323,7 +323,9 @@ def run(self, arguments=None):
'required to run the stop command '
'(requires --auth, default="%s")'
% ' '.join(self._default_auth_roles)))
init_parser.add_argument('--auth-role-docs', action='store_true', default=False, help='auth-roles are json documents')
init_parser.add_argument('--auth-role-docs', action='store_true',
default=False,
help='auth-roles are json documents')
init_parser.add_argument('--no-initial-user', action='store_false',
default=True, dest='initial-user',
help=('create an initial user if auth is '
Expand Down Expand Up @@ -579,7 +581,7 @@ def init(self):

# add the 'csrs' parameter as default for MongoDB >= 3.3.0
if (LooseVersion(current_version) >= LooseVersion("3.3.0") or
LooseVersion(current_version) == LooseVersion("0.0.0")):
LooseVersion(current_version) == LooseVersion("0.0.0")):
self.args['csrs'] = True

# construct startup strings
Expand Down Expand Up @@ -664,7 +666,7 @@ def init(self):
print("adding shards.")

shard_conns_and_names = list(zip(self.shard_connection_str,
shard_names))
shard_names))
while True:
try:
nshards = con['config']['shards'].count()
Expand Down Expand Up @@ -733,7 +735,6 @@ def init(self):
raise RuntimeError("can't connect to server, so adding admin "
"user isn't possible")


roles = []
found_cluster_admin = False
if self.args['auth_role_docs']:
Expand Down Expand Up @@ -762,8 +763,11 @@ def init(self):
members = sorted(self.get_tagged([shard]))
if self.args['verbose']:
print("adding users to %s" % shard)
self._add_user(members[0], name=self.args['username'], password=self.args['password'],
database=self.args['auth_db'], roles=self.args['auth_roles'])
self._add_user(members[0],
name=self.args['username'],
password=self.args['password'],
database=self.args['auth_db'],
roles=self.args['auth_roles'])

if self.args['verbose']:
print("added user %s on %s database" % (self.args['username'],
Expand Down Expand Up @@ -1172,7 +1176,7 @@ def discover(self):

# tag all nodes with 'all'
self.cluster_tags['all'].extend(list(range(current_port,
current_port + num_nodes)))
current_port + num_nodes)))

# tag all nodes with their port number (as string) and whether
# they are running
Expand All @@ -1199,8 +1203,8 @@ def discover(self):
shard_names = [None]

for shard in shard_names:
port_range = list(range(current_port, current_port +
num_nodes_per_shard))
port_range = list(range(current_port,
current_port + num_nodes_per_shard))

# all of these are mongod nodes
self.cluster_tags['mongod'].extend(port_range)
Expand Down Expand Up @@ -1228,7 +1232,7 @@ def discover(self):
(itemgetter(1),
mrsc.secondaries)))
self.cluster_tags['arbiter'].extend(list(map(itemgetter(1),
mrsc.arbiters)))
mrsc.arbiters)))

# secondaries in cluster_tree (order is now important)
self.cluster_tree.setdefault('secondary', [])
Expand Down Expand Up @@ -1507,21 +1511,23 @@ def _filter_valid_arguments(self, arguments, binary="mongod",
result = []
for i, arg in enumerate(arguments):
if arg.startswith('-'):
# check if the binary accepts this argument or special case -vvv for any number of v
# check if the binary accepts this argument
# or special case -vvv for any number of v
if arg in accepted_arguments or re.match(r'-v+', arg):
result.append(arg)
elif binary.endswith('mongod') and arg in self.UNDOCUMENTED_MONGOD_ARGS:
elif (binary.endswith('mongod') and
arg in self.UNDOCUMENTED_MONGOD_ARGS):
result.append(arg)
elif self.ignored_arguments.get(binary + arg) is None:
# warn once for each combination of binary and unknown arg
self.ignored_arguments[binary + arg] = True
print("warning: ignoring unknown argument %s for %s" % (arg, binary))
print("warning: ignoring unknown argument %s for %s" %
(arg, binary))
elif i > 0 and arguments[i - 1] in result:
# if it doesn't start with a '-', it could be the value of
# the last argument, e.g. `--slowms 1000`
result.append(arg)


# return valid arguments as joined string
return ' '.join(result)

Expand Down Expand Up @@ -1618,16 +1624,13 @@ def _start_on_ports(self, ports, wait=False, override_auth=False):

try:
if os.name == 'nt':
ret = subprocess.check_call(command_str,
shell=True)
subprocess.check_call(command_str, shell=True)
# create sub process on windows doesn't wait for output,
# wait a few seconds for mongod instance up
time.sleep(5)
else:
ret = subprocess.check_output([command_str],
stderr=subprocess.STDOUT,
shell=True)

subprocess.check_output([command_str], shell=True,
stderr=subprocess.STDOUT)

binary = command_str.split()[0]
if '--configsvr' in command_str:
Expand All @@ -1648,7 +1651,6 @@ def _start_on_ports(self, ports, wait=False, override_auth=False):
if wait:
self.wait_for(ports)


def _initiate_replset(self, port, name, maxwait=30):
"""Initiate replica set."""
if not self.args['replicaset'] and name != 'configRepl':
Expand Down Expand Up @@ -1677,7 +1679,7 @@ def _initiate_replset(self, port, name, maxwait=30):
print("replica set '%s' initialized." % name)

def _add_user(self, port, name, password, database, roles):
con = self.client('localhost:%i'%port)
con = self.client('localhost:%i' % port)
v = con['admin'].command('isMaster').get('maxWireVersion', 0)
if v >= 7:
# Until drivers have implemented SCRAM-SHA-256, use old mechanism.
Expand All @@ -1689,7 +1691,8 @@ def _add_user(self, port, name, password, database, roles):
password = None

try:
con[database].add_user(name, password=password, roles=roles, **opts)
con[database].add_user(name, password=password, roles=roles,
**opts)
except OperationFailure as e:
raise e
except TypeError as e:
Expand Down Expand Up @@ -1754,9 +1757,9 @@ def _wait_for_primary(self):
# update cluster tags now that we have a primary
self.cluster_tags['primary'].append(mrsc.primary[1])
self.cluster_tags['secondary'].extend(list(map(itemgetter(1),
mrsc.secondaries)))
mrsc.secondaries)))
self.cluster_tags['arbiter'].extend(list(map(itemgetter(1),
mrsc.arbiters)))
mrsc.arbiters)))

# secondaries in cluster_tree (order is now important)
self.cluster_tree.setdefault('secondary', [])
Expand Down Expand Up @@ -1812,20 +1815,25 @@ def _construct_sharded(self):
# create shards as stand-alones or replica sets
nextport = self.args['port'] + num_mongos
for shard in shard_names:
if (self.args['single']
and LooseVersion(current_version) >= LooseVersion("3.6.0")):
errmsg = " \n * In MongoDB 3.6 and above a Shard must be made up of a replica set. Please use --replicaset option when starting a sharded cluster.*"
if (self.args['single'] and
LooseVersion(current_version) >= LooseVersion("3.6.0")):
errmsg = " \n * In MongoDB 3.6 and above a Shard must be " \
"made up of a replica set. Please use --replicaset " \
"option when starting a sharded cluster.*"
raise SystemExit(errmsg)

elif (self.args['single']
and LooseVersion(current_version) < LooseVersion("3.6.0")):
elif (self.args['single'] and
LooseVersion(current_version) < LooseVersion("3.6.0")):
self.shard_connection_str.append(
self._construct_single(self.dir, nextport, name=shard, extra='--shardsvr'))
self._construct_single(
self.dir, nextport, name=shard, extra='--shardsvr'))
nextport += 1
elif self.args['replicaset']:
self.shard_connection_str.append(
self._construct_replset(self.dir, nextport, shard, num_nodes=list(range(self.args['nodes'])),
arbiter=self.args['arbiter'], extra='--shardsvr'))
self._construct_replset(
self.dir, nextport, shard,
num_nodes=list(range(self.args['nodes'])),
arbiter=self.args['arbiter'], extra='--shardsvr'))
nextport += self.args['nodes']
if self.args['arbiter']:
nextport += 1
Expand Down Expand Up @@ -1926,8 +1934,8 @@ def _construct_config(self, basedir, port, name=None, isreplset=False):
if isreplset:
return self._construct_replset(basedir=basedir, portstart=port,
name=name,
num_nodes=list(range(self
.args['config'])),
num_nodes=list(range(
self.args['config'])),
arbiter=False, extra='--configsvr')
else:
datapath = self._create_paths(basedir, name)
Expand Down Expand Up @@ -1980,7 +1988,10 @@ def _construct_mongod(self, dbpath, logpath, port, replset=None, extra=''):
and (self.args['sharded'] or self.args['replicaset'])
and '--bind_ip' not in extra):
os.removedirs(dbpath)
errmsg = " \n * If hostname is specified, please include '--bind_ip_all' or '--bind_ip' options when deploying replica sets or sharded cluster with MongoDB version 3.6.0 or greater"
errmsg = " \n * If hostname is specified, please include "\
"'--bind_ip_all' or '--bind_ip' options when deploying "\
"replica sets or sharded cluster with MongoDB version 3.6.0 "\
"or greater"
raise SystemExit(errmsg)

extra += self._get_ssl_server_args()
Expand All @@ -1989,12 +2000,14 @@ def _construct_mongod(self, dbpath, logpath, port, replset=None, extra=''):
if os.name == 'nt':
newdbpath = dbpath.replace('\\', '\\\\')
newlogpath = logpath.replace('\\', '\\\\')
command_str = ("start /b \"\" \"%s\" %s --dbpath \"%s\" --logpath \"%s\" --port %i "
command_str = ("start /b \"\" \"%s\" %s --dbpath \"%s\" "
" --logpath \"%s\" --port %i "
"%s %s" % (os.path.join(path, 'mongod.exe'),
rs_param, newdbpath, newlogpath, port,
auth_param, extra))
else:
command_str = ("\"%s\" %s --dbpath \"%s\" --logpath \"%s\" --port %i --fork "
command_str = ("\"%s\" %s --dbpath \"%s\" --logpath \"%s\" "
"--port %i --fork "
"%s %s" % (os.path.join(path, 'mongod'), rs_param,
dbpath, logpath, port, auth_param,
extra))
Expand Down
22 changes: 11 additions & 11 deletions mtools/mlogfilter/mlogfilter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,8 @@ def _outputLine(self, logevent, length=None, human=False):

if length:
if len(line) > length:
line = line[:int(length / 2 - 2)] + '...' + line[int(-length/2 + 1):]
line = (line[:int(length / 2 - 2)] + '...' +
line[int(-length / 2 + 1):])
if human:
line = self._changeMs(line)
line = self._formatNumbers(line)
Expand Down Expand Up @@ -196,20 +197,19 @@ def _merge_logfiles(self):

while any(lines):
min_line = min(lines, key=self._datetime_key_for_merge)
min_index = lines.index(min_line)
min_idx = lines.index(min_line)

if self.args['markers'][min_index]:
min_line.merge_marker_str = self.args['markers'][min_index]
if self.args['markers'][min_idx]:
min_line.merge_marker_str = self.args['markers'][min_idx]

yield min_line

# update lines array with a new line from the min_index'th logfile
lines[min_index] = next(iter(self.args['logfile'][min_index]), None)
if lines[min_index] and lines[min_index].datetime:
lines[min_index]._datetime = (lines[min_index].datetime +
timedelta(hours=self
.args['timezone']
[min_index]))
# update lines array with a new line from the min_idx'th logfile
lines[min_idx] = next(iter(self.args['logfile'][min_idx]), None)
if lines[min_idx] and lines[min_idx].datetime:
lines[min_idx]._datetime = (
lines[min_idx].datetime +
timedelta(hours=self.args['timezone'][min_idx]))

def logfile_generator(self):
"""Yield each line of the file, or the next line if several files."""
Expand Down
22 changes: 16 additions & 6 deletions mtools/mloginfo/sections/connection_section.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,6 @@ def run(self):

connections_start[connid] = dt


pos = line.find('end connection')
if pos != -1:
# connection was closed, increase counter
Expand Down Expand Up @@ -193,11 +192,22 @@ def run(self):
closed = ip_closed[ip] if ip in ip_closed else 0

if genstats:
covered_count = ipwise_count[ip] if ip in ipwise_count else 1
connection_duration_ip = (ipwise_sum_durations[ip]
if ip in ipwise_sum_durations else 0)
ipwise_min_connection_duration_final = ipwise_min_connection_duration[ip] if ipwise_min_connection_duration[ip] != MIN_DURATION_EMPTY else 0
ipwise_max_connection_duration_final = ipwise_max_connection_duration[ip] if ipwise_max_connection_duration[ip] != MAX_DURATION_EMPTY else 0
covered_count = (
ipwise_count[ip]
if ip in ipwise_count
else 1)
connection_duration_ip = (
ipwise_sum_durations[ip]
if ip in ipwise_sum_durations
else 0)
ipwise_min_connection_duration_final = (
ipwise_min_connection_duration[ip]
if ipwise_min_connection_duration[ip] != MIN_DURATION_EMPTY
else 0)
ipwise_max_connection_duration_final = (
ipwise_max_connection_duration[ip]
if ipwise_max_connection_duration[ip] != MAX_DURATION_EMPTY
else 0)

print("%-15s opened: %-8i closed: %-8i dur-avg(s): %-8i "
"dur-min(s): %-8i dur-max(s): %-8i"
Expand Down
9 changes: 6 additions & 3 deletions mtools/mloginfo/sections/rs_info_section.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,15 @@ def run(self):
print(" rs name: %s" % self.mloginfo.logfile.repl_set)
print(" rs members: %s"
% (self.mloginfo.logfile.repl_set_members
if self.mloginfo.logfile.repl_set_members else "unknown"))
if self.mloginfo.logfile.repl_set_members
else "unknown"))
print(" rs version: %s"
% (self.mloginfo.logfile.repl_set_version
if self.mloginfo.logfile.repl_set_version else "unknown"))
if self.mloginfo.logfile.repl_set_version
else "unknown"))
print("rs protocol: %s"
% (self.mloginfo.logfile.repl_set_protocol
if self.mloginfo.logfile.repl_set_protocol else "unknown"))
if self.mloginfo.logfile.repl_set_protocol
else "unknown"))
else:
print(" no rs info changes found")
7 changes: 5 additions & 2 deletions mtools/mplotqueries/mplotqueries.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,10 @@ def parse_logevents(self):
multiple_files = True
self.args['group'] = 'filename'

self.plot_instance = self.plot_types[self.args['type']](args=self.args, unknown_args=self.unknown_args)
self.plot_instance = self.plot_types[
self.args['type']](
args=self.args,
unknown_args=self.unknown_args)

for logfile in self.logfiles:

Expand Down Expand Up @@ -586,7 +589,7 @@ def plot(self):
# use timezone of first log file (may not always be what user wants
# but must make a choice)
tz = self.logfiles[0].timezone
tzformat = '%b %d\n%H:%M:%S' if tz == tzutc() else '%b %d\n%H:%M:%S%z'
# tzformat='%b %d\n%H:%M:%S' if tz == tzutc() else '%b %d\n%H:%M:%S%z'

locator = AutoDateLocator(tz=tz, minticks=5, maxticks=10)
formatter = AutoDateFormatter(locator, tz=tz)
Expand Down
2 changes: 1 addition & 1 deletion mtools/mplotqueries/plottypes/range_type.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ def plot_group(self, group, idx, axis):
return artists

def clicked(self, event):
group = event.artist._mt_group
# group = event.artist._mt_group
print(num2date(event.artist._mt_left).strftime("%a %b %d %H:%M:%S") +
' - ' +
num2date(event.artist._mt_right).strftime("%a %b %d %H:%M:%S"))
1 change: 1 addition & 0 deletions mtools/mplotqueries/plottypes/scatter_type.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

try:
from matplotlib import __version__ as mpl_version
import matplotlib.pyplot as plt
from matplotlib.dates import date2num
from matplotlib.patches import Polygon

Expand Down
Loading