Merge branch 'release-1.1.6'

CHANGES.md

@@ -1,6 +1,25 @@
 Changes to mtools
 =================
 
+#### version 1.1.6
+
+  * mlogfilter: `--thread` now also matches "connection accepted" lines for that connection (#218, #219)
+  * mlogfilter: fixed bug that would print milliseconds in timestamp twice in 2.6 format for UTC timezone (#241)
+  * mlaunch: allow overriding hostname for replica set setup (#256)
+  * mlaunch: added a `restart` command (#253)
+  * mlaunch: added `--startup` to `list` command to show all startup strings (#257) 
+  * mlaunch: aliased `--verbose` (now depricated) as `--tags` (#257)
+  * mloginfo: added `--rsstate` option to show all replica set state changes in log file. Added by @jimoleary (#228)
+  * mloginfo: fixed issues with 95-percentile calculation. Added by @gianpaj (#238)
+  * mloginfo: show host name and port if available (#247)
+  * mloginfo: fixed bug where empty lines can't be parsed (#213)
+  * mloginfo: show milliseconds for start/end (#245)
+  * mloginfo: made numpy dependency optional for mloginfo. Added by @brondsem (#216)
+  * mplotqueries: option to write output to image file instead of interactive mode. Added by @dpercy (#266)
+  * mplotqueries: show correct timezone for time axis (#274)
+  * mplotqueries: added option to allow fixing y-axis to specific min/max values (#214)
+
+
 #### version 1.1.5
 
   * added workaround for compile errors with XCode 5.1 / clang 3.4 (#203)

README.md

@@ -46,7 +46,7 @@ instructions for these modules.
 Recent Changes
 --------------
 
-The current version of mtools is 1.1.5. See [CHANGES.md](./CHANGES.md) for a list of recent changes from previous versions of mtools.
+The current version of mtools is 1.1.6. See [CHANGES.md](./CHANGES.md) for a list of recent changes from previous versions of mtools.
 
 
 Contribute to mtools

mtools/mgenerate/mgenerate.py

@@ -4,6 +4,7 @@
 import bson
 import sys
 import inspect 
+from datetime import datetime
 from multiprocessing import Process, cpu_count
 
 try:
@@ -20,17 +21,31 @@
 from mtools.util.cmdlinetool import BaseCmdLineTool
 
 
+class DateTimeEncoder(json.JSONEncoder):
+    """ custom datetime encoder for json output. """
+    def default(self, obj):
+        if isinstance(obj, datetime):
+            return obj.isoformat()
+        try:
+            res = json.JSONEncoder.default(self, obj)
+        except TypeError:
+            res = str(obj)
+        return res
+
+
 class InsertProcess(Process):
 
-    def __init__(self, number, template, collection, stdout):
+    operator_classes = inspect.getmembers(operators, inspect.isclass)
+
+    def __init__(self, number, template, collection, args):
         Process.__init__(self)
         self.number = number
         self.template = template
         self.collection = collection
-        self.stdout = stdout
+        self.args = args
 
         # add all operators classes from the operators module, pass in _decode method
-        self.operators = [c[1](self._decode) for c in inspect.getmembers(operators, inspect.isclass)]
+        self.operators = [c[1](self._decode) for c in self.operator_classes]
 
         self.string_operators = {}
         self.dict_operators = {}
@@ -52,8 +67,10 @@ def run(self):
             # decode the template
             doc = self._decode(self.template)
 
-            if self.stdout:
-                print doc
+            if not self.collection:
+                indent = 4 if self.args['pretty'] else None
+                print json.dumps(doc, cls=DateTimeEncoder, indent=indent, ensure_ascii=False) 
+
             else:
                 batch.append(doc)
                 batchsize += self.bsonsize(doc)
@@ -63,7 +80,7 @@ def run(self):
                     batch = []
                     batchsize = 0
 
-        if not self.stdout:
+        if self.collection:
             if batch:
                 self.collection.insert(batch)
 
@@ -145,8 +162,9 @@ def __init__(self):
         self.argparser.add_argument('--collection', '-c', action='store', metavar='C', default='mgendata', help='collection C to import data, default=mgendata')
         self.argparser.add_argument('--drop', action='store_true', default=False, help='drop collection before inserting data')
         self.argparser.add_argument('--stdout', action='store_true', default=False, help='prints data to stdout instead of inserting to mongod/s instance.')
+        self.argparser.add_argument('--pretty', action='store_true', default=False, help="if set, prettyfies the output to stdout (indented), requires --stdout")
         self.argparser.add_argument('--write-concern', '-w', action='store', metavar="W", default=1, help='write concern for inserts, default=1')
-
+        self.argparser.add_argument('--processes', '-p', action='store', type=int, default=0, help='specify number of processes (# cpus by default)')
 
     def run(self, arguments=None):
         BaseCmdLineTool.run(self, arguments)
@@ -174,16 +192,24 @@ def run(self, arguments=None):
             col = mc[self.args['database']][self.args['collection']]
             if self.args['drop']:
                 col.drop()
+        else:
+            col = None
 
         # divide work over number of cores
-        num_cores = 1 if self.args['stdout'] else cpu_count()
+        if self.args['stdout']:
+            num_cores = 1
+        elif self.args['processes'] > 0:
+            num_cores = self.args['processes']
+        else:
+            num_cores = cpu_count()
+
         num_list = [self.args['number'] // num_cores] * num_cores
         num_list[0] += self.args['number'] % num_cores
 
         processes = []
 
         for n in num_list:
-            p = InsertProcess(n, template, col, self.args['stdout'])
+            p = InsertProcess(n, template, col, self.args)
             p.start()
             processes.append(p)
 

mtools/mgenerate/operators/__init__.py

@@ -1 +1 @@
-from operators import ChooseOperator, MissingOperator, NumberOperator, FloatOperator, ObjectIdOperator, ArrayOperator, DateTimeOperator, StringOperator, IncOperator
+from operators import ChooseOperator, MissingOperator, NumberOperator, FloatOperator, ObjectIdOperator, ArrayOperator, DateTimeOperator, StringOperator, IncOperator, CoordinateOperator, PointOperator

mtools/mgenerate/operators/operators.py

@@ -8,6 +8,7 @@
 
 import time
 import string
+import itertools
 
 
 class BaseOperator(object):
@@ -19,7 +20,6 @@ class BaseOperator(object):
     def __init__(self, decode_method):
         self._decode = decode_method
 
-
     def _parse_options(self, options={}):
         parsed = self.defaults.copy()
 
@@ -84,19 +84,25 @@ def __call__(self, options=None):
         return val
 
 
-
 class IncOperator(BaseOperator):
 
-    dict_format = False
+    dict_format = True
     string_format = True
     names = ['$inc']
-    value = -1
+    defaults = OrderedDict([ ('start', 0), ('step', 1) ])
+
+    def __init__(self, decode_method):
+        self.counter = None
+        BaseOperator.__init__(self, decode_method)
 
     def __call__(self, options=None):
         options = self._parse_options(options)
+
+        # initialize counter on first use (not threadsafe!)
+        if self.counter == None:
+            self.counter = itertools.count(options['start'], options['step'])
 
-        self.value += 1
-        return self.value
+        return self.counter.next()
 
 
 class StringOperator(BaseOperator):
@@ -193,6 +199,42 @@ def __call__(self, options=None):
         return [ options['of'] ] * number
 
 
+class CoordinateOperator(BaseOperator):
+
+    dict_format = True
+    string_format = True
+    names = ['$coordinates', '$coordinate', '$coord']
+    defaults = OrderedDict([ ('long_lim', [-180, 180]), ('lat_lim', [-90, 90]) ])
+
+    def __call__(self, options=None):
+        options = self._parse_options(options)
+
+        # evaluate limits
+        long_lim = self._decode(options['long_lim'])
+        lat_lim = self._decode(options['lat_lim'])
+
+        # return coordinate by using random numbers between limits
+        return [ {"$float": long_lim }, {"$float": lat_lim }]
+
+
+class PointOperator(BaseOperator):
+
+    dict_format = True
+    string_format = True
+    names = ['$point']
+    defaults = OrderedDict([ ('long_lim', [-180, 180]), ('lat_lim', [-90, 90]) ])
+
+    def __call__(self, options=None):
+        options = self._parse_options(options)
+
+        # evaluate limits
+        long_lim = self._decode(options['long_lim'])
+        lat_lim = self._decode(options['lat_lim'])
+
+        # return coordinate by using random numbers between limits
+        return { "type": "Point", "coordinates": { "$coord": [long_lim, lat_lim] } }
+
+
 class DateTimeOperator(BaseOperator):
 
     dict_format = True