Dropping support for python 3<x<3.6

pyleus/__init__.py

@@ -2,7 +2,7 @@
 
 import pkg_resources
 
-__version__ = '0.3.0'
+__version__ = '0.3.5'
 
 BASE_JAR = "pyleus-base.jar"
 BASE_JAR_PATH = pkg_resources.resource_filename('pyleus', BASE_JAR)

pyleus/cli/build.py

@@ -13,6 +13,7 @@
 import tempfile
 import yaml
 import zipfile
+import sys
 
 from pyleus import __version__
 from pyleus.cli.topology_spec import TopologySpec
@@ -27,6 +28,7 @@
 YAML_FILENAME = "pyleus_topology.yaml"
 DEFAULT_REQUIREMENTS_FILENAME = "requirements.txt"
 VIRTUALENV_NAME = "pyleus_venv"
+SYS_ENCODING = sys.getdefaultencoding()
 
 log = logging.getLogger(__name__)
 
@@ -96,7 +98,7 @@ def _remove_pyleus_base_jar(venv):
     virtualenv, and we don't want to attempt its removal in that case.
     """
     base_jar_path = venv.execute_module("pyleus._base_jar",
-                                        cwd=venv.path).strip()
+                                        cwd=venv.path).decode(SYS_ENCODING).strip()
     if _path_contained_by(venv.path, base_jar_path):
         os.remove(base_jar_path)
 

pyleus/cli/virtualenv_proxy.py

@@ -35,7 +35,8 @@ class VirtualenvProxy(object):
     def __init__(self, path,
                  system_site_packages=False,
                  pypi_index_url=None,
-                 use_wheel=True,
+                 # Default to false since --use-wheel flag is deprecated since pip 9.0.1
+                 use_wheel=False,
                  python_interpreter=None,
                  verbose=False):
         """Creates the virtualenv with the options specified"""
@@ -55,7 +56,7 @@ def __init__(self, path,
 
     def _create_virtualenv(self):
         """Creates the actual virtualenv"""
-        cmd = ["virtualenv", self.path]
+        cmd = ["virtualenv", "--always-copy", self.path]
         if self._system_site_packages:
             cmd.append("--system-site-packages")
 

pyleus/storm/component.py

@@ -146,7 +146,13 @@ def __init__(self, input_stream=None, output_stream=None):
             input_stream = sys.stdin
 
         if output_stream is None:
-            output_stream = sys.stdout
+            # Pyleus serializer outputs byte sequence; however, Python 3.6 sys.stdout expects unicode stream inputs.
+            # In order to write binary data to stdout, we use the underlying binary buffer object.
+            # Ref: https://docs.python.org/3/library/sys.html#sys.stderr
+            try:
+                output_stream = sys.stdout.buffer
+            except AttributeError:
+                output_stream = sys.stdout
 
         self._input_stream = input_stream
         self._output_stream = output_stream

pyleus/storm/serializers/json_serializer.py

@@ -1,10 +1,13 @@
 """JSON implementation of Pyleus serializer"""
 
 try:
-    import simplejson as json
-    _ = json # pyflakes
+    import ujson as json
 except ImportError:
-    import json
+    try:
+        import simplejson as json
+        _ = json # pyflakes
+    except ImportError:
+        import json
 
 from pyleus.storm import StormWentAwayError
 from pyleus.storm.serializers.serializer import Serializer

pyleus/storm/serializers/msgpack_serializer.py

@@ -9,7 +9,11 @@
 
 
 def _messages_generator(input_stream):
-    unpacker = msgpack.Unpacker()
+    # Python 3.6 strings are in unicode and are different from generic byte sequence (binary).
+    # As of v0.4.8, msgpack-python supports the distinction between str and bin types, but it is not enabled by default.
+    # To enable it requires using use_bin_type=True option in Packer and encoding="utf-8" option in Unpacker
+    # Ref: https://github.com/msgpack/msgpack-python#string-and-binary-type
+    unpacker = msgpack.Unpacker(encoding='utf-8')
     while True:
         # f.read(n) on sys.stdin blocks until n bytes are read, causing
         # serializer to hang.
@@ -46,5 +50,6 @@ def send_msg(self, msg_dict):
         """"Messages are delimited by msgapck itself, no need for Storm
         multilang end line.
         """
+        # Don't set use_bin_type=True because the receiving end (Java component) does not expect unicode strings
         msgpack.pack(msg_dict, self._output_stream)
         self._output_stream.flush()

tests/cli/build_test.py

@@ -149,7 +149,7 @@ def test__remove_pyleus_base_jar(self):
         mock_venv_path = "/path/to/venv"
 
         def mock_execute_module(module, cwd):
-            return "/path/to/venv/inside.jar"
+            return b"/path/to/venv/inside.jar"
 
         mock_venv = mock.Mock(
             path=mock_venv_path,
@@ -166,7 +166,7 @@ def test__remove_pyleus_base_jar_no_remove(self):
         mock_venv_path = "/path/to/venv"
 
         def mock_execute_module(module, cwd):
-            return "/foo/bar/outside.jar"
+            return b"/foo/bar/outside.jar"
 
         mock_venv = mock.Mock(
             path=mock_venv_path,

tests/cli/virtualenv_proxy_test.py

@@ -58,7 +58,7 @@ def test__create_virtualenv_no_options(
                                system_site_packages=False,
                                verbose=True)
         mock_cmd.assert_called_once_with(
-            ["virtualenv", VENV_PATH],
+            ["virtualenv", "--always-copy", VENV_PATH],
             stdout=venv._out_stream,
             stderr=venv._err_stream,
             err_msg=mock.ANY
@@ -72,22 +72,21 @@ def test__create_virtualenv_system_site_packages(
                                system_site_packages=True,
                                verbose=True)
         mock_cmd.assert_called_once_with(
-            ["virtualenv", VENV_PATH, "--system-site-packages"],
+            ["virtualenv", "--always-copy", VENV_PATH, "--system-site-packages"],
             stdout=venv._out_stream,
             stderr=venv._err_stream,
             err_msg=mock.ANY
         )
 
     @mock.patch.object(builtins, 'open', autospec=True)
     @mock.patch.object(virtualenv_proxy, '_exec_shell_cmd', autospec=True)
-    def test__create_virtualenv_python_interpreter(
+    def test__create_virtualenv_no_system_site_packages(
             self, mock_cmd, mock_open):
-        path = "python2.7"
         venv = VirtualenvProxy(VENV_PATH,
-                               python_interpreter=path,
+                               system_site_packages=False,
                                verbose=True)
         mock_cmd.assert_called_once_with(
-            ["virtualenv", VENV_PATH, "--python", path],
+            ["virtualenv", "--always-copy", VENV_PATH],
             stdout=venv._out_stream,
             stderr=venv._err_stream,
             err_msg=mock.ANY

tests/storm/serializers/msgpack_serializer_test.py

@@ -14,7 +14,7 @@ class TestMsgpackSerializer(SerializerTestCase):
 
     def test_read_msg_dict(self):
         msg_dict = {
-            b'hello': b"world",
+            'hello': "world",
         }
 
         encoded_msg = msgpack.packb(msg_dict)

topology_builder/src/main/java/com/yelp/pyleus/PythonComponentsFactory.java

@@ -11,6 +11,16 @@
 public class PythonComponentsFactory {
 
     public static final String VIRTUALENV_INTERPRETER = "pyleus_venv/bin/python";
+    // A few notes on how this works:
+    // (1) The developer of the topology is responsible for putting a newrelic.ini at the root of the topology folder,
+    // alongside with the topology definition file (.yml).
+    // (2) newrelic-admin only works when the python process it monitors uses the same python installation that
+    // installed newrelic-admin, as described here:
+    // https://docs.newrelic.com/docs/agents/python-agent/installation-configuration/python-agent-admin-script
+    // (3) newrelic-admin installed by Pyleus in the virtualenv actually points to the wrong python installation in a
+    // temporary folder. We work around that issue by run it with a specified python installation instead of
+    // running it directly as a self-executable script.
+    public static final String NEW_RELIC_PREFIX = String.format("NEW_RELIC_CONFIG_FILE=newrelic.ini %s pyleus_venv/bin/newrelic-admin run-program", VIRTUALENV_INTERPRETER);
     public static final String MODULE_OPTION = "-m";
 
     private String[] buildCommand(final String module, final Map<String, Object> argumentsMap,
@@ -23,7 +33,7 @@ private String[] buildCommand(final String module, final Map<String, Object> arg
 
         StringBuilder strBuf = new StringBuilder();
         // Done before launching any spout or bolt in order to cope with Storm permissions bug
-        strBuf.append(String.format("chmod 755 %s; %s", VIRTUALENV_INTERPRETER, VIRTUALENV_INTERPRETER));
+        strBuf.append(String.format("chmod 755 %s; %s %s", VIRTUALENV_INTERPRETER, NEW_RELIC_PREFIX, VIRTUALENV_INTERPRETER));
         strBuf.append(String.format(" %s %s", MODULE_OPTION, module));
 
         if (argumentsMap != null) {

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py26, py27, py31, py32, py33, py34
+envlist = py26, py27, py36
 # tox has no use_wheel option
 install_cmd = pip install --pre --use-wheel {opts} {packages}