Replace python mechanize with SideFX download API

.travis.yml

@@ -73,6 +73,8 @@ env:
   - ABI=5 BLOSC=no RELEASE=no HOUDINI_MAJOR=none
   - ABI=4 BLOSC=no RELEASE=no HOUDINI_MAJOR=none
   - ABI=3 BLOSC=no RELEASE=no HOUDINI_MAJOR=none
+  - secure: bTVyqXEIfdhhjJiHI1ELq766B711W41/2fHKRjFjPUIeEYeD8LrG/MUrGc9Jst/OYWdEcFN329/vjjTNYMKk7DOaOMwlpPap1XmdKSvk0S1Zq80M1j8XWyClkW5UEk5/DZAdtw18k5P8rnjptzxro6BoxPIaQ3P+0UUAOW3pASWGbV8LvNHfzTdTj8OV8RIxm5MV5dlYOj7fDjEViBWjopJ+gcnBBDrtGOpCWSGbnz1WBolym41cMPc8SAJXXzDK6mXy2k4Gpuk23kkKqmZJA9GbPnv/E5VITG2CTzu9sE3XQpPuHa1x7JFvSXm62eo+Gex6d1uixXsrP5rH59eMqPCIAVrmCCfwNn/JBmGNSob1Hp03TDOJ4cw1B69+WI+CrnqNDUAL/FviUIC4mpVNj/Xdoe4fWrCmg6GXEVYbacQcTHmXz+LwjJUDNKnMFwb3w0uimPMCiWUFHr6l73rdixICxYOIWMPZeSzsftV2WidteZPPx9nByGLIDfldXhUuBfOCgxfTYwKiHRExlV2ShHSN38XNwA2qVcuD4JH1DOhi29Pb0+MiJZmGKr6PwMPPtfkzJkQs5FhlUSW1exU4CzPm0gbSd2fGdmRq3nWTk3wPq5cuNc017LAX5Mu9k0DLevTLw5NFwmn17+tDF0Nab6RiA2T1nOMuHFSkeDYzdSY=
+  - secure: qmyZOe7zNIdkY5102R4BA5LzB2Jv3byH45TfGkkeWXhernECAef5CEAydMxHg+YShNX5KDUzGi5c0RrECxYpFSgL2Zvh58GSROXsZ22xpAE/HxmZRSFfpVsnfOqQZu3hWa+9zIUzVjawpaPsnCeuBedDwQd/q9AZccQJFVRDirLGNeSWyiLX719f02ogxPyxorwrb4DDGSPT4tXRyAseKnczDoCl68gTnF9ucfaaLAVAeG6dkLTyX+tcaYSxfVrKz8pJJiP/MdRqlRvOAsO5zqnEKevKkEU7+J1/QkK4yUjfWrT/eZeZ8nTVFR8QM2srbN+niyObFUZTohcSzs69vdHpl0myxiRy89Z5cg1mRzuWJBqqxBLh4ducco52UIGkZDko5yXLmbTLcHy4Zv4JnL5xJM4j7azUO0oizmA3WqKicxOVB0llmPVlaQLJ/xJK/VP1lsQAU1HfFV53W0Qs/GY98FsaOKn/SVl8fbhcBRon50PR3afoI11MiBddK5LeVazIGpeV0jCqaLkJNlqKSuEwiZUezt9MgEToQDdAxC1y3vFn42SMstQhkzgo5GHWXjTs4uICjwsnebhkYF1/nyzZNWruFWqX3PeljtyqTs30X6zWA3rpcbXpnxEeS4sk2IVWaTRP7WGzb0sDyHAxqaOlq6RgJ95vm74qoZrZMQw=
 
 # Build and install all library dependencies for OpenVDB
 # (build will error if this stage does not succeed)

travis/sidefx_api.py

@@ -0,0 +1,143 @@
+import time
+import json
+import base64
+try:
+    import html.parser as HTMLParser
+except ImportError:
+    import HTMLParser
+import requests
+
+
+# Code that provides convenient Python wrappers to call into the API:
+
+def service(
+        access_token_url, client_id, client_secret_key, endpoint_url,
+        access_token=None, access_token_expiry_time=None):
+    if (access_token is None or
+            access_token_expiry_time is None or
+            access_token_expiry_time < time.time()):
+        access_token, access_token_expiry_time = (
+            get_access_token_and_expiry_time(
+                access_token_url, client_id, client_secret_key))
+
+    return _Service(
+        endpoint_url, access_token, access_token_expiry_time)
+
+
+class _Service(object):
+    def __init__(
+            self, endpoint_url, access_token, access_token_expiry_time):
+        self.endpoint_url = endpoint_url
+        self.access_token = access_token
+        self.access_token_expiry_time = access_token_expiry_time
+
+    def __getattr__(self, attr_name):
+        return _APIFunction(attr_name, self)
+
+
+class _APIFunction(object):
+    def __init__(self, function_name, service):
+        self.function_name = function_name
+        self.service = service
+
+    def __getattr__(self, attr_name):
+        # This isn't actually an API function, but a family of them.  Append
+        # the requested function name to our name.
+        return _APIFunction(
+            "{0}.{1}".format(self.function_name, attr_name), self.service)
+
+    def __call__(self, *args, **kwargs):
+        return call_api_with_access_token(
+            self.service.endpoint_url, self.service.access_token,
+            self.function_name, args, kwargs)
+
+#---------------------------------------------------------------------------
+# Code that implements authentication and raw calls into the API:
+
+
+def get_access_token_and_expiry_time(
+        access_token_url, client_id, client_secret_key):
+    """Given an API client (id and secret key) that is allowed to make API
+    calls, return an access token that can be used to make calls.
+    """
+    response = requests.post(
+        access_token_url,
+        headers={
+            "Authorization": u"Basic {0}".format(
+                base64.b64encode(
+                    "{0}:{1}".format(
+                        client_id, client_secret_key
+                    ).encode()
+                ).decode('utf-8')
+            ),
+        })
+    if response.status_code != 200:
+        raise AuthorizationError(
+            response.status_code,
+            "{0}: {1}".format(
+                response.status_code,
+                _extract_traceback_from_response(response)))
+
+    response_json = response.json()
+    access_token_expiry_time = time.time() - 2 + response_json["expires_in"]
+    return response_json["access_token"], access_token_expiry_time
+
+
+class AuthorizationError(Exception):
+    """Raised from the client if the server generated an error while generating
+    an access token.
+    """
+    def __init__(self, http_code, message):
+        super(AuthorizationError, self).__init__(message)
+        self.http_code = http_code
+
+
+def call_api_with_access_token(
+        endpoint_url, access_token, function_name, args, kwargs):
+    """Call into the API using an access token that was returned by
+    get_access_token.
+    """
+    response = requests.post(
+        endpoint_url,
+        headers={
+            "Authorization": "Bearer " + access_token,
+        },
+        data=dict(
+            json=json.dumps([function_name, args, kwargs]),
+        ))
+    if response.status_code == 200:
+        return response.json()
+
+    raise APIError(
+        response.status_code,
+        "{0}".format(_extract_traceback_from_response(response)))
+
+
+class APIError(Exception):
+    """Raised from the client if the server generated an error while calling
+    into the API.
+    """
+    def __init__(self, http_code, message):
+        super(APIError, self).__init__(message)
+        self.http_code = http_code
+
+
+def _extract_traceback_from_response(response):
+    """Helper function to extract a traceback from the web server response
+    if an API call generated a server-side exception
+    """
+    error_message = response.text
+    if response.status_code != 500:
+        return error_message
+
+    traceback = ""
+    for line in error_message.split("\n"):
+        if len(traceback) != 0 and line == "</textarea>":
+            break
+        if line == "Traceback:" or len(traceback) != 0:
+            traceback += line + "\n"
+
+    if len(traceback) == 0:
+        traceback = error_message
+
+    return HTMLParser.HTMLParser().unescape(traceback)

travis/travis.py

@@ -28,55 +28,41 @@
 #
 # Author: Dan Bailey
 
-import mechanize
+import requests
 import sys
 import re
-import exceptions
+import shutil
+from sidefx_api import service
+
 
 # this argument is for the major.minor version of Houdini to download (such as 15.0, 15.5, 16.0)
 version = sys.argv[1]
 
 if not re.match('[0-9][0-9]\.[0-9]$', version):
     raise IOError('Invalid Houdini Version "%s", expecting in the form "major.minor" such as "16.0"' % version)
 
-br = mechanize.Browser()
-br.set_handle_robots(False)
-
-# login to sidefx.com as openvdb
-br.open('https://www.sidefx.com/login/?next=/download/daily-builds')
-br.select_form(nr=0)
-br.form['username'] = 'openvdb'
-br.form['password'] = 'L3_M2f2W'
-br.submit()
+sidefx_client_id = sys.argv[2]
+sidefx_secret_key = sys.argv[3]
 
-# retrieve download id
-br.open('http://www.sidefx.com/download/daily-builds/')
+sidefx_service = service(
+    access_token_url='https://www.sidefx.com/oauth2/application_token',
+    client_id=sidefx_client_id,
+    client_secret_key=sidefx_secret_key,
+    endpoint_url='https://www.sidefx.com/api/',
+)
 
-for link in br.links():
-    if not link.url.startswith('/download/download-houdini'):
-        continue
-    if link.text.startswith('houdini-%s' % version) and 'linux_x86_64' in link.text:
-        response = br.follow_link(text=link.text, nr=0)
-        url = response.geturl()
-        id = url.split('/download-houdini/')[-1]
-        break
 
-# accept eula terms
-#url = 'https://www.sidefx.com/download/eula/accept/?next=/download/download-houdini/%sget/' % id
-#br.open(url)
-#br.select_form(nr=0)
-#br.form.find_control('terms').items[1].selected=True
-#br.submit()
+release_list = service.download.get_daily_builds_list(
+    product='houdini', version=version, platform='linux')
+latest_release = service.download.get_daily_builds_download(
+    product='houdini', version=version, build=release_list[0]['build'], platform='linux')
 
-# download houdini tarball in 50MB chunks
-url = 'https://www.sidefx.com/download/download-houdini/%sget/' % id
-response = br.open(url)
-mb = 1024*1024
-chunk = 50
-size = 0
-file = open('hou.tar.gz', 'wb')
-for bytes in iter((lambda: response.read(chunk*mb)), ''):
-    size += 50
-    print 'Read: %sMB' % size
-    file.write(bytes)
-file.close()
+# Download the file
+local_filename = latest_release['filename']
+r = requests.get(latest_release['download_url'], stream=True)
+if r.status_code == 200:
+    with open(local_filename, 'wb') as f:
+        r.raw.decode_content = True
+        shutil.copyfileobj(r.raw, f)
+else:
+    raise Exception('Error downloading file!')

travis/travis.run

@@ -117,10 +117,10 @@ if [ "$TASK" = "install" ]; then
         # install houdini pre-requisites
         sudo apt-get install -y libxi-dev
         sudo apt-get install -y csh
-        sudo apt-get install python-mechanize
+        sudo apt-get install python-requests
         export PYTHONPATH=${PYTHONPATH}:/usr/lib/python2.7/dist-packages
         # download and unpack latest houdini headers and libraries from daily-builds
-        python travis/travis.py $HOUDINI_MAJOR
+        python travis/travis.py $HOUDINI_MAJOR $SIDEFX_CLIENT_ID $SIDEFX_SECRET_KEY
         tar -xzf hou.tar.gz
         ln -s houdini* hou
         cd hou