From 0ff297ccd29c2edb3f89ad6bd0b119deb19c17c9 Mon Sep 17 00:00:00 2001 From: Remi Date: Mon, 11 Feb 2019 17:59:15 -0500 Subject: [PATCH] Replace python mechanize with SideFX download API - Retrieve daily build from Download API using requests - Store client/secret in Travis encrypted global environment variables --- .travis.yml | 2 + travis/sidefx_api.py | 143 +++++++++++++++++++++++++++++++++++++++++++ travis/travis.py | 64 ++++++++----------- travis/travis.run | 4 +- 4 files changed, 172 insertions(+), 41 deletions(-) create mode 100644 travis/sidefx_api.py diff --git a/.travis.yml b/.travis.yml index 7e99ac4558..ffe59bb3f3 100755 --- a/.travis.yml +++ b/.travis.yml @@ -73,6 +73,8 @@ env: - ABI=5 BLOSC=no RELEASE=no HOUDINI_MAJOR=none - ABI=4 BLOSC=no RELEASE=no HOUDINI_MAJOR=none - ABI=3 BLOSC=no RELEASE=no HOUDINI_MAJOR=none + - secure: bTVyqXEIfdhhjJiHI1ELq766B711W41/2fHKRjFjPUIeEYeD8LrG/MUrGc9Jst/OYWdEcFN329/vjjTNYMKk7DOaOMwlpPap1XmdKSvk0S1Zq80M1j8XWyClkW5UEk5/DZAdtw18k5P8rnjptzxro6BoxPIaQ3P+0UUAOW3pASWGbV8LvNHfzTdTj8OV8RIxm5MV5dlYOj7fDjEViBWjopJ+gcnBBDrtGOpCWSGbnz1WBolym41cMPc8SAJXXzDK6mXy2k4Gpuk23kkKqmZJA9GbPnv/E5VITG2CTzu9sE3XQpPuHa1x7JFvSXm62eo+Gex6d1uixXsrP5rH59eMqPCIAVrmCCfwNn/JBmGNSob1Hp03TDOJ4cw1B69+WI+CrnqNDUAL/FviUIC4mpVNj/Xdoe4fWrCmg6GXEVYbacQcTHmXz+LwjJUDNKnMFwb3w0uimPMCiWUFHr6l73rdixICxYOIWMPZeSzsftV2WidteZPPx9nByGLIDfldXhUuBfOCgxfTYwKiHRExlV2ShHSN38XNwA2qVcuD4JH1DOhi29Pb0+MiJZmGKr6PwMPPtfkzJkQs5FhlUSW1exU4CzPm0gbSd2fGdmRq3nWTk3wPq5cuNc017LAX5Mu9k0DLevTLw5NFwmn17+tDF0Nab6RiA2T1nOMuHFSkeDYzdSY= + - secure: qmyZOe7zNIdkY5102R4BA5LzB2Jv3byH45TfGkkeWXhernECAef5CEAydMxHg+YShNX5KDUzGi5c0RrECxYpFSgL2Zvh58GSROXsZ22xpAE/HxmZRSFfpVsnfOqQZu3hWa+9zIUzVjawpaPsnCeuBedDwQd/q9AZccQJFVRDirLGNeSWyiLX719f02ogxPyxorwrb4DDGSPT4tXRyAseKnczDoCl68gTnF9ucfaaLAVAeG6dkLTyX+tcaYSxfVrKz8pJJiP/MdRqlRvOAsO5zqnEKevKkEU7+J1/QkK4yUjfWrT/eZeZ8nTVFR8QM2srbN+niyObFUZTohcSzs69vdHpl0myxiRy89Z5cg1mRzuWJBqqxBLh4ducco52UIGkZDko5yXLmbTLcHy4Zv4JnL5xJM4j7azUO0oizmA3WqKicxOVB0llmPVlaQLJ/xJK/VP1lsQAU1HfFV53W0Qs/GY98FsaOKn/SVl8fbhcBRon50PR3afoI11MiBddK5LeVazIGpeV0jCqaLkJNlqKSuEwiZUezt9MgEToQDdAxC1y3vFn42SMstQhkzgo5GHWXjTs4uICjwsnebhkYF1/nyzZNWruFWqX3PeljtyqTs30X6zWA3rpcbXpnxEeS4sk2IVWaTRP7WGzb0sDyHAxqaOlq6RgJ95vm74qoZrZMQw= # Build and install all library dependencies for OpenVDB # (build will error if this stage does not succeed) diff --git a/travis/sidefx_api.py b/travis/sidefx_api.py new file mode 100644 index 0000000000..4c0f6b337b --- /dev/null +++ b/travis/sidefx_api.py @@ -0,0 +1,143 @@ +import time +import json +import base64 +try: + import html.parser as HTMLParser +except ImportError: + import HTMLParser +import requests + + +# Code that provides convenient Python wrappers to call into the API: + +def service( + access_token_url, client_id, client_secret_key, endpoint_url, + access_token=None, access_token_expiry_time=None): + if (access_token is None or + access_token_expiry_time is None or + access_token_expiry_time < time.time()): + access_token, access_token_expiry_time = ( + get_access_token_and_expiry_time( + access_token_url, client_id, client_secret_key)) + + return _Service( + endpoint_url, access_token, access_token_expiry_time) + + +class _Service(object): + def __init__( + self, endpoint_url, access_token, access_token_expiry_time): + self.endpoint_url = endpoint_url + self.access_token = access_token + self.access_token_expiry_time = access_token_expiry_time + + def __getattr__(self, attr_name): + return _APIFunction(attr_name, self) + + +class _APIFunction(object): + def __init__(self, function_name, service): + self.function_name = function_name + self.service = service + + def __getattr__(self, attr_name): + # This isn't actually an API function, but a family of them. Append + # the requested function name to our name. + return _APIFunction( + "{0}.{1}".format(self.function_name, attr_name), self.service) + + def __call__(self, *args, **kwargs): + return call_api_with_access_token( + self.service.endpoint_url, self.service.access_token, + self.function_name, args, kwargs) + +#--------------------------------------------------------------------------- +# Code that implements authentication and raw calls into the API: + + +def get_access_token_and_expiry_time( + access_token_url, client_id, client_secret_key): + """Given an API client (id and secret key) that is allowed to make API + calls, return an access token that can be used to make calls. + """ + response = requests.post( + access_token_url, + headers={ + "Authorization": u"Basic {0}".format( + base64.b64encode( + "{0}:{1}".format( + client_id, client_secret_key + ).encode() + ).decode('utf-8') + ), + }) + if response.status_code != 200: + raise AuthorizationError( + response.status_code, + "{0}: {1}".format( + response.status_code, + _extract_traceback_from_response(response))) + + response_json = response.json() + access_token_expiry_time = time.time() - 2 + response_json["expires_in"] + return response_json["access_token"], access_token_expiry_time + + +class AuthorizationError(Exception): + """Raised from the client if the server generated an error while generating + an access token. + """ + def __init__(self, http_code, message): + super(AuthorizationError, self).__init__(message) + self.http_code = http_code + + +def call_api_with_access_token( + endpoint_url, access_token, function_name, args, kwargs): + """Call into the API using an access token that was returned by + get_access_token. + """ + response = requests.post( + endpoint_url, + headers={ + "Authorization": "Bearer " + access_token, + }, + data=dict( + json=json.dumps([function_name, args, kwargs]), + )) + if response.status_code == 200: + return response.json() + + raise APIError( + response.status_code, + "{0}".format(_extract_traceback_from_response(response))) + + +class APIError(Exception): + """Raised from the client if the server generated an error while calling + into the API. + """ + def __init__(self, http_code, message): + super(APIError, self).__init__(message) + self.http_code = http_code + + +def _extract_traceback_from_response(response): + """Helper function to extract a traceback from the web server response + if an API call generated a server-side exception + """ + error_message = response.text + if response.status_code != 500: + return error_message + + traceback = "" + for line in error_message.split("\n"): + if len(traceback) != 0 and line == "": + break + if line == "Traceback:" or len(traceback) != 0: + traceback += line + "\n" + + if len(traceback) == 0: + traceback = error_message + + return HTMLParser.HTMLParser().unescape(traceback) diff --git a/travis/travis.py b/travis/travis.py index e1e2a7fabd..37092c114a 100755 --- a/travis/travis.py +++ b/travis/travis.py @@ -28,10 +28,12 @@ # # Author: Dan Bailey -import mechanize +import requests import sys import re -import exceptions +import shutil +from sidefx_api import service + # this argument is for the major.minor version of Houdini to download (such as 15.0, 15.5, 16.0) version = sys.argv[1] @@ -39,44 +41,28 @@ if not re.match('[0-9][0-9]\.[0-9]$', version): raise IOError('Invalid Houdini Version "%s", expecting in the form "major.minor" such as "16.0"' % version) -br = mechanize.Browser() -br.set_handle_robots(False) - -# login to sidefx.com as openvdb -br.open('https://www.sidefx.com/login/?next=/download/daily-builds') -br.select_form(nr=0) -br.form['username'] = 'openvdb' -br.form['password'] = 'L3_M2f2W' -br.submit() +sidefx_client_id = sys.argv[2] +sidefx_secret_key = sys.argv[3] -# retrieve download id -br.open('http://www.sidefx.com/download/daily-builds/') +sidefx_service = service( + access_token_url='https://www.sidefx.com/oauth2/application_token', + client_id=sidefx_client_id, + client_secret_key=sidefx_secret_key, + endpoint_url='https://www.sidefx.com/api/', +) -for link in br.links(): - if not link.url.startswith('/download/download-houdini'): - continue - if link.text.startswith('houdini-%s' % version) and 'linux_x86_64' in link.text: - response = br.follow_link(text=link.text, nr=0) - url = response.geturl() - id = url.split('/download-houdini/')[-1] - break -# accept eula terms -#url = 'https://www.sidefx.com/download/eula/accept/?next=/download/download-houdini/%sget/' % id -#br.open(url) -#br.select_form(nr=0) -#br.form.find_control('terms').items[1].selected=True -#br.submit() +release_list = service.download.get_daily_builds_list( + product='houdini', version=version, platform='linux') +latest_release = service.download.get_daily_builds_download( + product='houdini', version=version, build=release_list[0]['build'], platform='linux') -# download houdini tarball in 50MB chunks -url = 'https://www.sidefx.com/download/download-houdini/%sget/' % id -response = br.open(url) -mb = 1024*1024 -chunk = 50 -size = 0 -file = open('hou.tar.gz', 'wb') -for bytes in iter((lambda: response.read(chunk*mb)), ''): - size += 50 - print 'Read: %sMB' % size - file.write(bytes) -file.close() +# Download the file +local_filename = latest_release['filename'] +r = requests.get(latest_release['download_url'], stream=True) +if r.status_code == 200: + with open(local_filename, 'wb') as f: + r.raw.decode_content = True + shutil.copyfileobj(r.raw, f) +else: + raise Exception('Error downloading file!') diff --git a/travis/travis.run b/travis/travis.run index 2bec17d333..215cb5921f 100755 --- a/travis/travis.run +++ b/travis/travis.run @@ -117,10 +117,10 @@ if [ "$TASK" = "install" ]; then # install houdini pre-requisites sudo apt-get install -y libxi-dev sudo apt-get install -y csh - sudo apt-get install python-mechanize + sudo apt-get install python-requests export PYTHONPATH=${PYTHONPATH}:/usr/lib/python2.7/dist-packages # download and unpack latest houdini headers and libraries from daily-builds - python travis/travis.py $HOUDINI_MAJOR + python travis/travis.py $HOUDINI_MAJOR $SIDEFX_CLIENT_ID $SIDEFX_SECRET_KEY tar -xzf hou.tar.gz ln -s houdini* hou cd hou