build.py

#!/usr/bin/env python

import glob
import time
import re
import io
import base64
import sys
import os
import shutil
import json
import shelve
import hashlib

try:
  import urllib2
except ImportError:
  import urllib.request as urllib2

# load settings file
from buildsettings import buildSettings

# load option local settings file
try:
    from localbuildsettings import buildSettings as localBuildSettings
    buildSettings.update(localBuildSettings)
except ImportError:
    pass

# load default build
try:
    from localbuildsettings import defaultBuild
except ImportError:
    defaultBuild = None


buildName = defaultBuild

# build name from command line
if len(sys.argv) == 2:	# argv[0] = program, argv[1] = buildname, len=2
    buildName = sys.argv[1]


if buildName is None or not buildName in buildSettings:
    print ("Usage: build.py buildname")
    print (" available build names: %s" % ', '.join(buildSettings.keys()))
    sys.exit(1)

settings = buildSettings[buildName]

# set up vars used for replacements

utcTime = time.gmtime()
buildDate = time.strftime('%Y-%m-%d-%H%M%S',utcTime)
# userscripts have specific specifications for version numbers - the above date format doesn't match
dateTimeVersion = time.strftime('%Y%m%d.',utcTime) + time.strftime('%H%M%S',utcTime).lstrip('0')

# extract required values from the settings entry
resourceUrlBase = settings.get('resourceUrlBase')
distUrlBase = settings.get('distUrlBase')
buildMobile = settings.get('buildMobile')
antOptions = settings.get('antOptions','')
antBuildFile = settings.get('antBuildFile', 'mobile/build.xml');


# plugin wrapper code snippets. handled as macros, to ensure that
# 1. indentation caused by the "function wrapper()" doesn't apply to the plugin code body
# 2. the wrapper is formatted correctly for removal by the IITC Mobile android app
pluginWrapperStart = """
function wrapper(plugin_info) {
// ensure plugin framework is there, even if iitc is not yet loaded
if(typeof window.plugin !== 'function') window.plugin = function() {};

//PLUGIN AUTHORS: writing a plugin outside of the IITC build environment? if so, delete these lines!!
//(leaving them in place might break the 'About IITC' page or break update checks)
plugin_info.buildName = '@@BUILDNAME@@';
plugin_info.dateTimeVersion = '@@DATETIMEVERSION@@';
plugin_info.pluginId = '@@PLUGINNAME@@';
//END PLUGIN AUTHORS NOTE

"""

pluginWrapperEnd = """
setup.info = plugin_info; //add the script info data to the function as a property
if(!window.bootPlugins) window.bootPlugins = [];
window.bootPlugins.push(setup);
// if IITC has already booted, immediately run the 'setup' function
if(window.iitcLoaded && typeof setup === 'function') setup();
} // wrapper end
// inject code into site context
var script = document.createElement('script');
var info = {};
if (typeof GM_info !== 'undefined' && GM_info && GM_info.script) info.script = { version: GM_info.script.version, name: GM_info.script.name, description: GM_info.script.description };
script.appendChild(document.createTextNode('('+ wrapper +')('+JSON.stringify(info)+');'));
(document.body || document.head || document.documentElement).appendChild(script);

"""


def readfile(fn):
    with io.open(fn, 'Ur', encoding='utf8') as f:
        return f.read()

def loaderString(var):
    fn = var.group(1)
    return readfile(fn).replace('\n', '\\n').replace('\'', '\\\'')

def loaderRaw(var):
    fn = var.group(1)
    return readfile(fn)

def loaderMD(var):
    fn = var.group(1)
    # use different MD.dat's for python 2 vs 3 incase user switches versions, as they are not compatible
    db = shelve.open('build/MDv' + str(sys.version_info[0]) + '.dat')
    if 'files' in db:
      files = db['files']
    else:
      files = {}
    file = readfile(fn)
    filemd5 = hashlib.md5(file.encode('utf8')).hexdigest()
    # check if file has already been parsed by the github api
    if fn in files and filemd5 in files[fn]:
      # use the stored copy if nothing has changed to avoid hitting the api more then the 60/hour when not signed in
      db.close()
      return files[fn][filemd5]
    else:
      url = 'https://api.github.com/markdown'
      payload = {'text': file, 'mode': 'markdown'}
      headers = {'Content-Type': 'application/json'}
      req = urllib2.Request(url, json.dumps(payload).encode('utf8'), headers)
      md = urllib2.urlopen(req).read().decode('utf8').replace('\n', '\\n').replace('\'', '\\\'')
      files[fn] = {}
      files[fn][filemd5] = md
      db['files'] = files
      db.close()
      return md

def loaderImage(var):
    fn = var.group(1)
    return 'data:image/png;base64,{0}'.format(base64.encodestring(open(fn, 'rb').read()).decode('utf8').replace('\n', ''))

def loadCode(ignore):
    return '\n\n;\n\n'.join(map(readfile, sorted(glob.glob('code/*.js'))))


def extractUserScriptMeta(var):
    m = re.search ( r"//[ \t]*==UserScript==\n.*?//[ \t]*==/UserScript==\n", var, re.MULTILINE|re.DOTALL )
    return m.group(0)



def doReplacements(script,updateUrl,downloadUrl,pluginName=None):

    script = re.sub('@@INJECTCODE@@',loadCode,script)

    script = script.replace('@@PLUGINSTART@@', pluginWrapperStart)
    script = script.replace('@@PLUGINEND@@', pluginWrapperEnd)

    script = re.sub('@@INCLUDERAW:([0-9a-zA-Z_./-]+)@@', loaderRaw, script)
    script = re.sub('@@INCLUDESTRING:([0-9a-zA-Z_./-]+)@@', loaderString, script)
    script = re.sub('@@INCLUDEMD:([0-9a-zA-Z_./-]+)@@', loaderMD, script)
    script = re.sub('@@INCLUDEIMAGE:([0-9a-zA-Z_./-]+)@@', loaderImage, script)

    script = script.replace('@@BUILDDATE@@', buildDate)
    script = script.replace('@@DATETIMEVERSION@@', dateTimeVersion)

    if resourceUrlBase:
        script = script.replace('@@RESOURCEURLBASE@@', resourceUrlBase)
    else:
        if '@@RESOURCEURLBASE@@' in script:
            raise Exception("Error: '@@RESOURCEURLBASE@@' found in script, but no replacement defined")

    script = script.replace('@@BUILDNAME@@', buildName)

    script = script.replace('@@UPDATEURL@@', updateUrl)
    script = script.replace('@@DOWNLOADURL@@', downloadUrl)

    if (pluginName):
        script = script.replace('@@PLUGINNAME@@', pluginName);

    return script


def saveScriptAndMeta(script,ourDir,filename,oldDir=None):
    # TODO: if oldDir is set, compare files. if only data/time-based version strings are different
    # copy from there instead of saving a new file

    fn = os.path.join(outDir,filename)
    with io.open(fn, 'w', encoding='utf8') as f:
        f.write(script)

    metafn = fn.replace('.user.js', '.meta.js')
    if metafn != fn:
        with io.open(metafn, 'w', encoding='utf8') as f:
            meta = extractUserScriptMeta(script)
            f.write(meta)


outDir = os.path.join('build', buildName)


# create the build output

# first, delete any existing build - but keep it in a temporary folder for now
oldDir = None
if os.path.exists(outDir):
    oldDir = outDir+'~';
    if os.path.exists(oldDir):
        shutil.rmtree(oldDir)
    os.rename(outDir, oldDir)


# copy the 'dist' folder, if it exists
if os.path.exists('dist'):
    # this creates the target directory (and any missing parent dirs)
    # FIXME? replace with manual copy, and any .css and .js files are parsed for replacement tokens?
    shutil.copytree('dist', outDir)
else:
    # no 'dist' folder - so create an empty target folder
    os.makedirs(outDir)


# run any preBuild commands
for cmd in settings.get('preBuild',[]):
    os.system ( cmd )


# load main.js, parse, and create main total-conversion-build.user.js
main = readfile('main.js')

downloadUrl = distUrlBase and distUrlBase + '/total-conversion-build.user.js' or 'none'
updateUrl = distUrlBase and distUrlBase + '/total-conversion-build.meta.js' or 'none'
main = doReplacements(main,downloadUrl=downloadUrl,updateUrl=updateUrl)

saveScriptAndMeta(main, outDir, 'total-conversion-build.user.js', oldDir)

with io.open(os.path.join(outDir, '.build-timestamp'), 'w') as f:
    f.write(time.strftime('%Y-%m-%d %H:%M:%S UTC', utcTime))


# for each plugin, load, parse, and save output
os.mkdir(os.path.join(outDir,'plugins'))

for fn in glob.glob("plugins/*.user.js"):
    script = readfile(fn)

    downloadUrl = distUrlBase and distUrlBase + '/' + fn.replace("\\","/") or 'none'
    updateUrl = distUrlBase and downloadUrl.replace('.user.js', '.meta.js') or 'none'
    pluginName = os.path.splitext(os.path.splitext(os.path.basename(fn))[0])[0]
    script = doReplacements(script, downloadUrl=downloadUrl, updateUrl=updateUrl, pluginName=pluginName)

    saveScriptAndMeta(script, outDir, fn, oldDir)

# if we're building mobile too
if buildMobile:
    if buildMobile not in ['debug','release','copyonly']:
        raise Exception("Error: buildMobile must be 'debug' or 'release' or 'copyonly'")

    # compile the user location script
    fn = "user-location.user.js"
    script = readfile("mobile/plugins/" + fn)
    downloadUrl = distUrlBase and distUrlBase + '/' + fn.replace("\\","/") or 'none'
    updateUrl = distUrlBase and downloadUrl.replace('.user.js', '.meta.js') or 'none'
    script = doReplacements(script, downloadUrl=downloadUrl, updateUrl=updateUrl, pluginName='user-location')

    saveScriptAndMeta(script, outDir, fn)

    # copy the IITC script into the mobile folder. create the folder if needed
    try:
        os.makedirs("mobile/assets")
    except:
        pass
    shutil.copy(os.path.join(outDir,"total-conversion-build.user.js"), "mobile/assets/total-conversion-build.user.js")
    # copy the user location script into the mobile folder.
    shutil.copy(os.path.join(outDir,"user-location.user.js"), "mobile/assets/user-location.user.js")
    # also copy plugins
    try:
        shutil.rmtree("mobile/assets/plugins")
    except:
        pass
    shutil.copytree(os.path.join(outDir,"plugins"), "mobile/assets/plugins",
            # do not include desktop-only plugins to mobile assets
            ignore=shutil.ignore_patterns('*.meta.js',
            'force-https*', 'speech-search*', 'basemap-cloudmade*',
            'scroll-wheel-zoom-disable*'))


    if buildMobile != 'copyonly':
        # now launch 'ant' to build the mobile project
        retcode = os.system("ant %s -buildfile %s %s" % (antOptions, antBuildFile, buildMobile))

        if retcode != 0:
            print ("Error: mobile app failed to build. ant returned %d" % retcode)
            exit(1) # ant may return 256, but python seems to allow only values <256
        else:
            shutil.copy("mobile/bin/IITC_Mobile-%s.apk" % buildMobile, os.path.join(outDir,"IITC_Mobile-%s.apk" % buildMobile) )


# run any postBuild commands
for cmd in settings.get('postBuild',[]):
    os.system ( cmd )


# vim: ai si ts=4 sw=4 sts=4 et