build.py

#!/usr/bin/python
# Compresses the core Blockly files into a single JavaScript file.
#
# Copyright 2012 Google Inc.
# http://blockly.googlecode.com/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script generates two files:
#   blockly_compressed.js
#   blockly_uncompressed.js
# The compressed file is a concatenation of all of Blockly's core files which
# have been run through Google's Closure Compiler.  This is done using the
# online API (which takes a few seconds and requires an Internet connection).
# The uncompressed file is a script that loads in each of Blockly's core files
# one by one.  This takes much longer for a browser to load, but is useful
# when debugging code since line numbers are meaningful and variables haven't
# been renamed.  The uncompressed file also allows for a faster developement
# cycle since there is no need to rebuild or recompile, just reload.

import glob, httplib, json, os, re, sys, threading, urllib

def import_path(fullpath):
  """Import a file with full path specification.
  Allows one to import from any directory, something __import__ does not do.

  Args:
      fullpath:  Path and filename of import.

  Returns:
      An imported module.
  """
  path, filename = os.path.split(fullpath)
  filename, ext = os.path.splitext(filename)
  sys.path.append(path)
  module = __import__(filename)
  reload(module) # Might be out of date
  del sys.path[-1]
  return module


HEADER = ('// Do not edit this file; automatically generated by build.py.\n'
          '"use strict";\n')


class Gen_uncompressed(threading.Thread):
  """Generate a JavaScript file that loads Blockly's raw files.
  Runs in a separate thread.
  """
  def __init__(self, search_paths):
    threading.Thread.__init__(self)
    self.search_paths = search_paths

  def run(self):
    target_filename = 'blockly_uncompressed.js'
    f = open(target_filename, 'w')
    f.write(HEADER)
    f.write("""
window.BLOCKLY_DIR = (function() {
  // Find name of current directory.
  var scripts = document.getElementsByTagName('script');
  var re = new RegExp('(.+)[\/]blockly_uncompressed\.js$');
  for (var x = 0, script; script = scripts[x]; x++) {
    var match = re.exec(script.src);
    if (match) {
      return match[1];
    }
  }
  alert('Could not detect Blockly\\'s directory name.');
  return '';
})();

window.BLOCKLY_BOOT = function() {
// Execute after Closure has loaded.
if (!window.goog) {
  alert('Error: Closure not found.  Read this:\\n' +
        'http://code.google.com/p/blockly/wiki/Closure\\n');
}

// Build map of all dependencies (used and unused).
var dir = window.BLOCKLY_DIR.match(/[^\\/]+$/)[0];
""")
    add_dependency = []
    base_path = calcdeps.FindClosureBasePath(self.search_paths)
    for dep in calcdeps.BuildDependenciesFromFiles(self.search_paths):
      add_dependency.append(calcdeps.GetDepsLine(dep, base_path))
    add_dependency = '\n'.join(add_dependency)
    # Find the Blockly directory name and replace it with a JS variable.
    # This allows blockly_uncompressed.js to be compiled on one computer and be
    # used on another, even if the directory name differs.
    m = re.search('[\\/]([^\\/]+)[\\/]core[\\/]blockly.js', add_dependency)
    add_dependency = re.sub('([\\/])' + re.escape(m.group(1)) +
        '([\\/]core[\\/])', '\\1" + dir + "\\2', add_dependency)
    f.write(add_dependency + '\n')

    provides = []
    for dep in calcdeps.BuildDependenciesFromFiles(self.search_paths):
      if not dep.filename.startswith(os.pardir + os.sep):  # '../'
        provides.extend(dep.provides)
    provides.sort()
    f.write('\n')
    f.write('// Load Blockly.\n')
    for provide in provides:
      f.write('goog.require(\'%s\');\n' % provide)

    f.write("""
delete window.BLOCKLY_DIR;
delete window.BLOCKLY_BOOT;
};

// Delete any existing Closure (e.g. Soy's nogoog_shim).
document.write('<script type="text/javascript">var goog = undefined;</script>');
// Load fresh Closure Library.
document.write('<script type="text/javascript" src="' + window.BLOCKLY_DIR +
    '/../closure-library-read-only/closure/goog/base.js"></script>');
document.write('<script type="text/javascript">window.BLOCKLY_BOOT()</script>');
""")
    f.close()
    print('SUCCESS: ' + target_filename)


class Gen_compressed(threading.Thread):
  """Generate a JavaScript file that contains all of Blockly's core and all
  required parts of Closure, compiled together.
  Uses the Closure Compiler's online API.
  Runs in a separate thread.
  """
  def __init__(self, search_paths):
    threading.Thread.__init__(self)
    self.search_paths = search_paths

  def run(self):
    self.gen_core()
    self.gen_generator('javascript')
    self.gen_generator('python')
    self.gen_language('de')
    self.gen_language('en')
    self.gen_language('pt_br')
    self.gen_language('vi')
    self.gen_language('zh_tw')

  def gen_core(self):
    target_filename = 'blockly_compressed.js'
    # Define the parameters for the POST request.
    params = [
        ('compilation_level', 'SIMPLE_OPTIMIZATIONS'),
        ('use_closure_library', 'true'),
        ('output_format', 'json'),
        ('output_info', 'compiled_code'),
        ('output_info', 'warnings'),
        ('output_info', 'errors'),
        ('output_info', 'statistics'),
      ]

    # Read in all the source files.
    filenames = calcdeps.CalculateDependencies(self.search_paths,
        [os.path.join('core', 'blockly.js')])
    for filename in filenames:
      # Filter out the Closure files (the compiler will add them).
      if filename.startswith(os.pardir + os.sep):  # '../'
        continue
      f = open(filename)
      params.append(('js_code', ''.join(f.readlines())))
      f.close()

    self.do_compile(params, target_filename, filenames)

  def gen_generator(self, language):
    target_filename = language + '_compressed.js'
    # Define the parameters for the POST request.
    params = [
        ('compilation_level', 'SIMPLE_OPTIMIZATIONS'),
        ('output_format', 'json'),
        ('output_info', 'compiled_code'),
        ('output_info', 'warnings'),
        ('output_info', 'errors'),
        ('output_info', 'statistics'),
      ]

    # Read in all the source files.
    filenames = glob.glob(
        os.path.join('generators', language, '*.js'))
    filenames.insert(0, os.path.join('generators', language + '.js'))
    for filename in filenames:
      f = open(filename)
      params.append(('js_code', ''.join(f.readlines())))
      f.close()

    self.do_compile(params, target_filename, filenames)

  def gen_language(self, language):
    target_filename = language + '_compressed.js'
    # Define the parameters for the POST request.
    params = [
        ('compilation_level', 'SIMPLE_OPTIMIZATIONS'),
        ('output_format', 'json'),
        ('output_info', 'compiled_code'),
        ('output_info', 'warnings'),
        ('output_info', 'errors'),
        ('output_info', 'statistics'),
      ]

    # Read in all the source files.
    filenames = glob.glob(os.path.join('language', 'common', '*.js'))
    filenames += glob.glob(os.path.join('language', language, '*.js'))
    filenames.remove(os.path.join('language', language, '_messages.js'))
    filenames.insert(0, os.path.join('language', language, '_messages.js'))
    for filename in filenames:
      f = open(filename)
      params.append(('js_code', ''.join(f.readlines())))
      f.close()

    self.do_compile(params, target_filename, filenames)

  def do_compile(self, params, target_filename, filenames):
    # Send the request to Google.
    headers = { "Content-type": "application/x-www-form-urlencoded" }
    conn = httplib.HTTPConnection('closure-compiler.appspot.com')
    conn.request('POST', '/compile', urllib.urlencode(params), headers)
    response = conn.getresponse()
    json_str = response.read()
    conn.close()

    # Parse the JSON response.
    json_data = json.loads(json_str)

    def file_lookup(name):
      if not name.startswith('Input_'):
        return '???'
      n = int(name[6:])
      return filenames[n]

    if json_data.has_key('serverErrors'):
      errors = json_data['serverErrors']
      for error in errors:
        print 'SERVER ERROR: %s' % target_filename
        print error['error']
    elif json_data.has_key('errors'):
      errors = json_data['errors']
      for error in errors:
        print('FATAL ERROR')
        print(error['error'])
        print('%s at line %d:' % (
            file_lookup(error['file']), error['lineno']))
        print(error['line'])
        print((' ' * error['charno']) + '^')
        sys.exit(1)
    else:
      if json_data.has_key('warnings'):
        warnings = json_data['warnings']
        for warning in warnings:
          print('WARNING')
          print(warning['warning'])
          print('%s at line %d:' % (
              file_lookup(warning['file']), warning['lineno']))
          print(warning['line'])
          print((' ' * warning['charno']) + '^')
        print()

      if not json_data.has_key('compiledCode'):
        print('FATAL ERROR: Compiler did not return compiledCode.')
        sys.exit(1)

      code = HEADER + '\n' + json_data['compiledCode']

      stats = json_data['statistics']
      original_b = stats['originalSize']
      compressed_b = stats['compressedSize']
      if original_b > 0 and compressed_b > 0:
        f = open(target_filename, 'w')
        f.write(code)
        f.close()

        original_kb = int(original_b / 1024 + 0.5)
        compressed_kb = int(compressed_b / 1024 + 0.5)
        ratio = int(float(compressed_b) / float(original_b) * 100 + 0.5)
        print('SUCCESS: ' + target_filename)
        print('Size changed from %d KB to %d KB (%d%%).' % (
            original_kb, compressed_kb, ratio))
      else:
        print 'UNKNOWN ERROR'


if __name__ == '__main__':
  try:
    calcdeps = import_path(os.path.join(os.path.pardir,
          'closure-library-read-only', 'closure', 'bin', 'calcdeps.py'))
  except ImportError:
    print("""Error: Closure not found.  Read this:
http://code.google.com/p/blockly/wiki/Closure""")
    sys.exit(1)
  search_paths = calcdeps.ExpandDirectories(
      ['core', os.path.join(os.path.pardir, 'closure-library-read-only')])

  # Run both tasks in parallel threads.
  # Uncompressed is limited by processor speed.
  # Compressed is limited by network and server speed.
  Gen_uncompressed(search_paths).start()
  Gen_compressed(search_paths).start()