diff --git a/requirements.txt b/requirements.txt index 16db371..560cc0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ Cerberus>=1.1 click>=6.2 -Jinja2>=2.8 +Jinja2>=3.1 PyYAML>=3.1 docker-py>=1.6.0 requests>2.11 requests-toolbelt mock>=2.0.0 -pexpect>=4.2.1 \ No newline at end of file +pexpect>=4.2.1 diff --git a/tile_generator/erb.py b/tile_generator/erb.py index 7927da8..9638e9a 100644 --- a/tile_generator/erb.py +++ b/tile_generator/erb.py @@ -27,117 +27,128 @@ import string from . import build -from jinja2 import Environment, FileSystemLoader, exceptions, contextfilter +from jinja2 import Environment, FileSystemLoader, exceptions, pass_context PATH = os.path.dirname(os.path.realpath(__file__)) TEMPLATE_PATH = os.path.realpath(os.path.join(PATH, '..', 'templates')) + def render_hyphens(input): - return input.replace('_','-') + return input.replace('_', '-') + -@contextfilter +@pass_context def render_shell_string(context, input): - expression = context.environment.compile_expression(input, undefined_to_none=False) - return "'" + str(expression(context)).replace("'", "'\\''") + "'" + expression = context.environment.compile_expression(input, undefined_to_none=False) + return "'" + str(expression(context)).replace("'", "'\\''") + "'" + -@contextfilter +@pass_context def render_plans_json(context, input): - name = 'missing.' + input - form = context.environment.compile_expression(name, undefined_to_none=False)(context) - plans = {} - for p in form: - plans[p['name']] = p - return input.upper() + "='" + json.dumps(plans).replace("'", "'\\''") + "'" + name = 'missing.' + input + form = context.environment.compile_expression(name, undefined_to_none=False)(context) + plans = {} + for p in form: + plans[p['name']] = p + return input.upper() + "='" + json.dumps(plans).replace("'", "'\\''") + "'" + TEMPLATE_ENVIRONMENT = Environment( - trim_blocks=True, lstrip_blocks=True, - comment_start_string='<%', comment_end_string='%>', # To completely ignore Ruby code blocks + trim_blocks=True, lstrip_blocks=True, + comment_start_string='<%', comment_end_string='%>', # To completely ignore Ruby code blocks ) TEMPLATE_ENVIRONMENT.loader = FileSystemLoader(TEMPLATE_PATH) TEMPLATE_ENVIRONMENT.filters['hyphens'] = render_hyphens TEMPLATE_ENVIRONMENT.filters['shell_string'] = render_shell_string TEMPLATE_ENVIRONMENT.filters['plans_json'] = render_plans_json + def render(errand_name, config_dir): - template_file = os.path.join('jobs', errand_name + '.sh.erb') - config = compile_config(config_dir) - target_path = errand_name + '.sh' - with open(target_path, 'wb') as target: - target.write(TEMPLATE_ENVIRONMENT.get_template(template_file).render(config)) - return target_path + template_file = os.path.join('jobs', errand_name + '.sh.erb') + config = compile_config(config_dir) + target_path = errand_name + '.sh' + with open(target_path, 'wb') as target: + target.write(TEMPLATE_ENVIRONMENT.get_template(template_file).render(config)) + return target_path + def mkdir_p(dir): - try: - os.makedirs(dir) - except os.error as e: - if e.errno != errno.EEXIST: - raise + try: + os.makedirs(dir) + except os.error as e: + if e.errno != errno.EEXIST: + raise + def get_file_properties(filename): - try: - with open(filename) as f: - properties = yaml.safe_load(f) - if properties is None: - return {} - else: - return properties - except IOError as e: - print(filename, 'not found', file=sys.stderr) - sys.exit(1) + try: + with open(filename) as f: + properties = yaml.safe_load(f) + if properties is None: + return {} + else: + return properties + except IOError as e: + print(filename, 'not found', file=sys.stderr) + sys.exit(1) + def get_cf_properties(): - cf = opsmgr.get_cfinfo() - properties = {} - properties['cf'] = { - 'admin_user': cf['system_services_username'], - 'admin_password': cf['system_services_password'], - } - properties['domain'] = cf['system_domain'] - properties['app_domains'] = [ cf['apps_domain'] ] - properties['ssl'] = { 'skip_cert_verify': True } - properties['security'] = { - 'user': 'admin', - 'password': ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(12)) - } - return properties + cf = opsmgr.get_cfinfo() + properties = {} + properties['cf'] = { + 'admin_user': cf['system_services_username'], + 'admin_password': cf['system_services_password'], + } + properties['domain'] = cf['system_domain'] + properties['app_domains'] = [cf['apps_domain']] + properties['ssl'] = {'skip_cert_verify': True} + properties['security'] = { + 'user': 'admin', + 'password': ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(12)) + } + return properties + def merge_properties(properties, new_properties): - for p in new_properties: - if properties.get(p, None) is None: - properties[p] = new_properties[p] + for p in new_properties: + if properties.get(p, None) is None: + properties[p] = new_properties[p] + def merge_property_array(properties, new_properties): - for p in new_properties: - name = p['name'] - if properties.get(name, None) is None: - value = p.get('value', p.get('default', None)) - if value is not None: - properties[name] = value + for p in new_properties: + name = p['name'] + if properties.get(name, None) is None: + value = p.get('value', p.get('default', None)) + if value is not None: + properties[name] = value + def compile_config(config_dir): - context = get_file_properties(os.path.join(config_dir, 'tile.yml')) - build.validate_config(context) - build.add_defaults(context) - build.upgrade_config(context) - - properties = {} - missing = get_file_properties(os.path.join(config_dir, 'missing-properties.yml')) - merge_properties(properties, get_cf_properties()) - merge_properties(properties, context) - merge_property_array(properties, context.get('properties', [])) - for form in context.get('forms', []): - merge_property_array(properties, form.get('properties', [])) - for package in context.get('packages', []): - merge_properties(package, missing.get(package['name'], {})) - merge_properties(package, properties['security']) - merge_properties(properties, { package['name'].replace('-','_'): package }) - merge_properties(properties, missing) - - properties['org'] = properties.get('org', properties['name'] + '-org') - properties['space'] = properties.get('space', properties['name'] + '-space') - - return { - 'context': context, - 'properties': properties, - 'missing': missing, - } + context = get_file_properties(os.path.join(config_dir, 'tile.yml')) + build.validate_config(context) + build.add_defaults(context) + build.upgrade_config(context) + + properties = {} + missing = get_file_properties(os.path.join(config_dir, 'missing-properties.yml')) + merge_properties(properties, get_cf_properties()) + merge_properties(properties, context) + merge_property_array(properties, context.get('properties', [])) + for form in context.get('forms', []): + merge_property_array(properties, form.get('properties', [])) + for package in context.get('packages', []): + merge_properties(package, missing.get(package['name'], {})) + merge_properties(package, properties['security']) + merge_properties(properties, {package['name'].replace('-', '_'): package}) + merge_properties(properties, missing) + + properties['org'] = properties.get('org', properties['name'] + '-org') + properties['space'] = properties.get('space', properties['name'] + '-space') + + return { + 'context': context, + 'properties': properties, + 'missing': missing, + } diff --git a/tile_generator/template.py b/tile_generator/template.py index 0c18ce9..eb82912 100644 --- a/tile_generator/template.py +++ b/tile_generator/template.py @@ -23,160 +23,179 @@ import errno import yaml -from jinja2 import Template, Environment, FileSystemLoader, exceptions, contextfilter +from jinja2 import Template, Environment, FileSystemLoader, exceptions, pass_context PATH = os.path.dirname(os.path.realpath(__file__)) TEMPLATE_PATH = os.path.realpath(os.path.join(PATH, 'templates')) PROPERTY_FIELDS = { - 'simple_credentials': ['identity', 'password'], - 'rsa_cert_credentials': [ 'private_key_pem', 'cert_pem', 'public_key_pem', 'cert_and_private_key_pems' ], - 'rsa_pkey_credentials': [ 'private_key_pem', 'public_key_pem', 'public_key_openssh', 'public_key_fingerprint' ], - 'salted_credentials': [ 'salt', 'identity', 'password' ], - 'selector': [ 'value', ('selected_option', 'selected_option.parsed_manifest(manifest_snippet)') ], + 'simple_credentials': ['identity', 'password'], + 'rsa_cert_credentials': ['private_key_pem', 'cert_pem', 'public_key_pem', 'cert_and_private_key_pems'], + 'rsa_pkey_credentials': ['private_key_pem', 'public_key_pem', 'public_key_openssh', 'public_key_fingerprint'], + 'salted_credentials': ['salt', 'identity', 'password'], + 'selector': ['value', ('selected_option', 'selected_option.parsed_manifest(manifest_snippet)')], } + def render_hyphens(input): - return input.replace('_','-') + return input.replace('_', '-') -def expand_selector(input): - if input.get('type', None) == 'selector': - for option in input.get('option_templates', []): - properties = '' - for p in option.get('property_blueprints', []): - if p['type'] in PROPERTY_FIELDS: - properties += p['name'] + ': { ' - subproperties = [] - for subproperty in PROPERTY_FIELDS[p['type']]: - if type(subproperty) is tuple: - subproperties.append('{}: (( .properties.{}.{}.{}.{} ))'.format(subproperty[0], input['name'], option['name'], p['name'], subproperty[1])) - else: - subproperties.append('{}: (( .properties.{}.{}.{}.{} ))'.format(subproperty, input['name'], option['name'], p['name'], subproperty)) - properties += ', '.join(subproperties) + ' }\r\n' - else: - properties += p['name'] + ': (( .properties.' + input['name'] + '.' + option['name'] + '.' + p['name'] + '.value ))\r\n' - option['named_manifests'] = option.get("named_manifests", []) - option['named_manifests'].append({ - 'name': 'manifest_snippet', - 'manifest': properties - }) - return input +def expand_selector(input): + if input.get('type', None) == 'selector': + for option in input.get('option_templates', []): + properties = '' + for p in option.get('property_blueprints', []): + if p['type'] in PROPERTY_FIELDS: + properties += p['name'] + ': { ' + subproperties = [] + for subproperty in PROPERTY_FIELDS[p['type']]: + if type(subproperty) is tuple: + subproperties.append( + '{}: (( .properties.{}.{}.{}.{} ))'.format(subproperty[0], input['name'], + option['name'], p['name'], subproperty[1])) + else: + subproperties.append( + '{}: (( .properties.{}.{}.{}.{} ))'.format(subproperty, input['name'], option['name'], + p['name'], subproperty)) + properties += ', '.join(subproperties) + ' }\r\n' + else: + properties += p['name'] + ': (( .properties.' + input['name'] + '.' + option['name'] + '.' + p[ + 'name'] + '.value ))\r\n' + option['named_manifests'] = option.get("named_manifests", []) + option['named_manifests'].append({ + 'name': 'manifest_snippet', + 'manifest': properties + }) + return input def render_yaml(input): - # Inspired by https://stackoverflow.com/questions/50519454/python-yaml-dump-using-block-style-without-quotes - def multiline_representer(dumper, data): - return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|" if "\n" in data else None) - yaml.SafeDumper.add_representer(str, multiline_representer) - yaml.SafeDumper.ignore_aliases = lambda *args : True - return yaml.safe_dump(input, default_flow_style=False, width=float("inf")) + # Inspired by https://stackoverflow.com/questions/50519454/python-yaml-dump-using-block-style-without-quotes + def multiline_representer(dumper, data): + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|" if "\n" in data else None) + + yaml.SafeDumper.add_representer(str, multiline_representer) + yaml.SafeDumper.ignore_aliases = lambda *args: True + return yaml.safe_dump(input, default_flow_style=False, width=float("inf")) + def render_yaml_literal(input): - return yaml.safe_dump(input, default_flow_style=False, default_style='|', width=float("inf")) + return yaml.safe_dump(input, default_flow_style=False, default_style='|', width=float("inf")) + def render_shell_string(input): - return '<%= Shellwords.escape ' + input + ' %>' + return '<%= Shellwords.escape ' + input + ' %>' + def render_plans_json(input, escape=True, export=True): - property_name = input['name'] - variable_name = input.get('variable_name', property_name.upper()) - value = '=<%= Shellwords.escape JSON.dump(plans) %>' if escape else '=<%= JSON.dump(plans) %>' - export = 'export ' if export else '' - return ('<%\n' - ' plans = { }\n' - ' p("' + property_name + '").each do |plan|\n' - ' plan_name = plan[\'name\']\n' - ' plans[plan_name] = plan\n' - ' end\n' - '%>\n' - '' + export + variable_name + value) + property_name = input['name'] + variable_name = input.get('variable_name', property_name.upper()) + value = '=<%= Shellwords.escape JSON.dump(plans) %>' if escape else '=<%= JSON.dump(plans) %>' + export = 'export ' if export else '' + return ('<%\n' + ' plans = { }\n' + ' p("' + property_name + '").each do |plan|\n' + ' plan_name = plan[\'name\']\n' + ' plans[plan_name] = plan\n' + ' end\n' + '%>\n' + '' + export + variable_name + value) + def render_selector_json(input, escape=True, export=True): - value = '=<%= Shellwords.escape JSON.dump(hash) %>' if escape else '=<%= JSON.dump(hash) %>' - export = 'export ' if export else '' - return ('<%\n' - ' hash = { }\n' - ' hash["value"] = p("' + input + '")["value"]\n' - ' hash["selected_option"] = { }\n' - ' if p("' + input + '")["selected_option"]\n' - ' p("' + input + '")["selected_option"].each_pair do |key, value|\n' - ' hash["selected_option"][key] = value\n' - ' end\n' - ' end\n' - '%>\n' - '' + export + input.upper() + value) + value = '=<%= Shellwords.escape JSON.dump(hash) %>' if escape else '=<%= JSON.dump(hash) %>' + export = 'export ' if export else '' + return ('<%\n' + ' hash = { }\n' + ' hash["value"] = p("' + input + '")["value"]\n' + ' hash["selected_option"] = { }\n' + ' if p("' + input + '")["selected_option"]\n' + ' p("' + input + '")["selected_option"].each_pair do |key, value|\n' + ' hash["selected_option"][key] = value\n' + ' end\n' + ' end\n' + '%>\n' + '' + export + input.upper() + value) + def render_collection_json(input, escape=True, export=True): - value = '=<%= Shellwords.escape JSON.dump(array) %>' if escape else '=<%= JSON.dump(array) %>' - export = 'export ' if export else '' - return ('<%\n' - ' array = [ ]\n' - ' p("' + input + '").each do |m|\n' - ' member = { }\n' - ' m.each_pair do |key, value|\n' - ' member[key] = value\n' - ' end\n' - ' array << member\n' - ' end\n' - '%>\n' - '' + export + input.upper() + value) + value = '=<%= Shellwords.escape JSON.dump(array) %>' if escape else '=<%= JSON.dump(array) %>' + export = 'export ' if export else '' + return ('<%\n' + ' array = [ ]\n' + ' p("' + input + '").each do |m|\n' + ' member = { }\n' + ' m.each_pair do |key, value|\n' + ' member[key] = value\n' + ' end\n' + ' array << member\n' + ' end\n' + '%>\n' + '' + export + input.upper() + value) + def render_property_json(input, escape=True, export=True): - escape = ' Shellwords.escape' if escape else '' - export = 'export ' if export else '' - return(export + '{}=<%={} properties.{}.marshal_dump.to_json %>'.format(input.upper(), escape, input)) + escape = ' Shellwords.escape' if escape else '' + export = 'export ' if export else '' + return (export + '{}=<%={} properties.{}.marshal_dump.to_json %>'.format(input.upper(), escape, input)) + def render_property_value(input, escape=True, export=True): - escape = ' Shellwords.escape' if escape else '' - export = 'export ' if export else '' - return(export + '{}=<%={} properties.{} %>'.format(input.upper(), escape, input)) + escape = ' Shellwords.escape' if escape else '' + export = 'export ' if export else '' + return (export + '{}=<%={} properties.{} %>'.format(input.upper(), escape, input)) + def render_env_variable(property, escape=True, export=True): - complex_types = ( - 'simple_credentials', - 'rsa_cert_credentials', - 'rsa_pkey_credentials', - 'salted_credentials', - 'selector', - ) - if property['type'] in [ 'selector' ]: - return render_selector_json(property['name'], escape, export) - elif property['type'] in [ 'collection' ]: - return render_collection_json(property['name'], escape, export) - elif property['type'] in complex_types: - return render_property_json(property['name'], escape, export) - else: - return render_property_value(property['name'], escape, export) + complex_types = ( + 'simple_credentials', + 'rsa_cert_credentials', + 'rsa_pkey_credentials', + 'salted_credentials', + 'selector', + ) + if property['type'] in ['selector']: + return render_selector_json(property['name'], escape, export) + elif property['type'] in ['collection']: + return render_collection_json(property['name'], escape, export) + elif property['type'] in complex_types: + return render_property_json(property['name'], escape, export) + else: + return render_property_value(property['name'], escape, export) + def render_property(property): - """Render a property for bosh manifest, according to its type.""" - # This ain't the prettiest thing, but it should get the job done. - # I don't think we have anything more elegant available at bosh-manifest-generation time. - # See https://docs.pivotal.io/partners/product-template-reference.html for list. - if 'type' in property and property['type'] in PROPERTY_FIELDS: - fields = {} - for field in PROPERTY_FIELDS[property['type']]: - if type(field) is tuple: - fields[field[0]] = '(( .properties.{}.{} ))'.format(property['name'], field[1]) - else: - fields[field] = '(( .properties.{}.{} ))'.format(property['name'], field) - out = { property['name']: fields } - else: - if property.get('is_reference', False): - out = { property['name']: property['default'] } - else: - out = { property['name']: '(( .properties.{}.value ))'.format(property['name']) } - return out + """Render a property for bosh manifest, according to its type.""" + # This ain't the prettiest thing, but it should get the job done. + # I don't think we have anything more elegant available at bosh-manifest-generation time. + # See https://docs.pivotal.io/partners/product-template-reference.html for list. + if 'type' in property and property['type'] in PROPERTY_FIELDS: + fields = {} + for field in PROPERTY_FIELDS[property['type']]: + if type(field) is tuple: + fields[field[0]] = '(( .properties.{}.{} ))'.format(property['name'], field[1]) + else: + fields[field] = '(( .properties.{}.{} ))'.format(property['name'], field) + out = {property['name']: fields} + else: + if property.get('is_reference', False): + out = {property['name']: property['default']} + else: + out = {property['name']: '(( .properties.{}.value ))'.format(property['name'])} + return out + def render_shell_variable_name(s): - """Convert s to a shell variable identifier.""" - return re.sub(r'[^a-zA-Z0-9]+', '_', s).upper() + """Convert s to a shell variable identifier.""" + return re.sub(r'[^a-zA-Z0-9]+', '_', s).upper() + -@contextfilter +@pass_context def render(context, input): - template = Template(input) - return template.render(context) + template = Template(input) + return template.render(context) + TEMPLATE_ENVIRONMENT = Environment(trim_blocks=True, lstrip_blocks=True, extensions=['jinja2.ext.do']) TEMPLATE_ENVIRONMENT.loader = FileSystemLoader(TEMPLATE_PATH) @@ -191,22 +210,26 @@ def render(context, input): TEMPLATE_ENVIRONMENT.filters['env_variable'] = render_env_variable TEMPLATE_ENVIRONMENT.filters['render'] = render + def render(target_path, template_file, config): - target_dir = os.path.dirname(target_path) - if target_dir != '': - mkdir_p(target_dir) - with open(target_path, 'wb') as target: - target.write(bytes(TEMPLATE_ENVIRONMENT.get_template(template_file).render(config), 'utf-8')) + target_dir = os.path.dirname(target_path) + if target_dir != '': + mkdir_p(target_dir) + with open(target_path, 'wb') as target: + target.write(bytes(TEMPLATE_ENVIRONMENT.get_template(template_file).render(config), 'utf-8')) + def exists(template_file): - return os.exists(path(template_file)) + return os.exists(path(template_file)) + def path(template_file): - return os.path.join(TEMPLATE_PATH, template_file) + return os.path.join(TEMPLATE_PATH, template_file) + def mkdir_p(dir): - try: - os.makedirs(dir) - except os.error as e: - if e.errno != errno.EEXIST: - raise + try: + os.makedirs(dir) + except os.error as e: + if e.errno != errno.EEXIST: + raise