Skip to content

Commit

Permalink
Job Specific Blueprints (#315)
Browse files Browse the repository at this point in the history 
* add job specific blue prints feature

* * Job specific props not getting removed soon enough
* Job specific props duplicated when more than one BOSH release
* Remove job specific props from getting dumped into manifest

* Docker-bosh manifest fix for job specific props

* Python 3 fix (mutating map while iterating)
  • Loading branch information
@cholick @keshto
cholick authored and keshto committed Jul 30, 2019
1 parent 78740ad commit c013a47
Show file tree
Hide file tree
Showing 3 changed files with 45 additions and 35 deletions.
3 changes: 2 additions & 1 deletion tile_generator/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -376,7 +376,8 @@ def build_job_manifest(self, job):
'pks_password': '(( ..pivotal-container-service.properties.pks_basic_auth.password ))',
})
merge_dict(manifest, job['properties'])
for property in self.get('all_properties', []):
all_properties = self.get('all_properties', [])
for property in [p for p in all_properties if 'job' not in p]:
merge_dict(manifest, template.render_property(property))
for service_plan_form in self.get('service_plan_forms', []):
manifest[service_plan_form['name']] = '(( .properties.{}.value ))'.format(service_plan_form['name'])
Expand Down
1 change: 1 addition & 0 deletions tile_generator/template.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ def render_yaml(input):
def multiline_representer(dumper, data):
return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|" if "\n" in data else None)
yaml.SafeDumper.add_representer(str, multiline_representer)
yaml.SafeDumper.ignore_aliases = lambda *args : True
return yaml.safe_dump(input, default_flow_style=False, width=float("inf"))

def render_yaml_literal(input):
Expand Down
76 changes: 42 additions & 34 deletions tile_generator/tile_metadata.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import yaml
import copy
from . import template as template_helper


Expand Down Expand Up @@ -172,8 +173,9 @@ def _build_property_blueprints(self):
# Custom properties from the tile.yml file
#
for prop in self.config['all_properties']:
prop = template_helper.expand_selector(prop)
self.tile_metadata['property_blueprints'] += [prop]
if 'job' not in prop:
prop = template_helper.expand_selector(prop)
self.tile_metadata['property_blueprints'] += [prop]

def _build_form_types(self):
form_types = list()
Expand All @@ -191,7 +193,7 @@ def _build_form_types(self):
custom_form["property_inputs"] = list()
for prop in form.get('properties', []):
prop_input = {
"reference": ".properties." + prop['name'],
"reference": "." + prop.get('job', "properties") + "." + prop['name'],
"label": prop['label'],
"description": prop.get('description') or prop.get('label')
}
Expand Down Expand Up @@ -384,6 +386,26 @@ def _build_job_types(self):

job_types.append(compilation_job)

prop_blueprints = [{'default': {'identity': 'vcap'},
'name': 'vm_credentials',
'type': 'salted_credentials'}]

# Grab all job specific property blueprints
job_specific_prop_blueprints = [prop for prop in self.config['all_properties'] if 'job' in prop]
for prop in job_specific_prop_blueprints:
keys = list(prop.keys())
for k in keys:
# Remove any keys not specified here https://docs.pivotal.io/tiledev/2-2/property-reference.html#common-attributes
# but keep `job` key because it is needed for matching
if k not in ['job', 'name', 'type', 'optional', 'configurable', 'freeze_on_deploy']: prop.pop(k)

def match_job_specific_prop_blueprints(job_name, prop_blueprints):
for prop in job_specific_prop_blueprints:
if prop['job'] == job_name:
tmp_prop = dict(prop)
tmp_prop.pop('job')
prop_blueprints.append(tmp_prop)


#
# {{ job.template }} job for {{ job.package.name }}
Expand Down Expand Up @@ -432,17 +454,10 @@ def _build_job_types(self):
'name': 'route_registrar',
'release': 'routing'}
],
'property_blueprints': [
{
'default': {'identity': 'vcap'},
'name': 'vm_credentials',
'type': 'salted_credentials'},
{
'name': 'app_credentials',
'type': 'salted_credentials'
}
]
'property_blueprints': prop_blueprints + [{'name': 'app_credentials',
'type': 'salted_credentials'}]
}
match_job_specific_prop_blueprints(release_job['name'], release_job['property_blueprints'])

release_job['manifest'] = dict()
release_job['manifest'].update(job.get('manifest', {}))
Expand All @@ -461,8 +476,9 @@ def _build_job_types(self):
release_job_manifest['route_registrar'] = routes

for prop in self.config.get('all_properties'):
prop = template_helper.render_property(prop)
release_job_manifest.update(prop)
if 'job' not in prop:
prop = template_helper.render_property(prop)
release_job_manifest.update(prop)

for service_plan_form in self.config.get('service_plan_forms', []):
form_name = service_plan_form.get('name')
Expand Down Expand Up @@ -497,10 +513,10 @@ def _build_job_types(self):
release_job['manifest'] = literal_unicode(template_helper.render_yaml(release_job_manifest))

instance_def = {
'configurable': True,
'default': job.get('instances', 1),
'name': 'instances',
'type': 'integer'
'configurable': True,
'default': job.get('instances', 1),
'name': 'instances',
'type': 'integer'
}
if self.config['metadata_version'] < 1.7:
release_job['instance_definitions'] = [instance_def]
Expand All @@ -518,13 +534,11 @@ def _build_job_types(self):
'max_in_flight': job.get('max_in_flight', 1),
'single_az_only': job.get('single_az_only', True if job.get('lifecycle') == 'errand' else False),
'static_ip': job.get('static_ip', 0),
'property_blueprints': [{
'default': {'identity': 'vcap'},
'name': 'vm_credentials',
'type': 'salted_credentials'
}],
'property_blueprints': copy.copy(prop_blueprints),
'manifest': literal_unicode(template_helper.render_yaml(job.get('manifest'))),
}
match_job_specific_prop_blueprints(bosh_release_job['name'], bosh_release_job['property_blueprints'])

if job.get('lifecycle') == 'errand':
bosh_release_job['errand'] = True
if job.get('run_post_deploy_errand_default'):
Expand Down Expand Up @@ -637,18 +651,12 @@ def _build_job_types(self):
"dynamic_ip": 1,
"max_in_flight": 1,
"single_az_only": True,
"property_blueprints": [{
"default": {"identity": "vcap"},
"type": "salted_credentials",
"name": "vm_credentials"
},
{
"type": "salted_credentials",
"name": "app_credentials"
}
],
"property_blueprints": prop_blueprints + [{"type": "salted_credentials",
"name": "app_credentials"}],
"manifest": literal_unicode(template_helper.render_yaml(job.get('manifest')))
}
match_job_specific_prop_blueprints(job_type['name'], job_type['property_blueprints'])

if job['name'] == 'deploy-all' or job['name'] == 'delete-all':
job_type['templates'].append({
"release": 'cf-cli',
Expand Down

0 comments on commit c013a47

Please sign in to comment.