forked from simple-framework/simple_grid_yaml_compiler
-
Notifications
You must be signed in to change notification settings - Fork 0
/
simple_grid_yaml_compiler.py
131 lines (113 loc) · 6.36 KB
/
simple_grid_yaml_compiler.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
from compiler import lexemes, semantics, yaml_augmentation, repo_processor, runtime_variables, processor_config_schemas
import argparse
from ruamel.yaml import YAML
from shutil import copyfile
# fetch repos, add include statement for the downloaded default_values.yaml
# INPUT: raw site-level-config file filled by site admin
# PROCESS: extract repo_urls, download and save default files for repositories
# OUTPUT: include statements for default files of repositories + raw site level-config file
def phase_1(site_level_configuration_file):
# fetch repo and get default_values.yaml
main_default_values_file = repo_processor.get_default_values("https://github.com/WLCG-Lightweight-Sites/simple_grid_site_defaults", "site_level_configuration_defaults.yaml" )
repo_urls = lexemes.get_repo_list(site_level_configuration_file)
print(repo_urls)
file_names_repository_default = [main_default_values_file]
for url in repo_urls:
file_names_repository_default.append(repo_processor.get_default_values(url, 'default-data.yaml'))
default_includes_yaml_file = yaml_augmentation.add_include_statements_for_default_files(file_names_repository_default, site_level_configuration_file)
return default_includes_yaml_file, repo_urls
# add data from all includes
# INPUT: include statements for default files of repositories + raw site level-config file
# PROCESS: recursively replace include statements with contents of the files that are to be included.
# OUTPUT: include statements replaced by contents of referred files + raw site-level-config file
def phase_2(default_includes_yaml_file):
return yaml_augmentation.add_included_files(default_includes_yaml_file)
def phase_3(include_made):
runtime_vars, config_file = runtime_variables.extract_runtime_variables(include_made)
return runtime_vars, runtime_variables.add_runtime_variables(runtime_vars, config_file)
# syntax_checking
def phase_4(final_yaml_file):
return semantics.check_yaml_syntax(final_yaml_file)
def phase_5(phase_4_output, runtime_vars, yaml):
phase_4_output_file = open(phase_4_output.name, 'r')
phase_5_output_file = open("./.temp/phase_5_output.yaml", 'w')
data = yaml.load(phase_4_output_file)
phase_5_output = {}
for data_section in data:
if data_section == 'lightweight_components':
updated_components = []
for lightweight_component in data[data_section]:
print "Component: " + lightweight_component['name']
updated_component = {}
for component_section in lightweight_component:
if component_section == 'config':
repo_url = lightweight_component['repository_url']
repo_processor.get_config_schema(repo_url)
repo_processor.get_meta_info(repo_url)
repo_info = repo_processor.analyse_repo_url(repo_url)
config_schema_file_name = repo_processor.generate_config_schema_file_name(repo_info)
config_schema_file = open(config_schema_file_name, 'r')
meta_info_file_name = repo_processor.generate_meta_info_file_name(repo_info)
meta_info_file = open(meta_info_file_name, 'r')
default_data_file_name = repo_processor.generate_default_file_name(repo_info)
default_data_runtime_file = open(default_data_file_name + ".runtime", 'w')
default_data_file = open(default_data_file_name, 'r')
default_data_runtime_file.write(runtime_vars)
for l in default_data_file.readlines():
default_data_runtime_file.write(l)
default_data_file.close()
default_data_runtime_file.close()
config_schema_file.close()
meta_info_file.close()
augmented_config = processor_config_schemas.process_config_schema(lightweight_component[component_section], config_schema_file, default_data_runtime_file, meta_info_file)
updated_component[component_section] = augmented_config
else:
updated_component[component_section] = lightweight_component[component_section]
updated_components.append(updated_component)
phase_5_output[data_section] = updated_components
else:
phase_5_output[data_section] = data[data_section]
yaml.dump(phase_5_output, phase_5_output_file)
phase_5_output_file.close()
return phase_5_output_file
def phase_6(phase_5_output_file, yaml):
phase_5_output = open(phase_5_output_file.name, 'r')
input_data = yaml.load(phase_5_output)
phase_6_output_file = open("./.temp/phase_6_output.yaml", 'w')
#pass 1
variable_hierarchies = lexemes.parse_for_variable_hierarchies(input_data, "__from__")
#pass 2
split_config = yaml_augmentation.split_component_config(variable_hierarchies)
#pass 3
with_ids = yaml_augmentation.add_component_ids(split_config)
print with_ids
yaml.dump(with_ids, phase_6_output_file)
return phase_6_output_file
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('filename')
parser.add_argument('-o', '--output')
args = parser.parse_args()
return {
'site_level_configuration_file': args.filename,
'output': args.output
}
if __name__ == "__main__":
args = parse_args()
site_level_configuration_file = open(args['site_level_configuration_file'], 'r')
output = open(args['output'], 'w')
yaml = YAML()
phase_1_output, repo_urls = phase_1(site_level_configuration_file)
phase_2_output = phase_2(phase_1_output)
runtime_vars, phase_3_output = phase_3(phase_2_output)
phase_4_output = phase_4(phase_3_output)
# augmented_yaml_file = phase_3(open('./.temp/runtime.yaml', 'r'), output)
augmented_yaml_file = open(phase_4_output.name, 'r')
# data =yaml.load(augmented_yaml_file)
# print data['lightweight_components'][0]['config']['cream-info']['ce_cpu_model']
# print data['supported_virtual_organizations']
# print data['site']['latitude']
phase_5_output_file = phase_5(phase_4_output, runtime_vars, yaml)
phase_6_output = phase_6(phase_5_output_file, yaml)
copyfile(phase_6_output.name, output.name)
output.close()