Use utf-8 when opening text files

All of our source files are encoded in UTF-8. We have other scripts that
make this assumption. We don't want to depend on the platform encoding.

This fixes a bug whereby non-ASCII characters in .function files led to an
error when writing the generated C file. Fix #64.

Signed-off-by: Gilles Peskine <[email protected]>

scripts/generate_test_code.py

@@ -1092,8 +1092,8 @@ def read_code_from_input_files(platform_file, helpers_file,
     :return:
     """
     # Read helpers
-    with open(helpers_file, 'r') as help_f, open(platform_file, 'r') as \
-            platform_f:
+    with open(helpers_file, 'r', encoding='utf-8') as help_f, \
+         open(platform_file, 'r', encoding='utf-8') as platform_f:
         snippets['test_common_helper_file'] = helpers_file
         snippets['test_common_helpers'] = help_f.read()
         snippets['test_platform_file'] = platform_file
@@ -1123,7 +1123,8 @@ def write_test_source_file(template_file, c_file, snippets):
     invalid = "(?P<invalid>__MBEDTLS_TEST_TEMPLATE__)"
     placeholder_pattern = re.compile("|".join([escaped, named, braced, invalid]))
 
-    with open(template_file, 'r') as template_f, open(c_file, 'w') as c_f:
+    with open(template_file, 'r', encoding='utf-8') as template_f, \
+         open(c_file, 'w', encoding='utf-8') as c_f:
         for line_no, line in enumerate(template_f.readlines(), 1):
             # Update line number. +1 as #line directive sets next line number
             snippets['line_no'] = line_no + 1
@@ -1166,7 +1167,7 @@ def generate_intermediate_data_file(data_file, out_data_file,
     :return:
     """
     with FileWrapper(data_file) as data_f, \
-            open(out_data_file, 'w') as out_data_f:
+         open(out_data_file, 'w', encoding='utf-8') as out_data_f:
         dep_check_code, expression_code = gen_from_test_data(
             data_f, out_data_f, func_info, suite_dependencies)
         snippets['dep_check_code'] = dep_check_code