Update Stanford i18n tooling to extract all strings

Previously platform mako strings had custom extraction config but we
don't need it since strings we want to ignore are segmented into smaller
files using the default paver i18n_extract command.

.tx/config

@@ -67,27 +67,57 @@ source_file = conf/locale/en/LC_MESSAGES/djangojs.po
 source_lang = en
 type = PO
 
-[stanford-openedx.django]
-file_filter = conf/locale/<lang>/LC_MESSAGES/stanford_django.po
-source_file = conf/locale/en/LC_MESSAGES/stanford_django.po
+[stanford-openedx.django-partial]
+file_filter = conf/locale/<lang>/LC_MESSAGES/stanford-django-partial.po
+source_file = conf/locale/en/LC_MESSAGES/stanford-django-partial.po
 source_lang = en
 type = PO
 
-[stanford-openedx.djangojs]
-file_filter = conf/locale/<lang>/LC_MESSAGES/stanford_djangojs.po
-source_file = conf/locale/en/LC_MESSAGES/stanford_djangojs.po
+[stanford-openedx.django-studio]
+file_filter = conf/locale/<lang>/LC_MESSAGES/stanford-django-studio.po
+source_file = conf/locale/en/LC_MESSAGES/stanford-django-studio.po
+source_lang = en
+type = PO
+
+[stanford-openedx.djangojs-partial]
+file_filter = conf/locale/<lang>/LC_MESSAGES/stanford-djangojs-partial.po
+source_file = conf/locale/en/LC_MESSAGES/stanford-djangojs-partial.po
+source_lang = en
+type = PO
+
+[stanford-openedx.djangojs-studio]
+file_filter = conf/locale/<lang>/LC_MESSAGES/stanford-djangojs-studio.po
+source_file = conf/locale/en/LC_MESSAGES/stanford-djangojs-studio.po
 source_lang = en
 type = PO
 
 [stanford-openedx.mako]
-file_filter = conf/locale/<lang>/LC_MESSAGES/stanford_mako.po
-source_file = conf/locale/en/LC_MESSAGES/stanford_mako.po
+file_filter = conf/locale/<lang>/LC_MESSAGES/stanford-mako.po
+source_file = conf/locale/en/LC_MESSAGES/stanford-mako.po
+source_lang = en
+type = PO
+
+[stanford-openedx.mako-studio]
+file_filter = conf/locale/<lang>/LC_MESSAGES/stanford-mako-studio.po
+source_file = conf/locale/en/LC_MESSAGES/stanford-mako-studio.po
 source_lang = en
 type = PO
 
 [stanford-openedx.underscore]
-file_filter = conf/locale/<lang>/LC_MESSAGES/stanford_underscore.po
-source_file = conf/locale/en/LC_MESSAGES/stanford_underscore.po
+file_filter = conf/locale/<lang>/LC_MESSAGES/stanford-underscore.po
+source_file = conf/locale/en/LC_MESSAGES/stanford-underscore.po
+source_lang = en
+type = PO
+
+[stanford-openedx.underscore-studio]
+file_filter = conf/locale/<lang>/LC_MESSAGES/stanford-underscore-studio.po
+source_file = conf/locale/en/LC_MESSAGES/stanford-underscore-studio.po
+source_lang = en
+type = PO
+
+[stanford-openedx.wiki]
+file_filter = conf/locale/<lang>/LC_MESSAGES/stanford-wiki.po
+source_file = conf/locale/en/LC_MESSAGES/stanford-wiki.po
 source_lang = en
 type = PO
 

conf/locale/stanford_config.yaml

@@ -132,8 +132,13 @@ generate_merge:
         - privacy.po
         - copyright.po
         - theme.po
-        - stanford_django.po
-        - stanford_mako.po
+        - stanford-django-partial.po
+        - stanford-django-studio.po
+        - stanford-mako.po
+        - stanford-mako-studio.po
+        - stanford-wiki.po
     djangojs.po:
-        - stanford_djangojs.po
-        - stanford_underscore.po
+        - stanford-djangojs-partial.po
+        - stanford-djangojs-studio.po
+        - stanford-underscore.po
+        - stanford-underscore-studio.po

openedx/stanford/pavelib/i18n.py

@@ -12,10 +12,6 @@
 from .utils.i18n_helpers import (
     CONFIG,
     clean_pofile,
-    extract_platform_django,
-    extract_platform_djangojs,
-    extract_platform_mako,
-    extract_platform_underscore,
     extract_theme_mako,
     extract_theme_tos,
     fix_privacy,
@@ -24,6 +20,7 @@
     merge_translations,
     pull,
     push,
+    rename_platform_pofiles,
 )
 
 
@@ -34,21 +31,14 @@
 
 @task
 @needs(
-    'pavelib.prereqs.install_prereqs',
+    'pavelib.i18n.i18n_extract',
 )
 def stanfordi18n_extract_platform():
     """
     Extract platform strings that need translation.
     """
-    LOG.info('Extracting platform...')
-    helpers = [
-        extract_platform_mako,
-        extract_platform_underscore,
-        extract_platform_django,
-        extract_platform_djangojs,
-    ]
-    for helper in helpers:
-        filename = helper()
+    renamed_files = rename_platform_pofiles()
+    for filename in renamed_files:
         clean_pofile(filename)
     LOG.info('Extracted platform.')
 

openedx/stanford/pavelib/utils/i18n_helpers.py

@@ -27,6 +27,7 @@
     "--output={output} {template_directory}"
 )
 BASE_DIR = Path('.').abspath()
+DEFAULT_CONFIG = Configuration()
 CONFIG = Configuration(filename=BASE_DIR / 'conf/locale/stanford_config.yaml')
 LOG = logging.getLogger(__name__)
 
@@ -53,45 +54,6 @@ def _extract_babel(file_config, file_output, template_directory='.', working_dir
     return file_output
 
 
-def extract_platform_mako():
-    mako_config = CONFIG.locale_dir / 'stanford_mako.cfg'
-    mako_file = CONFIG.source_messages_dir / 'stanford_mako.po'
-    output = _extract_babel(mako_config, mako_file)
-    return output
-
-
-def extract_platform_underscore():
-    underscore_config = CONFIG.locale_dir / 'babel_underscore.cfg'
-    underscore_file = CONFIG.source_messages_dir / 'stanford_underscore.po'
-    output = _extract_babel(underscore_config, underscore_file)
-    return output
-
-
-def extract_platform_django(file_base='django'):
-    filename = CONFIG.source_messages_dir / file_base + '.po'
-    filename_backup = filename + '.backup'
-    filename_stanford = CONFIG.source_messages_dir / 'stanford_' + file_base + '.po'
-    os.rename(filename, filename_backup)
-    makemessages = 'django-admin.py makemessages -l en'
-    ignores = ' '.join([
-        '--ignore="{}/*"'.format(directory)
-        for directory in CONFIG.ignore_dirs
-    ])
-    if ignores:
-        makemessages += ' ' + ignores
-    if file_base == 'djangojs':
-        makemessages += ' -d djangojs'
-    execute(makemessages)
-    os.rename(filename, filename_stanford)
-    os.rename(filename_backup, filename)
-    return filename_stanford
-
-
-def extract_platform_djangojs():
-    output = extract_platform_django('djangojs')
-    return output
-
-
 def extract_theme_mako():
     theme_dir = get_theme_dir()
     template_directory = 'lagunita/lms/templates/'
@@ -153,7 +115,6 @@ def get_theme_dir():
 
 
 def git_add():
-    sh('git add conf/locale')
     sh('git add lms/static/js/i18n')
     subprocess.check_call(
         'git add conf/locale',
@@ -254,25 +215,15 @@ def fix_header(pofile):
     """
     pofile.metadata_is_fuzzy = []
     header = pofile.header
-    fixes = (
-        ('SOME DESCRIPTIVE TITLE', 'Stanford OpenEdX translation file'),
-        ('Translations template for PROJECT.', 'Stanford OpenEdX translation file'),
-        ('YEAR', str(datetime.utcnow().year)),
-        ('ORGANIZATION', 'Stanford University'),
-        ("THE PACKAGE'S COPYRIGHT HOLDER", 'Stanford University'),
-        (
-            'This file is distributed under the same license as the PROJECT project.',
-            'This file is distributed under the GNU AFFERO GENERAL PUBLIC LICENSE.'
-        ),
-        (
-            'This file is distributed under the same license as the PACKAGE package.',
-            'This file is distributed under the GNU AFFERO GENERAL PUBLIC LICENSE.'
-        ),
-        ('FIRST AUTHOR <EMAIL@ADDRESS>', 'Stanford OpenEdX Team'),
-    )
-    for src, dest in fixes:
-        header = header.replace(src, dest)
-    pofile.header = header
+    year_string = str(datetime.utcnow().year)
+    header_data = [
+        'Stanford OpenEdX translation file',
+        'Copyright (C) ' + year_string + ' Stanford University',
+        'This file is distributed under the GNU AFFERO GENERAL PUBLIC LICENSE.',
+        'Stanford OpenEdX Team, ' + year_string + '.',
+        ''
+    ]
+    pofile.header = '\n'.join(header_data)
 
 
 def fix_metadata(pofile):
@@ -286,6 +237,7 @@ def fix_metadata(pofile):
         'Language': 'en',
         'Last-Translator': '',
         'Language-Team': 'English',
+        'Plural-Forms': 'nplurals=2; plural=(n != 1);',
     }
     pofile.metadata.update(fixes)
 
@@ -297,11 +249,7 @@ def merge_translations():
             # Language not yet available in code, fetch from upstream Transifex
             LOG.warn("Fetch upstream translations manually for %s", language)
             break
-        merge_mappings = {
-            'django.po': ['django', 'mako'],
-            'djangojs.po': ['djangojs', 'underscore'],
-        }
-        for existing, targets in merge_mappings.items():
+        for existing, targets in DEFAULT_CONFIG.generate_merge.items():
             for target in targets:
                 _merge_existing_translations(existing, target, language)
         LOG.info("Merged language: %s.", language)
@@ -340,17 +288,17 @@ def _merge_existing_translations(existing, target, language):
     Merge translations from existing file to target file for locale
     language and push up to Transifex
     """
-    target_filename = "stanford_" + target + '.po'
+    target_filename = "stanford-" + target
     target_source = CONFIG.source_messages_dir / target_filename
     common_file = CONFIG.get_messages_dir(language) / 'common.po'
     _merge_pull(target_source, common_file, language, existing)
     _merge_combine(target_source, target_filename, common_file, language)
-    _merge_push(target, language)
+    _merge_push(target.split('.')[0], language)
 
 
 def pull(language):
     LOG.info("Pulling language: %s...", language)
-    command = 'tx pull -l {language} -r "stanford-openedx.*"'.format(
+    command = 'tx pull -f -l {language} -r "stanford-openedx.*"'.format(
         language=language,
     )
     execute(command)
@@ -361,3 +309,24 @@ def push():
     LOG.info('Pushing source files to Transifex...')
     execute('tx push -s -r "stanford-openedx.*"')
     LOG.info('Pushed source files to Transifex.')
+
+
+def rename_platform_pofiles():
+    """
+    Prefix source files with 'stanford-' to push to Stanford Transifex project
+    rather than Open edX. Uses filenames specified in the generate_merge
+    mappings in the default config.yaml, which should match what was extracted.
+    """
+    merge_mapping_values = DEFAULT_CONFIG.generate_merge.itervalues()
+    renamed_files = set()
+    for mapping_filenames in merge_mapping_values:
+        for filename in mapping_filenames:
+            src = DEFAULT_CONFIG.source_messages_dir / filename
+            dst = DEFAULT_CONFIG.source_messages_dir / 'stanford-' + filename
+            if src.exists():
+                LOG.info("Renaming %s", filename)
+                os.rename(src, dst)
+                renamed_files.add(dst)
+            else:
+                LOG.error("%s doesn't exist, check extract command output files.", filename)
+    return renamed_files