From a2f22ebd77f6f741f5ad297f7388e1b780649945 Mon Sep 17 00:00:00 2001 From: Adam Stankiewicz Date: Tue, 11 Jun 2024 10:47:23 -0400 Subject: [PATCH] feat: upload source maps to Datadog during MFE deploy (#3) --- tubular/scripts/frontend_deploy.py | 15 ++- tubular/scripts/frontend_multi_deploy.py | 15 ++- tubular/scripts/frontend_utils.py | 138 ++++++++++++++++------- 3 files changed, 119 insertions(+), 49 deletions(-) diff --git a/tubular/scripts/frontend_deploy.py b/tubular/scripts/frontend_deploy.py index 316437c3..306307f1 100755 --- a/tubular/scripts/frontend_deploy.py +++ b/tubular/scripts/frontend_deploy.py @@ -22,6 +22,10 @@ @click.command("frontend_deploy") +@click.option( + '--common-config-file', + help='File from which common configuration variables are read.', +) @click.option( '--env-config-file', help='File from which to read environment configuration variables.', @@ -40,25 +44,28 @@ is_flag=True, help='Boolean to decide if Cloudflare cache needs to be purged or not.', ) -def frontend_deploy(env_config_file, app_name, app_dist, purge_cache): +def frontend_deploy(common_config_file, env_config_file, app_name, app_dist, purge_cache): """ Copies a frontend application to an s3 bucket. Args: + common_config_file (str): Path to a YAML file containing common configuration variables. env_config_file (str): Path to a YAML file containing environment configuration variables. app_name (str): Name of the frontend app. app_dist (str): Path to frontend application dist directory. purge_cache (bool): Should Cloudflare cache needs to be purged. """ - if not env_config_file: - FAIL(1, 'Environment config file was not specified.') if not app_name: FAIL(1, 'Frontend application name was not specified.') if not app_dist: FAIL(1, 'Frontend application dist path was not specified.') + if not common_config_file: + FAIL(1, 'Common config file was not specified.') + if not env_config_file: + FAIL(1, 'Environment config file was not specified.') - deployer = FrontendDeployer(env_config_file, app_name) + deployer = FrontendDeployer(common_config_file, env_config_file, app_name) bucket_name = deployer.env_cfg.get('S3_BUCKET_NAME') if not bucket_name: FAIL(1, 'No S3 bucket name configured for {}.'.format(app_name)) diff --git a/tubular/scripts/frontend_multi_deploy.py b/tubular/scripts/frontend_multi_deploy.py index a6b57d5b..da60c10e 100755 --- a/tubular/scripts/frontend_multi_deploy.py +++ b/tubular/scripts/frontend_multi_deploy.py @@ -22,6 +22,10 @@ @click.command("frontend_deploy") +@click.option( + '--common-config-file', + help='File from which common configuration variables are read.', +) @click.option( '--env-config-file', help='File from which to read environment configuration variables.', @@ -40,27 +44,30 @@ is_flag=True, help='Boolean to decide if Cloudflare cache needs to be purged or not.', ) -def frontend_deploy(env_config_file, app_name, app_dist, purge_cache): +def frontend_deploy(common_config_file, env_config_file, app_name, app_dist, purge_cache): """ Copies a frontend application to an s3 bucket. Args: + common_config_file (str): Path to a YAML file containing common configuration variables. env_config_file (str): Path to a YAML file containing environment configuration variables. app_name (str): Name of the frontend app. app_dist (str): Path to frontend application dist directory. purge_cache (bool): Should Cloudflare cache needs to be purged. """ - if not env_config_file: - FAIL(1, 'Environment config file was not specified.') if not app_name: FAIL(1, 'Frontend application name was not specified.') if not app_dist: FAIL(1, 'Frontend application dist path was not specified.') + if not common_config_file: + FAIL(1, 'Common config file was not specified.') + if not env_config_file: + FAIL(1, 'Environment config file was not specified.') # We are deploying ALL sites to a single bucket so they live at # // within the global bucket. - deployer = FrontendDeployer(env_config_file, app_name) + deployer = FrontendDeployer(common_config_file, env_config_file, app_name) bucket_name = deployer.env_cfg.get('BUCKET_NAME') if not bucket_name: FAIL(1, 'No S3 bucket name configured for {}.'.format(app_name)) diff --git a/tubular/scripts/frontend_utils.py b/tubular/scripts/frontend_utils.py index 64d61df5..476be189 100755 --- a/tubular/scripts/frontend_utils.py +++ b/tubular/scripts/frontend_utils.py @@ -26,21 +26,29 @@ MAX_TRIES = 3 -class FrontendBuilder: - """ Utility class for building frontends. """ - SCRIPT_SHORTNAME = 'Build frontend' - LOG = partial(_log, SCRIPT_SHORTNAME) - FAIL = partial(_fail, SCRIPT_SHORTNAME) - def __init__(self, common_config_file, env_config_file, app_name, version_file): +class FrontendUtils: + """ + Base class for frontend utilities used for both building and + deploying frontends, i.e. `FrontendBuilder` and `FrontendDeployer`. + """ + + def __init__(self, common_config_file, env_config_file, app_name): self.common_config_file = common_config_file self.env_config_file = env_config_file self.app_name = app_name - self.version_file = version_file self.common_cfg, self.env_cfg = self._get_configs() + def FAIL(self): + """ Placeholder for failure method """ + raise NotImplementedError + + def LOG(self): + """ Placeholder for logging method """ + raise NotImplementedError + def _get_configs(self): - """Loads configs from their paths""" + """ Loads configs from their paths """ try: with io.open(self.common_config_file, 'r') as contents: common_vars = yaml.safe_load(contents) @@ -55,6 +63,35 @@ def _get_configs(self): return (common_vars, env_vars) + def get_app_config(self): + """ Combines the common and environment configs APP_CONFIG data """ + app_config = self.common_cfg.get('APP_CONFIG', {}) + app_config.update(self.env_cfg.get('APP_CONFIG', {})) + app_config['APP_VERSION'] = self.get_version_commit_sha() + if not app_config: + self.LOG('Config variables do not exist for app {}.'.format(self.app_name)) + return app_config + + def get_version_commit_sha(self): + """ Returns the commit SHA of the current HEAD """ + return LocalGitAPI(Repo(self.app_name)).get_head_sha() + + +class FrontendBuilder(FrontendUtils): + """ Utility class for building frontends. """ + + SCRIPT_SHORTNAME = 'Build frontend' + LOG = partial(_log, SCRIPT_SHORTNAME) + FAIL = partial(_fail, SCRIPT_SHORTNAME) + + def __init__(self, common_config_file, env_config_file, app_name, version_file): + super().__init__( + common_config_file=common_config_file, + env_config_file=env_config_file, + app_name=app_name, + ) + self.version_file = version_file + def install_requirements(self): """ Install requirements for app to build """ proc = subprocess.Popen(['npm install'], cwd=self.app_name, shell=True) @@ -104,21 +141,12 @@ def install_requirements_npm_private(self): install_list, self.app_name )) - def get_app_config(self): - """ Combines the common and environment configs APP_CONFIG data """ - app_config = self.common_cfg.get('APP_CONFIG', {}) - app_config.update(self.env_cfg.get('APP_CONFIG', {})) - app_config['APP_VERSION'] = self.get_version_commit_sha() - if not app_config: - self.LOG('Config variables do not exist for app {}.'.format(self.app_name)) - return app_config - def get_npm_aliases_config(self): """ Combines the common and environment configs NPM_ALIASES data """ npm_aliases_config = self.common_cfg.get('NPM_ALIASES', {}) npm_aliases_config.update(self.env_cfg.get('NPM_ALIASES', {})) if not npm_aliases_config: - self.LOG('No npm package aliases defined in config.') + self.LOG('No NPM package aliases defined in config.') return npm_aliases_config def get_npm_private_config(self): @@ -126,11 +154,11 @@ def get_npm_private_config(self): npm_private_config = self.common_cfg.get('NPM_PRIVATE', []) npm_private_config.extend(self.env_cfg.get('NPM_PRIVATE', [])) if not npm_private_config: - self.LOG('No npm private packages defined in config.') + self.LOG('No NPM private packages defined in config.') return list(set(npm_private_config)) def build_app(self, env_vars, fail_msg): - """ Builds the app with environment variable.""" + """ Builds the app with environment variable. """ proc = subprocess.Popen( ' '.join(env_vars + ['npm run build']), cwd=self.app_name, @@ -140,10 +168,6 @@ def build_app(self, env_vars, fail_msg): if build_return_code != 0: self.FAIL(1, fail_msg) - def get_version_commit_sha(self): - """ Returns the commit SHA of the current HEAD """ - return LocalGitAPI(Repo(self.app_name)).get_head_sha() - def create_version_file(self): """ Creates a version.json file to be deployed with frontend """ # Add version.json file to build. @@ -176,29 +200,15 @@ def copy_js_config_file_to_app_root(self, app_config, app_name): self.FAIL(1, f"Could not copy '{source}' to '{destination}', due to destination not writable.") -class FrontendDeployer: +class FrontendDeployer(FrontendUtils): """ Utility class for deploying frontends. """ SCRIPT_SHORTNAME = 'Deploy frontend' LOG = partial(_log, SCRIPT_SHORTNAME) FAIL = partial(_fail, SCRIPT_SHORTNAME) - def __init__(self, env_config_file, app_name): - self.env_config_file = env_config_file - self.app_name = app_name - self.env_cfg = self._get_config() - - def _get_config(self): - """Loads config from it's path""" - try: - with io.open(self.env_config_file, 'r') as contents: - env_vars = yaml.safe_load(contents) - except IOError: - self.FAIL(1, 'Environment config file {} could not be opened.'.format(self.env_config_file)) - return env_vars - - def deploy_site(self, bucket_name, app_path): - """Deploy files to bucket.""" + def _deploy_to_s3(self, bucket_name, app_path): + """ Deploy files to S3 bucket. """ bucket_uri = 's3://{}'.format(bucket_name) proc = subprocess.Popen( ' '.join(['aws s3 sync', app_path, bucket_uri, '--delete']), @@ -207,6 +217,52 @@ def deploy_site(self, bucket_name, app_path): return_code = proc.wait() if return_code != 0: self.FAIL(1, 'Could not sync app {} with S3 bucket {}.'.format(self.app_name, bucket_uri)) + + def _upload_js_sourcemaps(self, app_path): + """ Upload JavaScript sourcemaps to Datadog. """ + app_config = self.get_app_config() + datadog_api_key = os.environ.get('DATADOG_API_KEY') + if not datadog_api_key: + # Can't upload source maps without ``DATADOG_API_KEY``, which must be set as an environment variable + # before executing the Datadog CLI. The Datadog documentation suggests using a dedicated Datadog API key: + # https://docs.datadoghq.com/real_user_monitoring/guide/upload-javascript-source-maps/ + self.LOG('Could not find DATADOG_API_KEY environment variable while uploading source maps.') + return + + service = app_config.get('DATADOG_SERVICE') + if not service: + self.LOG('Could not find DATADOG_SERVICE for app {} while uploading source maps.'.format(self.app_name)) + + # Prioritize app-specific version override, if any, before default APP_VERSION commit SHA version + version = app_config.get('DATADOG_VERSION') or app_config.get('APP_VERSION') + if not version: + self.LOG('Could not find version for app {} while uploading source maps.'.format(self.app_name)) + return + + command_args = ' '.join([ + f'--service="{service}"', + f'--release-version="{version}"' + '--minified-path-prefix="/"', # Sourcemaps are relative to the root when deployed + ]) + self.LOG('Uploading source maps to Datadog for app {}.'.format(self.app_name)) + proc = subprocess.Popen( + ' '.join([ + './node_modules/.bin/datadog-ci sourcemaps upload', + app_path, + command_args, + ]), + cwd=self.app_name, + shell=True, + ) + return_code = proc.wait() + if return_code != 0: + # If failure occurs, log the error and but don't fail the deployment. + self.LOG('Could not upload source maps to Datadog for app {}.'.format(self.app_name)) + + def deploy_site(self, bucket_name, app_path): + """ Deploy files to bucket. """ + self._deploy_to_s3(bucket_name, app_path) + self._upload_js_sourcemaps(app_path) self.LOG('Frontend application {} successfully deployed to {}.'.format(self.app_name, bucket_name)) @backoff.on_exception(backoff.expo,