#2: Implemented lifecycle of an EC2 instance

.github/workflows/check_ci.yaml

@@ -4,6 +4,7 @@ on: [push]
 
 jobs:
   check_setup_py:
+    environment: AWS
     runs-on: ubuntu-latest
 
     steps:
@@ -19,4 +20,8 @@ jobs:
 
     - name: Run pytest
       run: poetry run pytest
+      env: # Set the secret as an env variable
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }}
+        AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
 

.gitignore

@@ -0,0 +1,3 @@
+.idea
+.pytest_cache
+dist

dependencies.md

@@ -2,4 +2,5 @@
 
 - Python 3.8
 - Poetry
+- AWS
 

doc/changes/changes_0.1.0.md

@@ -12,7 +12,7 @@ n/a
 
 ## Features / Enhancements
 
-n/a
+ - #2: Implemented launch of an EC2 instance
 
 ## Documentation
 

exasol_script_languages_developer_sandbox/__init__.py

@@ -0,0 +1,4 @@
+
+from exasol_script_languages_developer_sandbox.cli.commands import (
+    setup_ec2,
+)

exasol_script_languages_developer_sandbox/cli/cli.py

@@ -0,0 +1,6 @@
+import click
+
+
+@click.group()
+def cli():
+    pass

exasol_script_languages_developer_sandbox/cli/commands/setup_ec2.py

@@ -0,0 +1,26 @@
+from typing import Optional
+
+import click
+
+from exasol_script_languages_developer_sandbox.cli.cli import cli
+from exasol_script_languages_developer_sandbox.cli.common import add_options
+from exasol_script_languages_developer_sandbox.cli.options.aws_options import aws_options
+from exasol_script_languages_developer_sandbox.cli.options.logging import logging_options, set_log_level
+from exasol_script_languages_developer_sandbox.lib.aws_access import AwsAccess
+from exasol_script_languages_developer_sandbox.lib.run_setup_ec2 import run_setup_ec2
+
+
+@cli.command()
+@add_options(aws_options)
+@add_options(logging_options)
+@click.option('--ec2-key-file', required=False, type=click.Path(exists=True, file_okay=True, dir_okay=False),
+                 default=None, help="The EC2 key-pair-file to use. If not given a temporary key-pair-file will be created.")
+@click.option('--ec2-key-name', required=False, type=str,
+             default=None, help="The EC2 key-pair-name to use. Only needs to be set together with ec2-key-file.")
+def setup_ec2(
+            aws_profile: str,
+            ec2_key_file: Optional[str],
+            ec2_key_name: Optional[str],
+            log_level: str):
+    set_log_level(log_level)
+    run_setup_ec2(AwsAccess(aws_profile), ec2_key_file, ec2_key_name)

exasol_script_languages_developer_sandbox/cli/common.py

@@ -0,0 +1,8 @@
+
+def add_options(options):
+    def _add_options(func):
+        for option in reversed(options):
+            func = option(func)
+        return func
+
+    return _add_options

exasol_script_languages_developer_sandbox/cli/options/aws_options.py

@@ -0,0 +1,6 @@
+import click
+
+aws_options = [
+    click.option('--aws-profile', required=False, type=str,
+                 help="Id of the AWS profile to use."),
+]

exasol_script_languages_developer_sandbox/cli/options/logging.py

@@ -0,0 +1,20 @@
+import logging
+
+import click
+
+SUPPORTED_LOG_LEVELS = {"normal": logging.WARNING, "info": logging.INFO, "debug": logging.DEBUG}
+
+logging_options = [
+    click.option('--log-level', type=click.Choice(list(SUPPORTED_LOG_LEVELS.keys())), default="normal",
+                 show_default=True,
+                 help="Level of information printed out. "
+                      "'Normal' prints only necessary information. "
+                      "'Info' prints also internal status info. 'Debug' prints detailed information."),
+]
+
+
+def set_log_level(level: str):
+    try:
+        logging.basicConfig(level=SUPPORTED_LOG_LEVELS[level])
+    except KeyError as ex:
+        raise ValueError(f"log level {level} is not supported!") from ex

exasol_script_languages_developer_sandbox/lib/aws_access.py

@@ -0,0 +1,120 @@
+import logging
+from typing import Optional, Any, List, Dict
+
+import boto3
+
+from exasol_script_languages_developer_sandbox.lib.deployer import Deployer
+
+
+class AwsAccess(object):
+    def __init__(self, aws_profile: Optional[str]):
+        self._aws_profile = aws_profile
+
+    @property
+    def aws_profile_for_logging(self) -> str:
+        if self._aws_profile is not None:
+            return self._aws_profile
+        else:
+            return "{default}"
+
+    @property
+    def aws_profile(self) -> Optional[str]:
+        return self._aws_profile
+
+    def create_new_ec2_key_pair(self, key_name: str) -> str:
+        """
+        Create an EC-2 Key-Pair, identified by parameter 'key_name'
+        """
+        logging.debug(f"Running create_new_ec2_key_pair for aws profile {self.aws_profile_for_logging}")
+        cloud_client = self._get_aws_client("ec2")
+        key_pair = cloud_client.create_key_pair(KeyName=key_name)
+        return str(key_pair['KeyMaterial'])
+
+    def delete_ec2_key_pair(self, key_name: str) -> None:
+        """
+        Delete the EC-2 Key-Pair, given by parameter 'key_name'
+        """
+        logging.debug(f"Running delete_ec2_key_pair for aws profile {self.aws_profile_for_logging}")
+        cloud_client = self._get_aws_client("ec2")
+        cloud_client.delete_key_pair(KeyName=key_name)
+
+    def upload_cloudformation_stack(self, yml: str, stack_name: str):
+        """
+        Deploy the cloudformation stack.
+        """
+        logging.debug(f"Running upload_cloudformation_stack for aws profile {self.aws_profile_for_logging}")
+        cloud_client = self._get_aws_client("cloudformation")
+        try:
+            cfn_deployer = Deployer(cloudformation_client=cloud_client)
+            result = cfn_deployer.create_and_wait_for_changeset(stack_name=stack_name, cfn_template=yml,
+                                                                parameter_values=[],
+                                                                capabilities=(), role_arn=None,
+                                                                notification_arns=None, tags=tuple())
+        except Exception as e:
+            logging.error(f"Error creating changeset for cloud formation template: {e}")
+            raise e
+        try:
+            cfn_deployer.execute_changeset(changeset_id=result.changeset_id, stack_name=stack_name)
+            cfn_deployer.wait_for_execute(stack_name=stack_name, changeset_type=result.changeset_type)
+        except Exception as e:
+            logging.error(f"Error executing changeset for cloud formation template: {e}")
+            logging.error(f"Run 'aws cloudformation describe-stack-events --stack-name {stack_name}' to get details.")
+            raise e
+
+    def validate_cloudformation_template(self, cloudformation_yml) -> None:
+        """
+        This function pushes the YAML to AWS Cloudformation for validation
+        (see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-validate-template.html)
+        Pitfall: Boto3 expects the YAML string as parameter, whereas the AWS CLI expects the file URL as parameter.
+        It requires to have the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY env variables set correctly.
+        """
+        logging.debug(f"Running validate_cloudformation_template for aws profile {self.aws_profile_for_logging}")
+        cloud_client = self._get_aws_client("cloudformation")
+        cloud_client.validate_template(TemplateBody=cloudformation_yml)
+
+    def get_all_stack_resources(self, stack_name: str) -> List[Dict[str, str]]:
+        """
+        This functions uses Boto3 to get all AWS Cloudformation resources for a specific Cloudformation stack,
+        identified by parameter `stack_name`.
+        The AWS API truncates at a size of 1MB, and in order to get all chunks the method must be called
+        passing the previous retrieved token until no token is returned.
+        """
+        logging.debug(f"Running get_all_codebuild_projects for aws profile {self.aws_profile_for_logging}")
+        cf_client = self._get_aws_client('cloudformation')
+        current_result = cf_client.list_stack_resources(StackName=stack_name)
+        result = current_result["StackResourceSummaries"]
+
+        while "nextToken" in current_result:
+            current_result = cf_client.list_projects(StackName=stack_name, nextToken=current_result["nextToken"])
+            result.extend(current_result["StackResourceSummaries"])
+        return result
+
+    def delete_stack(self, stack_name: str) -> None:
+        """
+        This functions uses Boto3 to delete a stack identified by parameter "stack_name".
+        """
+        logging.debug(f"Running delete_stack for aws profile {self.aws_profile_for_logging}")
+        cf_client = self._get_aws_client('cloudformation')
+        cf_client.delete_stack(StackName=stack_name)
+
+    def describe_instance(self, instance_id: str):
+        """
+        Describes an AWS instance identified by parameter instance_id
+        """
+        logging.debug(f"Running delete_ec2_key_pair for aws profile {self.aws_profile_for_logging}")
+        cloud_client = self._get_aws_client("ec2")
+        return cloud_client.describe_instances(InstanceIds=[instance_id])["Reservations"][0]["Instances"][0]
+
+    def get_user(self) -> str:
+        """
+        Return the current IAM user name.
+        """
+        iam_client = self._get_aws_client("iam")
+        cu = iam_client.get_user()
+        return cu["User"]["UserName"]
+
+    def _get_aws_client(self, service_name: str) -> Any:
+        if self._aws_profile is None:
+            return boto3.client(service_name)
+        aws_session = boto3.session.Session(profile_name=self._aws_profile)
+        return aws_session.client(service_name)

exasol_script_languages_developer_sandbox/lib/cf_stack.py

@@ -0,0 +1,69 @@
+import logging
+from typing import Optional
+
+from exasol_script_languages_developer_sandbox.lib.aws_access import AwsAccess
+from exasol_script_languages_developer_sandbox.lib.random_string_generator import get_random_str
+from exasol_script_languages_developer_sandbox.lib.render_template import render_template
+
+
+class CloudformationStack:
+    """
+    This class provides instantiation and destruction  of an AWS Cloudformation stack.
+    It is implemented as ContextManager, so that if it enters the context, the instance will be created,
+    and when exiting the stack will be destroyed.
+    """
+
+    def __init__(self, aws_access: AwsAccess, ec2_key_name: str, user_name: str):
+        self._aws_access = aws_access
+        self._stack_name = None
+        self._ec2_key_name = ec2_key_name
+        self._user_name = user_name
+
+    @staticmethod
+    def _generate_stack_name() -> str:
+        """
+        Create a new stack name. We append a random number as suffix,
+        so that in theory multiple instances can be created.
+        """
+        return f"EC2-SLC-DEV-SANDBOX-{get_random_str(5)}"
+
+    @property
+    def stack_name(self) -> Optional[str]:
+        return self._stack_name
+
+    def upload_cloudformation_stack(self):
+        yml = render_template("ec2_cloudformation.jinja.yaml", key_name=self._ec2_key_name, user_name=self._user_name)
+        self._stack_name = self._generate_stack_name()
+        self._aws_access.upload_cloudformation_stack(yml, self._stack_name)
+        logging.info(f"Deployed cloudformation stack {self._stack_name}")
+        return self
+
+    def get_ec2_instance_id(self) -> str:
+        stack_resources = self._aws_access.get_all_stack_resources(self._stack_name)
+        ec2_instance = [i for i in stack_resources if i["ResourceType"] == "AWS::EC2::Instance"]
+        if len(ec2_instance) == 0:
+            raise RuntimeError("Error starting or retrieving ec2 instance of stack %s" % self._stack_name)
+        elif len(ec2_instance) > 1:
+            raise RuntimeError("Multiple ec2 instances of stack %s" % self._stack_name)
+        ec2_instance_id = ec2_instance[0]["PhysicalResourceId"]
+        logging.info(f"Started EC2 with physical id {ec2_instance_id}")
+        return ec2_instance_id
+
+    def close(self) -> None:
+        if self._stack_name is not None:
+            self._aws_access.delete_stack(self._stack_name)
+
+
+class CloudformationStackContextManager:
+    """
+    The ContextManager-wrapper for CloudformationStack
+    """
+    def __init__(self, cf_stack: CloudformationStack):
+        self._cf_stack = cf_stack
+
+    def __enter__(self) -> CloudformationStack:
+        self._cf_stack.upload_cloudformation_stack()
+        return self._cf_stack
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        self._cf_stack.close()