Add workflow for AWS tests & document the setup

- Add a new workflow integration-aws for running the AWS integration tests. - Add setup docs for configuring the AWS account and GitHub repository for running the tests. Signed-off-by: Sunny <[email protected]>
fluxcd · May 28, 2024 · 6ff943f · 6ff943f
1 parent dbf9c82
commit 6ff943f
Show file tree

Hide file tree

Showing 2 changed files with 349 additions and 0 deletions.
diff --git a/.github/workflows/integration-aws.yaml b/.github/workflows/integration-aws.yaml
@@ -0,0 +1,77 @@
+name: integration-aws
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 6 * * *"
+  # push:
+  #   branches:
+  #     - main
+
+permissions:
+  contents: read
+  id-token: write # Required for obtaining AWS OIDC federated credential.
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./tests/integration
+    steps:
+      - name: Checkout
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      - name: Set up Go
+        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
+        with:
+          go-version-file: 'tests/integration/go.mod'
+          cache-dependency-path: tests/integration/go.sum
+      - name: configure aws credentials
+        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
+        with:
+          role-to-assume: arn:aws:iam::${{ secrets.IRC_E2E_AWS_ACCOUNT_ID }}:role/${{ secrets.IRC_E2E_AWS_ASSUME_ROLE_NAME }}
+          role-session-name: GitHub_to_AWS_via_FederatedOIDC
+          aws-region: ${{ vars.IRC_E2E_AWS_REGION }}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb  # v3.3.0
+      - name: Cache Docker layers
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        id: cache
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-ghcache-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-ghcache-
+      - name: Set dynamic variables in .env
+        run: |
+          cat > .env <<EOF
+          export TF_VAR_rand=${RANDOM}
+          export TF_VAR_tags='{"environment"="github", "ci"="true", "repo"="image-reflector-controller", "createdat"="$(date -u +x%Y-%m-%d_%Hh%Mm%Ss)"}'
+          EOF
+      - name: Print .env for dynamic tag value reference
+        run: cat .env
+      - name: Build controller
+        run: |
+          make docker-build IMG=fluxcd/image-reflector-controller:dev \
+            BUILD_PLATFORMS=linux/amd64 \
+            BUILD_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache \
+              --cache-to=type=local,dest=/tmp/.buildx-cache-new,mode=max"
+        working-directory: ./
+      - # Temp fix
+        # https://github.com/docker/build-push-action/issues/252
+        # https://github.com/moby/buildkit/issues/1896
+        name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+      - name: Run tests
+        run: . .env && make test-aws TEST_IMG=fluxcd/image-reflector-controller:dev
+        env:
+          AWS_REGION: ${{ vars.IRC_E2E_AWS_REGION }}
+      - name: Ensure resource cleanup
+        if: ${{ always() }}
+        run: . .env && make destroy-aws
+        env:
+          AWS_REGION: ${{ vars.IRC_E2E_AWS_REGION }}
diff --git a/tests/integration/README.md b/tests/integration/README.md
@@ -9,6 +9,278 @@
 - AWS CLI, does not need to be configured with the AWS account.
 - Docker CLI for registry login.
 - kubectl for applying certain install manifests.
+- jq for parsing JSON response from AWS.
+
+#### Permissions
+
+The following policy document can be used to create an IAM Policy for
+provisioning the infrastructure and running the tests:
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "testinfra",
+            "Effect": "Allow",
+            "Action": [
+                "ec2:AllocateAddress",
+                "ec2:AssociateRouteTable",
+                "ec2:AttachInternetGateway",
+                "ec2:AuthorizeSecurityGroupEgress",
+                "ec2:AuthorizeSecurityGroupIngress",
+                "ec2:CreateInternetGateway",
+                "ec2:CreateLaunchTemplate",
+                "ec2:CreateLaunchTemplateVersion",
+                "ec2:CreateNatGateway",
+                "ec2:CreateNetworkAcl",
+                "ec2:CreateNetworkAclEntry",
+                "ec2:CreateRoute",
+                "ec2:CreateRouteTable",
+                "ec2:CreateSecurityGroup",
+                "ec2:CreateSubnet",
+                "ec2:CreateTags",
+                "ec2:CreateVpc",
+                "ec2:DeleteInternetGateway",
+                "ec2:DeleteLaunchTemplate",
+                "ec2:DeleteNatGateway",
+                "ec2:DeleteNetworkAcl",
+                "ec2:DeleteNetworkAclEntry",
+                "ec2:DeleteRoute",
+                "ec2:DeleteRouteTable",
+                "ec2:DeleteSecurityGroup",
+                "ec2:DeleteSubnet",
+                "ec2:DeleteTags",
+                "ec2:DeleteVolume",
+                "ec2:DeleteVpc",
+                "ec2:DescribeAddresses",
+                "ec2:DescribeAddressesAttribute",
+                "ec2:DescribeAvailabilityZones",
+                "ec2:DescribeInternetGateways",
+                "ec2:DescribeLaunchTemplates",
+                "ec2:DescribeLaunchTemplateVersions",
+                "ec2:DescribeNatGateways",
+                "ec2:DescribeNetworkAcls",
+                "ec2:DescribeNetworkInterfaces",
+                "ec2:DescribeRouteTables",
+                "ec2:DescribeSecurityGroupRules",
+                "ec2:DescribeSecurityGroups",
+                "ec2:DescribeSubnets",
+                "ec2:DescribeTags",
+                "ec2:DescribeVpcAttribute",
+                "ec2:DescribeVpcs",
+                "ec2:DetachInternetGateway",
+                "ec2:DisassociateAddress",
+                "ec2:DisassociateRouteTable",
+                "ec2:ModifyVpcAttribute",
+                "ec2:ReleaseAddress",
+                "ec2:RevokeSecurityGroupEgress",
+                "ec2:RevokeSecurityGroupIngress",
+                "ec2:RunInstances",
+                "ecr:BatchGetImage",
+                "ecr:BatchCheckLayerAvailability",
+                "ecr:CreateRepository",
+                "ecr:CompleteLayerUpload",
+                "ecr:DeleteRepository",
+                "ecr:DescribeRepositories",
+                "ecr:GetAuthorizationToken",
+                "ecr:InitiateLayerUpload",
+                "ecr:ListTagsForResource",
+                "ecr:PutImage",
+                "ecr:TagResource",
+                "ecr:UploadLayerPart",
+                "eks:AssociateAccessPolicy",
+                "eks:CreateAccessEntry",
+                "eks:CreateAddon",
+                "eks:CreateCluster",
+                "eks:CreateNodegroup",
+                "eks:DeleteAccessEntry",
+                "eks:DeleteAddon",
+                "eks:DeleteCluster",
+                "eks:DeleteNodegroup",
+                "eks:DescribeAccessEntry",
+                "eks:DescribeAddon",
+                "eks:DescribeAddonVersions",
+                "eks:DescribeCluster",
+                "eks:DescribeNodegroup",
+                "eks:DisassociateAccessPolicy",
+                "eks:ListAssociatedAccessPolicies",
+                "eks:ListNodegroups",
+                "eks:TagResource",
+                "eks:UpdateNodegroupConfig",
+                "eks:UpdateNodegroupVersion",
+                "iam:AttachRolePolicy",
+                "iam:CreateOpenIDConnectProvider",
+                "iam:CreateRole",
+                "iam:DeleteOpenIDConnectProvider",
+                "iam:DeleteRole",
+                "iam:DetachRolePolicy",
+                "iam:GetOpenIDConnectProvider",
+                "iam:GetRole",
+                "iam:ListAttachedRolePolicies",
+                "iam:ListInstanceProfilesForRole",
+                "iam:ListRolePolicies",
+                "iam:TagOpenIDConnectProvider",
+                "iam:TagRole",
+                "ssm:GetParameters"
+            ],
+            "Resource": "*"
+        },
+        {
+            "Sid": "clusterperms",
+            "Effect": "Allow",
+            "Action": [
+                "iam:PassRole"
+            ],
+            "Resource": [
+                "arn:aws:iam::*:role/flux-test-*",
+                "arn:aws:iam::*:role/blue-eks-node-group-*",
+                "arn:aws:iam::*:role/green-eks-node-group-*"
+            ]
+        }
+    ]
+}
+```
+
+#### IAM and CI Setup
+
+To create all the necessary IAM role and policy with all the permissions, set up
+CI secrets and variables using
+[aws-gh-actions](https://github.com/fluxcd/test-infra/tree/main/tf-modules/aws/github-actions)
+with the terraform configuration below. Please make sure all the requirements of
+aws-gh-actions are followed before running it, especially registering GitHub
+OIDC as an identity provider in the AWS account.
+
+**NOTE:** When running the following for a repo under an organization, set the
+environment variable `GITHUB_ORGANIZATION` if setting the `owner` in the
+`github` provider doesn't work.
+
+```hcl
+module "aws_gh_actions" {
+  source = "git::https://github.com/fluxcd/test-infra.git//tf-modules/aws/github-actions"
+
+  aws_region             = "us-east-2"
+  aws_policy_name        = "irc-e2e"
+  aws_policy_description = "policy for image-reflector-controller e2e tests"
+  aws_provision_perms = [
+    "ec2:AllocateAddress",
+    "ec2:AssociateRouteTable",
+    "ec2:AttachInternetGateway",
+    "ec2:AuthorizeSecurityGroupEgress",
+    "ec2:AuthorizeSecurityGroupIngress",
+    "ec2:CreateInternetGateway",
+    "ec2:CreateLaunchTemplate",
+    "ec2:CreateLaunchTemplateVersion",
+    "ec2:CreateNatGateway",
+    "ec2:CreateNetworkAcl",
+    "ec2:CreateNetworkAclEntry",
+    "ec2:CreateRoute",
+    "ec2:CreateRouteTable",
+    "ec2:CreateSecurityGroup",
+    "ec2:CreateSubnet",
+    "ec2:CreateTags",
+    "ec2:CreateVpc",
+    "ec2:DeleteInternetGateway",
+    "ec2:DeleteLaunchTemplate",
+    "ec2:DeleteNatGateway",
+    "ec2:DeleteNetworkAcl",
+    "ec2:DeleteNetworkAclEntry",
+    "ec2:DeleteRoute",
+    "ec2:DeleteRouteTable",
+    "ec2:DeleteSecurityGroup",
+    "ec2:DeleteSubnet",
+    "ec2:DeleteTags",
+    "ec2:DeleteVolume",
+    "ec2:DeleteVpc",
+    "ec2:DescribeAddresses",
+    "ec2:DescribeAddressesAttribute",
+    "ec2:DescribeAvailabilityZones",
+    "ec2:DescribeInternetGateways",
+    "ec2:DescribeLaunchTemplates",
+    "ec2:DescribeLaunchTemplateVersions",
+    "ec2:DescribeNatGateways",
+    "ec2:DescribeNetworkAcls",
+    "ec2:DescribeNetworkInterfaces",
+    "ec2:DescribeRouteTables",
+    "ec2:DescribeSecurityGroupRules",
+    "ec2:DescribeSecurityGroups",
+    "ec2:DescribeSubnets",
+    "ec2:DescribeTags",
+    "ec2:DescribeVpcAttribute",
+    "ec2:DescribeVpcs",
+    "ec2:DetachInternetGateway",
+    "ec2:DisassociateAddress",
+    "ec2:DisassociateRouteTable",
+    "ec2:ModifyVpcAttribute",
+    "ec2:ReleaseAddress",
+    "ec2:RevokeSecurityGroupEgress",
+    "ec2:RevokeSecurityGroupIngress",
+    "ec2:RunInstances",
+    "ecr:BatchGetImage",
+    "ecr:BatchCheckLayerAvailability",
+    "ecr:CreateRepository",
+    "ecr:CompleteLayerUpload",
+    "ecr:DeleteRepository",
+    "ecr:DescribeRepositories",
+    "ecr:GetAuthorizationToken",
+    "ecr:InitiateLayerUpload",
+    "ecr:ListTagsForResource",
+    "ecr:PutImage",
+    "ecr:TagResource",
+    "ecr:UploadLayerPart",
+    "eks:AssociateAccessPolicy",
+    "eks:CreateAccessEntry",
+    "eks:CreateAddon",
+    "eks:CreateCluster",
+    "eks:CreateNodegroup",
+    "eks:DeleteAccessEntry",
+    "eks:DeleteAddon",
+    "eks:DeleteCluster",
+    "eks:DeleteNodegroup",
+    "eks:DescribeAccessEntry",
+    "eks:DescribeAddon",
+    "eks:DescribeAddonVersions",
+    "eks:DescribeCluster",
+    "eks:DescribeNodegroup",
+    "eks:DisassociateAccessPolicy",
+    "eks:ListAssociatedAccessPolicies",
+    "eks:ListNodegroups",
+    "eks:TagResource",
+    "eks:UpdateNodegroupConfig",
+    "eks:UpdateNodegroupVersion",
+    "iam:AttachRolePolicy",
+    "iam:CreateOpenIDConnectProvider",
+    "iam:CreateRole",
+    "iam:DeleteOpenIDConnectProvider",
+    "iam:DeleteRole",
+    "iam:DetachRolePolicy",
+    "iam:GetOpenIDConnectProvider",
+    "iam:GetRole",
+    "iam:ListAttachedRolePolicies",
+    "iam:ListInstanceProfilesForRole",
+    "iam:ListRolePolicies",
+    "iam:TagOpenIDConnectProvider",
+    "iam:TagRole",
+    "ssm:GetParameters"
+  ]
+  aws_cluster_role_prefix = [
+    "flux-test-",
+    "blue-eks-node-group-",
+    "green-eks-node-group-"
+  ]
+  aws_role_name          = "irc-e2e"
+  aws_role_description   = "role to assume in image-reflector-controller e2e test"
+  github_repo_owner      = "fluxcd"
+  github_project         = "image-reflector-controller"
+  github_repo_branch_ref = "*"
+
+  github_secret_accound_id_name  = "IRC_E2E_AWS_ACCOUNT_ID"
+  github_secret_assume_role_name = "IRC_E2E_AWS_ASSUME_ROLE_NAME"
+  github_variable_region_name    = "IRC_E2E_AWS_REGION"
+}
+```
+
+**NOTE:** Change the various names and environment variables above as necessary.
 
 ### Microsoft Azure