Skip to content

Commit

Permalink
Merge pull request #3165 from raft-tech/2458-nexus-mvp
Browse files Browse the repository at this point in the history
Nexus MVP
  • Loading branch information
elipe17 authored Oct 2, 2024
2 parents 5b8feae + c15cb7c commit fab247d
Show file tree
Hide file tree
Showing 13 changed files with 218 additions and 25 deletions.
4 changes: 2 additions & 2 deletions .circleci/base_config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@ executors:
machine-executor:
machine:
docker_layer_caching: false
image: ubuntu-2204:2024.01.1
image: ubuntu-2204:2024.05.1
large-machine-executor:
machine:
docker_layer_caching: false
image: ubuntu-2204:2024.01.1
image: ubuntu-2204:2024.05.1
resource_class: large

parameters:
Expand Down
9 changes: 8 additions & 1 deletion .circleci/build-and-test/commands.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,13 @@
- run:
name: Disable npm audit warnings in CI
command: npm set audit false

# This allows us to use the node orb to install packages within other commands
install-nodejs-packages: node/install-packages

docker-login:
steps:
- run:
name: Docker login
command: |
echo "$CIRCI_DOCKER_LOGIN" | docker login https://tdp-docker.dev.raftlabs.tech -u tdp-circi --password-stdin
2 changes: 2 additions & 0 deletions .circleci/build-and-test/jobs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
steps:
- checkout
- docker-compose-check
- docker-login
- docker-compose-up-backend
- run:
name: Run Unit Tests And Create Code Coverage Report
Expand Down Expand Up @@ -46,6 +47,7 @@
steps:
- checkout
- docker-compose-check
- docker-login
- docker-compose-up-backend
- docker-compose-up-frontend
- install-nodejs-machine
Expand Down
32 changes: 29 additions & 3 deletions .circleci/deployment/commands.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,33 @@
# commands:
init-deploy:
steps:
- checkout
- sudo-check
- cf-check

build-and-tag-images:
parameters:
backend-appname:
default: tdp-backend
type: string
frontend-appname:
default: tdp-frontend
type: string
steps:
- run:
name: Update Docker daemon
command: |
sudo echo '{"max-concurrent-uploads": 1}' | sudo tee /etc/docker/daemon.json
sudo service docker restart
- run:
name: Create builder
command: |
docker buildx create --name container-builder --driver docker-container --use --bootstrap
- run:
name: Build and tag images
command: |
./scripts/build-and-tag-images.sh <<parameters.backend-appname>> <<parameters.frontend-appname>> ./tdrs-backend ./tdrs-frontend $CIRCLE_BUILD_NUM $CIRCLE_SHA1 "$CIRCI_DOCKER_LOGIN" tdp-circi
deploy-cloud-dot-gov:
parameters:
environment:
Expand All @@ -25,9 +54,6 @@
default: tdp-frontend
type: string
steps:
- checkout
- sudo-check
- cf-check
- login-cloud-dot-gov:
cf-password: <<parameters.cf-password>>
cf-org: <<parameters.cf-org>>
Expand Down
34 changes: 34 additions & 0 deletions .circleci/deployment/jobs.yml
Original file line number Diff line number Diff line change
@@ -1,10 +1,41 @@
build-and-tag-develop:
executor: large-machine-executor
working_directory: ~/tdp-deploy
steps:
- checkout
- sudo-check
- build-and-tag-images:
backend-appname: tdp-backend-develop
frontend-appname: tdp-frontend-develop

build-and-tag-staging:
executor: large-machine-executor
working_directory: ~/tdp-deploy
steps:
- checkout
- sudo-check
- build-and-tag-images:
backend-appname: tdp-backend-staging
frontend-appname: tdp-frontend-staging

build-and-tag-production:
executor: large-machine-executor
working_directory: ~/tdp-deploy
steps:
- checkout
- sudo-check
- build-and-tag-images:
backend-appname: tdp-backend-production
frontend-appname: tdp-frontend-production

deploy-dev:
parameters:
target_env:
type: string
executor: docker-executor
working_directory: ~/tdp-deploy
steps:
- init-deploy
- deploy-cloud-dot-gov:
backend-appname: tdp-backend-<< parameters.target_env >>
frontend-appname: tdp-frontend-<< parameters.target_env >>
Expand All @@ -13,6 +44,7 @@
executor: docker-executor
working_directory: ~/tdp-deploy
steps:
- init-deploy
- deploy-cloud-dot-gov:
backend-appname: tdp-backend-staging
frontend-appname: tdp-frontend-staging
Expand All @@ -24,6 +56,7 @@
executor: docker-executor
working_directory: ~/tdp-deploy
steps:
- init-deploy
- deploy-cloud-dot-gov:
backend-appname: tdp-backend-develop
frontend-appname: tdp-frontend-develop
Expand Down Expand Up @@ -133,6 +166,7 @@
executor: docker-executor
working_directory: ~/tdp-deploy
steps:
- init-deploy
- deploy-cloud-dot-gov:
environment: production
backend-appname: tdp-backend-prod
Expand Down
27 changes: 24 additions & 3 deletions .circleci/deployment/workflows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -93,27 +93,48 @@
- develop
- main
- master
- deploy-develop:
- build-and-tag-develop:
requires:
- deploy-infrastructure-staging
filters:
branches:
only:
- develop
- deploy-staging:
- deploy-develop:
requires:
- build-and-tag-develop
filters:
branches:
only:
- develop
- build-and-tag-staging:
requires:
- deploy-infrastructure-staging
filters:
branches:
only:
- main
- deploy-production:
- deploy-staging:
requires:
- build-and-tag-staging
filters:
branches:
only:
- main
- build-and-tag-production:
requires:
- deploy-infrastructure-production
filters:
branches:
only:
- master
- deploy-production:
requires:
- build-and-tag-production
filters:
branches:
only:
- master
- test-deployment-e2e:
requires:
- deploy-develop
Expand Down
Empty file modified .circleci/generate_config.sh
100644 → 100755
Empty file.
3 changes: 3 additions & 0 deletions .circleci/owasp/jobs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
steps:
- checkout
- docker-compose-check
- docker-login
- docker-compose-up-backend
- docker-compose-up-frontend
- run:
Expand All @@ -26,6 +27,7 @@
steps:
- checkout
- docker-compose-check
- docker-login
- docker-compose-up-backend
- docker-compose-up-frontend
- run:
Expand Down Expand Up @@ -66,6 +68,7 @@
- sudo-check
- cf-check
- docker-compose-check
- docker-login
- login-cloud-dot-gov:
cf-password: <<parameters.cf_password>>
cf-space: <<parameters.cf_space>>
Expand Down
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
79 changes: 72 additions & 7 deletions docs/Technical-Documentation/nexus-repo.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,29 +40,94 @@ After logging in as root for the first time, you will be taken to a page to set

In order to use Nexus as a Docker repository, the DNS for the repo needs to be able to terminate https. We are currently using cloudflare to do this.

When creating the repository (must be signed in with admin privileges), since the nexus server isn't actually terminating the https, select the HTTP repository connector. The port can be anything you assign, as long as the tool used to terminate the https connection forwards the traffic to that port.
When creating the repository (must be signed in with admin privileges), since the nexus server isn't actually terminating the https, select the HTTP repository connector. The port can be anything you assign, as long as the tool used to terminate the https connection forwards the traffic to that port.

In order to allow [Docker client login and connections](https://help.sonatype.com/repomanager3/nexus-repository-administration/formats/docker-registry/docker-authentication) you must set up the Docker Bearer Token Realm in Settings -> Security -> Realms -> and move the Docker Bearer Token Realm over to Active.
Also, any users will need nx-repository-view-docker-#{RepoName}-(browse && read) at a minimum and (add and edit) in order to push images.

We have a separate endpoint to connect specifically to the docker repository.
[https://tdp-docker.dev.raftlabs.tech](tdp-docker.dev.raftlabs.tech)

e.g. `docker login https://tdp-docker.dev.raftlabs.tech`
e.g.
```
docker login https://tdp-docker.dev.raftlabs.tech
```

### Pushing Images

Before an image can be pushed to the nexus repository, it must be tagged for that repo:

`docker image tag ${ImageId} tdp-docker.dev.raftlabs.tech/${ImageName}:${Version}`
```
docker image tag ${ImageId} tdp-docker.dev.raftlabs.tech/${ImageName}:${Version}
```

then you can push:

`docker push tdp-docker.dev.raftlabs.tech/${ImageName}:${Version}`
```
docker push tdp-docker.dev.raftlabs.tech/${ImageName}:${Version}
```

### Pulling Images

We have set up a proxy mirror to dockerhub that can pull and cache DockerHub images.
Then we have created a group docker repository that can be pulled from. If the container is in our hosted repo, the group will return that container. If not, it will see if we have a cached version of that container in our proxy repo and, if not, pull that from dockerhub, cache it and allow the docker pull to happen.
We do not allow anonymous access on our Nexus instance. With that said, if you have not [logged in with Docker](#docker-login) you will not be able to pull. If you are logged in:

```
docker pull tdp-docker.dev.raftlabs.tech/${ImageName}:${Version}
```

## Nexus Administration

### UI Admin Login
To administer Nexus via the UI, you will need to access the service key in our dev cloud.gov environment.

Log in with CloudFoundry
```
cf login --sso
```
Be sure to specify the space as `tanf-dev`

`docker pull https://tdp-docker-store.dev.raftlabs.tech/${ImageName}:${Version}`
After you've authenticated you can grab the password from the key:
```
cf service-key tanf-keys nexus-dev-admin
```

The key returns a username and a password:
```
{
"credentials": {
"password": REDACTED,
"username": REDACTED
}
}
```
Copy the `password` to your clipboard and login into the Nexus UI with the `tdp-dev-admin` user. See below:

![Nexus Dev Admin Login](./images/nexus-dev-admin-login.png)

### VM Login
To access the VM running Nexus, you will need to gain access to the Raft internal network. To do this, you will need to install CloudFlare's WARP zero trust VPN. Follow the instructions [here](https://gorafttech-my.sharepoint.com/:w:/g/personal/tradin_teamraft_com/EZePOTv0dbdBguHITcoXQF0Bd5JAcqeLsJTlEOktTfIXHA?e=34WqB4) to get setup. From there, reach out to Eric Lipe or Connor Meehan for the IP, username, and password to access the VM. Once you have the credentials, you can login with SSH:
```
ssh username@IP_Address
```

Once logged in, you can run `docker ps` or other docker commands to view and administer the Nexus container as necessary. You should also consider generating an ssh key to avoid having to enter the password each time you login. To do so, run the following commands on your local machine.
```
ssh-keygen
```

```
ssh-copy-id username@IP_Address
```
Now you will no longer have to enter the password when logging in.

## Local Docker Login
After logging into the `tanf-dev` space with the `cf` cli, execute the following commands to authenticate your local docker daemon
```
export NEXUS_DOCKER_PASSWORD=`cf service-key tanf-keys nexus-dev | tail -n +2 | jq .credentials.password`
echo "$NEXUS_DOCKER_PASSWORD" | docker login https://tdp-docker.dev.raftlabs.tech -u tdp-dev --password-stdin
```

Sometimes the `docker login...` command above doesn't work. If that happens, just copy the content of `NEXUS_DOCKER_PASSWORD` to your clipboard and paste it when prompted for the password after executing the command below.
```
docker login https://tdp-docker.dev.raftlabs.tech -u tdp-dev
```
34 changes: 34 additions & 0 deletions scripts/build-and-tag-images.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#!/bin/bash

if [ "$#" -ne 8 ]; then
echo "Error, this script expects 8 parameters."
echo "I.e: ./build-tag-images.sh BACKEND_APP_NAME FRONTEND_APP_NAME BACKEND_PATH FRONTEND_PATH BUILD_NUM COMMIT_HASH DOCKER_LOGIN DOCKER_USER"
exit 1
fi

BACKEND_APP_NAME=$1
FRONTEND_APP_NAME=$2
BACKEND_PATH=$3
FRONTEND_PATH=$4
BUILD_NUM=$5
COMMIT_HASH=$6
DOCKER_LOGIN=$7
DOCKER_USER=$8
BUILD_DATE=`date +%F`
TAG="${BUILD_DATE}_build-${BUILD_NUM}_${COMMIT_HASH}"

export DOCKER_CLI_EXPERIMENTAL=enabled

build_and_tag() {
echo "$DOCKER_LOGIN" | docker login https://tdp-docker.dev.raftlabs.tech -u $DOCKER_USER --password-stdin
docker buildx build --load --platform linux/amd64 -t tdp-docker.dev.raftlabs.tech/$BACKEND_APP_NAME:$TAG -t tdp-docker.dev.raftlabs.tech/$BACKEND_APP_NAME:latest "$BACKEND_PATH"
docker buildx build --load --platform linux/arm64 -t tdp-docker.dev.raftlabs.tech/$BACKEND_APP_NAME:$TAG -t tdp-docker.dev.raftlabs.tech/$BACKEND_APP_NAME:latest "$BACKEND_PATH"
docker push --all-tags tdp-docker.dev.raftlabs.tech/$BACKEND_APP_NAME
docker buildx build --load --platform linux/amd64 -t tdp-docker.dev.raftlabs.tech/$FRONTEND_APP_NAME:$TAG -t tdp-docker.dev.raftlabs.tech/$FRONTEND_APP_NAME:latest "$FRONTEND_PATH"
docker buildx build --load --platform linux/arm64 -t tdp-docker.dev.raftlabs.tech/$FRONTEND_APP_NAME:$TAG -t tdp-docker.dev.raftlabs.tech/$FRONTEND_APP_NAME:latest "$FRONTEND_PATH"
docker push --all-tags tdp-docker.dev.raftlabs.tech/$FRONTEND_APP_NAME
docker logout
}

echo "Building and Tagging images for $BACKEND_APP_NAME and $FRONTEND_APP_NAME"
build_and_tag
Loading

0 comments on commit fab247d

Please sign in to comment.