Merge branch 'main' into production

.github/workflows/test-and-deploy.yml

@@ -0,0 +1,121 @@
+name: Unit Tests
+
+on: [push]
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set Node version
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: '.nvmrc'
+      - name: npm install
+        run: npm ci
+      - name: Unit Tests
+        run: npm test
+  deploy-qa:
+    permissions:
+      id-token: write
+      contents: read
+    runs-on: ubuntu-latest
+    needs: tests
+    if: github.ref == 'refs/heads/qa'
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: arn:aws:iam::946183545209:role/GithubActionsDeployerRole
+          aws-region: us-east-1
+
+      - name: Log in to ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Build, tag, and push image to Amazon ECR
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: discovery-api
+          IMAGE_TAG: ${{ github.sha }}
+        run: |
+          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:qa-latest
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:qa-latest
+
+      - name: Force ECS Update
+        run: |
+          aws ecs update-service --cluster discovery-api-qa --service discovery-api-qa --force-new-deployment
+  deploy-qa2:
+    permissions:
+      id-token: write
+      contents: read
+    runs-on: ubuntu-latest
+    needs: tests
+    if: github.ref == 'refs/heads/qa2'
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: arn:aws:iam::946183545209:role/GithubActionsDeployerRole
+          aws-region: us-east-1
+
+      - name: Log in to ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Build, tag, and push image to Amazon ECR
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: discovery-api
+          IMAGE_TAG: ${{ github.sha }}
+        run: |
+          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:qa2-latest
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:qa2-latest
+
+      - name: Force ECS Update
+        run: |
+          aws ecs update-service --cluster discovery-api-qa2 --service discovery-api-qa2 --force-new-deployment
+  deploy-production:
+    permissions:
+      id-token: write
+      contents: read
+    runs-on: ubuntu-latest
+    needs: tests
+    if: github.ref == 'refs/heads/production'
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: arn:aws:iam::946183545209:role/GithubActionsDeployerRole
+          aws-region: us-east-1
+
+      - name: Log in to ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Build, tag, and push image to Amazon ECR
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: discovery-api
+          IMAGE_TAG: ${{ github.sha }}
+        run: |
+          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:production-latest
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:production-latest
+      - name: Force ECS Update
+        run: |
+          aws ecs update-service --cluster discovery-api-production --service discovery-api-production --force-new-deployment

.nvmrc

@@ -1 +1 @@
-6.10.3
+20

Dockerfile

@@ -0,0 +1,27 @@
+FROM node:20-bullseye as production
+
+RUN apt-get update
+RUN apt-get upgrade -y
+
+# Create app directory
+WORKDIR /usr/src/app
+
+# Install app dependencies
+COPY package-lock.json ./
+COPY package.json ./
+RUN npm cache verify
+RUN npm install
+
+# Add app source
+COPY . .
+
+# Remove any unneeded files
+RUN rm -rf /usr/src/app/.DS_Store
+RUN rm -rf /usr/src/app/.ebextensions
+RUN rm -rf /usr/src/app/.elasticbeanstalk
+RUN rm -rf /usr/src/app/.git
+RUN rm -rf /usr/src/app/.gitignore
+
+ENV LOG_STYLE=json
+
+CMD [ "npm", "start" ]

README.md

@@ -1,51 +1,46 @@
 [![Build Status](https://travis-ci.com/NYPL/discovery-api.svg?branch=main)](https://travis-ci.com/NYPL/discovery-api)
 
-# Discovery API
+# Discovery API (aka Research Catalog API)
 
 This is the API providing most of bibliographic data to the [NYPL Research Catalog front-end](https://github.com/NYPL/discovery-front-end). Check the [current swagger](https://github.com/NYPL/discovery-api/blob/main/swagger.v1.1.x.json) for the machine readable api contract.
 
 ## Installing & Running Locally
 
-Use [nvm](https://github.com/creationix/nvm) to set your Node version:
+For local development, it's easiest to just use local node binaries:
 
 ```
-nvm use
+nvm use; npm i
+nvm use; ENV=qa npm start
 ```
 
-Install dependencies:
+Note that when developing locally, if connecting to a IP ACL protected index (a practice we're currently deprecating), you may need to [add your IP to the access control policy of the relevant ES domain](https://github.com/NYPL/aws/blob/b5c0af0ec8357af9a645d8b47a5dbb0090966071/common/elasticsearch.md#2-make-the-domain-public-restrict-by-ip). If your IP has not been authorized, you will see errors such as the following in the application logs:
 
 ```
-npm i
+error: Error connecting to index: 403: {"Message":"User: anonymous is not authorized to perform: es:ESHttpPost because no resource-based policy allows the es:ESHttpPost action"}
 ```
 
-Create a `.env` based on `.env.example`. Fill it with values from the appropriate `config/[environment].env` file (`qa.env` is probably sensible). Note - if using values from `config/[environment].env` - you must decrypt the following keys using [`aws kms decrypt`](https://github.com/NYPL/engineering-general/blob/main/security/secrets.md#encryptingdecrypting) (or [kms-util](https://github.com/NYPL-discovery/kms-util)) (i.e. all values in `.env` must be decrypted):
- * `SCSB_URL`
- * `SCSCB_API_KEY`
- * `NYPL_OAUTH_SECRET`
+### Using Docker
 
-Now start the app:
+Docker files are included for deployment and can be used locally.
+
+To start the container with AWS creds so that the app can decrypt config from `config/*`:
 
 ```
-npm start
+ AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... docker-compose up
 ```
 
-Note that when developing locally, you may need to [add your IP to the access control policy of the relevant ES domain](https://github.com/NYPL/aws/blob/b5c0af0ec8357af9a645d8b47a5dbb0090966071/common/elasticsearch.md#2-make-the-domain-public-restrict-by-ip).
-
-If you are adding a dependency or need to rebuild package-lock.json, run this script:
-
+After making changes, rebuild the image:
 ```
-nvm use; npm i -g [email protected]
+docker-compose build
 ```
 
-## About Environment Variables
-
-See `.env.example` for a description the variables. **If you're adding new variables please add them to .env.example and `./lib/preflight_check.js`**
+Or, equivalently, to build and run the image and container directly:
 
-Note that config variables are maintained in `./config/*.env` as a formality and to assist onboarding; Changes made to those files will not effect deployed config. Config changes must be applied to the deployed Beanstalk app manually. Note that sensitive values are encrypted in `./config/*.env` whereas they are not in the deployed Beanstalk.
-
-### Changes to SCSB/UAT endpoints
-
-When HTC changes the SCSB endpoint, apply changes to the relevant environment file in `config/` (encrypting the value if sensitive) and also manually apply the change to the running Beanstalk app (without encrypting the value).
+```
+docker image build -t discovery-api:local .
+docker container rm discovery-api
+docker run --name discovery-api -e ENV=qa -e AWS_ACCESS_KEY_ID=... -e AWS_SECRET_ACCESS_KEY=... -p 8082:8082 -it discovery-api:local
+```
 
 ## Contributing
 
@@ -63,36 +58,12 @@ This app uses a [PRs Target Main, Merge to Deployment Branches](https://github.c
 1. Confirm app deploys to QA and run appropriate testing
 1. Merge `main` > `production`
 
-## Deployment
-
-This app is deployed via a deployment hook in `.travis.yml`. Pushes to `qa` deploy to our QA deployment; Pushes to `production` deploy to our Production deployment. We generally don't update `development` or use the Development deployment.
-
-### Initial Creation / Deployment to Elastic Beanstalk
+### Deploying
 
-1. `.ebextensions` directory needed at application's root directory
-2. `.ebextensions/00_environment.config` to store environment variables. For environment variables that needs to be hidden,
-3. `.ebextensions/03_nodecommand.config` to start node app after deployment.
-4. `eb init -i --profile <<your AWS profile>>`
-5. Initial creation of instance on Beanstalk:
-
-Please use the instance profile of _cloudwatchable-beanstalk_.
-Which has all the permissions needed for a traditional or Docker-flavored Beanstalk
-machine that wants to log to CloudWatch.
-
-```bash
-eb create discovery-api-[environmentname] \
-    --instance_type t2.small \
-    --instance_profile cloudwatchable-beanstalk \
-    --cname discovery-api-[environmentname] \
-    --vpc.id vpc-1e293a7b \
-    --vpc.elbsubnets public-subnet-id-1,public-subnet-id-2 \
-    --vpc.ec2subnets private-subnet-id-1,private-subnet-id-2 \
-    --vpc.elbpublic \
-    --tags Project=Discovery, Foo=Bar \
-    --keyname dgdvteam \
-    --scale 2 \
-    --envvars VAR_NAME_1="xxx",VAR_NAME_2="xxx"
-```
+App deploys through [GH Actions](./.github/workflows/test-and-deploy.yml) to ECS when updates are made to deployment branches:
+ - `qa`: discovery-api-qa.nypl.org
+ - `qa2`: discovery-api-qa2.nypl.org
+ - `production`: discovery-api-production.nypl.org
 
 ## Testing