From 3f0fe556b00b5d2f7c1c457c051a96a00b43c763 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Rame=CC=81?= Date: Tue, 20 Feb 2024 10:04:21 +0100 Subject: [PATCH] aaa --- .env.model | 1 - .env.test | 1 - Dockerfile | 22 +++++++++++++++++++++- Dockerfile.dev.clevercloud | 3 +++ Dockerfile.prod.clevercloud | 3 +++ README.md | 17 ++++------------- next.config.js | 2 +- src/semgrep/requirements.txt | 1 + src/utils/app-version.js | 10 +++++----- 9 files changed, 38 insertions(+), 22 deletions(-) create mode 100644 Dockerfile.dev.clevercloud create mode 100644 Dockerfile.prod.clevercloud create mode 100644 src/semgrep/requirements.txt diff --git a/.env.model b/.env.model index 8564ffe..3a4130a 100644 --- a/.env.model +++ b/.env.model @@ -3,7 +3,6 @@ APP_BASE_URL= DATABASE_URL= LLM_MANAGER_MOCK=false MAINTENANCE_API_KEY= -NEXT_AUTH_SECRET= NEXT_PUBLIC_CRISP_WEBSITE_ID= NEXT_PUBLIC_SENTRY_DSN= NEXT_PUBLIC_MATOMO_URL= diff --git a/.env.test b/.env.test index 15e604e..a641fb7 100644 --- a/.env.test +++ b/.env.test @@ -2,7 +2,6 @@ PORT=3000 DATABASE_URL=postgresql://postgres:changeme@localhost:5432/postgres LLM_MANAGER_MOCK=true MAINTENANCE_API_KEY=random -NEXT_AUTH_SECRET=seed-to-forge-jwt-tokens-not-sensitive-in-temporary-environments NEXT_PUBLIC_CRISP_WEBSITE_ID=random-one-since-cannot-work-without-a-remote-crisp-account NEXT_PUBLIC_SENTRY_DSN= NEXT_PUBLIC_MATOMO_URL= diff --git a/Dockerfile b/Dockerfile index 10242a0..5b67ab8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,28 @@ # [IMPORTANT] Must be built from the root of the project for the COPY/paths to work +ARG NODE_VERSION=18.19.0 +ARG RUBY_VERSION=3.2.2-r1 +ARG PIP_VERSION=23.3.1-r0 ARG PORT=3000 -FROM ghcr.io/betagouv/etabli/etabli-${BRANCH}:latest +FROM node:${NODE_VERSION}-alpine + +RUN apk add --no-cache 'ruby=${RUBY_VERSION}' 'py3-pip=${PIP_VERSION}' +RUN apk update + +# Copy manifest files + +COPY "src/bibliothecary/Gemfile" "src/bibliothecary/Gemfile.lock" ./ +COPY "src/semgrep/requirements" ./ + +# Install tools + +RUN gem install bundler +RUN bundle --gemfile src/bibliothecary/Gemfile + +RUN pip install -r src/semgrep/requirements.txt + +# Manage the final server build RUN addgroup --system --gid 1001 nodejs RUN adduser --system --uid 1001 nextjs diff --git a/Dockerfile.dev.clevercloud b/Dockerfile.dev.clevercloud new file mode 100644 index 0000000..f81efb8 --- /dev/null +++ b/Dockerfile.dev.clevercloud @@ -0,0 +1,3 @@ +# We didn't want to rebuild everything in the Clever Cloud pipeline so we pushed an image from GitHub to be retrieved here without much processing + +FROM ghcr.io/betagouv/etabli/etabli-dev:latest diff --git a/Dockerfile.prod.clevercloud b/Dockerfile.prod.clevercloud new file mode 100644 index 0000000..863d8c9 --- /dev/null +++ b/Dockerfile.prod.clevercloud @@ -0,0 +1,3 @@ +# We didn't want to rebuild everything in the Clever Cloud pipeline so we pushed an image from GitHub to be retrieved here without much processing + +FROM ghcr.io/betagouv/etabli/etabli-main:latest diff --git a/README.md b/README.md index 81d0296..ed17f13 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,8 @@ If you want to only launch one of the two: - the database - [Pandoc](https://pandoc.org/) (to format legal documents, can be installed through a manager like `brew`) -- [Semgrep](https://semgrep.dev/) (to analyze code files, can be installed through a manager like `brew`) -- [Bibliothecary](https://github.com/librariesio/bibliothecary) (to analyze code dependencies, can be installed through the Ruby package manager `bundle` by running `bundle` command inside `./src/bibliothecary/`) +- [Semgrep](https://semgrep.dev/) (to analyze code files, can be installed through the Python package manager `pip` by running ``) +- [Bibliothecary](https://github.com/librariesio/bibliothecary) (to analyze code dependencies, can be installed through the Ruby package manager `bundle` by running `bundle --gemfile src/bibliothecary/Gemfile`) Note for long-running tools the easiest way to use them is to use `docker-compose`. In another terminal just set up all tools: @@ -192,11 +192,11 @@ Clever Cloud is used as a PaaS to host our builds. For each build and runtime (since they are shared), you should have set some environment variables. +- `CC_DOCKERFILE`: [TO_DEFINE] _(should be `Dockerfile.prod.clevercloud` for the production environment and `Dockerfile.dev.clevercloud` for the development one. It's used during the build stage on the Clever Cloud side)_ +- `CC_DOCKER_EXPOSED_HTTP_PORT`: `3000` _(it tells to Clever Cloud the port we are listening on)_ - `APP_MODE`: `prod` _(can be `dev` in case you would like to deploy a development environment)_ -- `GITHUB_TOKEN`: [SECRET] \_(limited GitHub fine-grained personal access tokens scoped to this repository, see the `Clever Cloud` section) - `DATABASE_URL`: `$POSTGRESQL_ADDON_URI` _(you must copy/paste the value provided by Clever Cloud into `$POSTGRESQL_ADDON_URI`, and note you must add as query parameter `sslmode=prefer`)_ - `MAINTENANCE_API_KEY`: [SECRET] _(random string that can be generated with `openssl rand -base64 32`. Note this is needed to perform maintenance through dedicated API endpoints)_ -- `NEXT_AUTH_SECRET`: [SECRET] _(random string that can be generated with `openssl rand -base64 32`. Note that if this secret is lost, all users will have to log in again)_ - `NEXT_PUBLIC_APP_BASE_URL`: [TO_DEFINE] _(must be the root URL to access the application, format `https://xxx.yyy.zzz`)_ - `NEXT_PUBLIC_CRISP_WEBSITE_ID`: [TO_DEFINE] _(this ID is defined in your Crisp account and depends on the development or production environment)_ - `NEXT_PUBLIC_SENTRY_DSN`: [SECRET] _(format `https://xxx.yyy.zzz/nn`)_ @@ -206,15 +206,6 @@ For each build and runtime (since they are shared), you should have set some env _Note: `OPENAI_API_KEY` variable can be found in the code even if not used in production. It remains for comparing purposes, but also as a legacy since the proof of concept of Établi was based on GPT models._ -### GitHub access - -During the build we get some repository information from GitHub to enhance Sentry metadata. It went well for some time but randomly we got the error: -`API rate limit exceeded for ${IP} (...) But here's the good news: Authenticated requests get a higher rate limit.` - -By default the `@octokit/rest` client will fetch data while not being authenticated so we share the quota with others, which may fail. We decided to create a fine-grained personal access token scoped to the current repository with only the scope `Read-Only` on `Contents` to be used as `GITHUB_TOKEN` in Clever Cloud. The only drawback is we need to specify an expiration with 1 year as maximum (we hope they will change it in the future to allow "no expiration" as for classic tokens). - -_Note that the other way is to create a GitHub App, connect it to the repository, manage the token... which is way more complicated! Even with the current 1 year expiration we are fine since builds are done only if people is working on the project :)_ - #### Monitoring It's important to be aware of some events, for this we decided to monitor: diff --git a/next.config.js b/next.config.js index 8c24ef3..2a3a294 100644 --- a/next.config.js +++ b/next.config.js @@ -38,7 +38,7 @@ const moduleExports = async () => { let standardModuleExports = { reactStrictMode: true, swcMinify: true, - output: process.env.NEXTJS_BUILD_OUTPUT || 'standalone', // To debug locally the `next start` comment this line (it will avoid trying to mess with the assembling folders logic of standalone mode) + output: process.env.NEXTJS_BUILD_OUTPUT_MODE || 'standalone', // To debug locally the `next start` comment this line (it will avoid trying to mess with the assembling folders logic of standalone mode) env: { // Those will replace `process.env.*` with hardcoded values (useful when the value is calculated during the build time) SENTRY_RELEASE_TAG: appHumanVersion, diff --git a/src/semgrep/requirements.txt b/src/semgrep/requirements.txt new file mode 100644 index 0000000..513c9a1 --- /dev/null +++ b/src/semgrep/requirements.txt @@ -0,0 +1 @@ +semgrep==1.48.0 diff --git a/src/utils/app-version.js b/src/utils/app-version.js index 041e366..c1df1d5 100644 --- a/src/utils/app-version.js +++ b/src/utils/app-version.js @@ -3,6 +3,7 @@ const gitRevision = require('git-rev-sync'); const parseGithubUrl = require('parse-github-url'); const octokit = new Octokit({ + // Octokit was used when the local Git repository is not present, but for Établi we managed all build stages into GitHub Actions so it should work with no workaround auth: process.env.GITHUB_TOKEN || undefined, // If not specified it uses public shared quota based on IP }); @@ -11,12 +12,11 @@ function getRepositoryInformation() { } function getFallbackCommitSha() { - // Heroku/Scalingo remove the `.git` folder so during the build time we are unable to use `git-rev-sync` - // we work around this by doing GET requests since they at least provide `$SOURCE_VERSION` that equals the commit SHA - // during the build phase, and `$CONTAINER_VERSION` during the runtime phase (that should be most of the time the commit SHA). - let commitSha = process.env.SOURCE_VERSION || process.env.CONTAINER_VERSION; + // This should not be used since all our Next.js build stages are done in GitHub Actions where + // we can use `git-rev-sync`. Just in case leaving the fallback for Clever Cloud (originally it was for Scalingo that doesn't support Docker and forces the whole build in their custom pipeline) + let commitSha = process.env.CC_COMMIT_ID; if (commitSha === undefined) { - throw new Error('`$SOURCE_VERSION` or `$CONTAINER_VERSION` environment variable must be provided to use the fallback'); + throw new Error('`$CC_COMMIT_ID` environment variable must be provided to use the fallback'); } return commitSha;