Merge pull request #2195 from scpwiki/WJ-1286-install-dir

[WJ-1286] Reorganize install/ directory

.do/app.yaml

@@ -19,7 +19,7 @@ domains:
 services:
   - name: framerail
     source_dir: .
-    dockerfile_path: install/dev/digitalocean/web/Dockerfile
+    dockerfile_path: install/dev/web/Dockerfile
     github:
       repo: scpwiki/wikijump
       branch: develop
@@ -34,7 +34,7 @@ services:
 
   - name: deepwell
     source_dir: .
-    dockerfile_path: install/dev/digitalocean/api/Dockerfile
+    dockerfile_path: install/dev/api/Dockerfile
     github:
       repo: scpwiki/wikijump
       branch: develop

.github/workflows/deepwell.yaml

@@ -9,10 +9,9 @@ on:
       - '.github/workflows/deepwell.yaml'
       - '.github/codecov.yml'
       - 'deepwell/config.example.toml'
-      - 'install/files/local/deepwell.toml'
-      # TODO: add install/files/dev/deepwell.toml
-      #       and install/files/prod/deepwell.toml
-      #       both here and below
+      - 'install/local/api/deepwell.toml'
+      - 'install/dev/api/deepwell.toml'
+      - 'install/prod/api/deepwell.toml'
   push:
     branches:
       - develop
@@ -52,7 +51,7 @@ jobs:
         run: cd deepwell && cargo build --all-features
 
       - name: Check Configuration
-        run: cd deepwell && cargo run -- config.example.toml ../install/files/local/deepwell.toml
+        run: cd deepwell && cargo run -- config.example.toml ../install/local/api/deepwell.toml ../install/dev/api/deepwell.toml ../install/prod/api/deepwell.toml
         env:
           DEEPWELL_SPECIAL_ACTION: validate-config
 

.github/workflows/docker-build-api.dev.yaml

@@ -4,7 +4,7 @@ on:
   pull_request:
     paths:
       - 'deepwell/**'
-      - 'install/dev/digitalocean/api/Dockerfile'
+      - 'install/dev/api/Dockerfile'
       - '.github/workflows/docker-build-api.dev.yaml'
 
 jobs:
@@ -15,6 +15,6 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Build image
-        run: docker build -f install/dev/digitalocean/api/Dockerfile .
+        run: docker build -f install/dev/api/Dockerfile .
         env:
           DOCKER_BUILDKIT: 1

.github/workflows/docker-build-api.local.yaml

@@ -4,7 +4,7 @@ on:
   pull_request:
     paths:
       - 'deepwell/**'
-      - 'install/local/dev/api/Dockerfile'
+      - 'install/local/api/Dockerfile'
       - '.github/workflows/docker-build-api.local.yaml'
 
 jobs:
@@ -15,6 +15,6 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Build image
-        run: docker build -f install/local/dev/api/Dockerfile .
+        run: docker build -f install/local/api/Dockerfile .
         env:
           DOCKER_BUILDKIT: 1

.github/workflows/docker-build-api.prod.yaml

@@ -4,7 +4,7 @@ on:
   pull_request:
     paths:
       - 'deepwell/**'
-      - 'install/aws/prod/docker/api/Dockerfile'
+      - 'install/prod/api/Dockerfile'
       - '.github/workflows/docker-build-api.prod.yaml'
 
 jobs:
@@ -15,6 +15,6 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Build image
-        run: docker build -f install/aws/prod/docker/api/Dockerfile .
+        run: docker build -f install/prod/api/Dockerfile .
         env:
           DOCKER_BUILDKIT: 1

.github/workflows/docker-build-minio.local.yaml

@@ -3,7 +3,7 @@ name: '[backend] Docker build Minio (local)'
 on:
   pull_request:
     paths:
-      - 'install/local/dev/minio/*'
+      - 'install/local/minio/*'
       - '.github/workflows/docker-build-minio.local.yaml'
 
 jobs:
@@ -14,6 +14,6 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Build image
-        run: cd install/local/dev/minio && docker build .
+        run: cd install/local/minio && docker build .
         env:
           DOCKER_BUILDKIT: 1

.github/workflows/docker-build-postgres.local.yaml

@@ -4,8 +4,8 @@ on:
   pull_request:
     paths:
       - 'web/database/**'
-      - 'install/files/postgres/**'
-      - 'install/local/dev/postgres/Dockerfile'
+      - 'install/common/postgres/**'
+      - 'install/local/postgres/Dockerfile'
       - '.github/workflows/docker-build-postgres.local.yaml'
 
 jobs:
@@ -16,6 +16,6 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Build image
-        run: docker build -f install/local/dev/postgres/Dockerfile .
+        run: docker build -f install/local/postgres/Dockerfile .
         env:
           DOCKER_BUILDKIT: 1

.github/workflows/docker-build-postgres.prod.yaml

@@ -4,8 +4,8 @@ on:
   pull_request:
     paths:
       - 'web/database/**'
-      - 'install/files/postgres/**'
-      - 'install/aws/prod/docker/postgres/Dockerfile'
+      - 'install/common/postgres/**'
+      - 'install/prod/postgres/Dockerfile'
       - '.github/workflows/docker-build-postgres.prod.yaml'
 
 jobs:
@@ -16,6 +16,6 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Build image
-        run: docker build -f install/aws/prod/docker/postgres/Dockerfile .
+        run: docker build -f install/prod/postgres/Dockerfile .
         env:
           DOCKER_BUILDKIT: 1

.github/workflows/docker-build-redis.local.yaml

@@ -3,7 +3,7 @@ name: '[backend] Docker build Redis (local)'
 on:
   pull_request:
     paths:
-      - 'install/local/dev/redis/*'
+      - 'install/local/redis/*'
       - '.github/workflows/docker-build-redis.local.yaml'
 
 jobs:
@@ -14,6 +14,6 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Build image
-        run: cd install/local/dev/redis && docker build .
+        run: cd install/local/redis && docker build .
         env:
           DOCKER_BUILDKIT: 1

.github/workflows/docker-build-web.dev.yaml

@@ -4,7 +4,7 @@ on:
   pull_request:
     paths:
       - 'framerail/**'
-      - 'install/dev/digitalocean/web/Dockerfile'
+      - 'install/dev/web/Dockerfile'
       - '.github/workflows/docker-build-web.dev.yaml'
 
 jobs:
@@ -15,6 +15,6 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Build image
-        run: docker build -f install/dev/digitalocean/web/Dockerfile .
+        run: docker build -f install/dev/web/Dockerfile .
         env:
           DOCKER_BUILDKIT: 1

.github/workflows/docker-build-web.local.yaml

@@ -4,7 +4,7 @@ on:
   pull_request:
     paths:
       - 'framerail/**'
-      - 'install/local/dev/web/Dockerfile'
+      - 'install/local/web/Dockerfile'
       - '.github/workflows/docker-build-web.local.yaml'
 
 jobs:
@@ -15,6 +15,6 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Build image
-        run: docker build -f install/local/dev/web/Dockerfile .
+        run: docker build -f install/local/web/Dockerfile .
         env:
           DOCKER_BUILDKIT: 1

.github/workflows/docker-build-web.prod.yaml

@@ -4,7 +4,7 @@ on:
   pull_request:
     paths:
       - 'framerail/**'
-      - 'install/aws/prod/docker/web/Dockerfile'
+      - 'install/prod/web/Dockerfile'
       - '.github/workflows/docker-build-web.prod.yaml'
 
 jobs:
@@ -15,6 +15,6 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Build image
-        run: docker build -f install/aws/prod/docker/web/Dockerfile .
+        run: docker build -f install/prod/web/Dockerfile .
         env:
           DOCKER_BUILDKIT: 1

deepwell/src/config/special_action.rs

@@ -66,6 +66,6 @@ fn validate_config() -> i32 {
         }
     }
 
-    println!("All passed files checked, exiting");
+    println!("All config files have been checked, {return_code} failed");
     return_code
 }

docs/development.md

@@ -47,7 +47,7 @@ Node, NPM, and PNPM are well-behaved on Windows and Linux, and the difference in
 
 There are a couple of configuration files that need to be initialized prior to running your instance of Wikijump. These will be the `config.toml` and `.env` files, both located in the `deepwell` subdirectory. For a typical local deploy, both of these files can be copied from their `.example` counterparts without changing them, though it is worth looking through them briefly to understand what can be configurated for your instance.
 
-There is also a Docker configuration file that configures the various containers that host Wikijump in the local environment. You can find this file in `install/local/dev/`, named `docker-compose.yaml`, alongside `docker-compose.dev.yaml` (which provides various helpful tools for developing locally).
+There is also a Docker configuration file that configures the various containers that host Wikijump in the local environment. You can find this file in `install/local`, named `docker-compose.yaml`, alongside `docker-compose.dev.yaml` (which provides various helpful tools for developing locally).
 
 Notice that in `docker-compose.yaml`, there are configuration options for the domains to use. For development purposes, these are set to `wikijump.localhost`. This is the domain you will be connecting to, e.g. `https://www.wikijump.localhost`. The TLD `.localhost` is just like the usual `localhost` domain. Even when running locally, HTTPS is used. Because this certificate is self-signed, you will need to dismiss the certificate warning.
 
@@ -61,13 +61,30 @@ $ pnpm install
 
 ## Building
 
-You can now finally build the Docker images. Navigate to `install/local/dev` and run the following:
+You can now start up the Docker images. The basic command used to run Wikijump locally is:
 
 ```sh
-$ docker-compose -p wikijump -f docker-compose.yaml -f docker-compose.dev.yaml up
+$ [sudo] docker-compose -p wikijump -f docker-compose.yaml [-f docker-compose.dev.yaml] <action>
 ```
 
-_This might take some time_. Thankfully, Docker's build step is _heavily_ cached. 
+* On some systems, `sudo` is required to run Docker, but on others it is not.
+* The `-f docker-compose.dev.yaml` configuration file provides container bindings for development. In other words, if you modify `deepwell/src/` files locally, then those changes will be reflected in the container.
+
+The `install/local/deploy.py` script is meant as a convenience here. You can pass in `--sudo` or `--no-dev` to make either of the above modifications for you.
+
+The "action" corresponds to actions that `docker-compose` can do. Some common commands are:
+* `up` &mdash; Create and start containers for Wikijump.
+* `up --build` &mdash; Like the above but it first builds new images before creating containers.
+* `start` &mdash; Start any already-existing containers for Wikijump.
+* `stop` &mdash; Stop currently-running containers for Wikijump.
+
+To start the project, you want to run a command that probably looks something like:
+
+```
+$ install/local/deploy.py up
+```
+
+_This might take some time_. Thankfully, Docker's build step is heavily cached. You will generally not need to rerun this build often.
 
 > ### For Windows:
 >
@@ -114,4 +131,4 @@ you can enter this command to sync your time up:
 
 ```
 $ ntpd -d -q -n -p 0.pool.ntp.org
-```
+```

install/dev/digitalocean/api/Dockerfile → install/dev/api/Dockerfile

@@ -40,10 +40,10 @@ COPY --from=rust /usr/local/cargo/bin/sqlx /usr/local/cargo/bin/sqlx
 COPY --from=rust /src/deepwell/target/release/deepwell /usr/local/bin/deepwell
 COPY --from=rust /src/deepwell/migrations /opt/database/migrations
 COPY --from=rust /src/deepwell/seeder /opt/database/seeder
-COPY install/files/dev/deepwell.toml /etc/deepwell.toml
-COPY install/files/dev/deepwell-start /usr/local/bin/wikijump-deepwell-start
-COPY install/files/api/health-check.sh /usr/local/bin/wikijump-health-check
-COPY locales/fluent /opt/locales/fluent
+COPY ./install/dev/api/deepwell.toml /etc/deepwell.toml
+COPY ./install/dev/api/deepwell-start /usr/local/bin/wikijump-deepwell-start
+COPY ./install/dev/api/health-check.sh /usr/local/bin/wikijump-health-check
+COPY ./locales/fluent /opt/locales/fluent
 
 USER daemon
 EXPOSE 2747

install/dev/api/health-check.sh

@@ -0,0 +1 @@
+../../common/api/health-check.sh

install/dev/digitalocean/redis/Dockerfile → install/dev/redis/Dockerfile

@@ -1,5 +1,5 @@
 FROM redis:alpine
 
 EXPOSE 6379
-COPY ./install/dev/digitalocean/redis/redis.conf /usr/local/etc/redis.conf
+COPY ./install/dev/redis/redis.conf /usr/local/etc/redis.conf
 CMD ["redis-server", "/usr/local/etc/redis.conf"]

install/local/dev/api/Dockerfile → install/local/api/Dockerfile

@@ -16,9 +16,9 @@ RUN apt install -y libmagic-dev
 RUN cargo install cargo-watch sqlx-cli
 
 # Install files
-COPY install/files/local/deepwell.toml /etc/deepwell.toml
-COPY install/files/local/deepwell-start /usr/local/bin/wikijump-deepwell-start
-COPY install/files/api/health-check.sh /usr/local/bin/wikijump-health-check
+COPY ./install/local/api/deepwell.toml /etc/deepwell.toml
+COPY ./install/local/api/deepwell-start /usr/local/bin/wikijump-deepwell-start
+COPY ./install/local/api/health-check.sh /usr/local/bin/wikijump-health-check
 
 # /opt/locales is provided via docker-compose.dev.yaml
 

install/local/api/health-check.sh

@@ -0,0 +1 @@
+../../common/api/health-check.sh

install/local/deploy.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+
+
+def chdir_wikijump():
+    # Determine where this script is
+    full_path = __file__
+    if not os.path.abspath(full_path):
+        full_path = os.path.join(os.getcwd(), full_path)
+
+    # The directory it's located in should be install/local/
+    # So we should change to that
+    install_local_dir = os.path.dirname(full_path)
+    os.chdir(install_local_dir)
+
+
+if __name__ == "__main__":
+    # Parse arguments
+    argparser = argparse.ArgumentParser()
+    argparser.add_argument(
+        "-S",
+        "--sudo",
+        action="store_true",
+        help="Runs the docker-compose command using sudo",
+    )
+    argparser.add_argument(
+        "-d",
+        "--no-dev",
+        action="store_true",
+        help="Do not use the docker-compose.dev.yaml convenience bindings",
+    )
+    argparser.add_argument(
+        "-x",
+        "--no-chdir",
+        action="store_true",
+        help="Do not change directory before attempting to run docker-compose",
+    )
+    argparser.add_argument("action", nargs=argparse.REMAINDER)
+    args = argparser.parse_args()
+
+    # Get into the right context
+    if not args.no_chdir:
+        chdir_wikijump()
+
+    # Build command-line argument list
+    cmdline = [
+        "docker-compose",
+        "-p",
+        "wikijump",
+        "-f",
+        "docker-compose.yaml",
+    ]
+
+    if args.sudo:
+        cmdline.insert(0, "sudo")
+
+    if not args.no_dev:
+        cmdline.extend(("-f", "docker-compose.dev.yaml"))
+
+    cmdline.extend(args.action)
+
+    # We use exec instead of subprocess since this is just a launch script,
+    # it doesn't need to do any work after starting docker-compose.
+    print(" ".join(cmdline))
+    os.execvp(cmdline[0], cmdline)