feat: v4 (#548)

* feat(ent): sessions table

* feat(auth): use sessions for local login instead of jwt

* start standardizing http responses

* fix(ent): remove omitempty for watched channels

* rename env var

* feat(auth): refactor oauth to use sessions

* additional standardizing responses

* update river

* feat: wip new frontend

* api updates

* On v4: wip-rebase

* rebase

* rebase issue

* remove unused tests

* misc fixes

* disable arm for quicker dev builds

* fix

* push

* fix

* remove unused jwt logic

* add supervisord to container

* fix channel get

* use inter font

* fix theme flicker

* maybe fix styling?

* fix archive buttons

* more frontend changes

* fix tsc

* use web thumbnail for queue header

* frontend cleanup

* convert more api responses to standard format

* feat(ent): clip support

* clip support

* tests will be fixed later

* add clips section below video if video has clips

* fix build issue

* add index to contrain vod_id and user_id

* revert

* feat: sprite thumbnails

Twitch VODs that have sprite thumbnails will download them. Live streams and other videos can optionally create sprite thumbnails in the video menu

Still need to tinker with the quality settings

* fix lint

* run on start

* only download thumbnail sprites if archive

* move live stream check to task system

* bump packge versions

* bump docker go version

* add task to generate sprites automatically if enabled

* fix lint

* mark sprite thumbnails as experimental + add task to generate for all videos

* improve mobile layout for watching videos

* fix(tasks): use the generate thumbnail queue

* reimplement multistream view

* re-enable arm64 docker images

* feat: embed riverui in server

* feat: chat histogram

* do not show histogram for clips

* remove unused env vars

* docs update

* set dev image tag

* remove duplicate push

.devcontainer/Dockerfile

@@ -1,15 +1,11 @@
 FROM mcr.microsoft.com/devcontainers/go:1
 
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-  && apt-get -y install --no-install-recommends ffmpeg python3 python3-pip \
+  && apt-get -y install --no-install-recommends ffmpeg \
   && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
-RUN pip3 install --no-cache --upgrade --break-system-packages pip streamlink chat-downloader
-
 WORKDIR /tmp
 
-RUN wget https://github.com/rsms/inter/releases/download/v4.0-beta7/Inter-4.0-beta7.zip && unzip Inter-4.0-beta7.zip && mkdir -p /usr/share/fonts/opentype/inter/ && cp /tmp/Desktop/Inter-*.otf /usr/share/fonts/opentype/inter/ && fc-cache -f -v
+RUN wget https://github.com/rsms/inter/releases/download/v4.1/Inter-4.1.zip && unzip Inter-4.1.zip && mkdir -p /usr/share/fonts/opentype/inter/ && cp /tmp/extras/otf/Inter-*.otf /usr/share/fonts/opentype/inter/ && fc-cache -f -v
 
 RUN wget https://github.com/lay295/TwitchDownloader/releases/download/1.55.0/TwitchDownloaderCLI-1.55.0-Linux-x64.zip && unzip TwitchDownloaderCLI-1.55.0-Linux-x64.zip && mv TwitchDownloaderCLI /usr/local/bin/ && chmod +x /usr/local/bin/TwitchDownloaderCLI && rm TwitchDownloaderCLI-1.55.0-Linux-x64.zip
-
-RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.50.1

.devcontainer/devcontainer.json

@@ -3,39 +3,53 @@
   "features": {
     "ghcr.io/jungaretti/features/make:1": {},
     "ghcr.io/devcontainers/features/github-cli:1": {},
-    "ghcr.io/devcontainers/features/docker-in-docker:2": {}
+    "ghcr.io/devcontainers/features/docker-in-docker:2": {},
+    "ghcr.io/devcontainers/features/node:1": {},
+    "ghcr.io/guiyomh/features/golangci-lint:0": {},
+    "ghcr.io/devcontainers/features/python:1": {}
   },
   "customizations": {
     "vscode": {
       "extensions": [
         "dbaeumer.vscode-eslint",
         "esbenp.prettier-vscode",
         "eamodio.gitlens",
-        "github.copilot",
-        "yzhang.markdown-all-in-one"
+        "yzhang.markdown-all-in-one",
+        "esbenp.prettier-vscode",
+        "naumovs.color-highlight",
+        "ms-vscode-remote.remote-containers",
+        "burkeholland.simple-react-snippets",
+        "vunguyentuan.vscode-css-variables",
+        "vunguyentuan.vscode-postcss"
       ]
     }
   },
   // "forwardPorts": [4000],
   "appPort": ["0.0.0.0:4000:4000"],
   "mounts": [
     {
-      "source": "${localWorkspaceFolder}/dev/vods",
-      "target": "/vods",
+      "source": "${localWorkspaceFolder}/dev/videos",
+      "target": "/data/videos",
       "type": "bind"
     },
     {
-      "source": "${localWorkspaceFolder}/dev/data",
-      "target": "/data",
+      "source": "${localWorkspaceFolder}/dev/temp",
+      "target": "/data/temp",
       "type": "bind"
     },
     {
       "source": "${localWorkspaceFolder}/dev/logs",
-      "target": "/logs",
+      "target": "/data/logs",
+      "type": "bind"
+    },
+    {
+      "source": "${localWorkspaceFolder}/dev/config",
+      "target": "/data/config",
       "type": "bind"
     }
   ],
   "workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=cached",
   "workspaceFolder": "/workspace",
+  "postCreateCommand": "pip install streamlink chat-downloader --break-system-packages",
   "postAttachCommand": "sudo chown -R vscode:vscode /go && make dev_setup"
 }

.dockerignore

@@ -1,4 +1,22 @@
 .github
 dev
 tmp
-bin
+bin
+/frontend/node_modules
+/frontend/.pnp
+/frontend/.pnp.*
+/frontend/.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/versions
+/coverage
+/frontend/.next/
+/out/
+/build
+/frontend/npm-debug.log*
+/frontend/yarn-debug.log*
+/frontend/yarn-error.log*
+/frontend/.env*
+/frontend/*.tsbuildinfo
+/frontend/next-env.d.ts

.github/workflows/docker-publish.yml

@@ -9,9 +9,7 @@ on:
     branches: ["main"]
 
 env:
-  # Use docker.io for Docker Hub if empty
   REGISTRY: ghcr.io
-  # github.repository as <account>/<repo>
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
@@ -60,3 +58,8 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           context: .
+          # cache builds in github actions
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          # better compression
+          outputs: type=image,oci-mediatypes=true,compression=zstd,compression-level=3,force-compression=true

.gitignore

@@ -1,7 +1,3 @@
-# If you prefer the allow list template instead of the deny list, see community template:
-# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
-#
-# Binaries for programs and plugins
 *.exe
 *.exe~
 *.dll
@@ -14,9 +10,6 @@
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 
-# Dependency directories (remove the comment below to include it)
-# vendor/
-
 # Go workspace file
 go.work
 
@@ -26,4 +19,36 @@ go.work
 dev
 tmp
 **/__debug
-__debug*
+__debug*
+
+# dependencies
+/frontend/node_modules
+/frontend/.pnp
+/frontend/.pnp.*
+/frontend/.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/versions
+
+# testing
+/coverage
+
+# next.js
+/frontend/.next/
+/out/
+
+# production
+/build
+
+# debug
+/frontend/npm-debug.log*
+/frontend/yarn-debug.log*
+/frontend/yarn-error.log*
+
+# env files (can opt-in for committing if needed)
+/frontend/.env*
+
+# typescript
+/frontend/*.tsbuildinfo
+/frontend/next-env.d.ts

.server.air.toml

@@ -7,7 +7,7 @@ tmp_dir = "tmp"
   bin = "./tmp/server"
   cmd = "make build_dev_server"
   delay = 1000
-  exclude_dir = ["tmp", "dev"]
+  exclude_dir = ["tmp", "dev", "frontend"]
   exclude_file = []
   exclude_regex = ["_test.go"]
   exclude_unchanged = false

.worker.air.toml

@@ -7,7 +7,7 @@ tmp_dir = "tmp"
   bin = "./tmp/worker"
   cmd = "make build_dev_worker"
   delay = 1000
-  exclude_dir = ["tmp", "dev"]
+  exclude_dir = ["tmp", "dev", "frontend"]
   exclude_file = []
   exclude_regex = ["_test.go"]
   exclude_unchanged = false

Dockerfile

@@ -1,25 +1,29 @@
 ARG TWITCHDOWNLOADER_VERSION="1.55.0"
 
-# Build stage
-FROM golang:1.22-bookworm AS build
+#
+# API Build
+#
+FROM golang:1.23-bookworm AS build-api
 WORKDIR /app
 COPY . .
 RUN make build_server build_worker
 
-# Tools stage
+#
+# API Tools
+#
 FROM debian:bookworm-slim AS tools
 WORKDIR /tmp
 RUN apt-get update && apt-get install -y --no-install-recommends \
-unzip git ca-certificates curl \
-&& rm -rf /var/lib/apt/lists/*
+    unzip git ca-certificates curl \
+    && rm -rf /var/lib/apt/lists/*
 
 # Download TwitchDownloader for the correct platform
 ARG TWITCHDOWNLOADER_VERSION
 ENV TWITCHDOWNLOADER_URL=https://github.com/lay295/TwitchDownloader/releases/download/${TWITCHDOWNLOADER_VERSION}/TwitchDownloaderCLI-${TWITCHDOWNLOADER_VERSION}-Linux-x64.zip
 
 
 RUN if [ "$(uname -m)" = "aarch64" ]; then \
-TWITCHDOWNLOADER_URL=https://github.com/lay295/TwitchDownloader/releases/download/${TWITCHDOWNLOADER_VERSION}/TwitchDownloaderCLI-${TWITCHDOWNLOADER_VERSION}-LinuxArm64.zip; \
+    TWITCHDOWNLOADER_URL=https://github.com/lay295/TwitchDownloader/releases/download/${TWITCHDOWNLOADER_VERSION}/TwitchDownloaderCLI-${TWITCHDOWNLOADER_VERSION}-LinuxArm64.zip; \
     fi && \
     echo "Download URL: $TWITCHDOWNLOADER_URL" && \
     curl -L $TWITCHDOWNLOADER_URL -o twitchdownloader.zip && \
@@ -28,13 +32,48 @@ TWITCHDOWNLOADER_URL=https://github.com/lay295/TwitchDownloader/releases/downloa
 
 RUN git clone --depth 1 https://github.com/xenova/chat-downloader.git
 
+#
+# Frontend base
+#
+FROM node:22-alpine AS base-frontend
+
+# Install dependencies only when needed
+FROM node:22-alpine AS deps
+RUN apk add --no-cache libc6-compat
+WORKDIR /app
+
+COPY frontend/package.json frontend/package-lock.json* ./
+RUN \
+    if [ -f yarn.lock ]; then yarn --frozen-lockfile; \
+    elif [ -f package-lock.json ]; then npm ci --force; \
+    elif [ -f pnpm-lock.yaml ]; then corepack enable pnpm && pnpm i --frozen-lockfile; \
+    else echo "Lockfile not found." && exit 1; \
+    fi
+
+#
+# Frontend build
+#
+FROM node:22-alpine AS build-frontend
+WORKDIR /app
+COPY --from=deps /app/node_modules ./node_modules
+COPY frontend/. .
+
+ENV NEXT_TELEMETRY_DISABLED=1
+
+RUN \
+    if [ -f yarn.lock ]; then yarn run build; \
+    elif [ -f package-lock.json ]; then npm run build; \
+    elif [ -f pnpm-lock.yaml ]; then corepack enable pnpm && pnpm run build; \
+    else echo "Lockfile not found." && exit 1; \
+    fi
+
 # Production stage
 FROM debian:bookworm-slim
 WORKDIR /opt/app
 
 # Install dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    python3 python3-pip fontconfig ffmpeg tzdata procps \
+    python3 python3-pip fontconfig ffmpeg tzdata procps supervisor \
     fonts-noto-core fonts-noto-cjk fonts-noto-extra fonts-inter \
     curl \
     && rm -rf /var/lib/apt/lists/* \
@@ -48,6 +87,20 @@ RUN curl -LO https://github.com/tianon/gosu/releases/latest/download/gosu-$(dpkg
     && chmod 0755 gosu-$(dpkg --print-architecture | awk -F- '{ print $NF }') \
     && mv gosu-$(dpkg --print-architecture | awk -F- '{ print $NF }') /usr/local/bin/gosu
 
+# Install node for frontend
+ENV NODE_VERSION=22.x \
+    DEBIAN_FRONTEND=noninteractive
+
+# Install required packages, add NodeSource repository, and install Node.js
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    curl \
+    ca-certificates \
+    gnupg \
+    && curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION} | bash - \
+    && apt-get install -y --no-install-recommends nodejs \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN node --version && npm --version
+
 # Setup user
 RUN useradd -u 911 -d /data abc && usermod -a -G users abc
 
@@ -62,13 +115,30 @@ RUN chmod 644 /usr/share/fonts/* && chmod -R a+rX /usr/share/fonts
 COPY --from=tools /tmp/TwitchDownloaderCLI /usr/local/bin/
 RUN chmod +x /usr/local/bin/TwitchDownloaderCLI
 
-# Copy application files
-COPY --from=build /app/ganymede-api .
-COPY --from=build /app/ganymede-worker .
+# Copy api and worker builds
+COPY --from=build-api /app/ganymede-api .
+COPY --from=build-api /app/ganymede-worker .
+
+# Setup frontend
+ENV NODE_ENV=production
+ENV NEXT_TELEMETRY_DISABLED=1
+RUN addgroup --system --gid 1001 nodejs
+RUN adduser --system --uid 1001 nextjs
+
+COPY --from=build-frontend /app/public ./public
+
+RUN mkdir .next
+RUN chown nextjs:nodejs .next
+
+COPY --from=build-frontend --chown=nextjs:nodejs /app/.next/standalone ./
+COPY --from=build-frontend --chown=nextjs:nodejs /app/.next/static ./.next/static
+ENV HOSTNAME="0.0.0.0" 
+
 
 # Setup entrypoint
 COPY entrypoint.sh /usr/local/bin/
 RUN chmod +x /usr/local/bin/entrypoint.sh
+COPY supervisord.conf /opt/app/supervisord.conf
 
 EXPOSE 4000
 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

Makefile

@@ -1,6 +1,7 @@
 dev_setup:
 	go install github.com/joho/godotenv/cmd/godotenv@latest
 	go install github.com/air-verse/air@latest
+	cd frontend && npm install --force
 
 build_server:
 	go build -ldflags='-X github.com/zibbp/ganymede/internal/utils.Commit=$(shell git rev-parse HEAD) -X github.com/zibbp/ganymede/internal/utils.BuildTime=$(shell date -u "+%Y-%m-%d_%H:%M:%S")' -o ganymede-api cmd/server/main.go
@@ -22,9 +23,16 @@ dev_worker:
 	rm -f ./tmp/worker
 	air -c ./.worker.air.toml
 
+dev_web:
+	cd frontend && npm run dev
+
 ent_generate:
 	go run -mod=mod entgo.io/ent/cmd/ent generate --feature sql/upsert ./ent/schema
 
+ent_new_schema:
+	@read -p "Enter schema name:" schema; \
+	go run -mod=mod entgo.io/ent/cmd/ent new $$schema
+
 go_update_packages:
 	go get -u ./... && go mod tidy