From 4d66767c2bd9f822fcc2544de898e463bcb08579 Mon Sep 17 00:00:00 2001
From: Jim Nielsen
Date: Wed, 11 Dec 2024 14:12:29 -0700
Subject: [PATCH 001/155] initial commit
---
quadratic-client/src/router.tsx | 12 ++++++++++++
quadratic-client/src/routes/teams.$teamUuid.tsx | 4 +++-
2 files changed, 15 insertions(+), 1 deletion(-)
diff --git a/quadratic-client/src/router.tsx b/quadratic-client/src/router.tsx
index ca1dc74a8a..edffb5fc60 100644
--- a/quadratic-client/src/router.tsx
+++ b/quadratic-client/src/router.tsx
@@ -1,5 +1,6 @@
import { BrowserCompatibilityLayoutRoute } from '@/dashboard/components/BrowserCompatibilityLayoutRoute';
import * as Page404 from '@/routes/404';
+import { useDashboardRouteLoaderData } from '@/routes/_dashboard';
import * as Login from '@/routes/login';
import * as LoginResult from '@/routes/login-result';
import * as Logout from '@/routes/logout';
@@ -87,6 +88,17 @@ export const router = createBrowserRouter(
/>
import('./routes/labs')} />
+ {
+ const data = useDashboardRouteLoaderData();
+ // This renders
+ return Hello
;
+ // This triggers a route boundary error
+ return ;
+ }}
+ />
+
import('./routes/teams.create')} />
import('./routes/teams.$teamUuid')}>
diff --git a/quadratic-client/src/routes/teams.$teamUuid.tsx b/quadratic-client/src/routes/teams.$teamUuid.tsx
index dce927472d..e3ee823ede 100644
--- a/quadratic-client/src/routes/teams.$teamUuid.tsx
+++ b/quadratic-client/src/routes/teams.$teamUuid.tsx
@@ -5,7 +5,7 @@ import { Button } from '@/shared/shadcn/ui/button';
import { ExclamationTriangleIcon } from '@radix-ui/react-icons';
import mixpanel from 'mixpanel-browser';
import { ApiTypes } from 'quadratic-shared/typesAndSchemas';
-import { ActionFunctionArgs, Link, Outlet, redirectDocument } from 'react-router-dom';
+import { ActionFunctionArgs, Link, Outlet, redirectDocument, useRouteError } from 'react-router-dom';
export type TeamAction = {
'request.update-team': ReturnType;
@@ -139,6 +139,8 @@ export const Component = () => {
};
export const ErrorBoundary = () => {
+ const error = useRouteError();
+ console.error(error);
// Maybe we log this to Sentry?
return (
Date: Thu, 12 Dec 2024 03:20:22 +0530
Subject: [PATCH 002/155] fix pixiAppSettings circular dependency
---
quadratic-client/src/app/atoms/codeEditorAtom.ts | 11 +----------
quadratic-client/src/app/gridGL/PixiAppEffects.tsx | 7 +++++++
2 files changed, 8 insertions(+), 10 deletions(-)
diff --git a/quadratic-client/src/app/atoms/codeEditorAtom.ts b/quadratic-client/src/app/atoms/codeEditorAtom.ts
index 50fd8b348c..688b0e0499 100644
--- a/quadratic-client/src/app/atoms/codeEditorAtom.ts
+++ b/quadratic-client/src/app/atoms/codeEditorAtom.ts
@@ -1,6 +1,5 @@
import { getPromptMessages } from '@/app/ai/tools/message.helper';
import { events } from '@/app/events/events';
-import { pixiAppSettings } from '@/app/gridGL/pixiApp/PixiAppSettings';
import { CodeCell } from '@/app/gridGL/types/codeCell';
import { Coordinate } from '@/app/gridGL/types/size';
import { focusGrid } from '@/app/helpers/focusGrid';
@@ -189,15 +188,7 @@ export const codeEditorUnsavedChangesAtom = selector({
key: 'codeEditorUnsavedChangesAtom',
get: ({ get }) => {
const { editorContent, codeString } = get(codeEditorAtom);
- const unsavedChanges = editorContent !== codeString;
-
- if (unsavedChanges) {
- pixiAppSettings.unsavedEditorChanges = editorContent;
- } else {
- pixiAppSettings.unsavedEditorChanges = undefined;
- }
-
- return unsavedChanges;
+ return editorContent !== codeString;
},
});
diff --git a/quadratic-client/src/app/gridGL/PixiAppEffects.tsx b/quadratic-client/src/app/gridGL/PixiAppEffects.tsx
index 5f1f14cff7..606b263b92 100644
--- a/quadratic-client/src/app/gridGL/PixiAppEffects.tsx
+++ b/quadratic-client/src/app/gridGL/PixiAppEffects.tsx
@@ -42,6 +42,13 @@ export const PixiAppEffects = () => {
const [codeEditorState, setCodeEditorState] = useRecoilState(codeEditorAtom);
useEffect(() => {
pixiAppSettings.updateCodeEditorState(codeEditorState, setCodeEditorState);
+
+ const unsavedChanges = codeEditorState.editorContent !== codeEditorState.codeString;
+ if (unsavedChanges) {
+ pixiAppSettings.unsavedEditorChanges = codeEditorState.editorContent;
+ } else {
+ pixiAppSettings.unsavedEditorChanges = undefined;
+ }
}, [codeEditorState, setCodeEditorState]);
const { addGlobalSnackbar } = useGlobalSnackbar();
From d3f3466d980fea89ed6cf11d0031a31d78f9efbc Mon Sep 17 00:00:00 2001
From: Jim Nielsen
Date: Wed, 11 Dec 2024 17:19:16 -0700
Subject: [PATCH 003/155] more tweaks
---
quadratic-client/src/app/actions.ts | 2 +-
.../dashboard/components/DashboardSidebar.tsx | 2 +-
.../components/FilesListEmptyState.tsx | 2 +-
.../dashboard/components/NewFileButton.tsx | 8 +--
.../src/dashboard/shared/getActiveTeam.ts | 62 +++++++++++++++++++
quadratic-client/src/router.tsx | 24 +++----
quadratic-client/src/routes/_dashboard.tsx | 57 +++++------------
.../src/routes/_teams-redirect.tsx | 14 +++++
quadratic-client/src/routes/new.tsx | 15 +++++
.../routes/teams.$teamUuid.files.create.tsx | 15 ++++-
.../src/shared/constants/routes.ts | 29 +++++++--
.../src/shared/hooks/useNewFileFromState.ts | 8 +--
12 files changed, 160 insertions(+), 78 deletions(-)
create mode 100644 quadratic-client/src/dashboard/shared/getActiveTeam.ts
create mode 100644 quadratic-client/src/routes/_teams-redirect.tsx
create mode 100644 quadratic-client/src/routes/new.tsx
diff --git a/quadratic-client/src/app/actions.ts b/quadratic-client/src/app/actions.ts
index ca0232a6de..440b9a1a23 100644
--- a/quadratic-client/src/app/actions.ts
+++ b/quadratic-client/src/app/actions.ts
@@ -67,7 +67,7 @@ export const createNewFileAction = {
label: 'New',
isAvailable: isAvailableBecauseFileLocationIsAccessibleAndWriteable,
run({ teamUuid }: { teamUuid: string }) {
- window.location.href = ROUTES.CREATE_FILE_PRIVATE(teamUuid);
+ window.location.href = ROUTES.CREATE_FILE(teamUuid, { private: true });
},
};
diff --git a/quadratic-client/src/dashboard/components/DashboardSidebar.tsx b/quadratic-client/src/dashboard/components/DashboardSidebar.tsx
index 85ac190aae..2d5d2117eb 100644
--- a/quadratic-client/src/dashboard/components/DashboardSidebar.tsx
+++ b/quadratic-client/src/dashboard/components/DashboardSidebar.tsx
@@ -193,7 +193,7 @@ function SidebarNavLinkCreateButton({
size="icon-sm"
className="absolute right-2 top-1 ml-auto !bg-transparent opacity-30 hover:opacity-100"
>
-
+
diff --git a/quadratic-client/src/dashboard/components/FilesListEmptyState.tsx b/quadratic-client/src/dashboard/components/FilesListEmptyState.tsx
index b4d8b38906..ee49d4be7c 100644
--- a/quadratic-client/src/dashboard/components/FilesListEmptyState.tsx
+++ b/quadratic-client/src/dashboard/components/FilesListEmptyState.tsx
@@ -21,7 +21,7 @@ export const FilesListEmptyState = ({ isPrivate = false }: { isPrivate?: boolean
<>
You don’t have any files yet.{' '}
{
diff --git a/quadratic-client/src/dashboard/components/NewFileButton.tsx b/quadratic-client/src/dashboard/components/NewFileButton.tsx
index a568937a91..f856f91975 100644
--- a/quadratic-client/src/dashboard/components/NewFileButton.tsx
+++ b/quadratic-client/src/dashboard/components/NewFileButton.tsx
@@ -36,7 +36,7 @@ export default function NewFileButton({ isPrivate }: { isPrivate: boolean }) {
return (
-
+
New file
diff --git a/quadratic-client/src/dashboard/shared/getActiveTeam.ts b/quadratic-client/src/dashboard/shared/getActiveTeam.ts
new file mode 100644
index 0000000000..7ad25e3f52
--- /dev/null
+++ b/quadratic-client/src/dashboard/shared/getActiveTeam.ts
@@ -0,0 +1,62 @@
+import { ACTIVE_TEAM_UUID_KEY } from '@/routes/_dashboard';
+import { apiClient } from '@/shared/api/apiClient';
+import * as Sentry from '@sentry/react';
+
+// TODO: explain this
+// It's implicit what team is currently active. This is the function that
+// tells us what is most likely the currently active team.
+// Only once we do a get of the team do we know for sure the user has access to it.
+export default async function getActiveTeam(
+ teams: Awaited>['teams'],
+ teamUuidFromUrl: string | undefined
+) {
+ let teamCreated = false;
+
+ /**
+ * Determine what the active team is
+ */
+ let initialActiveTeamUuid = undefined;
+ // const uuidFromUrl = params.teamUuid;
+ const uuidFromLocalStorage = localStorage.getItem(ACTIVE_TEAM_UUID_KEY);
+
+ // FYI: if you have a UUID in the URL or localstorage, it doesn’t mean you
+ // have access to it (maybe you were removed from a team, so it’s a 404)
+ // So we have to ensure we A) have a UUID, and B) it's in the list of teams
+ // we have access to from the server.
+
+ // 1) Check the URL for a team UUID. If there's one, use that as that's
+ // explicitly what the user is trying to look at
+ if (teamUuidFromUrl) {
+ initialActiveTeamUuid = teamUuidFromUrl;
+
+ // 2) Check localstorage for a team UUID
+ // If what's in localstorage is not in the list of teams from the server —
+ // e.g. you lost access to a team — we'll skip this
+ } else if (uuidFromLocalStorage && teams.find((team) => team.team.uuid === uuidFromLocalStorage)) {
+ initialActiveTeamUuid = uuidFromLocalStorage;
+
+ // 3) There's no default preference (yet), so pick the 1st one in the API
+ } else if (teams.length > 0) {
+ initialActiveTeamUuid = teams[0].team.uuid;
+
+ // 4) There are no teams in the API, so we will create one
+ } else if (teams.length === 0) {
+ const newTeam = await apiClient.teams.create({ name: 'My Team' });
+ initialActiveTeamUuid = newTeam.uuid;
+ teamCreated = true;
+ }
+
+ // This should never happen, but if it does, we'll log it to sentry
+ if (initialActiveTeamUuid === undefined) {
+ Sentry.captureEvent({
+ message: 'No active team was found or could be created.',
+ level: 'fatal',
+ });
+ throw new Error('No active team could be found or created.');
+ }
+
+ return {
+ teamUuid: initialActiveTeamUuid,
+ teamCreated,
+ };
+}
diff --git a/quadratic-client/src/router.tsx b/quadratic-client/src/router.tsx
index edffb5fc60..1c84a5806a 100644
--- a/quadratic-client/src/router.tsx
+++ b/quadratic-client/src/router.tsx
@@ -1,6 +1,5 @@
import { BrowserCompatibilityLayoutRoute } from '@/dashboard/components/BrowserCompatibilityLayoutRoute';
import * as Page404 from '@/routes/404';
-import { useDashboardRouteLoaderData } from '@/routes/_dashboard';
import * as Login from '@/routes/login';
import * as LoginResult from '@/routes/login-result';
import * as Logout from '@/routes/logout';
@@ -69,6 +68,10 @@ export const router = createBrowserRouter(
/>
+ {/* Helper routes, e.g. /connections -> /teams/:uuid/connections */}
+
+ import('./routes/new')} />
+
{/* Dashboard UI routes */}
import('./routes/_dashboard')}>
import('./routes/labs')} />
- {
- const data = useDashboardRouteLoaderData();
- // This renders
- return Hello
;
- // This triggers a route boundary error
- return ;
- }}
- />
+ {/* TODO: handle these
+ import('./routes/_teams-redirect')} />
+ import('./routes/_teams-redirect')} />
+ import('./routes/_teams-redirect')} />
+
+ import('./routes/_teams-redirect')} />
+ import('./routes/files.create')} />
+ import('./routes/_teams-redirect')} />
+ */}
import('./routes/teams.create')} />
diff --git a/quadratic-client/src/routes/_dashboard.tsx b/quadratic-client/src/routes/_dashboard.tsx
index 8cee523e35..4960387017 100644
--- a/quadratic-client/src/routes/_dashboard.tsx
+++ b/quadratic-client/src/routes/_dashboard.tsx
@@ -3,6 +3,7 @@ import { DashboardSidebar } from '@/dashboard/components/DashboardSidebar';
import { EducationDialog } from '@/dashboard/components/EducationDialog';
import { Empty } from '@/dashboard/components/Empty';
import { ImportProgressList } from '@/dashboard/components/ImportProgressList';
+import getActiveTeam from '@/dashboard/shared/getActiveTeam';
import { apiClient } from '@/shared/api/apiClient';
import { MenuIcon } from '@/shared/components/Icons';
import { ROUTES, ROUTE_LOADER_IDS, SEARCH_PARAMS } from '@/shared/constants/routes';
@@ -12,7 +13,6 @@ import { Sheet, SheetContent, SheetTrigger } from '@/shared/shadcn/ui/sheet';
import { TooltipProvider } from '@/shared/shadcn/ui/tooltip';
import { cn } from '@/shared/shadcn/utils';
import { ExclamationTriangleIcon, InfoCircledIcon } from '@radix-ui/react-icons';
-import * as Sentry from '@sentry/react';
import { ApiTypes } from 'quadratic-shared/typesAndSchemas';
import { useEffect, useRef, useState } from 'react';
import { isMobile } from 'react-device-detect';
@@ -54,10 +54,10 @@ type LoaderData = {
teams: ApiTypes['/v0/teams.GET.response']['teams'];
userMakingRequest: ApiTypes['/v0/teams.GET.response']['userMakingRequest'];
eduStatus: ApiTypes['/v0/education.GET.response']['eduStatus'];
- initialActiveTeamUuid: string;
activeTeam: ApiTypes['/v0/teams/:uuid.GET.response'];
};
+// getActiveTeam()
export const loader = async ({ params, request }: LoaderFunctionArgs): Promise => {
/**
* Get the initial data
@@ -68,69 +68,40 @@ export const loader = async ({ params, request }: LoaderFunctionArgs): Promise team.team.uuid === uuidFromLocalStorage)) {
- initialActiveTeamUuid = uuidFromLocalStorage;
-
- // 3) there's no default preference (yet), so pick the 1st one in the API
- } else if (teams.length > 0) {
- initialActiveTeamUuid = teams[0].team.uuid;
-
- // 4) there's no teams in the API, so create one
- } else if (teams.length === 0) {
- const newTeam = await apiClient.teams.create({ name: 'My Team' });
- // Send user to team dashboard if mobile, otherwise to a new file
- return isMobile ? redirect(ROUTES.TEAM(newTeam.uuid)) : redirect(ROUTES.CREATE_FILE(newTeam.uuid));
- }
-
- // This should never happen, but if it does, we'll log it to sentry
- if (initialActiveTeamUuid === undefined) {
- Sentry.captureEvent({
- message: 'No active team was found or could be created.',
- level: 'fatal',
- });
- throw new Error('No active team could be found or created.');
+ // If a team was created, it was probably a first time user so send them to
+ // the team dashboard if mobile, otherwise to a new file
+ if (teamCreated) {
+ return isMobile ? redirect(ROUTES.TEAM(teamUuid)) : redirect(ROUTES.CREATE_FILE(teamUuid));
}
// If this was a request to the root of the app, re-route to the active team
const url = new URL(request.url);
if (url.pathname === '/') {
// If there are search params, keep 'em
- return redirect(ROUTES.TEAM(initialActiveTeamUuid) + url.search);
+ return redirect(ROUTES.TEAM(teamUuid) + url.search);
}
+ // TODO: replace this with /connections -> /teams/:uuid/connections
// If it was a shortcut team route, redirect there
// e.g. /?team-shortcut=connections
const teamShortcut = url.searchParams.get('team-shortcut');
if (teamShortcut) {
url.searchParams.delete('team-shortcut');
- return redirect(ROUTES.TEAM_CONNECTIONS(initialActiveTeamUuid) + url.search);
+ return redirect(ROUTES.TEAM_CONNECTIONS(teamUuid) + url.search);
}
/**
* Get data for the active team
*/
const activeTeam = await apiClient.teams
- .get(initialActiveTeamUuid)
+ .get(teamUuid)
.then((data) => {
// If we got to here, we successfully loaded the active team so now this is
// the one we keep in localstorage for when the page loads anew
- localStorage.setItem(ACTIVE_TEAM_UUID_KEY, initialActiveTeamUuid);
+ localStorage.setItem(ACTIVE_TEAM_UUID_KEY, teamUuid);
// Sort the users so the logged-in user is first in the list
data.users.sort((a, b) => {
@@ -154,7 +125,7 @@ export const loader = async ({ params, request }: LoaderFunctionArgs): Promise useRouteLoaderData(ROUTE_LOADER_IDS.DASHBOARD) as LoaderData;
diff --git a/quadratic-client/src/routes/_teams-redirect.tsx b/quadratic-client/src/routes/_teams-redirect.tsx
new file mode 100644
index 0000000000..d7413a9f73
--- /dev/null
+++ b/quadratic-client/src/routes/_teams-redirect.tsx
@@ -0,0 +1,14 @@
+import { useDashboardRouteLoaderData } from '@/routes/_dashboard';
+import { Navigate, useLocation } from 'react-router-dom';
+
+export const Component = () => {
+ const {
+ activeTeam: {
+ team: { uuid },
+ },
+ } = useDashboardRouteLoaderData();
+ const { pathname } = useLocation();
+ console.log(pathname);
+
+ return ;
+};
diff --git a/quadratic-client/src/routes/new.tsx b/quadratic-client/src/routes/new.tsx
new file mode 100644
index 0000000000..4500653083
--- /dev/null
+++ b/quadratic-client/src/routes/new.tsx
@@ -0,0 +1,15 @@
+import getActiveTeam from '@/dashboard/shared/getActiveTeam';
+import { apiClient } from '@/shared/api/apiClient';
+import { ROUTES } from '@/shared/constants/routes';
+import { LoaderFunctionArgs, redirect } from 'react-router-dom';
+
+export const loader = async ({ request }: LoaderFunctionArgs) => {
+ const url = new URL(request.url);
+ const prompt = url.searchParams.get('prompt');
+
+ const { teams } = await apiClient.teams.list();
+ const { teamUuid } = await getActiveTeam(teams, undefined);
+
+ const redirectUrl = ROUTES.CREATE_FILE(teamUuid, { prompt, private: true });
+ return redirect(redirectUrl);
+};
diff --git a/quadratic-client/src/routes/teams.$teamUuid.files.create.tsx b/quadratic-client/src/routes/teams.$teamUuid.files.create.tsx
index e5ba8b3c10..973be81db6 100644
--- a/quadratic-client/src/routes/teams.$teamUuid.files.create.tsx
+++ b/quadratic-client/src/routes/teams.$teamUuid.files.create.tsx
@@ -66,10 +66,19 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
const {
file: { uuid },
} = await apiClient.files.create({ teamUuid, isPrivate });
- // If there's a `state=...` query param, for starting a file in a specific
- // state, pass that along
+
+ // Pass along a few of the search params
+ let searchParamsToPass = new URLSearchParams();
const state = searchParams.get('state');
- return replace(ROUTES.FILE(uuid) + (state ? `?state=${state}` : ''));
+ if (state) {
+ searchParamsToPass.set('state', state);
+ }
+ const prompt = searchParams.get('prompt');
+ if (prompt) {
+ searchParamsToPass.set('prompt', prompt);
+ }
+
+ return replace(ROUTES.FILE(uuid) + (searchParamsToPass ? '?' + searchParamsToPass.toString() : ''));
} catch (error) {
return replace(getFailUrl(ROUTES.TEAM(teamUuid)));
}
diff --git a/quadratic-client/src/shared/constants/routes.ts b/quadratic-client/src/shared/constants/routes.ts
index 5db407031e..454ec221e6 100644
--- a/quadratic-client/src/shared/constants/routes.ts
+++ b/quadratic-client/src/shared/constants/routes.ts
@@ -12,14 +12,30 @@ export const ROUTES = {
FILES_SHARED_WITH_ME: '/files/shared-with-me',
FILE: (uuid: string) => `/file/${uuid}`,
- CREATE_FILE: (teamUuid: string, state?: UrlParamsDevState['insertAndRunCodeInNewSheet']) =>
- `/teams/${teamUuid}/files/create` +
- (state ? `?state=${btoa(JSON.stringify({ insertAndRunCodeInNewSheet: state }))}` : ''),
+ CREATE_FILE: (
+ teamUuid: string,
+ searchParams: {
+ state?: UrlParamsDevState['insertAndRunCodeInNewSheet'];
+ prompt?: string | null;
+ private?: boolean;
+ } = {}
+ ) => {
+ let url = new URL(window.location.origin + `/teams/${teamUuid}/files/create`);
+
+ if (searchParams.state) {
+ url.searchParams.set('state', btoa(JSON.stringify({ insertAndRunCodeInNewSheet: searchParams.state })));
+ }
+ if (searchParams.prompt) {
+ url.searchParams.set('prompt', searchParams.prompt);
+ }
+ if (searchParams.private) {
+ url.searchParams.set('private', 'true');
+ }
+
+ return url.toString();
+ },
CREATE_FILE_EXAMPLE: (teamUuid: string, publicFileUrlInProduction: string, isPrivate: boolean) =>
`/teams/${teamUuid}/files/create?example=${publicFileUrlInProduction}${isPrivate ? '&private' : ''}`,
- CREATE_FILE_PRIVATE: (teamUuid: string, state?: UrlParamsDevState['insertAndRunCodeInNewSheet']) =>
- `/teams/${teamUuid}/files/create?private` +
- (state ? `&state=${btoa(JSON.stringify({ insertAndRunCodeInNewSheet: state }))}` : ''),
TEAMS: `/teams`,
TEAMS_CREATE: `/teams/create`,
TEAM: (teamUuid: string) => `/teams/${teamUuid}`,
@@ -38,6 +54,7 @@ export const ROUTES = {
TEAM_SHORTCUT: {
CONNECTIONS: `/?team-shortcut=connections`,
},
+ CONNECTIONS: '/connections',
EDIT_TEAM: (teamUuid: string) => `/teams/${teamUuid}/edit`,
EXAMPLES: '/examples',
ACCOUNT: '/account',
diff --git a/quadratic-client/src/shared/hooks/useNewFileFromState.ts b/quadratic-client/src/shared/hooks/useNewFileFromState.ts
index 14d164342e..967286ab8b 100644
--- a/quadratic-client/src/shared/hooks/useNewFileFromState.ts
+++ b/quadratic-client/src/shared/hooks/useNewFileFromState.ts
@@ -12,9 +12,7 @@ export const useNewFileFromStatePythonApi = ({ isPrivate, teamUuid }: { isPrivat
language: 'Python' as CodeCellLanguage,
};
- const to = isPrivate
- ? ROUTES.CREATE_FILE_PRIVATE(teamUuid, stateUrlParam)
- : ROUTES.CREATE_FILE(teamUuid, stateUrlParam);
+ const to = ROUTES.CREATE_FILE(teamUuid, { state: stateUrlParam, private: isPrivate });
return to;
};
@@ -40,9 +38,7 @@ export const newNewFileFromStateConnection = ({
language: { Connection: { kind: connectionType, id: connectionUuid } },
};
- const to = isPrivate
- ? ROUTES.CREATE_FILE_PRIVATE(teamUuid, stateUrlParam)
- : ROUTES.CREATE_FILE(teamUuid, stateUrlParam);
+ const to = ROUTES.CREATE_FILE(teamUuid, { state: stateUrlParam, private: isPrivate });
return to;
};
From 15c437fd2733817ca35895d41f8cbece4f84d4a6 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Thu, 12 Dec 2024 06:45:54 +0530
Subject: [PATCH 004/155] submit analyst prompt from query params
---
.../src/app/atoms/aiAnalystAtom.ts | 2 ++
quadratic-client/src/app/events/events.ts | 2 ++
.../src/app/gridGL/PixiAppEffects.tsx | 6 +++--
.../src/app/gridGL/pixiApp/PixiAppSettings.ts | 9 ++++++-
.../gridGL/pixiApp/urlParams/UrlParamsUser.ts | 25 +++++++++++++++++--
.../hooks/useSubmitAIAnalystPrompt.tsx | 19 ++++++--------
6 files changed, 47 insertions(+), 16 deletions(-)
diff --git a/quadratic-client/src/app/atoms/aiAnalystAtom.ts b/quadratic-client/src/app/atoms/aiAnalystAtom.ts
index 4ba67d8dbc..c98a923ac6 100644
--- a/quadratic-client/src/app/atoms/aiAnalystAtom.ts
+++ b/quadratic-client/src/app/atoms/aiAnalystAtom.ts
@@ -1,6 +1,7 @@
import { aiAnalystOfflineChats } from '@/app/ai/offline/aiAnalystChats';
import { getPromptMessages } from '@/app/ai/tools/message.helper';
import { editorInteractionStateUserAtom, editorInteractionStateUuidAtom } from '@/app/atoms/editorInteractionStateAtom';
+import { events } from '@/app/events/events';
import { sheets } from '@/app/grid/controller/Sheets';
import { focusGrid } from '@/app/helpers/focusGrid';
import { Chat, ChatMessage } from 'quadratic-shared/typesAndSchemasAI';
@@ -62,6 +63,7 @@ export const aiAnalystAtom = atom({
console.error('[AIAnalystOfflineChats]: ', error);
}
}
+ events.emit('aiAnalystInitialized');
}
},
({ onSet }) => {
diff --git a/quadratic-client/src/app/events/events.ts b/quadratic-client/src/app/events/events.ts
index 7281499e92..743a20e04b 100644
--- a/quadratic-client/src/app/events/events.ts
+++ b/quadratic-client/src/app/events/events.ts
@@ -133,6 +133,8 @@ interface EventTypes {
hashContentChanged: (sheetId: string, hashX: number, hashY: number) => void;
codeEditorCodeCell: (codeCell?: CodeCell) => void;
+
+ aiAnalystInitialized: () => void;
}
export const events = new EventEmitter();
diff --git a/quadratic-client/src/app/gridGL/PixiAppEffects.tsx b/quadratic-client/src/app/gridGL/PixiAppEffects.tsx
index 606b263b92..863db83493 100644
--- a/quadratic-client/src/app/gridGL/PixiAppEffects.tsx
+++ b/quadratic-client/src/app/gridGL/PixiAppEffects.tsx
@@ -6,6 +6,7 @@ import { gridSettingsAtom, presentationModeAtom, showHeadingsAtom } from '@/app/
import { inlineEditorAtom } from '@/app/atoms/inlineEditorAtom';
import { pixiApp } from '@/app/gridGL/pixiApp/PixiApp';
import { pixiAppSettings } from '@/app/gridGL/pixiApp/PixiAppSettings';
+import { useSubmitAIAnalystPrompt } from '@/app/ui/menus/AIAnalyst/hooks/useSubmitAIAnalystPrompt';
import { useGlobalSnackbar } from '@/shared/components/GlobalSnackbarProvider';
import { useEffect } from 'react';
import { isMobile } from 'react-device-detect';
@@ -67,9 +68,10 @@ export const PixiAppEffects = () => {
}, [gridPanMode, setGridPanMode]);
const [aiAnalystState, setAIAnalystState] = useRecoilState(aiAnalystAtom);
+ const { submitPrompt } = useSubmitAIAnalystPrompt();
useEffect(() => {
- pixiAppSettings.updateAIAnalystState(aiAnalystState, setAIAnalystState);
- }, [aiAnalystState, setAIAnalystState]);
+ pixiAppSettings.updateAIAnalystState(aiAnalystState, setAIAnalystState, submitPrompt);
+ }, [aiAnalystState, setAIAnalystState, submitPrompt]);
useEffect(() => {
const handleMouseUp = () => {
diff --git a/quadratic-client/src/app/gridGL/pixiApp/PixiAppSettings.ts b/quadratic-client/src/app/gridGL/pixiApp/PixiAppSettings.ts
index 13572935e5..c325738098 100644
--- a/quadratic-client/src/app/gridGL/pixiApp/PixiAppSettings.ts
+++ b/quadratic-client/src/app/gridGL/pixiApp/PixiAppSettings.ts
@@ -9,6 +9,7 @@ import { sheets } from '@/app/grid/controller/Sheets';
import { inlineEditorHandler } from '@/app/gridGL/HTMLGrid/inlineEditor/inlineEditorHandler';
import { CursorMode } from '@/app/gridGL/HTMLGrid/inlineEditor/inlineEditorKeyboard';
import { pixiApp } from '@/app/gridGL/pixiApp/PixiApp';
+import { SubmitAIAnalystPromptArgs } from '@/app/ui/menus/AIAnalyst/hooks/useSubmitAIAnalystPrompt';
import { multiplayer } from '@/app/web-workers/multiplayerWebWorker/multiplayer';
import { GlobalSnackbar } from '@/shared/components/GlobalSnackbarProvider';
import { ApiTypes } from 'quadratic-shared/typesAndSchemas';
@@ -55,6 +56,7 @@ class PixiAppSettings {
aiAnalystState = defaultAIAnalystState;
setAIAnalystState?: SetterOrUpdater;
+ submitAIAnalystPrompt?: (prompt: SubmitAIAnalystPromptArgs) => Promise;
constructor() {
const settings = localStorage.getItem('viewSettings');
@@ -138,9 +140,14 @@ class PixiAppSettings {
this.setCodeEditorState = setCodeEditorState;
}
- updateAIAnalystState(aiAnalystState: AIAnalystState, setAIAnalystState: SetterOrUpdater): void {
+ updateAIAnalystState(
+ aiAnalystState: AIAnalystState,
+ setAIAnalystState: SetterOrUpdater,
+ submitAIAnalystPrompt: (prompt: SubmitAIAnalystPromptArgs) => Promise
+ ): void {
this.aiAnalystState = aiAnalystState;
this.setAIAnalystState = setAIAnalystState;
+ this.submitAIAnalystPrompt = submitAIAnalystPrompt;
}
get showGridLines(): boolean {
diff --git a/quadratic-client/src/app/gridGL/pixiApp/urlParams/UrlParamsUser.ts b/quadratic-client/src/app/gridGL/pixiApp/urlParams/UrlParamsUser.ts
index 25901ad188..5a0553b56e 100644
--- a/quadratic-client/src/app/gridGL/pixiApp/urlParams/UrlParamsUser.ts
+++ b/quadratic-client/src/app/gridGL/pixiApp/urlParams/UrlParamsUser.ts
@@ -13,7 +13,7 @@ export class UrlParamsUser {
this.loadSheet(params);
this.loadCursor(params);
this.loadCode(params);
- this.setupListeners();
+ this.setupListeners(params);
}
private loadSheet(params: URLSearchParams) {
@@ -65,10 +65,31 @@ export class UrlParamsUser {
}
}
- private setupListeners() {
+ private loadAIAnalystPrompt = (params: URLSearchParams) => {
+ const prompt = params.get('prompt');
+ if (!prompt) return;
+
+ const { submitAIAnalystPrompt } = pixiAppSettings;
+ if (!submitAIAnalystPrompt) {
+ throw new Error('Expected submitAIAnalystPrompt to be set in urlParams.loadAIAnalystPrompt');
+ }
+
+ submitAIAnalystPrompt({
+ userPrompt: prompt,
+ context: {
+ sheets: [],
+ currentSheet: sheets.sheet.name,
+ selection: undefined,
+ },
+ clearMessages: true,
+ });
+ };
+
+ private setupListeners(params: URLSearchParams) {
events.on('cursorPosition', this.setDirty);
events.on('changeSheet', this.setDirty);
events.on('codeEditor', this.setDirty);
+ events.on('aiAnalystInitialized', () => this.loadAIAnalystPrompt(params));
}
private setDirty = () => {
diff --git a/quadratic-client/src/app/ui/menus/AIAnalyst/hooks/useSubmitAIAnalystPrompt.tsx b/quadratic-client/src/app/ui/menus/AIAnalyst/hooks/useSubmitAIAnalystPrompt.tsx
index 8b29f9b037..a8eb0f1b24 100644
--- a/quadratic-client/src/app/ui/menus/AIAnalyst/hooks/useSubmitAIAnalystPrompt.tsx
+++ b/quadratic-client/src/app/ui/menus/AIAnalyst/hooks/useSubmitAIAnalystPrompt.tsx
@@ -28,6 +28,13 @@ import { useRecoilCallback } from 'recoil';
const MAX_TOOL_CALL_ITERATIONS = 5;
+export type SubmitAIAnalystPromptArgs = {
+ userPrompt: string;
+ context: Context;
+ messageIndex?: number;
+ clearMessages?: boolean;
+};
+
export function useSubmitAIAnalystPrompt() {
const { handleAIRequestToAPI } = useAIRequestToAPI();
const { getQuadraticContext } = useQuadraticContextMessages();
@@ -79,17 +86,7 @@ export function useSubmitAIAnalystPrompt() {
const submitPrompt = useRecoilCallback(
({ set, snapshot }) =>
- async ({
- userPrompt,
- context,
- messageIndex,
- clearMessages,
- }: {
- userPrompt: string;
- context: Context;
- messageIndex?: number;
- clearMessages?: boolean;
- }) => {
+ async ({ userPrompt, context, messageIndex, clearMessages }: SubmitAIAnalystPromptArgs) => {
set(showAIAnalystAtom, true);
set(aiAnalystShowChatHistoryAtom, false);
From abca5a1b871d360b2ea0910ec8c5ceb284781d0a Mon Sep 17 00:00:00 2001
From: Jim Nielsen
Date: Thu, 12 Dec 2024 17:01:33 -0700
Subject: [PATCH 005/155] remove url param
---
.../src/app/gridGL/pixiApp/urlParams/UrlParamsUser.ts | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/quadratic-client/src/app/gridGL/pixiApp/urlParams/UrlParamsUser.ts b/quadratic-client/src/app/gridGL/pixiApp/urlParams/UrlParamsUser.ts
index 5a0553b56e..e362913493 100644
--- a/quadratic-client/src/app/gridGL/pixiApp/urlParams/UrlParamsUser.ts
+++ b/quadratic-client/src/app/gridGL/pixiApp/urlParams/UrlParamsUser.ts
@@ -83,6 +83,12 @@ export class UrlParamsUser {
},
clearMessages: true,
});
+
+ // Remove the `prompt` param when we're done
+ const url = new URL(window.location.href);
+ params.delete('prompt');
+ url.search = params.toString();
+ window.history.replaceState(null, '', url.toString());
};
private setupListeners(params: URLSearchParams) {
From f4487f2ab4775cb51c186d828c02edcfaedf70d8 Mon Sep 17 00:00:00 2001
From: Jim Nielsen
Date: Thu, 12 Dec 2024 17:01:41 -0700
Subject: [PATCH 006/155] update route
---
quadratic-client/src/router.tsx | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/quadratic-client/src/router.tsx b/quadratic-client/src/router.tsx
index 1c84a5806a..1dd3de5c62 100644
--- a/quadratic-client/src/router.tsx
+++ b/quadratic-client/src/router.tsx
@@ -70,7 +70,7 @@ export const router = createBrowserRouter(
{/* Helper routes, e.g. /connections -> /teams/:uuid/connections */}
- import('./routes/new')} />
+ import('./routes/new')} />
{/* Dashboard UI routes */}
import('./routes/_dashboard')}>
From bcae9146838a63b03ca981540c00f9c1d2872191 Mon Sep 17 00:00:00 2001
From: Jim Nielsen
Date: Thu, 12 Dec 2024 17:02:36 -0700
Subject: [PATCH 007/155] first time user creation logic
---
quadratic-api/src/middleware/user.ts | 13 +++++++------
.../src/routes/v0/users.acknowledge.GET.ts | 2 +-
quadratic-api/src/types/Request.ts | 1 +
quadratic-client/src/routes/login-result.tsx | 15 ++++++++++++++-
quadratic-shared/typesAndSchemas.ts | 2 +-
5 files changed, 24 insertions(+), 9 deletions(-)
diff --git a/quadratic-api/src/middleware/user.ts b/quadratic-api/src/middleware/user.ts
index e2fe6f2fb9..61b8c9c996 100644
--- a/quadratic-api/src/middleware/user.ts
+++ b/quadratic-api/src/middleware/user.ts
@@ -61,7 +61,7 @@ const getOrCreateUser = async (auth0Id: string) => {
});
if (user) {
- return user;
+ return { user, userCreated: false };
}
// If they don't exist yet, create them
@@ -74,18 +74,19 @@ const getOrCreateUser = async (auth0Id: string) => {
await runFirstTimeUserLogic(newUser);
// Return the user
- return newUser;
+ return { user: newUser, userCreated: true };
};
export const userMiddleware = async (req: Request, res: Response, next: NextFunction) => {
const { auth } = req as RequestWithAuth;
- const user = await getOrCreateUser(auth.sub);
+ const { user, userCreated } = await getOrCreateUser(auth.sub);
if (!user) {
return res.status(500).json({ error: { message: 'Unable to get authenticated user' } });
}
(req as RequestWithUser).user = user;
+ (req as RequestWithUser).userCreated = userCreated === true;
next();
};
@@ -93,12 +94,12 @@ export const userOptionalMiddleware = async (req: Request, res: Response, next:
const { auth } = req as RequestWithOptionalAuth;
if (auth && auth.sub) {
- const user = await getOrCreateUser(auth.sub);
+ const { user } = await getOrCreateUser(auth.sub);
if (!user) {
return res.status(500).json({ error: { message: 'Unable to get authenticated user' } });
}
- // @ts-expect-error
- req.user = user;
+
+ (req as RequestWithUser).user = user;
}
next();
diff --git a/quadratic-api/src/routes/v0/users.acknowledge.GET.ts b/quadratic-api/src/routes/v0/users.acknowledge.GET.ts
index dce3263540..bc2a42ed83 100644
--- a/quadratic-api/src/routes/v0/users.acknowledge.GET.ts
+++ b/quadratic-api/src/routes/v0/users.acknowledge.GET.ts
@@ -19,5 +19,5 @@ export default [validateAccessToken, userMiddleware, handler];
* been created yet or associated with teams and/or files.
*/
async function handler(req: RequestWithUser, res: Response) {
- return res.status(200).json({ message: 'acknowledged' });
+ return res.status(200).json({ message: 'acknowledged', userCreated: req.userCreated });
}
diff --git a/quadratic-api/src/types/Request.ts b/quadratic-api/src/types/Request.ts
index a63ae6cb25..56cc2b1f6e 100644
--- a/quadratic-api/src/types/Request.ts
+++ b/quadratic-api/src/types/Request.ts
@@ -34,6 +34,7 @@ export type RequestWithAuth = JWTRequest & {
export type RequestWithUser = RequestWithAuth & {
user: User;
+ userCreated: boolean;
};
export type RequestWithOptionalUser = RequestWithOptionalAuth & {
diff --git a/quadratic-client/src/routes/login-result.tsx b/quadratic-client/src/routes/login-result.tsx
index 10678d1279..32bdc77e02 100644
--- a/quadratic-client/src/routes/login-result.tsx
+++ b/quadratic-client/src/routes/login-result.tsx
@@ -11,7 +11,20 @@ export const loader = async () => {
if (isAuthenticated) {
// Acknowledge the user has just logged in. The backend may need
// to run some logic before making any other API calls in parallel
- await apiClient.users.acknowledge();
+ const { userCreated } = await apiClient.users.acknowledge();
+
+ // Special case for first-time users
+ if (userCreated) {
+ console.log('First-time user created');
+ try {
+ // @ts-expect-error
+ window.dataLayer.push({
+ event: 'registrationComplete',
+ });
+ } catch (e) {
+ // No google analytics available
+ }
+ }
let redirectTo = new URLSearchParams(window.location.search).get('redirectTo') || '/';
return redirect(redirectTo);
diff --git a/quadratic-shared/typesAndSchemas.ts b/quadratic-shared/typesAndSchemas.ts
index 37aa5c1ceb..88b68572ed 100644
--- a/quadratic-shared/typesAndSchemas.ts
+++ b/quadratic-shared/typesAndSchemas.ts
@@ -377,7 +377,7 @@ export const ApiSchemas = {
* Users
* ===========================================================================
*/
- '/v0/users/acknowledge.GET.response': z.object({ message: z.string() }),
+ '/v0/users/acknowledge.GET.response': z.object({ message: z.string(), userCreated: z.boolean() }),
/**
* ===========================================================================
From c2b1bb1f4a3fd5acf0176b28633f442259185c48 Mon Sep 17 00:00:00 2001
From: Jim Nielsen
Date: Thu, 12 Dec 2024 20:09:34 -0700
Subject: [PATCH 008/155] Update login-result.tsx
---
quadratic-client/src/routes/login-result.tsx | 1 -
1 file changed, 1 deletion(-)
diff --git a/quadratic-client/src/routes/login-result.tsx b/quadratic-client/src/routes/login-result.tsx
index 32bdc77e02..570c87743b 100644
--- a/quadratic-client/src/routes/login-result.tsx
+++ b/quadratic-client/src/routes/login-result.tsx
@@ -15,7 +15,6 @@ export const loader = async () => {
// Special case for first-time users
if (userCreated) {
- console.log('First-time user created');
try {
// @ts-expect-error
window.dataLayer.push({
From 68c0e8b61c68b330cd3a1f3da767821542b25e4d Mon Sep 17 00:00:00 2001
From: Jim Nielsen
Date: Fri, 13 Dec 2024 16:07:29 -0700
Subject: [PATCH 009/155] cleanup
---
.../src/dashboard/shared/getActiveTeam.ts | 20 +++++++++----------
quadratic-client/src/router.tsx | 15 ++------------
quadratic-client/src/routes/_dashboard.tsx | 9 ---------
.../src/routes/{new.tsx => files.create.tsx} | 0
.../src/shared/constants/routes.ts | 7 -------
5 files changed, 11 insertions(+), 40 deletions(-)
rename quadratic-client/src/routes/{new.tsx => files.create.tsx} (100%)
diff --git a/quadratic-client/src/dashboard/shared/getActiveTeam.ts b/quadratic-client/src/dashboard/shared/getActiveTeam.ts
index 7ad25e3f52..2b3eb34dd9 100644
--- a/quadratic-client/src/dashboard/shared/getActiveTeam.ts
+++ b/quadratic-client/src/dashboard/shared/getActiveTeam.ts
@@ -2,9 +2,8 @@ import { ACTIVE_TEAM_UUID_KEY } from '@/routes/_dashboard';
import { apiClient } from '@/shared/api/apiClient';
import * as Sentry from '@sentry/react';
-// TODO: explain this
-// It's implicit what team is currently active. This is the function that
-// tells us what is most likely the currently active team.
+// When a user lands on the app, we don't necessarily know what their "active"
+// team is. It’s semi-implicit, but we can make a guess.
// Only once we do a get of the team do we know for sure the user has access to it.
export default async function getActiveTeam(
teams: Awaited>['teams'],
@@ -15,8 +14,7 @@ export default async function getActiveTeam(
/**
* Determine what the active team is
*/
- let initialActiveTeamUuid = undefined;
- // const uuidFromUrl = params.teamUuid;
+ let teamUuid = undefined;
const uuidFromLocalStorage = localStorage.getItem(ACTIVE_TEAM_UUID_KEY);
// FYI: if you have a UUID in the URL or localstorage, it doesn’t mean you
@@ -27,27 +25,27 @@ export default async function getActiveTeam(
// 1) Check the URL for a team UUID. If there's one, use that as that's
// explicitly what the user is trying to look at
if (teamUuidFromUrl) {
- initialActiveTeamUuid = teamUuidFromUrl;
+ teamUuid = teamUuidFromUrl;
// 2) Check localstorage for a team UUID
// If what's in localstorage is not in the list of teams from the server —
// e.g. you lost access to a team — we'll skip this
} else if (uuidFromLocalStorage && teams.find((team) => team.team.uuid === uuidFromLocalStorage)) {
- initialActiveTeamUuid = uuidFromLocalStorage;
+ teamUuid = uuidFromLocalStorage;
// 3) There's no default preference (yet), so pick the 1st one in the API
} else if (teams.length > 0) {
- initialActiveTeamUuid = teams[0].team.uuid;
+ teamUuid = teams[0].team.uuid;
// 4) There are no teams in the API, so we will create one
} else if (teams.length === 0) {
const newTeam = await apiClient.teams.create({ name: 'My Team' });
- initialActiveTeamUuid = newTeam.uuid;
+ teamUuid = newTeam.uuid;
teamCreated = true;
}
// This should never happen, but if it does, we'll log it to sentry
- if (initialActiveTeamUuid === undefined) {
+ if (teamUuid === undefined) {
Sentry.captureEvent({
message: 'No active team was found or could be created.',
level: 'fatal',
@@ -56,7 +54,7 @@ export default async function getActiveTeam(
}
return {
- teamUuid: initialActiveTeamUuid,
+ teamUuid,
teamCreated,
};
}
diff --git a/quadratic-client/src/router.tsx b/quadratic-client/src/router.tsx
index 1dd3de5c62..cce28b273b 100644
--- a/quadratic-client/src/router.tsx
+++ b/quadratic-client/src/router.tsx
@@ -68,9 +68,8 @@ export const router = createBrowserRouter(
/>
- {/* Helper routes, e.g. /connections -> /teams/:uuid/connections */}
-
- import('./routes/new')} />
+ {/* Route to redirect to a new file in the app */}
+ import('./routes/files.create')} />
{/* Dashboard UI routes */}
import('./routes/_dashboard')}>
@@ -91,16 +90,6 @@ export const router = createBrowserRouter(
/>
import('./routes/labs')} />
- {/* TODO: handle these
- import('./routes/_teams-redirect')} />
- import('./routes/_teams-redirect')} />
- import('./routes/_teams-redirect')} />
-
- import('./routes/_teams-redirect')} />
- import('./routes/files.create')} />
- import('./routes/_teams-redirect')} />
- */}
-
import('./routes/teams.create')} />
import('./routes/teams.$teamUuid')}>
diff --git a/quadratic-client/src/routes/_dashboard.tsx b/quadratic-client/src/routes/_dashboard.tsx
index 4960387017..8c811493f6 100644
--- a/quadratic-client/src/routes/_dashboard.tsx
+++ b/quadratic-client/src/routes/_dashboard.tsx
@@ -57,7 +57,6 @@ type LoaderData = {
activeTeam: ApiTypes['/v0/teams/:uuid.GET.response'];
};
-// getActiveTeam()
export const loader = async ({ params, request }: LoaderFunctionArgs): Promise => {
/**
* Get the initial data
@@ -84,14 +83,6 @@ export const loader = async ({ params, request }: LoaderFunctionArgs): Promise /teams/:uuid/connections
- // If it was a shortcut team route, redirect there
- // e.g. /?team-shortcut=connections
- const teamShortcut = url.searchParams.get('team-shortcut');
- if (teamShortcut) {
- url.searchParams.delete('team-shortcut');
- return redirect(ROUTES.TEAM_CONNECTIONS(teamUuid) + url.search);
- }
/**
* Get data for the active team
diff --git a/quadratic-client/src/routes/new.tsx b/quadratic-client/src/routes/files.create.tsx
similarity index 100%
rename from quadratic-client/src/routes/new.tsx
rename to quadratic-client/src/routes/files.create.tsx
diff --git a/quadratic-client/src/shared/constants/routes.ts b/quadratic-client/src/shared/constants/routes.ts
index 454ec221e6..a391bbf4d5 100644
--- a/quadratic-client/src/shared/constants/routes.ts
+++ b/quadratic-client/src/shared/constants/routes.ts
@@ -48,13 +48,6 @@ export const ROUTES = {
TEAM_FILES_PRIVATE: (teamUuid: string) => `/teams/${teamUuid}/files/private`,
TEAM_MEMBERS: (teamUuid: string) => `/teams/${teamUuid}/members`,
TEAM_SETTINGS: (teamUuid: string) => `/teams/${teamUuid}/settings`,
- // This is a way to navigate to a team route without necessariliy knowing
- // the teamUuid upfront. It’s useful from the app-side when you want to navigate
- // back to the dashboard.
- TEAM_SHORTCUT: {
- CONNECTIONS: `/?team-shortcut=connections`,
- },
- CONNECTIONS: '/connections',
EDIT_TEAM: (teamUuid: string) => `/teams/${teamUuid}/edit`,
EXAMPLES: '/examples',
ACCOUNT: '/account',
From b17ec37ffcf4f55e99fcddc0cb4a0a2d512e57eb Mon Sep 17 00:00:00 2001
From: Jim Nielsen
Date: Fri, 13 Dec 2024 16:11:41 -0700
Subject: [PATCH 010/155] Delete _teams-redirect.tsx
---
quadratic-client/src/routes/_teams-redirect.tsx | 14 --------------
1 file changed, 14 deletions(-)
delete mode 100644 quadratic-client/src/routes/_teams-redirect.tsx
diff --git a/quadratic-client/src/routes/_teams-redirect.tsx b/quadratic-client/src/routes/_teams-redirect.tsx
deleted file mode 100644
index d7413a9f73..0000000000
--- a/quadratic-client/src/routes/_teams-redirect.tsx
+++ /dev/null
@@ -1,14 +0,0 @@
-import { useDashboardRouteLoaderData } from '@/routes/_dashboard';
-import { Navigate, useLocation } from 'react-router-dom';
-
-export const Component = () => {
- const {
- activeTeam: {
- team: { uuid },
- },
- } = useDashboardRouteLoaderData();
- const { pathname } = useLocation();
- console.log(pathname);
-
- return ;
-};
From bb48954b52ef19ea8573a7f1372ffe645ce4d941 Mon Sep 17 00:00:00 2001
From: Jim Nielsen
Date: Fri, 13 Dec 2024 16:21:53 -0700
Subject: [PATCH 011/155] Update router.tsx
---
quadratic-client/src/router.tsx | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/quadratic-client/src/router.tsx b/quadratic-client/src/router.tsx
index cce28b273b..2d2112563b 100644
--- a/quadratic-client/src/router.tsx
+++ b/quadratic-client/src/router.tsx
@@ -101,9 +101,8 @@ export const router = createBrowserRouter(
+
-
-
From b0e890f0d5fb402ab80b0c29c447f71ad49e3fdc Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 14 Dec 2024 06:28:35 +0530
Subject: [PATCH 012/155] fix debug flag and improve initialisation trigger
---
quadratic-client/src/app/debugFlags.ts | 2 +-
quadratic-client/src/app/events/events.ts | 2 ++
.../src/app/gridGL/PixiAppEffects.tsx | 5 +++++
.../gridGL/pixiApp/urlParams/UrlParamsUser.ts | 20 ++++++++++++++++++-
.../app/gridGL/pixiApp/urlParams/urlParams.ts | 2 +-
5 files changed, 28 insertions(+), 3 deletions(-)
diff --git a/quadratic-client/src/app/debugFlags.ts b/quadratic-client/src/app/debugFlags.ts
index 2399a29d32..971f42c5d1 100644
--- a/quadratic-client/src/app/debugFlags.ts
+++ b/quadratic-client/src/app/debugFlags.ts
@@ -65,7 +65,7 @@ export const debugGridSettings = debug && false;
export const debugShowMultiplayer = debug && false;
-export const debugSaveURLState = (debug && false) || url.has('state');
+export const debugSaveURLState = debug && false;
// --------
// UI
diff --git a/quadratic-client/src/app/events/events.ts b/quadratic-client/src/app/events/events.ts
index 743a20e04b..e298033b45 100644
--- a/quadratic-client/src/app/events/events.ts
+++ b/quadratic-client/src/app/events/events.ts
@@ -135,6 +135,8 @@ interface EventTypes {
codeEditorCodeCell: (codeCell?: CodeCell) => void;
aiAnalystInitialized: () => void;
+
+ pixiAppSettingsInitialized: () => void;
}
export const events = new EventEmitter();
diff --git a/quadratic-client/src/app/gridGL/PixiAppEffects.tsx b/quadratic-client/src/app/gridGL/PixiAppEffects.tsx
index 863db83493..a881bf6e00 100644
--- a/quadratic-client/src/app/gridGL/PixiAppEffects.tsx
+++ b/quadratic-client/src/app/gridGL/PixiAppEffects.tsx
@@ -4,6 +4,7 @@ import { editorInteractionStateAtom } from '@/app/atoms/editorInteractionStateAt
import { gridPanModeAtom } from '@/app/atoms/gridPanModeAtom';
import { gridSettingsAtom, presentationModeAtom, showHeadingsAtom } from '@/app/atoms/gridSettingsAtom';
import { inlineEditorAtom } from '@/app/atoms/inlineEditorAtom';
+import { events } from '@/app/events/events';
import { pixiApp } from '@/app/gridGL/pixiApp/PixiApp';
import { pixiAppSettings } from '@/app/gridGL/pixiApp/PixiAppSettings';
import { useSubmitAIAnalystPrompt } from '@/app/ui/menus/AIAnalyst/hooks/useSubmitAIAnalystPrompt';
@@ -73,6 +74,10 @@ export const PixiAppEffects = () => {
pixiAppSettings.updateAIAnalystState(aiAnalystState, setAIAnalystState, submitPrompt);
}, [aiAnalystState, setAIAnalystState, submitPrompt]);
+ useEffect(() => {
+ events.emit('pixiAppSettingsInitialized');
+ }, []);
+
useEffect(() => {
const handleMouseUp = () => {
setGridPanMode((prev) => ({ ...prev, mouseIsDown: false }));
diff --git a/quadratic-client/src/app/gridGL/pixiApp/urlParams/UrlParamsUser.ts b/quadratic-client/src/app/gridGL/pixiApp/urlParams/UrlParamsUser.ts
index e362913493..5e436b274c 100644
--- a/quadratic-client/src/app/gridGL/pixiApp/urlParams/UrlParamsUser.ts
+++ b/quadratic-client/src/app/gridGL/pixiApp/urlParams/UrlParamsUser.ts
@@ -7,12 +7,17 @@ import { getLanguage } from '@/app/helpers/codeCellLanguage';
import { CodeCellLanguage } from '@/app/quadratic-core-types';
export class UrlParamsUser {
+ private pixiAppSettingsInitialized = false;
+ private aiAnalystInitialized = false;
+ private aiAnalystPromptLoaded = false;
+
dirty = false;
constructor(params: URLSearchParams) {
this.loadSheet(params);
this.loadCursor(params);
this.loadCode(params);
+ this.loadAIAnalystPrompt(params);
this.setupListeners(params);
}
@@ -66,6 +71,9 @@ export class UrlParamsUser {
}
private loadAIAnalystPrompt = (params: URLSearchParams) => {
+ if (!this.pixiAppSettingsInitialized || !this.aiAnalystInitialized) return;
+ if (this.aiAnalystPromptLoaded) return;
+
const prompt = params.get('prompt');
if (!prompt) return;
@@ -89,13 +97,23 @@ export class UrlParamsUser {
params.delete('prompt');
url.search = params.toString();
window.history.replaceState(null, '', url.toString());
+
+ this.aiAnalystPromptLoaded = true;
};
private setupListeners(params: URLSearchParams) {
events.on('cursorPosition', this.setDirty);
events.on('changeSheet', this.setDirty);
events.on('codeEditor', this.setDirty);
- events.on('aiAnalystInitialized', () => this.loadAIAnalystPrompt(params));
+
+ events.on('pixiAppSettingsInitialized', () => {
+ this.pixiAppSettingsInitialized = true;
+ this.loadAIAnalystPrompt(params);
+ });
+ events.on('aiAnalystInitialized', () => {
+ this.aiAnalystInitialized = true;
+ this.loadAIAnalystPrompt(params);
+ });
}
private setDirty = () => {
diff --git a/quadratic-client/src/app/gridGL/pixiApp/urlParams/urlParams.ts b/quadratic-client/src/app/gridGL/pixiApp/urlParams/urlParams.ts
index 19372007e8..5838ed15c3 100644
--- a/quadratic-client/src/app/gridGL/pixiApp/urlParams/urlParams.ts
+++ b/quadratic-client/src/app/gridGL/pixiApp/urlParams/urlParams.ts
@@ -19,7 +19,7 @@ class UrlParams {
show() {
const params = new URLSearchParams(window.location.search);
- if (debugSaveURLState) {
+ if (debugSaveURLState || params.has('state')) {
this.urlParamsDev = new UrlParamsDev(params);
if (this.urlParamsDev.noUpdates) return;
} else {
From 7ed75bc43150b5db1391db3291ceaff49d5b9d05 Mon Sep 17 00:00:00 2001
From: Jim Nielsen
Date: Thu, 19 Dec 2024 11:19:50 -0700
Subject: [PATCH 013/155] updates
---
.../app/gridGL/HTMLGrid/EmptyGridMessage.tsx | 150 ++++++++++++++++++
.../app/gridGL/HTMLGrid/HTMLGridContainer.tsx | 3 +
2 files changed, 153 insertions(+)
create mode 100644 quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
diff --git a/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx b/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
new file mode 100644
index 0000000000..f55b00b8f3
--- /dev/null
+++ b/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
@@ -0,0 +1,150 @@
+import {
+ editorInteractionStateShowCellTypeMenuAtom,
+ editorInteractionStateShowConnectionsMenuAtom,
+} from '@/app/atoms/editorInteractionStateAtom';
+import { events } from '@/app/events/events';
+import { sheets } from '@/app/grid/controller/Sheets';
+import { supportedFileTypes } from '@/app/helpers/files';
+import { useConnectionsFetcher } from '@/app/ui/hooks/useConnectionsFetcher';
+import { useFileImport } from '@/app/ui/hooks/useFileImport';
+import { CloseIcon } from '@/shared/components/Icons';
+import { useFileRouteLoaderData } from '@/shared/hooks/useFileRouteLoaderData';
+import { Button } from '@/shared/shadcn/ui/button';
+import { useEffect, useRef, useState } from 'react';
+import { useSetRecoilState } from 'recoil';
+
+const fileHasData = () => sheets.sheets.filter((sheet) => sheet.bounds.type === 'nonEmpty').length > 0;
+
+// When a file loads, if it's totally empty, show this message. Then once the
+// user has edited the file, we'll hide it permanently.
+export function EmptyGridMessage() {
+ const {
+ userMakingRequest: { filePermissions },
+ team: { uuid: teamUuid },
+ } = useFileRouteLoaderData();
+ const canEdit = filePermissions.includes('FILE_EDIT');
+ const [open, setOpen] = useState(fileHasData() ? false : true);
+ const showConnectionsMenu = useSetRecoilState(editorInteractionStateShowConnectionsMenuAtom);
+ const showCellTypeMenu = useSetRecoilState(editorInteractionStateShowCellTypeMenuAtom);
+ const { data } = useConnectionsFetcher();
+ const connections = data?.connections ?? [];
+
+ useEffect(() => {
+ const checkBounds = () => {
+ if (open && fileHasData()) {
+ setOpen(false);
+ }
+ };
+
+ events.on('hashContentChanged', checkBounds);
+ return () => {
+ events.off('hashContentChanged', checkBounds);
+ };
+ }, [open]);
+
+ if (!canEdit) {
+ return null;
+ }
+
+ if (!open) {
+ return null;
+ }
+
+ return (
+
+
+
Import data
+
+ Bring in your own data via a file (CSV, Excel, Parquet) or a connection (Postgres, MySQL, and more).
+
+
+
+
{
+ setOpen(false);
+ showConnectionsMenu(true);
+ }}
+ >
+ Create connection
+
+
{
+ setOpen(false);
+ showCellTypeMenu(true);
+ }}
+ >
+ Use connection
+
+
setOpen(false)}
+ className="absolute right-1 top-1 !bg-background text-muted-foreground"
+ >
+
+
+ {connections.length && false && (
+
+ Press /
to use an exisiting connection
+
+ )}
+
+
+ );
+}
+
+function UploadFileButton({ teamUuid }: { teamUuid: string }) {
+ const handleFileImport = useFileImport();
+ const fileInputRef = useRef(null);
+ return (
+ <>
+ fileInputRef.current?.click()}>
+ Upload file
+
+ {
+ const files = e.target.files;
+ if (files) {
+ handleFileImport({
+ files,
+ sheetId: sheets.sheet.id,
+ insertAt: { x: 1, y: 1 },
+ cursor: sheets.getCursorPosition(),
+ teamUuid,
+ });
+ }
+ }}
+ />
+ >
+ );
+}
diff --git a/quadratic-client/src/app/gridGL/HTMLGrid/HTMLGridContainer.tsx b/quadratic-client/src/app/gridGL/HTMLGrid/HTMLGridContainer.tsx
index 7a448d2e2a..cefdfa8be5 100644
--- a/quadratic-client/src/app/gridGL/HTMLGrid/HTMLGridContainer.tsx
+++ b/quadratic-client/src/app/gridGL/HTMLGrid/HTMLGridContainer.tsx
@@ -3,6 +3,7 @@ import { Annotations } from '@/app/gridGL/HTMLGrid/annotations/Annotations';
import { AskAISelection } from '@/app/gridGL/HTMLGrid/askAISelection/AskAISelection';
import { CodeHint } from '@/app/gridGL/HTMLGrid/CodeHint';
import { CodeRunning } from '@/app/gridGL/HTMLGrid/codeRunning/CodeRunning';
+import { EmptyGridMessage } from '@/app/gridGL/HTMLGrid/EmptyGridMessage';
import { GridContextMenu } from '@/app/gridGL/HTMLGrid/GridContextMenu';
import { HoverCell } from '@/app/gridGL/HTMLGrid/hoverCell/HoverCell';
import { HoverTooltip } from '@/app/gridGL/HTMLGrid/hoverTooltip/HoverTooltip';
@@ -123,6 +124,8 @@ export const HTMLGridContainer = (props: Props): ReactNode | null => {
+
+
{/* This is positioned on the grid over the headings and not zoomed. It comes
after the above, so it's above it on the grid. */}
Date: Thu, 19 Dec 2024 11:42:47 -0700
Subject: [PATCH 014/155] Update EmptyGridMessage.tsx
---
quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx b/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
index f55b00b8f3..399d94f9d2 100644
--- a/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
+++ b/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
@@ -109,7 +109,7 @@ export function EmptyGridMessage() {
>
- {connections.length && false && (
+ {connections.length && (
Press /
to use an exisiting connection
From 9c653a4562a1403bde40a76e25b082488f49a46b Mon Sep 17 00:00:00 2001
From: Jim Nielsen
Date: Thu, 19 Dec 2024 12:20:02 -0700
Subject: [PATCH 015/155] fix
---
.../app/gridGL/HTMLGrid/EmptyGridMessage.tsx | 27 +++++++++----------
1 file changed, 12 insertions(+), 15 deletions(-)
diff --git a/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx b/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
index 399d94f9d2..9904441c4e 100644
--- a/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
+++ b/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
@@ -91,16 +91,18 @@ export function EmptyGridMessage() {
>
Create connection
- {
- setOpen(false);
- showCellTypeMenu(true);
- }}
- >
- Use connection
-
+ {connections.length > 0 && (
+ {
+ setOpen(false);
+ showCellTypeMenu(true);
+ }}
+ >
+ Use connection
+
+ )}
- {connections.length && (
-
- Press /
to use an exisiting connection
-
- )}
);
From 14ea521ec1d193c86408bb0e762c766f5e712898 Mon Sep 17 00:00:00 2001
From: Jim Nielsen
Date: Fri, 20 Dec 2024 09:21:22 -0700
Subject: [PATCH 016/155] updates
---
.../app/gridGL/HTMLGrid/EmptyGridMessage.tsx | 42 +++++++------------
.../src/app/ui/hooks/useFileImport.tsx | 2 +
2 files changed, 18 insertions(+), 26 deletions(-)
diff --git a/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx b/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
index 9904441c4e..565fd78fda 100644
--- a/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
+++ b/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
@@ -7,7 +7,6 @@ import { sheets } from '@/app/grid/controller/Sheets';
import { supportedFileTypes } from '@/app/helpers/files';
import { useConnectionsFetcher } from '@/app/ui/hooks/useConnectionsFetcher';
import { useFileImport } from '@/app/ui/hooks/useFileImport';
-import { CloseIcon } from '@/shared/components/Icons';
import { useFileRouteLoaderData } from '@/shared/hooks/useFileRouteLoaderData';
import { Button } from '@/shared/shadcn/ui/button';
import { useEffect, useRef, useState } from 'react';
@@ -23,17 +22,16 @@ export function EmptyGridMessage() {
team: { uuid: teamUuid },
} = useFileRouteLoaderData();
const canEdit = filePermissions.includes('FILE_EDIT');
- const [open, setOpen] = useState(fileHasData() ? false : true);
+ const [open, setOpen] = useState(false);
const showConnectionsMenu = useSetRecoilState(editorInteractionStateShowConnectionsMenuAtom);
const showCellTypeMenu = useSetRecoilState(editorInteractionStateShowCellTypeMenuAtom);
const { data } = useConnectionsFetcher();
const connections = data?.connections ?? [];
+ // Show/hide depending on whether the file has any data in it
useEffect(() => {
const checkBounds = () => {
- if (open && fileHasData()) {
- setOpen(false);
- }
+ setOpen(fileHasData() ? false : true);
};
events.on('hashContentChanged', checkBounds);
@@ -81,36 +79,28 @@ export function EmptyGridMessage() {
- {
- setOpen(false);
- showConnectionsMenu(true);
- }}
- >
- Create connection
-
- {connections.length > 0 && (
+
+ {connections.length === 0 ? (
+ {
+ showConnectionsMenu(true);
+ }}
+ >
+ Create connection
+
+ ) : (
{
- setOpen(false);
showCellTypeMenu(true);
}}
>
Use connection
)}
- setOpen(false)}
- className="absolute right-1 top-1 !bg-background text-muted-foreground"
- >
-
-
);
diff --git a/quadratic-client/src/app/ui/hooks/useFileImport.tsx b/quadratic-client/src/app/ui/hooks/useFileImport.tsx
index b78535312d..e820337407 100644
--- a/quadratic-client/src/app/ui/hooks/useFileImport.tsx
+++ b/quadratic-client/src/app/ui/hooks/useFileImport.tsx
@@ -8,6 +8,7 @@ import { ApiError } from '@/shared/api/fetchFromApi';
import { useGlobalSnackbar } from '@/shared/components/GlobalSnackbarProvider';
import { ROUTES } from '@/shared/constants/routes';
import { Buffer } from 'buffer';
+import mixpanel from 'mixpanel-browser';
import { useLocation, useNavigate } from 'react-router-dom';
import { useSetRecoilState } from 'recoil';
@@ -35,6 +36,7 @@ export function useFileImport() {
isPrivate?: boolean;
teamUuid?: string;
}) => {
+ mixpanel.track('[ImportData].useFileImport');
quadraticCore.initWorker();
if (!files) files = await uploadFile(supportedFileTypes);
From 6325639663f8cc37f1b525d506298ed31b6091f4 Mon Sep 17 00:00:00 2001
From: Jim Nielsen
Date: Fri, 20 Dec 2024 10:00:13 -0700
Subject: [PATCH 017/155] Update EmptyGridMessage.tsx
---
quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx | 1 +
1 file changed, 1 insertion(+)
diff --git a/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx b/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
index 565fd78fda..b779dc9ae3 100644
--- a/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
+++ b/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
@@ -109,6 +109,7 @@ export function EmptyGridMessage() {
function UploadFileButton({ teamUuid }: { teamUuid: string }) {
const handleFileImport = useFileImport();
const fileInputRef = useRef(null);
+
return (
<>
fileInputRef.current?.click()}>
From cdbb72f5f54177cf5b1e26239bd1ca47fb2a36ea Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 01:56:17 +0530
Subject: [PATCH 018/155] build images on pr open/push
---
.../preview-branches-publish-images.yml | 93 +++++++++++++++++++
.../workflows/production-publish-images.yml | 85 +++++++++--------
2 files changed, 135 insertions(+), 43 deletions(-)
create mode 100644 .github/workflows/preview-branches-publish-images.yml
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
new file mode 100644
index 0000000000..6024f6af15
--- /dev/null
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -0,0 +1,93 @@
+name: Build Docker Images for Preview Branches
+
+on:
+ pull_request:
+ types: [opened, synchronize, reopened]
+
+concurrency:
+ group: ${{ github.head_ref }}-build-images
+ cancel-in-progress: true
+
+jobs:
+ build_images:
+ runs-on: ubuntu-latest-8-cores
+ permissions:
+ contents: read
+ timeout-minutes: 10
+
+ strategy:
+ matrix:
+ service: [multiplayer, files, connection, client, api]
+ fail-fast: true
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Generate Build Metadata
+ id: build-metadata
+ run: |
+ echo "BUILD_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
+ echo "GIT_SHA_SHORT=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+ - name: Configure AWS Credentials
+ uses: aws-actions/configure-aws-credentials@v4
+ with:
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEVELOPMENT }}
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }}
+ aws-region: ${{ secrets.AWS_REGION }}
+
+ - name: Login to Amazon ECR Public
+ id: login-ecr
+ uses: aws-actions/amazon-ecr-login@v2
+ with:
+ registry-type: private
+
+ - name: Define repository name
+ id: repo-name
+ run: |
+ echo "REPO_NAME=quadratic-${{ matrix.service }}" >> $GITHUB_OUTPUT
+
+ - name: Create Private ECR Repository if not exists
+ id: create-ecr
+ env:
+ REPO_NAME: ${{ steps.repo-name.outputs.REPO_NAME }}
+ run: |
+ aws ecr create-repository --repository-name $REPO_NAME || true
+ REPO_INFO=$(aws ecr describe-repositories --repository-names $REPO_NAME)
+ ECR_URL=$(echo $REPO_INFO | jq -r '.repositories[0].repositoryUri')
+ echo "ECR_URL=$ECR_URL" >> $GITHUB_OUTPUT
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Cache Docker layers
+ uses: actions/cache@v3
+ with:
+ path: /tmp/.buildx-cache
+ key: ${{ runner.os }}-buildx-${{ matrix.service }}-${{ github.sha }}
+ restore-keys: |
+ ${{ runner.os }}-buildx-${{ matrix.service }}-
+ ${{ runner.os }}-buildx-
+
+ - name: Build, Tag, and Push Image to Amazon ECR Private
+ env:
+ ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }}
+ run: |
+ docker buildx build \
+ --cache-from=type=local,src=/tmp/.buildx-cache \
+ --cache-to=type=local,dest=/tmp/.buildx-cache-new \
+ --build-arg BUILD_TIME="${{ steps.build-metadata.outputs.BUILD_TIME }}" \
+ --build-arg GIT_SHA="${{ github.sha }}" \
+ --build-arg PR_NUMBER="${{ github.event.pull_request.number }}" \
+ --label "org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}" \
+ --label "org.opencontainers.image.revision=${{ github.sha }}" \
+ --push \
+ -t $ECR_URL:pr-${{ github.event.pull_request.number }} \
+ -t $ECR_URL:${{ github.head_ref }} \
+ -t $ECR_URL:${{ steps.build-metadata.outputs.GIT_SHA_SHORT }} \
+ -f quadratic-${{ matrix.service }}/Dockerfile .
+
+ - name: Move cache
+ run: |
+ rm -rf /tmp/.buildx-cache
+ mv /tmp/.buildx-cache-new /tmp/.buildx-cache
diff --git a/.github/workflows/production-publish-images.yml b/.github/workflows/production-publish-images.yml
index 5e4315b3e9..d7e79df8aa 100644
--- a/.github/workflows/production-publish-images.yml
+++ b/.github/workflows/production-publish-images.yml
@@ -3,7 +3,6 @@ name: Build and Publish Images to ECR
on:
push:
branches:
- - self-hosting-setup #remove
- main
concurrency:
@@ -16,45 +15,45 @@ jobs:
matrix:
service: [multiplayer, files, connection, client, api]
steps:
- - uses: actions/checkout@v4
-
- - name: Configure AWS Credentials
- uses: aws-actions/configure-aws-credentials@v4
- with:
- aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
- aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- aws-region: us-east-1
-
- - name: Login to Amazon ECR Public
- id: login-ecr
- uses: aws-actions/amazon-ecr-login@v2
- with:
- registry-type: public
-
- - name: Define repository name
- id: repo-name
- run: |
- echo "REPO_NAME=quadratic-${{ matrix.service }}" >> $GITHUB_OUTPUT
-
- - name: Create Public ECR Repository if not exists
- id: create-ecr
- env:
- REPO_NAME: ${{ steps.repo-name.outputs.REPO_NAME }}
- run: |
- aws ecr-public create-repository --repository-name $REPO_NAME || true
- REPO_INFO=$(aws ecr-public describe-repositories --repository-names $REPO_NAME)
- ECR_URL=$(echo $REPO_INFO | jq -r '.repositories[0].repositoryUri')
- echo "ECR_URL=$ECR_URL" >> $GITHUB_OUTPUT
-
- - name: Read VERSION file
- id: version
- run: echo "VERSION=$(cat VERSION)" >> $GITHUB_OUTPUT
-
- - name: Build, Tag, and Push Image to Amazon ECR Public
- env:
- ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }}
- IMAGE_TAG: ${{ steps.version.outputs.VERSION }}
- run: |
- docker build -t $ECR_URL:$IMAGE_TAG -t $ECR_URL:latest -f quadratic-${{ matrix.service }}/Dockerfile .
- docker push $ECR_URL:$IMAGE_TAG
- docker push $ECR_URL:latest
\ No newline at end of file
+ - uses: actions/checkout@v4
+
+ - name: Configure AWS Credentials
+ uses: aws-actions/configure-aws-credentials@v4
+ with:
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ aws-region: ${{ secrets.AWS_REGION }}
+
+ - name: Login to Amazon ECR Public
+ id: login-ecr
+ uses: aws-actions/amazon-ecr-login@v2
+ with:
+ registry-type: public
+
+ - name: Define repository name
+ id: repo-name
+ run: |
+ echo "REPO_NAME=quadratic-${{ matrix.service }}" >> $GITHUB_OUTPUT
+
+ - name: Create Public ECR Repository if not exists
+ id: create-ecr
+ env:
+ REPO_NAME: ${{ steps.repo-name.outputs.REPO_NAME }}
+ run: |
+ aws ecr-public create-repository --repository-name $REPO_NAME || true
+ REPO_INFO=$(aws ecr-public describe-repositories --repository-names $REPO_NAME)
+ ECR_URL=$(echo $REPO_INFO | jq -r '.repositories[0].repositoryUri')
+ echo "ECR_URL=$ECR_URL" >> $GITHUB_OUTPUT
+
+ - name: Read VERSION file
+ id: version
+ run: echo "VERSION=$(cat VERSION)" >> $GITHUB_OUTPUT
+
+ - name: Build, Tag, and Push Image to Amazon ECR Public
+ env:
+ ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }}
+ IMAGE_TAG: ${{ steps.version.outputs.VERSION }}
+ run: |
+ docker build -t $ECR_URL:$IMAGE_TAG -t $ECR_URL:latest -f quadratic-${{ matrix.service }}/Dockerfile .
+ docker push $ECR_URL:$IMAGE_TAG
+ docker push $ECR_URL:latest
From f7d69104ac96bfffbe6b72316752c88f1c0e8aa1 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 02:22:00 +0530
Subject: [PATCH 019/155] fix branch name
---
.github/workflows/preview-branches-publish-images.yml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 6024f6af15..397561019d 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -28,6 +28,7 @@ jobs:
run: |
echo "BUILD_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
echo "GIT_SHA_SHORT=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+ echo "BRANCH_NAME=$(echo "${{ github.head_ref }}" | tr '/' '-')" >> $GITHUB_OUTPUT
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
@@ -83,7 +84,7 @@ jobs:
--label "org.opencontainers.image.revision=${{ github.sha }}" \
--push \
-t $ECR_URL:pr-${{ github.event.pull_request.number }} \
- -t $ECR_URL:${{ github.head_ref }} \
+ -t $ECR_URL:${{ steps.build-metadata.outputs.BRANCH_NAME }} \
-t $ECR_URL:${{ steps.build-metadata.outputs.GIT_SHA_SHORT }} \
-f quadratic-${{ matrix.service }}/Dockerfile .
From e80308ab6b686bc856bff55016ce36a1cd674375 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 02:30:17 +0530
Subject: [PATCH 020/155] create cache directory
---
.github/workflows/preview-branches-publish-images.yml | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 397561019d..b3495919b2 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -60,6 +60,8 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
+ with:
+ buildkitd-flags: --debug
- name: Cache Docker layers
uses: actions/cache@v3
@@ -70,6 +72,9 @@ jobs:
${{ runner.os }}-buildx-${{ matrix.service }}-
${{ runner.os }}-buildx-
+ - name: Create cache directory if it doesn't exist
+ run: mkdir -p /tmp/.buildx-cache
+
- name: Build, Tag, and Push Image to Amazon ECR Private
env:
ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }}
From 997c4f6d40ccb3bbe281983faf34504ff339ccde Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 02:35:41 +0530
Subject: [PATCH 021/155] init cache dir
---
.../workflows/preview-branches-publish-images.yml | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index b3495919b2..4b7ac4dc86 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -53,7 +53,13 @@ jobs:
env:
REPO_NAME: ${{ steps.repo-name.outputs.REPO_NAME }}
run: |
- aws ecr create-repository --repository-name $REPO_NAME || true
+ # Try to describe the repository first
+ if ! aws ecr describe-repositories --repository-names $REPO_NAME 2>/dev/null; then
+ # Repository doesn't exist, create it
+ aws ecr create-repository --repository-name $REPO_NAME
+ fi
+
+ # Get the repository URI either way
REPO_INFO=$(aws ecr describe-repositories --repository-names $REPO_NAME)
ECR_URL=$(echo $REPO_INFO | jq -r '.repositories[0].repositoryUri')
echo "ECR_URL=$ECR_URL" >> $GITHUB_OUTPUT
@@ -72,8 +78,10 @@ jobs:
${{ runner.os }}-buildx-${{ matrix.service }}-
${{ runner.os }}-buildx-
- - name: Create cache directory if it doesn't exist
- run: mkdir -p /tmp/.buildx-cache
+ - name: Create and initialize buildx cache
+ run: |
+ mkdir -p /tmp/.buildx-cache
+ echo '{"layers":{}}' > /tmp/.buildx-cache/index.json
- name: Build, Tag, and Push Image to Amazon ECR Private
env:
From 4cd48592a421d1f8061922457cb557bb9b57e20e Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 02:40:50 +0530
Subject: [PATCH 022/155] cache init bug
---
.github/workflows/preview-branches-publish-images.yml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 4b7ac4dc86..4441108241 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -81,7 +81,8 @@ jobs:
- name: Create and initialize buildx cache
run: |
mkdir -p /tmp/.buildx-cache
- echo '{"layers":{}}' > /tmp/.buildx-cache/index.json
+ CACHE_TAG="${{ runner.os }}-buildx-${{ matrix.service }}"
+ echo "{\"layers\":{},\"manifests\":{\"$CACHE_TAG\":{\"layers\":[]}}}" > /tmp/.buildx-cache/index.json
- name: Build, Tag, and Push Image to Amazon ECR Private
env:
From 51f633424c3f7725e83b561587d040f058009629 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 02:49:51 +0530
Subject: [PATCH 023/155] fix private ecr login
---
.github/workflows/preview-branches-publish-images.yml | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 4441108241..1aeca25800 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -37,11 +37,9 @@ jobs:
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }}
aws-region: ${{ secrets.AWS_REGION }}
- - name: Login to Amazon ECR Public
+ - name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v2
- with:
- registry-type: private
- name: Define repository name
id: repo-name
From 037d36f35180b28b962478213eeb0eb736dc290b Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 03:10:47 +0530
Subject: [PATCH 024/155] use existing ecr repo
---
.github/workflows/preview-branches-publish-images.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 1aeca25800..d1a4b9a0c7 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -44,7 +44,7 @@ jobs:
- name: Define repository name
id: repo-name
run: |
- echo "REPO_NAME=quadratic-${{ matrix.service }}" >> $GITHUB_OUTPUT
+ echo "REPO_NAME=quadratic-${{ matrix.service }}-development" >> $GITHUB_OUTPUT
- name: Create Private ECR Repository if not exists
id: create-ecr
From e297eb42a91f57d5879f6dea871f5cc9192d3c65 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 03:21:38 +0530
Subject: [PATCH 025/155] try ecr public
---
.../preview-branches-publish-images.yml | 97 ++++++++++---------
1 file changed, 53 insertions(+), 44 deletions(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index d1a4b9a0c7..6ac2415227 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -37,70 +37,79 @@ jobs:
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }}
aws-region: ${{ secrets.AWS_REGION }}
- - name: Login to Amazon ECR
+ - name: Login to Amazon ECR Public
id: login-ecr
uses: aws-actions/amazon-ecr-login@v2
+ with:
+ registry-type: public
- name: Define repository name
id: repo-name
run: |
echo "REPO_NAME=quadratic-${{ matrix.service }}-development" >> $GITHUB_OUTPUT
- - name: Create Private ECR Repository if not exists
+ - name: Create Public ECR Repository if not exists
id: create-ecr
env:
REPO_NAME: ${{ steps.repo-name.outputs.REPO_NAME }}
run: |
# Try to describe the repository first
- if ! aws ecr describe-repositories --repository-names $REPO_NAME 2>/dev/null; then
+ if ! aws ecr-public describe-repositories --repository-names $REPO_NAME 2>/dev/null; then
# Repository doesn't exist, create it
- aws ecr create-repository --repository-name $REPO_NAME
+ aws ecr-public create-repository --repository-name $REPO_NAME
fi
# Get the repository URI either way
- REPO_INFO=$(aws ecr describe-repositories --repository-names $REPO_NAME)
+ REPO_INFO=$(aws ecr-public describe-repositories --repository-names $REPO_NAME)
ECR_URL=$(echo $REPO_INFO | jq -r '.repositories[0].repositoryUri')
echo "ECR_URL=$ECR_URL" >> $GITHUB_OUTPUT
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v3
- with:
- buildkitd-flags: --debug
-
- - name: Cache Docker layers
- uses: actions/cache@v3
- with:
- path: /tmp/.buildx-cache
- key: ${{ runner.os }}-buildx-${{ matrix.service }}-${{ github.sha }}
- restore-keys: |
- ${{ runner.os }}-buildx-${{ matrix.service }}-
- ${{ runner.os }}-buildx-
-
- - name: Create and initialize buildx cache
- run: |
- mkdir -p /tmp/.buildx-cache
- CACHE_TAG="${{ runner.os }}-buildx-${{ matrix.service }}"
- echo "{\"layers\":{},\"manifests\":{\"$CACHE_TAG\":{\"layers\":[]}}}" > /tmp/.buildx-cache/index.json
-
- - name: Build, Tag, and Push Image to Amazon ECR Private
+ # - name: Set up Docker Buildx
+ # uses: docker/setup-buildx-action@v3
+ # with:
+ # buildkitd-flags: --debug
+
+ # - name: Cache Docker layers
+ # uses: actions/cache@v3
+ # with:
+ # path: /tmp/.buildx-cache
+ # key: ${{ runner.os }}-buildx-${{ matrix.service }}-${{ github.sha }}
+ # restore-keys: |
+ # ${{ runner.os }}-buildx-${{ matrix.service }}-
+ # ${{ runner.os }}-buildx-
+
+ # - name: Create and initialize buildx cache
+ # run: |
+ # mkdir -p /tmp/.buildx-cache
+ # CACHE_TAG="${{ runner.os }}-buildx-${{ matrix.service }}"
+ # echo "{\"layers\":{},\"manifests\":{\"$CACHE_TAG\":{\"layers\":[]}}}" > /tmp/.buildx-cache/index.json
+
+ # - name: Build, Tag, and Push Image to Amazon ECR Private
+ # env:
+ # ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }}
+ # run: |
+ # docker buildx build \
+ # --cache-from=type=local,src=/tmp/.buildx-cache \
+ # --cache-to=type=local,dest=/tmp/.buildx-cache-new \
+ # --build-arg BUILD_TIME="${{ steps.build-metadata.outputs.BUILD_TIME }}" \
+ # --build-arg GIT_SHA="${{ github.sha }}" \
+ # --build-arg PR_NUMBER="${{ github.event.pull_request.number }}" \
+ # --label "org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}" \
+ # --label "org.opencontainers.image.revision=${{ github.sha }}" \
+ # --push \
+ # -t $ECR_URL:pr-${{ github.event.pull_request.number }} \
+ # -t $ECR_URL:${{ steps.build-metadata.outputs.BRANCH_NAME }} \
+ # -t $ECR_URL:${{ steps.build-metadata.outputs.GIT_SHA_SHORT }} \
+ # -f quadratic-${{ matrix.service }}/Dockerfile .
+
+ # - name: Move cache
+ # run: |
+ # rm -rf /tmp/.buildx-cache
+ # mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+ - name: Build, Tag, and Push Image to Amazon ECR Public
env:
ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }}
run: |
- docker buildx build \
- --cache-from=type=local,src=/tmp/.buildx-cache \
- --cache-to=type=local,dest=/tmp/.buildx-cache-new \
- --build-arg BUILD_TIME="${{ steps.build-metadata.outputs.BUILD_TIME }}" \
- --build-arg GIT_SHA="${{ github.sha }}" \
- --build-arg PR_NUMBER="${{ github.event.pull_request.number }}" \
- --label "org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}" \
- --label "org.opencontainers.image.revision=${{ github.sha }}" \
- --push \
- -t $ECR_URL:pr-${{ github.event.pull_request.number }} \
- -t $ECR_URL:${{ steps.build-metadata.outputs.BRANCH_NAME }} \
- -t $ECR_URL:${{ steps.build-metadata.outputs.GIT_SHA_SHORT }} \
- -f quadratic-${{ matrix.service }}/Dockerfile .
-
- - name: Move cache
- run: |
- rm -rf /tmp/.buildx-cache
- mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+ docker build -t $ECR_URL:pr-${{ github.event.pull_request.number }} -f quadratic-${{ matrix.service }}/Dockerfile .
+ docker push $ECR_URL:pr-${{ github.event.pull_request.number }}
From c880b630e847a0d30e7952a0e21f156b40183f82 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 03:27:17 +0530
Subject: [PATCH 026/155] use us-east-1 for public gallery
---
.github/workflows/preview-branches-publish-images.yml | 2 +-
.github/workflows/production-publish-images.yml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 6ac2415227..daed3bb44d 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -35,7 +35,7 @@ jobs:
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEVELOPMENT }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }}
- aws-region: ${{ secrets.AWS_REGION }}
+ aws-region: us-east-1
- name: Login to Amazon ECR Public
id: login-ecr
diff --git a/.github/workflows/production-publish-images.yml b/.github/workflows/production-publish-images.yml
index d7e79df8aa..79c20b0abd 100644
--- a/.github/workflows/production-publish-images.yml
+++ b/.github/workflows/production-publish-images.yml
@@ -22,7 +22,7 @@ jobs:
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- aws-region: ${{ secrets.AWS_REGION }}
+ aws-region: us-east-1
- name: Login to Amazon ECR Public
id: login-ecr
From 8fc918b227e12ca202f47a355678d2f42bfa310a Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 03:40:21 +0530
Subject: [PATCH 027/155] try 64 cores
---
.github/workflows/preview-branches-publish-images.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index daed3bb44d..c57fdf299a 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -10,7 +10,7 @@ concurrency:
jobs:
build_images:
- runs-on: ubuntu-latest-8-cores
+ runs-on: ubuntu-latest-64-cores
permissions:
contents: read
timeout-minutes: 10
From 31344b7c058e714284d390379599789aed8028c3 Mon Sep 17 00:00:00 2001
From: Jim Nielsen
Date: Fri, 20 Dec 2024 15:33:44 -0700
Subject: [PATCH 028/155] check for permissions
---
.../src/app/gridGL/pixiApp/urlParams/UrlParamsUser.ts | 1 +
1 file changed, 1 insertion(+)
diff --git a/quadratic-client/src/app/gridGL/pixiApp/urlParams/UrlParamsUser.ts b/quadratic-client/src/app/gridGL/pixiApp/urlParams/UrlParamsUser.ts
index 5e436b274c..df0cdb1524 100644
--- a/quadratic-client/src/app/gridGL/pixiApp/urlParams/UrlParamsUser.ts
+++ b/quadratic-client/src/app/gridGL/pixiApp/urlParams/UrlParamsUser.ts
@@ -73,6 +73,7 @@ export class UrlParamsUser {
private loadAIAnalystPrompt = (params: URLSearchParams) => {
if (!this.pixiAppSettingsInitialized || !this.aiAnalystInitialized) return;
if (this.aiAnalystPromptLoaded) return;
+ if (!pixiAppSettings.permissions.includes('FILE_EDIT')) return;
const prompt = params.get('prompt');
if (!prompt) return;
From 6ea1712e7d464e41ae5bc0f2d89ee493cd6aa766 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 04:18:20 +0530
Subject: [PATCH 029/155] increase timeout
---
.../preview-branches-publish-images.yml | 18 +++++++++++++++---
1 file changed, 15 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index c57fdf299a..20465808e6 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -10,14 +10,26 @@ concurrency:
jobs:
build_images:
- runs-on: ubuntu-latest-64-cores
permissions:
contents: read
- timeout-minutes: 10
+
+ timeout-minutes: 30
+
+ runs-on: ${{ matrix.runner }}
strategy:
matrix:
- service: [multiplayer, files, connection, client, api]
+ include:
+ - service: api
+ runner: ubuntu-latest-8-cores
+ - service: client
+ runner: ubuntu-latest-8-cores
+ - service: connection
+ runner: ubuntu-latest-8-cores
+ - service: files
+ runner: ubuntu-latest-8-cores
+ - service: multiplayer
+ runner: ubuntu-latest-8-cores
fail-fast: true
steps:
From 24a44dc18703357cb7a822d1a82edcda5e5c7aa9 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 04:41:26 +0530
Subject: [PATCH 030/155] try 64 again
---
.github/workflows/preview-branches-publish-images.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 20465808e6..0cc59aea27 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -23,7 +23,7 @@ jobs:
- service: api
runner: ubuntu-latest-8-cores
- service: client
- runner: ubuntu-latest-8-cores
+ runner: ubuntu-latest-64-cores
- service: connection
runner: ubuntu-latest-8-cores
- service: files
From 7802f1767120fe25b14670e4d58cf6aa2b774a39 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 05:07:53 +0530
Subject: [PATCH 031/155] 4 core
---
.github/workflows/preview-branches-publish-images.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 0cc59aea27..ce27d0bcb0 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -23,7 +23,7 @@ jobs:
- service: api
runner: ubuntu-latest-8-cores
- service: client
- runner: ubuntu-latest-64-cores
+ runner: ubuntu-latest-4-cores
- service: connection
runner: ubuntu-latest-8-cores
- service: files
From 44a3861e6342482140da591294507cd02e36df4d Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 05:34:46 +0530
Subject: [PATCH 032/155] try blacksmith
---
.github/workflows/preview-branches-publish-images.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index ce27d0bcb0..1e9d1acd73 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -23,7 +23,7 @@ jobs:
- service: api
runner: ubuntu-latest-8-cores
- service: client
- runner: ubuntu-latest-4-cores
+ runner: blacksmith-8vcpu-ubuntu-2204
- service: connection
runner: ubuntu-latest-8-cores
- service: files
From 06eed50ec74657bd0ad3b34c162a8432530f0335 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 05:52:01 +0530
Subject: [PATCH 033/155] blacksmith 32 core
---
.github/workflows/preview-branches-publish-images.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 1e9d1acd73..5774ddea83 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -23,7 +23,7 @@ jobs:
- service: api
runner: ubuntu-latest-8-cores
- service: client
- runner: blacksmith-8vcpu-ubuntu-2204
+ runner: blacksmith-32vcpu-ubuntu-2204
- service: connection
runner: ubuntu-latest-8-cores
- service: files
From 6395c9f5d56e26060fb94fd9198f3bd011352998 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 06:12:41 +0530
Subject: [PATCH 034/155] try changing build script
---
quadratic-core/package.json | 2 +-
quadratic-rust-client/package.json | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/quadratic-core/package.json b/quadratic-core/package.json
index fd52cd67ca..3b76235638 100644
--- a/quadratic-core/package.json
+++ b/quadratic-core/package.json
@@ -5,7 +5,7 @@
"scripts": {
"start": "cargo watch -s 'wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs'",
"performance": "cargo watch -s 'wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs'",
- "build": "wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs",
+ "build": "wasm-pack build --release --target web --out-dir ../quadratic-client/src/app/quadratic-core",
"coverage": "npm run coverage:clean && npm run coverage:wasm:gen && npm run coverage:wasm:html && npm run coverage:wasm:view",
"coverage:wasm:gen": "CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='coverage/cargo-test-%p-%m.profraw' cargo test",
"coverage:wasm:html": "grcov . --binary-path ../target/debug/deps/ -s src -t html --branch --ignore-not-existing --ignore 'src/wasm_bindings/*' --ignore 'src/bin/*' --ignore '../*' --ignore '/*' -o coverage/html",
diff --git a/quadratic-rust-client/package.json b/quadratic-rust-client/package.json
index ae80d519d4..fbc498df49 100644
--- a/quadratic-rust-client/package.json
+++ b/quadratic-rust-client/package.json
@@ -2,7 +2,7 @@
"name": "quadratic-rust-client",
"private": true,
"scripts": {
- "build": "wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs",
+ "build": "wasm-pack build --release --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client",
"dev": "cargo watch -s 'wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs'",
"dev:perf": "cargo watch -s 'wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs'"
}
From a6bf30f281811c58038ba0e5de7cfbe8942a1c0e Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 06:48:07 +0530
Subject: [PATCH 035/155] revert changes, add cache
---
.../preview-branches-publish-images.yml | 77 ++++++-------------
quadratic-core/package.json | 2 +-
quadratic-rust-client/package.json | 2 +-
3 files changed, 25 insertions(+), 56 deletions(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 5774ddea83..2a2b9f871e 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -23,7 +23,7 @@ jobs:
- service: api
runner: ubuntu-latest-8-cores
- service: client
- runner: blacksmith-32vcpu-ubuntu-2204
+ runner: ubuntu-latest-64-cores
- service: connection
runner: ubuntu-latest-8-cores
- service: files
@@ -47,81 +47,50 @@ jobs:
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEVELOPMENT }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }}
- aws-region: us-east-1
+ aws-region: ${{ secrets.AWS_REGION }}
- - name: Login to Amazon ECR Public
+ - name: Login to Amazon ECR Private
id: login-ecr
uses: aws-actions/amazon-ecr-login@v2
- with:
- registry-type: public
- name: Define repository name
id: repo-name
run: |
echo "REPO_NAME=quadratic-${{ matrix.service }}-development" >> $GITHUB_OUTPUT
- - name: Create Public ECR Repository if not exists
+ - name: Create Private ECR Repository if not exists
id: create-ecr
env:
REPO_NAME: ${{ steps.repo-name.outputs.REPO_NAME }}
run: |
# Try to describe the repository first
- if ! aws ecr-public describe-repositories --repository-names $REPO_NAME 2>/dev/null; then
+ if ! aws ecr describe-repositories --repository-names $REPO_NAME 2>/dev/null; then
# Repository doesn't exist, create it
- aws ecr-public create-repository --repository-name $REPO_NAME
+ aws ecr create-repository --repository-name $REPO_NAME || true
fi
# Get the repository URI either way
- REPO_INFO=$(aws ecr-public describe-repositories --repository-names $REPO_NAME)
+ REPO_INFO=$(aws ecr describe-repositories --repository-names $REPO_NAME)
ECR_URL=$(echo $REPO_INFO | jq -r '.repositories[0].repositoryUri')
echo "ECR_URL=$ECR_URL" >> $GITHUB_OUTPUT
- # - name: Set up Docker Buildx
- # uses: docker/setup-buildx-action@v3
- # with:
- # buildkitd-flags: --debug
-
- # - name: Cache Docker layers
- # uses: actions/cache@v3
- # with:
- # path: /tmp/.buildx-cache
- # key: ${{ runner.os }}-buildx-${{ matrix.service }}-${{ github.sha }}
- # restore-keys: |
- # ${{ runner.os }}-buildx-${{ matrix.service }}-
- # ${{ runner.os }}-buildx-
-
- # - name: Create and initialize buildx cache
- # run: |
- # mkdir -p /tmp/.buildx-cache
- # CACHE_TAG="${{ runner.os }}-buildx-${{ matrix.service }}"
- # echo "{\"layers\":{},\"manifests\":{\"$CACHE_TAG\":{\"layers\":[]}}}" > /tmp/.buildx-cache/index.json
-
- # - name: Build, Tag, and Push Image to Amazon ECR Private
- # env:
- # ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }}
- # run: |
- # docker buildx build \
- # --cache-from=type=local,src=/tmp/.buildx-cache \
- # --cache-to=type=local,dest=/tmp/.buildx-cache-new \
- # --build-arg BUILD_TIME="${{ steps.build-metadata.outputs.BUILD_TIME }}" \
- # --build-arg GIT_SHA="${{ github.sha }}" \
- # --build-arg PR_NUMBER="${{ github.event.pull_request.number }}" \
- # --label "org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}" \
- # --label "org.opencontainers.image.revision=${{ github.sha }}" \
- # --push \
- # -t $ECR_URL:pr-${{ github.event.pull_request.number }} \
- # -t $ECR_URL:${{ steps.build-metadata.outputs.BRANCH_NAME }} \
- # -t $ECR_URL:${{ steps.build-metadata.outputs.GIT_SHA_SHORT }} \
- # -f quadratic-${{ matrix.service }}/Dockerfile .
-
- # - name: Move cache
- # run: |
- # rm -rf /tmp/.buildx-cache
- # mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+ with:
+ buildkitd-flags: --debug
- - name: Build, Tag, and Push Image to Amazon ECR Public
+ - name: Build, Tag, and Push Image to Amazon ECR Private
env:
ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }}
run: |
- docker build -t $ECR_URL:pr-${{ github.event.pull_request.number }} -f quadratic-${{ matrix.service }}/Dockerfile .
- docker push $ECR_URL:pr-${{ github.event.pull_request.number }}
+ docker buildx build \
+ --cache-from=type=registry,ref=$ECR_URL:buildcache \
+ --cache-to=type=registry,ref=$ECR_URL:buildcache,mode=max \
+ --build-arg BUILD_TIME="${{ steps.build-metadata.outputs.BUILD_TIME }}" \
+ --build-arg GIT_SHA="${{ github.sha }}" \
+ --build-arg PR_NUMBER="${{ github.event.pull_request.number }}" \
+ --label "org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}" \
+ --label "org.opencontainers.image.revision=${{ github.sha }}" \
+ --push \
+ -t $ECR_URL:pr-${{ github.event.pull_request.number }} \
+ -f quadratic-${{ matrix.service }}/Dockerfile .
diff --git a/quadratic-core/package.json b/quadratic-core/package.json
index 3b76235638..fd52cd67ca 100644
--- a/quadratic-core/package.json
+++ b/quadratic-core/package.json
@@ -5,7 +5,7 @@
"scripts": {
"start": "cargo watch -s 'wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs'",
"performance": "cargo watch -s 'wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs'",
- "build": "wasm-pack build --release --target web --out-dir ../quadratic-client/src/app/quadratic-core",
+ "build": "wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs",
"coverage": "npm run coverage:clean && npm run coverage:wasm:gen && npm run coverage:wasm:html && npm run coverage:wasm:view",
"coverage:wasm:gen": "CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='coverage/cargo-test-%p-%m.profraw' cargo test",
"coverage:wasm:html": "grcov . --binary-path ../target/debug/deps/ -s src -t html --branch --ignore-not-existing --ignore 'src/wasm_bindings/*' --ignore 'src/bin/*' --ignore '../*' --ignore '/*' -o coverage/html",
diff --git a/quadratic-rust-client/package.json b/quadratic-rust-client/package.json
index fbc498df49..ae80d519d4 100644
--- a/quadratic-rust-client/package.json
+++ b/quadratic-rust-client/package.json
@@ -2,7 +2,7 @@
"name": "quadratic-rust-client",
"private": true,
"scripts": {
- "build": "wasm-pack build --release --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client",
+ "build": "wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs",
"dev": "cargo watch -s 'wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs'",
"dev:perf": "cargo watch -s 'wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs'"
}
From ce65f365762eab5a80a85a7c2f659f4579c83554 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 07:00:07 +0530
Subject: [PATCH 036/155] try
---
.github/workflows/preview-branches-publish-images.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 2a2b9f871e..94ac2bcf5c 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -84,8 +84,8 @@ jobs:
ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }}
run: |
docker buildx build \
- --cache-from=type=registry,ref=$ECR_URL:buildcache \
- --cache-to=type=registry,ref=$ECR_URL:buildcache,mode=max \
+ # --cache-from=type=registry,ref=$ECR_URL:buildcache \
+ # --cache-to=type=registry,ref=$ECR_URL:buildcache,mode=max \
--build-arg BUILD_TIME="${{ steps.build-metadata.outputs.BUILD_TIME }}" \
--build-arg GIT_SHA="${{ github.sha }}" \
--build-arg PR_NUMBER="${{ github.event.pull_request.number }}" \
From cb04f4bde0d31579abca82ad387f71ab219a91b1 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 07:04:23 +0530
Subject: [PATCH 037/155] try
---
.github/workflows/preview-branches-publish-images.yml | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 94ac2bcf5c..5c5b56b0c9 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -84,8 +84,6 @@ jobs:
ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }}
run: |
docker buildx build \
- # --cache-from=type=registry,ref=$ECR_URL:buildcache \
- # --cache-to=type=registry,ref=$ECR_URL:buildcache,mode=max \
--build-arg BUILD_TIME="${{ steps.build-metadata.outputs.BUILD_TIME }}" \
--build-arg GIT_SHA="${{ github.sha }}" \
--build-arg PR_NUMBER="${{ github.event.pull_request.number }}" \
@@ -93,4 +91,5 @@ jobs:
--label "org.opencontainers.image.revision=${{ github.sha }}" \
--push \
-t $ECR_URL:pr-${{ github.event.pull_request.number }} \
- -f quadratic-${{ matrix.service }}/Dockerfile .
+ -f quadratic-${{ matrix.service }}/Dockerfile \
+ .
From 7dc5f3c5331c6162f831190de2e9a481a2c2018e Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 07:25:24 +0530
Subject: [PATCH 038/155] try caching again
---
.../preview-branches-publish-images.yml | 20 +++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 5c5b56b0c9..3fd50325be 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -79,11 +79,23 @@ jobs:
with:
buildkitd-flags: --debug
+ - name: Cache Docker layers
+ uses: actions/cache@v3
+ with:
+ path: /tmp/.buildx-cache
+ key: ${{ runner.os }}-buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }}
+ restore-keys: |
+ ${{ runner.os }}-buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }}
+ ${{ runner.os }}-buildx-${{ matrix.service }}-
+ ${{ runner.os }}-buildx-
+
- name: Build, Tag, and Push Image to Amazon ECR Private
env:
ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }}
run: |
docker buildx build \
+ --cache-from=type=local,src=/tmp/.buildx-cache \
+ --cache-to=type=local,dest=/tmp/.buildx-cache-new,mode=max \
--build-arg BUILD_TIME="${{ steps.build-metadata.outputs.BUILD_TIME }}" \
--build-arg GIT_SHA="${{ github.sha }}" \
--build-arg PR_NUMBER="${{ github.event.pull_request.number }}" \
@@ -93,3 +105,11 @@ jobs:
-t $ECR_URL:pr-${{ github.event.pull_request.number }} \
-f quadratic-${{ matrix.service }}/Dockerfile \
.
+
+ # Temp fix
+ # https://github.com/docker/build-push-action/issues/252
+ # https://github.com/moby/buildkit/issues/1896
+ - name: Move cache
+ run: |
+ rm -rf /tmp/.buildx-cache
+ mv /tmp/.buildx-cache-new /tmp/.buildx-cache
From 5e04cf7471e9c63a357ea099e1d6daffd82465fc Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 09:16:06 +0530
Subject: [PATCH 039/155] build image caching done
---
.github/workflows/preview-branches-publish-images.yml | 4 ++--
package.json | 1 -
quadratic-api/Dockerfile | 2 +-
quadratic-client/Dockerfile | 10 ++++------
4 files changed, 7 insertions(+), 10 deletions(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 3fd50325be..260ae7c2e6 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -23,7 +23,7 @@ jobs:
- service: api
runner: ubuntu-latest-8-cores
- service: client
- runner: ubuntu-latest-64-cores
+ runner: ubuntu-latest-8-cores
- service: connection
runner: ubuntu-latest-8-cores
- service: files
@@ -85,7 +85,6 @@ jobs:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }}
restore-keys: |
- ${{ runner.os }}-buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }}
${{ runner.os }}-buildx-${{ matrix.service }}-
${{ runner.os }}-buildx-
@@ -96,6 +95,7 @@ jobs:
docker buildx build \
--cache-from=type=local,src=/tmp/.buildx-cache \
--cache-to=type=local,dest=/tmp/.buildx-cache-new,mode=max \
+ --build-arg BUILDKIT_INLINE_CACHE=1 \
--build-arg BUILD_TIME="${{ steps.build-metadata.outputs.BUILD_TIME }}" \
--build-arg GIT_SHA="${{ github.sha }}" \
--build-arg PR_NUMBER="${{ github.event.pull_request.number }}" \
diff --git a/package.json b/package.json
index d685418d0a..55421dbd3c 100644
--- a/package.json
+++ b/package.json
@@ -63,7 +63,6 @@
},
"dependencies": {
"@ory/kratos-client": "^1.2.1",
- "tsc": "^2.0.4",
"vitest": "^1.5.0",
"zod": "^3.23.8"
},
diff --git a/quadratic-api/Dockerfile b/quadratic-api/Dockerfile
index 77a8ba4e16..ff5c16668d 100644
--- a/quadratic-api/Dockerfile
+++ b/quadratic-api/Dockerfile
@@ -2,9 +2,9 @@ FROM node:18-alpine AS builder
WORKDIR /app
COPY package.json .
COPY package-lock.json .
+RUN npm install
COPY quadratic-api ./quadratic-api
COPY quadratic-shared ./quadratic-shared
-RUN npm install
FROM node:18-slim AS runtime
WORKDIR /app
diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile
index e814a28455..ac575c408a 100644
--- a/quadratic-client/Dockerfile
+++ b/quadratic-client/Dockerfile
@@ -12,8 +12,8 @@ RUN echo 'wasm-pack version:' && wasm-pack --version
# Install wasm32-unknown-unknown target
# RUN rustup target add wasm32-unknown-unknown
-# Install python & clean up
-RUN apt-get update && apt-get install -y python-is-python3 python3-pip && apt-get clean && rm -rf /var/lib/apt/lists/*
+# Install python
+RUN apt-get update && apt-get install -y --no-install-recommends python-is-python3 python3-pip
# Install npm dependencies
WORKDIR /app
@@ -24,10 +24,7 @@ COPY ./quadratic-core/package*.json ./quadratic-core/
COPY ./quadratic-rust-client/package*.json ./quadratic-rust-client/
COPY ./quadratic-shared/package*.json ./quadratic-shared/
COPY ./quadratic-client/package*.json ./quadratic-client/
-RUN npm install
-
-# Install typescript
-RUN npm install -D typescript
+RUN npm install --no-audit --no-fund
# Copy the rest of the application
WORKDIR /app
@@ -51,6 +48,7 @@ WORKDIR /app/quadratic-core
RUN echo 'Exporting TS/Rust types...' && cargo run --bin export_types
# Build the quadratic-rust-client
+# Cache will not work beyond this point on CI because of GIT_COMMIT being different for each commit
WORKDIR /app
ARG GIT_COMMIT
ENV GIT_COMMIT=$GIT_COMMIT
From 23413df363bca7565a1752e93f5cfd23a1e99097 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 09:22:23 +0530
Subject: [PATCH 040/155] fix api dockerfile
---
quadratic-api/Dockerfile | 1 +
1 file changed, 1 insertion(+)
diff --git a/quadratic-api/Dockerfile b/quadratic-api/Dockerfile
index ff5c16668d..330ea76583 100644
--- a/quadratic-api/Dockerfile
+++ b/quadratic-api/Dockerfile
@@ -2,6 +2,7 @@ FROM node:18-alpine AS builder
WORKDIR /app
COPY package.json .
COPY package-lock.json .
+COPY ./quadratic-api/package*.json ./quadratic-api/
RUN npm install
COPY quadratic-api ./quadratic-api
COPY quadratic-shared ./quadratic-shared
From bbe0caba73f5e3f30dac8ae194a5da85485b72a3 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 09:29:47 +0530
Subject: [PATCH 041/155] really fix api dockerfile
---
quadratic-api/Dockerfile | 1 +
1 file changed, 1 insertion(+)
diff --git a/quadratic-api/Dockerfile b/quadratic-api/Dockerfile
index 330ea76583..69bec4f4ae 100644
--- a/quadratic-api/Dockerfile
+++ b/quadratic-api/Dockerfile
@@ -3,6 +3,7 @@ WORKDIR /app
COPY package.json .
COPY package-lock.json .
COPY ./quadratic-api/package*.json ./quadratic-api/
+COPY ./quadratic-shared/package*.json ./quadratic-shared/
RUN npm install
COPY quadratic-api ./quadratic-api
COPY quadratic-shared ./quadratic-shared
From 9d00b875044e3f24a676c8793dd418a9154beb15 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 11:57:36 +0530
Subject: [PATCH 042/155] try BuildKit inline cache
---
quadratic-client/Dockerfile | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile
index ac575c408a..3a65ef0954 100644
--- a/quadratic-client/Dockerfile
+++ b/quadratic-client/Dockerfile
@@ -24,7 +24,8 @@ COPY ./quadratic-core/package*.json ./quadratic-core/
COPY ./quadratic-rust-client/package*.json ./quadratic-rust-client/
COPY ./quadratic-shared/package*.json ./quadratic-shared/
COPY ./quadratic-client/package*.json ./quadratic-client/
-RUN npm install --no-audit --no-fund
+RUN --mount=type=cache,target=/root/.npm \
+ npm install --no-audit --no-fund
# Copy the rest of the application
WORKDIR /app
From 1f61596e2d0f486db6846b4fc9570441ae8569c1 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 21 Dec 2024 12:19:10 +0530
Subject: [PATCH 043/155] more cache trial and blacksmith 2 core
---
.../preview-branches-publish-images.yml | 44 +++++++++++--------
1 file changed, 25 insertions(+), 19 deletions(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 260ae7c2e6..47c4bfc56f 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -23,7 +23,7 @@ jobs:
- service: api
runner: ubuntu-latest-8-cores
- service: client
- runner: ubuntu-latest-8-cores
+ runner: blacksmith-2vcpu-ubuntu-2204
- service: connection
runner: ubuntu-latest-8-cores
- service: files
@@ -77,7 +77,9 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
- buildkitd-flags: --debug
+ driver-opts: |
+ image=moby/buildkit:latest
+ network=host
- name: Cache Docker layers
uses: actions/cache@v3
@@ -88,23 +90,27 @@ jobs:
${{ runner.os }}-buildx-${{ matrix.service }}-
${{ runner.os }}-buildx-
- - name: Build, Tag, and Push Image to Amazon ECR Private
- env:
- ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }}
- run: |
- docker buildx build \
- --cache-from=type=local,src=/tmp/.buildx-cache \
- --cache-to=type=local,dest=/tmp/.buildx-cache-new,mode=max \
- --build-arg BUILDKIT_INLINE_CACHE=1 \
- --build-arg BUILD_TIME="${{ steps.build-metadata.outputs.BUILD_TIME }}" \
- --build-arg GIT_SHA="${{ github.sha }}" \
- --build-arg PR_NUMBER="${{ github.event.pull_request.number }}" \
- --label "org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}" \
- --label "org.opencontainers.image.revision=${{ github.sha }}" \
- --push \
- -t $ECR_URL:pr-${{ github.event.pull_request.number }} \
- -f quadratic-${{ matrix.service }}/Dockerfile \
- .
+ - name: Build and push
+ uses: docker/build-push-action@v6
+ with:
+ context: .
+ file: quadratic-${{ matrix.service }}/Dockerfile
+ push: true
+ tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }}
+ cache-from: |
+ type=local,src=/tmp/.buildx-cache
+ type=registry,ref=${{ steps.create-ecr.outputs.ECR_URL }}:buildcache
+ cache-to: |
+ type=local,dest=/tmp/.buildx-cache-new,mode=max
+ type=registry,ref=${{ steps.create-ecr.outputs.ECR_URL }}:buildcache,mode=max
+ build-args: |
+ BUILDKIT_INLINE_CACHE=1
+ BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }}
+ GIT_SHA=${{ github.sha }}
+ PR_NUMBER=${{ github.event.pull_request.number }}
+ labels: |
+ org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}
+ org.opencontainers.image.revision=${{ github.sha }}
# Temp fix
# https://github.com/docker/build-push-action/issues/252
From 49bd52d5b1a272b758150eb09f3d259d85115799 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 22 Dec 2024 00:50:52 +0530
Subject: [PATCH 044/155] reduce docker layers, use package scripts
---
.../preview-branches-publish-images.yml | 34 ++++++++-----------
package.json | 1 +
quadratic-client/Dockerfile | 27 +++++----------
quadratic-core/package.json | 3 +-
4 files changed, 27 insertions(+), 38 deletions(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 47c4bfc56f..67777b9ee5 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -23,7 +23,7 @@ jobs:
- service: api
runner: ubuntu-latest-8-cores
- service: client
- runner: blacksmith-2vcpu-ubuntu-2204
+ runner: blacksmith-4vcpu-ubuntu-2204
- service: connection
runner: ubuntu-latest-8-cores
- service: files
@@ -81,14 +81,14 @@ jobs:
image=moby/buildkit:latest
network=host
- - name: Cache Docker layers
- uses: actions/cache@v3
- with:
- path: /tmp/.buildx-cache
- key: ${{ runner.os }}-buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }}
- restore-keys: |
- ${{ runner.os }}-buildx-${{ matrix.service }}-
- ${{ runner.os }}-buildx-
+ # - name: Cache Docker layers
+ # uses: actions/cache@v3
+ # with:
+ # path: /tmp/.buildx-cache
+ # key: ${{ runner.os }}-buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }}
+ # restore-keys: |
+ # ${{ runner.os }}-buildx-${{ matrix.service }}-
+ # ${{ runner.os }}-buildx-
- name: Build and push
uses: docker/build-push-action@v6
@@ -97,12 +97,8 @@ jobs:
file: quadratic-${{ matrix.service }}/Dockerfile
push: true
tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }}
- cache-from: |
- type=local,src=/tmp/.buildx-cache
- type=registry,ref=${{ steps.create-ecr.outputs.ECR_URL }}:buildcache
- cache-to: |
- type=local,dest=/tmp/.buildx-cache-new,mode=max
- type=registry,ref=${{ steps.create-ecr.outputs.ECR_URL }}:buildcache,mode=max
+ # cache-from: type=gha,scope=${{ matrix.service }}
+ # cache-to: type=gha,mode=max,scope=${{ matrix.service }}
build-args: |
BUILDKIT_INLINE_CACHE=1
BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }}
@@ -115,7 +111,7 @@ jobs:
# Temp fix
# https://github.com/docker/build-push-action/issues/252
# https://github.com/moby/buildkit/issues/1896
- - name: Move cache
- run: |
- rm -rf /tmp/.buildx-cache
- mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+ # - name: Move cache
+ # run: |
+ # rm -rf /tmp/.buildx-cache
+ # mv /tmp/.buildx-cache-new /tmp/.buildx-cache
diff --git a/package.json b/package.json
index 55421dbd3c..180cfea028 100644
--- a/package.json
+++ b/package.json
@@ -15,6 +15,7 @@
"quadratic-files",
"quadratic-multiplayer",
"quadratic-rust-client",
+ "quadratic-core",
"quadratic-client",
"quadratic-kernels/python-wasm"
],
diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile
index 3a65ef0954..260dc156ad 100644
--- a/quadratic-client/Dockerfile
+++ b/quadratic-client/Dockerfile
@@ -15,8 +15,9 @@ RUN echo 'wasm-pack version:' && wasm-pack --version
# Install python
RUN apt-get update && apt-get install -y --no-install-recommends python-is-python3 python3-pip
-# Install npm dependencies
WORKDIR /app
+
+# Copy all package.json files
COPY package.json .
COPY package-lock.json .
COPY ./quadratic-kernels/python-wasm/package*.json ./quadratic-kernels/python-wasm/
@@ -24,11 +25,11 @@ COPY ./quadratic-core/package*.json ./quadratic-core/
COPY ./quadratic-rust-client/package*.json ./quadratic-rust-client/
COPY ./quadratic-shared/package*.json ./quadratic-shared/
COPY ./quadratic-client/package*.json ./quadratic-client/
-RUN --mount=type=cache,target=/root/.npm \
- npm install --no-audit --no-fund
+
+# Install npm dependencies
+RUN npm install --no-audit --no-fund
# Copy the rest of the application
-WORKDIR /app
COPY updateAlertVersion.json .
COPY ./quadratic-kernels/python-wasm/. ./quadratic-kernels/python-wasm/
COPY ./quadratic-core/. ./quadratic-core/
@@ -37,38 +38,28 @@ COPY ./quadratic-shared/. ./quadratic-shared/
COPY ./quadratic-client/. ./quadratic-client/
# Run the packaging script for quadratic_py
-WORKDIR /app
RUN ./quadratic-kernels/python-wasm/package.sh --no-poetry
-# Build wasm
-WORKDIR /app/quadratic-core
-RUN echo 'Building wasm...' && wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs
-
-# Export TS/Rust types
-WORKDIR /app/quadratic-core
-RUN echo 'Exporting TS/Rust types...' && cargo run --bin export_types
+# Build wasm and export TS/Rust types
+RUN echo 'Building wasm...' && npm run build --workspace=quadratic-core
# Build the quadratic-rust-client
-# Cache will not work beyond this point on CI because of GIT_COMMIT being different for each commit
-WORKDIR /app
+# Layer caching will not work beyond this point on CI because of GIT_COMMIT being different for each commit
ARG GIT_COMMIT
ENV GIT_COMMIT=$GIT_COMMIT
RUN echo 'Building quadratic-rust-client...' && npm run build --workspace=quadratic-rust-client
# Build the quadratic-shared
-WORKDIR /app
RUN echo 'Building quadratic-shared...' && npm run compile --workspace=quadratic-shared
# Build the front-end
-WORKDIR /app
-RUN echo 'Building front-end...'
ENV VITE_DEBUG=VITE_DEBUG_VAL
ENV VITE_QUADRATIC_API_URL=VITE_QUADRATIC_API_URL_VAL
ENV VITE_QUADRATIC_MULTIPLAYER_URL=VITE_QUADRATIC_MULTIPLAYER_URL_VAL
ENV VITE_QUADRATIC_CONNECTION_URL=VITE_QUADRATIC_CONNECTION_URL_VAL
ENV VITE_AUTH_TYPE=VITE_AUTH_TYPE_VAL
ENV VITE_ORY_HOST=VITE_ORY_HOST_VAL
-RUN npm run build --workspace=quadratic-client
+RUN echo 'Building front-end...' && npm run build --workspace=quadratic-client
# The default command to run the application
# CMD ["npm", "run", "start:production"]
diff --git a/quadratic-core/package.json b/quadratic-core/package.json
index fd52cd67ca..0af92d0167 100644
--- a/quadratic-core/package.json
+++ b/quadratic-core/package.json
@@ -5,7 +5,8 @@
"scripts": {
"start": "cargo watch -s 'wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs'",
"performance": "cargo watch -s 'wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs'",
- "build": "wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs",
+ "build": "wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs",
+ "postbuild": "cargo run --bin export_types --features js",
"coverage": "npm run coverage:clean && npm run coverage:wasm:gen && npm run coverage:wasm:html && npm run coverage:wasm:view",
"coverage:wasm:gen": "CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='coverage/cargo-test-%p-%m.profraw' cargo test",
"coverage:wasm:html": "grcov . --binary-path ../target/debug/deps/ -s src -t html --branch --ignore-not-existing --ignore 'src/wasm_bindings/*' --ignore 'src/bin/*' --ignore '../*' --ignore '/*' -o coverage/html",
From 167c7024523006d513bec2fd1d3725d7d17387e0 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 22 Dec 2024 01:36:21 +0530
Subject: [PATCH 045/155] try blacksmith 2 core for other services
---
.github/workflows/preview-branches-publish-images.yml | 10 +++++-----
quadratic-client/Dockerfile | 6 +++++-
2 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 67777b9ee5..9f5125dcc4 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -21,15 +21,15 @@ jobs:
matrix:
include:
- service: api
- runner: ubuntu-latest-8-cores
+ runner: blacksmith-2vcpu-ubuntu-2204
- service: client
- runner: blacksmith-4vcpu-ubuntu-2204
+ runner: blacksmith-8vcpu-ubuntu-2204
- service: connection
- runner: ubuntu-latest-8-cores
+ runner: blacksmith-2vcpu-ubuntu-2204
- service: files
- runner: ubuntu-latest-8-cores
+ runner: blacksmith-2vcpu-ubuntu-2204
- service: multiplayer
- runner: ubuntu-latest-8-cores
+ runner: blacksmith-2vcpu-ubuntu-2204
fail-fast: true
steps:
diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile
index 260dc156ad..f765d3529c 100644
--- a/quadratic-client/Dockerfile
+++ b/quadratic-client/Dockerfile
@@ -4,13 +4,17 @@ FROM node:18 AS build
# Install rustup
RUN echo 'Installing rustup...' && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"
+ENV CARGO_TARGET_DIR=/app/target
+ENV CARGO_HOME=/app/.cargo
+ENV CARGO_BUILD_JOBS=64
+ENV RUSTFLAGS='-C codegen-units=64'
# Install wasm-pack
RUN echo 'Installing wasm-pack...' && curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
RUN echo 'wasm-pack version:' && wasm-pack --version
# Install wasm32-unknown-unknown target
-# RUN rustup target add wasm32-unknown-unknown
+RUN rustup target add wasm32-unknown-unknown
# Install python
RUN apt-get update && apt-get install -y --no-install-recommends python-is-python3 python3-pip
From 7b41e43a0613ebb8b90e942527903b1c6df72889 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 22 Dec 2024 02:30:01 +0530
Subject: [PATCH 046/155] try blacksmith cache
---
.../preview-branches-publish-images.yml | 30 +++++++++----------
.vscode/settings.json | 5 ++++
2 files changed, 20 insertions(+), 15 deletions(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 9f5125dcc4..599363c544 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -23,7 +23,7 @@ jobs:
- service: api
runner: blacksmith-2vcpu-ubuntu-2204
- service: client
- runner: blacksmith-8vcpu-ubuntu-2204
+ runner: blacksmith-4vcpu-ubuntu-2204 # no speed benefit of 4vcpu, 2vcpu runs out of memory
- service: connection
runner: blacksmith-2vcpu-ubuntu-2204
- service: files
@@ -81,14 +81,14 @@ jobs:
image=moby/buildkit:latest
network=host
- # - name: Cache Docker layers
- # uses: actions/cache@v3
- # with:
- # path: /tmp/.buildx-cache
- # key: ${{ runner.os }}-buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }}
- # restore-keys: |
- # ${{ runner.os }}-buildx-${{ matrix.service }}-
- # ${{ runner.os }}-buildx-
+ - name: Cache Docker layers
+ uses: useblacksmith/cache@v5
+ with:
+ path: /tmp/.buildx-cache
+ key: ${{ runner.os }}-buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }}
+ restore-keys: |
+ ${{ runner.os }}-buildx-${{ matrix.service }}-
+ ${{ runner.os }}-buildx-
- name: Build and push
uses: docker/build-push-action@v6
@@ -97,8 +97,8 @@ jobs:
file: quadratic-${{ matrix.service }}/Dockerfile
push: true
tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }}
- # cache-from: type=gha,scope=${{ matrix.service }}
- # cache-to: type=gha,mode=max,scope=${{ matrix.service }}
+ cache-from: type=local,src=/tmp/.buildx-cache
+ cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
build-args: |
BUILDKIT_INLINE_CACHE=1
BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }}
@@ -111,7 +111,7 @@ jobs:
# Temp fix
# https://github.com/docker/build-push-action/issues/252
# https://github.com/moby/buildkit/issues/1896
- # - name: Move cache
- # run: |
- # rm -rf /tmp/.buildx-cache
- # mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+ - name: Move cache
+ run: |
+ rm -rf /tmp/.buildx-cache
+ mv /tmp/.buildx-cache-new /tmp/.buildx-cache
diff --git a/.vscode/settings.json b/.vscode/settings.json
index 1d8a0dafbb..1d4922aaef 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -6,6 +6,8 @@
"bigdecimal",
"bincode",
"bindgen",
+ "buildkit",
+ "Buildx",
"CRPXNLSKVLJFHH",
"dashmap",
"dbgjs",
@@ -32,6 +34,7 @@
"MDSL",
"micropip",
"minmax",
+ "moby",
"msdf",
"nonblank",
"Northbridge",
@@ -59,7 +62,9 @@
"trackpad",
"undoable",
"unspill",
+ "useblacksmith",
"vals",
+ "vcpu",
"websockets",
"Westborough"
],
From f2a27f2c1a3b9aa83c0ccb989644b96a35db9007 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 22 Dec 2024 03:01:52 +0530
Subject: [PATCH 047/155] add cache compression
---
.github/workflows/preview-branches-publish-images.yml | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 599363c544..e2cfb2bccb 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -81,14 +81,13 @@ jobs:
image=moby/buildkit:latest
network=host
- - name: Cache Docker layers
+ - name: Set up Docker layer cache
uses: useblacksmith/cache@v5
with:
path: /tmp/.buildx-cache
- key: ${{ runner.os }}-buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }}
+ key: buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }}
restore-keys: |
- ${{ runner.os }}-buildx-${{ matrix.service }}-
- ${{ runner.os }}-buildx-
+ buildx-${{ matrix.service }}-
- name: Build and push
uses: docker/build-push-action@v6
@@ -98,7 +97,7 @@ jobs:
push: true
tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }}
cache-from: type=local,src=/tmp/.buildx-cache
- cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+ cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max,compression=zstd,compression-level=22,force-compression=true
build-args: |
BUILDKIT_INLINE_CACHE=1
BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }}
@@ -111,7 +110,7 @@ jobs:
# Temp fix
# https://github.com/docker/build-push-action/issues/252
# https://github.com/moby/buildkit/issues/1896
- - name: Move cache
+ - name: Update cache
run: |
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
From 3e137c46cce43fc00a08498cbe2932c92bdfaeaf Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 22 Dec 2024 03:25:40 +0530
Subject: [PATCH 048/155] try default compression level
---
.github/workflows/preview-branches-publish-images.yml | 2 +-
.vscode/settings.json | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index e2cfb2bccb..931902091e 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -97,7 +97,7 @@ jobs:
push: true
tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }}
cache-from: type=local,src=/tmp/.buildx-cache
- cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max,compression=zstd,compression-level=22,force-compression=true
+ cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max,compression=zstd
build-args: |
BUILDKIT_INLINE_CACHE=1
BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }}
diff --git a/.vscode/settings.json b/.vscode/settings.json
index 1d4922aaef..f0d8987b2b 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -66,7 +66,8 @@
"vals",
"vcpu",
"websockets",
- "Westborough"
+ "Westborough",
+ "zstd"
],
"editor.codeActionsOnSave": {
"source.organizeImports": "explicit"
From 92a8728a3d8097b876c42ee0a10eb126d81d7ac7 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 22 Dec 2024 03:49:03 +0530
Subject: [PATCH 049/155] revert to max cache compression
---
.github/workflows/preview-branches-publish-images.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 931902091e..e2cfb2bccb 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -97,7 +97,7 @@ jobs:
push: true
tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }}
cache-from: type=local,src=/tmp/.buildx-cache
- cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max,compression=zstd
+ cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max,compression=zstd,compression-level=22,force-compression=true
build-args: |
BUILDKIT_INLINE_CACHE=1
BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }}
From 1473500b432b7c2659bbd62d1afaa7a2b941adb4 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 22 Dec 2024 04:46:40 +0530
Subject: [PATCH 050/155] reduce build layer of connections, files and
multiplayer
---
.dockerignore | 43 ++++++++++++++++++++++++++++-
Cargo.lock | 1 -
quadratic-api/.dockerignore | 43 +++++++++++++++++++++++++++++
quadratic-client/.dockerignore | 43 +++++++++++++++++++++++++++++
quadratic-client/Dockerfile | 8 +++---
quadratic-connection/.dockerignore | 43 +++++++++++++++++++++++++++++
quadratic-connection/Cargo.toml | 34 ++++++++---------------
quadratic-connection/Dockerfile | 16 ++++++-----
quadratic-files/.dockerignore | 43 +++++++++++++++++++++++++++++
quadratic-files/Dockerfile | 20 ++++++++------
quadratic-multiplayer/.dockerignore | 43 +++++++++++++++++++++++++++++
quadratic-multiplayer/Dockerfile | 19 ++++++++-----
12 files changed, 305 insertions(+), 51 deletions(-)
create mode 100644 quadratic-api/.dockerignore
create mode 100644 quadratic-client/.dockerignore
create mode 100644 quadratic-connection/.dockerignore
create mode 100644 quadratic-files/.dockerignore
create mode 100644 quadratic-multiplayer/.dockerignore
diff --git a/.dockerignore b/.dockerignore
index 1f5f855728..5b5eab44ef 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,2 +1,43 @@
+# Environment files
+.env*
+**/.env*
+
+# Docker files
+.dockerignore
+**/.dockerignore
+Dockerfile*
+**/Dockerfile*
+docker-compose*
+**/docker-compose*
+
+# Build outputs
+build/
+**/build/
+dist/
+**/dist/
target/
-**/target/
\ No newline at end of file
+**/target/
+
+# Dependencies
+node_modules/
+**/node_modules/
+vendor/
+**/vendor/
+
+# Test and coverage
+coverage/
+**/coverage/
+*.test.*
+**/*.test.*
+
+# Version control
+.git/
+.gitignore
+.gitattributes
+
+# IDE and editor files
+.idea/
+.vscode/
+*.swp
+*.swo
+.DS_Store
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index 82e4ccbc2e..c06d42eaa6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4501,7 +4501,6 @@ dependencies = [
"log",
"openssl",
"parquet 51.0.0",
- "quadratic-core",
"quadratic-rust-shared",
"reqwest 0.11.27",
"serde",
diff --git a/quadratic-api/.dockerignore b/quadratic-api/.dockerignore
new file mode 100644
index 0000000000..5b5eab44ef
--- /dev/null
+++ b/quadratic-api/.dockerignore
@@ -0,0 +1,43 @@
+# Environment files
+.env*
+**/.env*
+
+# Docker files
+.dockerignore
+**/.dockerignore
+Dockerfile*
+**/Dockerfile*
+docker-compose*
+**/docker-compose*
+
+# Build outputs
+build/
+**/build/
+dist/
+**/dist/
+target/
+**/target/
+
+# Dependencies
+node_modules/
+**/node_modules/
+vendor/
+**/vendor/
+
+# Test and coverage
+coverage/
+**/coverage/
+*.test.*
+**/*.test.*
+
+# Version control
+.git/
+.gitignore
+.gitattributes
+
+# IDE and editor files
+.idea/
+.vscode/
+*.swp
+*.swo
+.DS_Store
\ No newline at end of file
diff --git a/quadratic-client/.dockerignore b/quadratic-client/.dockerignore
new file mode 100644
index 0000000000..5b5eab44ef
--- /dev/null
+++ b/quadratic-client/.dockerignore
@@ -0,0 +1,43 @@
+# Environment files
+.env*
+**/.env*
+
+# Docker files
+.dockerignore
+**/.dockerignore
+Dockerfile*
+**/Dockerfile*
+docker-compose*
+**/docker-compose*
+
+# Build outputs
+build/
+**/build/
+dist/
+**/dist/
+target/
+**/target/
+
+# Dependencies
+node_modules/
+**/node_modules/
+vendor/
+**/vendor/
+
+# Test and coverage
+coverage/
+**/coverage/
+*.test.*
+**/*.test.*
+
+# Version control
+.git/
+.gitignore
+.gitattributes
+
+# IDE and editor files
+.idea/
+.vscode/
+*.swp
+*.swo
+.DS_Store
\ No newline at end of file
diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile
index f765d3529c..8c2dfd58d2 100644
--- a/quadratic-client/Dockerfile
+++ b/quadratic-client/Dockerfile
@@ -4,8 +4,8 @@ FROM node:18 AS build
# Install rustup
RUN echo 'Installing rustup...' && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"
-ENV CARGO_TARGET_DIR=/app/target
-ENV CARGO_HOME=/app/.cargo
+ENV CARGO_TARGET_DIR=/quadratic/target
+ENV CARGO_HOME=/quadratic/.cargo
ENV CARGO_BUILD_JOBS=64
ENV RUSTFLAGS='-C codegen-units=64'
@@ -19,7 +19,7 @@ RUN rustup target add wasm32-unknown-unknown
# Install python
RUN apt-get update && apt-get install -y --no-install-recommends python-is-python3 python3-pip
-WORKDIR /app
+WORKDIR /quadratic
# Copy all package.json files
COPY package.json .
@@ -69,7 +69,7 @@ RUN echo 'Building front-end...' && npm run build --workspace=quadratic-client
# CMD ["npm", "run", "start:production"]
FROM nginx:stable-alpine
-COPY --from=build /app/build /usr/share/nginx/html
+COPY --from=build /quadratic/build /usr/share/nginx/html
EXPOSE 80 443 3000
diff --git a/quadratic-connection/.dockerignore b/quadratic-connection/.dockerignore
new file mode 100644
index 0000000000..5b5eab44ef
--- /dev/null
+++ b/quadratic-connection/.dockerignore
@@ -0,0 +1,43 @@
+# Environment files
+.env*
+**/.env*
+
+# Docker files
+.dockerignore
+**/.dockerignore
+Dockerfile*
+**/Dockerfile*
+docker-compose*
+**/docker-compose*
+
+# Build outputs
+build/
+**/build/
+dist/
+**/dist/
+target/
+**/target/
+
+# Dependencies
+node_modules/
+**/node_modules/
+vendor/
+**/vendor/
+
+# Test and coverage
+coverage/
+**/coverage/
+*.test.*
+**/*.test.*
+
+# Version control
+.git/
+.gitignore
+.gitattributes
+
+# IDE and editor files
+.idea/
+.vscode/
+*.swp
+*.swo
+.DS_Store
\ No newline at end of file
diff --git a/quadratic-connection/Cargo.toml b/quadratic-connection/Cargo.toml
index 601908079d..f138d0899f 100644
--- a/quadratic-connection/Cargo.toml
+++ b/quadratic-connection/Cargo.toml
@@ -15,10 +15,7 @@ chrono = { version = "0.4.31", features = ["serde"] }
dotenv = "0.15.0"
envy = "0.4.2"
futures = "0.3.29"
-futures-util = { version = "0.3.29", default-features = false, features = [
- "sink",
- "std",
-] }
+futures-util = { version = "0.3.29", default-features = false, features = ["sink", "std"] }
headers = "0.4.0"
http = "1.1.0"
http-body-util = "0.1.1"
@@ -27,17 +24,9 @@ hyper-util = { version = "0.1.5", features = ["service"] }
jsonwebtoken = "9.2.0"
log = "0.4.21"
openssl = { version = "0.10.66", features = ["vendored"] }
-parquet = { version = "51.0.0", default-features = false, features = [
- "arrow",
- "arrow-array",
-] }
+parquet = { version = "51.0.0", default-features = false, features = ["arrow", "arrow-array"] }
quadratic-rust-shared = { path = "../quadratic-rust-shared" }
-reqwest = { version = "0.11.22", features = [
- "cookies",
- "json",
- "serde_json",
- "stream",
-] }
+reqwest = { version = "0.11.22", features = ["cookies", "json", "serde_json", "stream"] }
serde = { version = "1.0.193", features = ["derive"] }
serde_json = "1.0.108"
strum = "0.26.3"
@@ -46,14 +35,14 @@ thiserror = "1.0.50"
tokio = { version = "1.34.0", features = ["full"] }
tower = { version = "0.4.13", features = ["util"] }
tower-http = { version = "0.5.0", features = [
- "auth",
- "compression-gzip",
- "cors",
- "fs",
- "sensitive-headers",
- "trace",
- "util",
- "validate-request",
+ "auth",
+ "compression-gzip",
+ "cors",
+ "fs",
+ "sensitive-headers",
+ "trace",
+ "util",
+ "validate-request",
] }
tracing = "0.1.40"
tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
@@ -61,7 +50,6 @@ uuid = { version = "1.6.1", features = ["serde", "v4"] }
[dev-dependencies]
fake = { version = "2.9.1", features = ["derive"] }
-quadratic-core = { path = "../quadratic-core" }
quadratic-rust-shared = { path = "../quadratic-rust-shared", features = ["test"] }
tracing-test = "0.2.4"
diff --git a/quadratic-connection/Dockerfile b/quadratic-connection/Dockerfile
index 73556ef906..832306702f 100644
--- a/quadratic-connection/Dockerfile
+++ b/quadratic-connection/Dockerfile
@@ -1,12 +1,14 @@
-FROM rust:latest as builder
+FROM rust:latest AS builder
-RUN USER=root cargo new --bin quadratic-connection
-COPY . /quadratic-connection
+WORKDIR /quadratic
+COPY ./quadratic-connection/. ./quadratic-connection/
+COPY ./quadratic-rust-shared/. ./quadratic-rust-shared/
RUN rustup component add rustfmt
-WORKDIR /quadratic-connection
-RUN cargo build --release --package quadratic-connection
-FROM debian:stable-slim as runtime
-COPY --from=builder /quadratic-connection/target/release/quadratic-connection .
+WORKDIR /quadratic/quadratic-connection
+RUN cargo build --release
+
+FROM debian:stable-slim AS runtime
+COPY --from=builder /quadratic/quadratic-connection/target/release/quadratic-connection .
RUN apt-get update && apt install -y ca-certificates
CMD ["./quadratic-connection"]
diff --git a/quadratic-files/.dockerignore b/quadratic-files/.dockerignore
new file mode 100644
index 0000000000..5b5eab44ef
--- /dev/null
+++ b/quadratic-files/.dockerignore
@@ -0,0 +1,43 @@
+# Environment files
+.env*
+**/.env*
+
+# Docker files
+.dockerignore
+**/.dockerignore
+Dockerfile*
+**/Dockerfile*
+docker-compose*
+**/docker-compose*
+
+# Build outputs
+build/
+**/build/
+dist/
+**/dist/
+target/
+**/target/
+
+# Dependencies
+node_modules/
+**/node_modules/
+vendor/
+**/vendor/
+
+# Test and coverage
+coverage/
+**/coverage/
+*.test.*
+**/*.test.*
+
+# Version control
+.git/
+.gitignore
+.gitattributes
+
+# IDE and editor files
+.idea/
+.vscode/
+*.swp
+*.swo
+.DS_Store
\ No newline at end of file
diff --git a/quadratic-files/Dockerfile b/quadratic-files/Dockerfile
index 723ca8660e..de44b5c073 100644
--- a/quadratic-files/Dockerfile
+++ b/quadratic-files/Dockerfile
@@ -1,12 +1,16 @@
-FROM rust:latest as builder
+FROM rust:latest AS builder
-RUN USER=root cargo new --bin quadratic-files
-COPY . /quadratic-files
+WORKDIR /quadratic
+COPY ./quadratic-files/. ./quadratic-files/
+COPY ./quadratic-core/. ./quadratic-core/
+COPY ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts
+COPY ./quadratic-rust-shared/. ./quadratic-rust-shared/
RUN rustup component add rustfmt
-WORKDIR /quadratic-files
-RUN cargo build --release --package quadratic-files
-FROM debian:stable-slim as runtime
-COPY --from=builder /quadratic-files/target/release/quadratic-files .
+WORKDIR /quadratic/quadratic-files
+RUN cargo build --release
+
+FROM debian:stable-slim AS runtime
+COPY --from=builder /quadratic/quadratic-files/target/release/quadratic-files .
RUN apt-get update && apt install -y ca-certificates
-CMD ["./quadratic-files"]
+CMD ["./quadratic-files"]
\ No newline at end of file
diff --git a/quadratic-multiplayer/.dockerignore b/quadratic-multiplayer/.dockerignore
new file mode 100644
index 0000000000..5b5eab44ef
--- /dev/null
+++ b/quadratic-multiplayer/.dockerignore
@@ -0,0 +1,43 @@
+# Environment files
+.env*
+**/.env*
+
+# Docker files
+.dockerignore
+**/.dockerignore
+Dockerfile*
+**/Dockerfile*
+docker-compose*
+**/docker-compose*
+
+# Build outputs
+build/
+**/build/
+dist/
+**/dist/
+target/
+**/target/
+
+# Dependencies
+node_modules/
+**/node_modules/
+vendor/
+**/vendor/
+
+# Test and coverage
+coverage/
+**/coverage/
+*.test.*
+**/*.test.*
+
+# Version control
+.git/
+.gitignore
+.gitattributes
+
+# IDE and editor files
+.idea/
+.vscode/
+*.swp
+*.swo
+.DS_Store
\ No newline at end of file
diff --git a/quadratic-multiplayer/Dockerfile b/quadratic-multiplayer/Dockerfile
index d2921d58ae..5005cf49b1 100644
--- a/quadratic-multiplayer/Dockerfile
+++ b/quadratic-multiplayer/Dockerfile
@@ -1,12 +1,17 @@
-FROM rust:latest as builder
+FROM rust:latest AS builder
-RUN USER=root cargo new --bin quadratic-multiplayer
-COPY . /quadratic-multiplayer
+WORKDIR /quadratic
+COPY updateAlertVersion.json .
+COPY ./quadratic-multiplayer/. ./quadratic-multiplayer/
+COPY ./quadratic-core/. ./quadratic-core/
+COPY ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts
+COPY ./quadratic-rust-shared/. ./quadratic-rust-shared/
RUN rustup component add rustfmt
-WORKDIR /quadratic-multiplayer
-RUN cargo build --release --package quadratic-multiplayer
-FROM debian:stable-slim as runtime
-COPY --from=builder /quadratic-multiplayer/target/release/quadratic-multiplayer .
+WORKDIR /quadratic/quadratic-multiplayer
+RUN cargo build --release
+
+FROM debian:stable-slim AS runtime
+COPY --from=builder /quadratic/quadratic-multiplayer/target/release/quadratic-multiplayer .
RUN apt-get update && apt install -y ca-certificates
CMD ["./quadratic-multiplayer"]
From 3cd98ee8a15c74b3a434e07ab9c320e84a28518e Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 22 Dec 2024 07:25:45 +0530
Subject: [PATCH 051/155] fix docker ignore
---
.dockerignore | 6 ---
.../preview-branches-publish-images.yml | 2 +-
quadratic-api/.dockerignore | 43 -------------------
quadratic-api/Dockerfile | 6 +--
quadratic-client/.dockerignore | 43 -------------------
quadratic-connection/.dockerignore | 43 -------------------
quadratic-files/.dockerignore | 43 -------------------
quadratic-multiplayer/.dockerignore | 43 -------------------
8 files changed, 4 insertions(+), 225 deletions(-)
delete mode 100644 quadratic-api/.dockerignore
delete mode 100644 quadratic-client/.dockerignore
delete mode 100644 quadratic-connection/.dockerignore
delete mode 100644 quadratic-files/.dockerignore
delete mode 100644 quadratic-multiplayer/.dockerignore
diff --git a/.dockerignore b/.dockerignore
index 5b5eab44ef..f95554c258 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -24,12 +24,6 @@ node_modules/
vendor/
**/vendor/
-# Test and coverage
-coverage/
-**/coverage/
-*.test.*
-**/*.test.*
-
# Version control
.git/
.gitignore
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index e2cfb2bccb..931902091e 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -97,7 +97,7 @@ jobs:
push: true
tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }}
cache-from: type=local,src=/tmp/.buildx-cache
- cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max,compression=zstd,compression-level=22,force-compression=true
+ cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max,compression=zstd
build-args: |
BUILDKIT_INLINE_CACHE=1
BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }}
diff --git a/quadratic-api/.dockerignore b/quadratic-api/.dockerignore
deleted file mode 100644
index 5b5eab44ef..0000000000
--- a/quadratic-api/.dockerignore
+++ /dev/null
@@ -1,43 +0,0 @@
-# Environment files
-.env*
-**/.env*
-
-# Docker files
-.dockerignore
-**/.dockerignore
-Dockerfile*
-**/Dockerfile*
-docker-compose*
-**/docker-compose*
-
-# Build outputs
-build/
-**/build/
-dist/
-**/dist/
-target/
-**/target/
-
-# Dependencies
-node_modules/
-**/node_modules/
-vendor/
-**/vendor/
-
-# Test and coverage
-coverage/
-**/coverage/
-*.test.*
-**/*.test.*
-
-# Version control
-.git/
-.gitignore
-.gitattributes
-
-# IDE and editor files
-.idea/
-.vscode/
-*.swp
-*.swo
-.DS_Store
\ No newline at end of file
diff --git a/quadratic-api/Dockerfile b/quadratic-api/Dockerfile
index 69bec4f4ae..8b57168303 100644
--- a/quadratic-api/Dockerfile
+++ b/quadratic-api/Dockerfile
@@ -1,5 +1,5 @@
FROM node:18-alpine AS builder
-WORKDIR /app
+WORKDIR /quadratic
COPY package.json .
COPY package-lock.json .
COPY ./quadratic-api/package*.json ./quadratic-api/
@@ -9,8 +9,8 @@ COPY quadratic-api ./quadratic-api
COPY quadratic-shared ./quadratic-shared
FROM node:18-slim AS runtime
-WORKDIR /app
-COPY --from=builder /app .
+WORKDIR /quadratic
+COPY --from=builder /quadratic .
RUN apt-get update && apt install -y openssl
RUN npm run postinstall --workspace=quadratic-api
RUN npm run build:prod --workspace=quadratic-api
diff --git a/quadratic-client/.dockerignore b/quadratic-client/.dockerignore
deleted file mode 100644
index 5b5eab44ef..0000000000
--- a/quadratic-client/.dockerignore
+++ /dev/null
@@ -1,43 +0,0 @@
-# Environment files
-.env*
-**/.env*
-
-# Docker files
-.dockerignore
-**/.dockerignore
-Dockerfile*
-**/Dockerfile*
-docker-compose*
-**/docker-compose*
-
-# Build outputs
-build/
-**/build/
-dist/
-**/dist/
-target/
-**/target/
-
-# Dependencies
-node_modules/
-**/node_modules/
-vendor/
-**/vendor/
-
-# Test and coverage
-coverage/
-**/coverage/
-*.test.*
-**/*.test.*
-
-# Version control
-.git/
-.gitignore
-.gitattributes
-
-# IDE and editor files
-.idea/
-.vscode/
-*.swp
-*.swo
-.DS_Store
\ No newline at end of file
diff --git a/quadratic-connection/.dockerignore b/quadratic-connection/.dockerignore
deleted file mode 100644
index 5b5eab44ef..0000000000
--- a/quadratic-connection/.dockerignore
+++ /dev/null
@@ -1,43 +0,0 @@
-# Environment files
-.env*
-**/.env*
-
-# Docker files
-.dockerignore
-**/.dockerignore
-Dockerfile*
-**/Dockerfile*
-docker-compose*
-**/docker-compose*
-
-# Build outputs
-build/
-**/build/
-dist/
-**/dist/
-target/
-**/target/
-
-# Dependencies
-node_modules/
-**/node_modules/
-vendor/
-**/vendor/
-
-# Test and coverage
-coverage/
-**/coverage/
-*.test.*
-**/*.test.*
-
-# Version control
-.git/
-.gitignore
-.gitattributes
-
-# IDE and editor files
-.idea/
-.vscode/
-*.swp
-*.swo
-.DS_Store
\ No newline at end of file
diff --git a/quadratic-files/.dockerignore b/quadratic-files/.dockerignore
deleted file mode 100644
index 5b5eab44ef..0000000000
--- a/quadratic-files/.dockerignore
+++ /dev/null
@@ -1,43 +0,0 @@
-# Environment files
-.env*
-**/.env*
-
-# Docker files
-.dockerignore
-**/.dockerignore
-Dockerfile*
-**/Dockerfile*
-docker-compose*
-**/docker-compose*
-
-# Build outputs
-build/
-**/build/
-dist/
-**/dist/
-target/
-**/target/
-
-# Dependencies
-node_modules/
-**/node_modules/
-vendor/
-**/vendor/
-
-# Test and coverage
-coverage/
-**/coverage/
-*.test.*
-**/*.test.*
-
-# Version control
-.git/
-.gitignore
-.gitattributes
-
-# IDE and editor files
-.idea/
-.vscode/
-*.swp
-*.swo
-.DS_Store
\ No newline at end of file
diff --git a/quadratic-multiplayer/.dockerignore b/quadratic-multiplayer/.dockerignore
deleted file mode 100644
index 5b5eab44ef..0000000000
--- a/quadratic-multiplayer/.dockerignore
+++ /dev/null
@@ -1,43 +0,0 @@
-# Environment files
-.env*
-**/.env*
-
-# Docker files
-.dockerignore
-**/.dockerignore
-Dockerfile*
-**/Dockerfile*
-docker-compose*
-**/docker-compose*
-
-# Build outputs
-build/
-**/build/
-dist/
-**/dist/
-target/
-**/target/
-
-# Dependencies
-node_modules/
-**/node_modules/
-vendor/
-**/vendor/
-
-# Test and coverage
-coverage/
-**/coverage/
-*.test.*
-**/*.test.*
-
-# Version control
-.git/
-.gitignore
-.gitattributes
-
-# IDE and editor files
-.idea/
-.vscode/
-*.swp
-*.swo
-.DS_Store
\ No newline at end of file
From 63d33b2b7f1f6c9a20374d0a0230df52c6626a33 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 22 Dec 2024 08:32:34 +0530
Subject: [PATCH 052/155] seperate TS and rust layers
---
.dockerignore | 7 ++++++-
quadratic-client/Dockerfile | 18 +++++++++---------
2 files changed, 15 insertions(+), 10 deletions(-)
diff --git a/.dockerignore b/.dockerignore
index f95554c258..271e5a1953 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -34,4 +34,9 @@ vendor/
.vscode/
*.swp
*.swo
-.DS_Store
\ No newline at end of file
+.DS_Store
+
+# TS/Rust types, this is generated during the build process
+quadratic-client/src/app/quadratic-core/
+quadratic-client/src/app/quadratic-core-types/
+quadratic-client/src/app/quadratic-rust-client/
diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile
index 8c2dfd58d2..5b57170790 100644
--- a/quadratic-client/Dockerfile
+++ b/quadratic-client/Dockerfile
@@ -21,6 +21,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends python-is-pytho
WORKDIR /quadratic
+# Copy updateAlertVersion.json
+COPY updateAlertVersion.json .
+
# Copy all package.json files
COPY package.json .
COPY package-lock.json .
@@ -33,30 +36,27 @@ COPY ./quadratic-client/package*.json ./quadratic-client/
# Install npm dependencies
RUN npm install --no-audit --no-fund
-# Copy the rest of the application
-COPY updateAlertVersion.json .
-COPY ./quadratic-kernels/python-wasm/. ./quadratic-kernels/python-wasm/
-COPY ./quadratic-core/. ./quadratic-core/
-COPY ./quadratic-rust-client/. ./quadratic-rust-client/
-COPY ./quadratic-shared/. ./quadratic-shared/
-COPY ./quadratic-client/. ./quadratic-client/
-
# Run the packaging script for quadratic_py
+COPY ./quadratic-kernels/python-wasm/. ./quadratic-kernels/python-wasm/
RUN ./quadratic-kernels/python-wasm/package.sh --no-poetry
# Build wasm and export TS/Rust types
+COPY ./quadratic-core/. ./quadratic-core/
+COPY ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts
RUN echo 'Building wasm...' && npm run build --workspace=quadratic-core
# Build the quadratic-rust-client
-# Layer caching will not work beyond this point on CI because of GIT_COMMIT being different for each commit
+COPY ./quadratic-rust-client/. ./quadratic-rust-client/
ARG GIT_COMMIT
ENV GIT_COMMIT=$GIT_COMMIT
RUN echo 'Building quadratic-rust-client...' && npm run build --workspace=quadratic-rust-client
# Build the quadratic-shared
+COPY ./quadratic-shared/. ./quadratic-shared/
RUN echo 'Building quadratic-shared...' && npm run compile --workspace=quadratic-shared
# Build the front-end
+COPY ./quadratic-client/. ./quadratic-client/
ENV VITE_DEBUG=VITE_DEBUG_VAL
ENV VITE_QUADRATIC_API_URL=VITE_QUADRATIC_API_URL_VAL
ENV VITE_QUADRATIC_MULTIPLAYER_URL=VITE_QUADRATIC_MULTIPLAYER_URL_VAL
From ebc70a56ff550c186528873862ff0aeb464dd137 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 22 Dec 2024 08:33:03 +0530
Subject: [PATCH 053/155] use blacksmith build and push action
---
.../preview-branches-publish-images.yml | 29 ++++++++++---------
1 file changed, 15 insertions(+), 14 deletions(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 931902091e..87012042a5 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -81,23 +81,24 @@ jobs:
image=moby/buildkit:latest
network=host
- - name: Set up Docker layer cache
- uses: useblacksmith/cache@v5
- with:
- path: /tmp/.buildx-cache
- key: buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }}
- restore-keys: |
- buildx-${{ matrix.service }}-
+ # - name: Set up Docker layer cache
+ # uses: useblacksmith/cache@v5
+ # with:
+ # path: /tmp/.buildx-cache
+ # key: buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }}
+ # restore-keys: |
+ # buildx-${{ matrix.service }}-
- name: Build and push
- uses: docker/build-push-action@v6
+ # uses: docker/build-push-action@v6
+ uses: useblacksmith/build-push-action@v1
with:
context: .
file: quadratic-${{ matrix.service }}/Dockerfile
push: true
tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }}
- cache-from: type=local,src=/tmp/.buildx-cache
- cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max,compression=zstd
+ # cache-from: type=local,src=/tmp/.buildx-cache
+ # cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max,compression=zstd
build-args: |
BUILDKIT_INLINE_CACHE=1
BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }}
@@ -110,7 +111,7 @@ jobs:
# Temp fix
# https://github.com/docker/build-push-action/issues/252
# https://github.com/moby/buildkit/issues/1896
- - name: Update cache
- run: |
- rm -rf /tmp/.buildx-cache
- mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+ # - name: Update cache
+ # run: |
+ # rm -rf /tmp/.buildx-cache
+ # mv /tmp/.buildx-cache-new /tmp/.buildx-cache
From b1f4876fdecedbb45a742c0d65de4d08de81454f Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 22 Dec 2024 09:04:27 +0530
Subject: [PATCH 054/155] prefer caching rust layers over npm dependencies
---
.../preview-branches-publish-images.yml | 21 +------------------
quadratic-client/Dockerfile | 20 +++++++-----------
2 files changed, 9 insertions(+), 32 deletions(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 87012042a5..772001b137 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -58,7 +58,7 @@ jobs:
run: |
echo "REPO_NAME=quadratic-${{ matrix.service }}-development" >> $GITHUB_OUTPUT
- - name: Create Private ECR Repository if not exists
+ - name: Create Private ECR Repository
id: create-ecr
env:
REPO_NAME: ${{ steps.repo-name.outputs.REPO_NAME }}
@@ -81,24 +81,13 @@ jobs:
image=moby/buildkit:latest
network=host
- # - name: Set up Docker layer cache
- # uses: useblacksmith/cache@v5
- # with:
- # path: /tmp/.buildx-cache
- # key: buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }}
- # restore-keys: |
- # buildx-${{ matrix.service }}-
-
- name: Build and push
- # uses: docker/build-push-action@v6
uses: useblacksmith/build-push-action@v1
with:
context: .
file: quadratic-${{ matrix.service }}/Dockerfile
push: true
tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }}
- # cache-from: type=local,src=/tmp/.buildx-cache
- # cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max,compression=zstd
build-args: |
BUILDKIT_INLINE_CACHE=1
BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }}
@@ -107,11 +96,3 @@ jobs:
labels: |
org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}
org.opencontainers.image.revision=${{ github.sha }}
-
- # Temp fix
- # https://github.com/docker/build-push-action/issues/252
- # https://github.com/moby/buildkit/issues/1896
- # - name: Update cache
- # run: |
- # rm -rf /tmp/.buildx-cache
- # mv /tmp/.buildx-cache-new /tmp/.buildx-cache
diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile
index 5b57170790..1a4a0d2f3e 100644
--- a/quadratic-client/Dockerfile
+++ b/quadratic-client/Dockerfile
@@ -21,20 +21,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends python-is-pytho
WORKDIR /quadratic
-# Copy updateAlertVersion.json
+# Copy root dependencies
+# Any changes to these files will trigger a full rebuild (version change)
COPY updateAlertVersion.json .
-
-# Copy all package.json files
COPY package.json .
-COPY package-lock.json .
-COPY ./quadratic-kernels/python-wasm/package*.json ./quadratic-kernels/python-wasm/
-COPY ./quadratic-core/package*.json ./quadratic-core/
-COPY ./quadratic-rust-client/package*.json ./quadratic-rust-client/
-COPY ./quadratic-shared/package*.json ./quadratic-shared/
-COPY ./quadratic-client/package*.json ./quadratic-client/
-
-# Install npm dependencies
-RUN npm install --no-audit --no-fund
# Run the packaging script for quadratic_py
COPY ./quadratic-kernels/python-wasm/. ./quadratic-kernels/python-wasm/
@@ -51,6 +41,12 @@ ARG GIT_COMMIT
ENV GIT_COMMIT=$GIT_COMMIT
RUN echo 'Building quadratic-rust-client...' && npm run build --workspace=quadratic-rust-client
+# Install npm dependencies
+COPY package-lock.json .
+COPY ./quadratic-shared/package*.json ./quadratic-shared/
+COPY ./quadratic-client/package*.json ./quadratic-client/
+RUN npm install --no-audit --no-fund
+
# Build the quadratic-shared
COPY ./quadratic-shared/. ./quadratic-shared/
RUN echo 'Building quadratic-shared...' && npm run compile --workspace=quadratic-shared
From b51c290dae1fffca52c0b0e75efdfb3af86889ba Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 22 Dec 2024 09:15:40 +0530
Subject: [PATCH 055/155] add service name in tag
---
.github/workflows/preview-branches-publish-images.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 772001b137..26bc8f4f96 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -87,7 +87,7 @@ jobs:
context: .
file: quadratic-${{ matrix.service }}/Dockerfile
push: true
- tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }}
+ tags: ${{ steps.create-ecr.outputs.ECR_URL }}:${{ matrix.service }}-pr-${{ github.event.pull_request.number }}
build-args: |
BUILDKIT_INLINE_CACHE=1
BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }}
From 70af95bc7d86ae6064c7c00e945edc173fdaf03b Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 22 Dec 2024 09:48:14 +0530
Subject: [PATCH 056/155] revert: add service name in tag, cache work without
it
---
.github/workflows/preview-branches-publish-images.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml
index 26bc8f4f96..772001b137 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-branches-publish-images.yml
@@ -87,7 +87,7 @@ jobs:
context: .
file: quadratic-${{ matrix.service }}/Dockerfile
push: true
- tags: ${{ steps.create-ecr.outputs.ECR_URL }}:${{ matrix.service }}-pr-${{ github.event.pull_request.number }}
+ tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }}
build-args: |
BUILDKIT_INLINE_CACHE=1
BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }}
From 8b612aeea9aa864a93a9080716b5d1bee5cbc346 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 22 Dec 2024 16:14:59 +0530
Subject: [PATCH 057/155] try cloudformation from github actions
---
.../preview-cloudformation-deploy.yml | 77 ++++++
...-images.yml => preview-publish-images.yml} | 3 +-
.vscode/settings.json | 8 +
infra/aws-cloudformation/preview.yml | 233 ++++++++++++++++++
4 files changed, 320 insertions(+), 1 deletion(-)
create mode 100644 .github/workflows/preview-cloudformation-deploy.yml
rename .github/workflows/{preview-branches-publish-images.yml => preview-publish-images.yml} (96%)
create mode 100644 infra/aws-cloudformation/preview.yml
diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml
new file mode 100644
index 0000000000..37e0c0e0aa
--- /dev/null
+++ b/.github/workflows/preview-cloudformation-deploy.yml
@@ -0,0 +1,77 @@
+name: Build Docker Images for Preview Branches
+
+on:
+ pull_request:
+ types: [opened, reopened]
+
+concurrency:
+ group: pr-${{ github.event.pull_request.number }}
+
+jobs:
+ deploy_cloudformation:
+ permissions:
+ contents: read
+ id-token: write
+
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+
+ env:
+ STACK_NAME: preview-pr-${{ github.event.pull_request.number }}
+ STACK_PARAMS: >-
+ ParameterKey=LicenseKey,ParameterValue=5a32bd8a-409e-4733-8846-1868c568a813
+ ParameterKey=ImageTag,ParameterValue=pr-${{ github.event.pull_request.number }}
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Configure AWS Credentials
+ uses: aws-actions/configure-aws-credentials@v4
+ with:
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEVELOPMENT }}
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }}
+ aws-region: ${{ secrets.AWS_REGION }}
+
+ - name: Deploy CloudFormation Stack
+ run: |
+ # Check if stack exists
+ if aws cloudformation describe-stacks --stack-name $STACK_NAME 2>/dev/null; then
+ # Update existing stack
+ aws cloudformation update-stack \
+ --stack-name $STACK_NAME \
+ --template-body file://infra/aws-cloudformation/preview.yml \
+ --parameters $STACK_PARAMS \
+ --capabilities CAPABILITY_IAM \
+ --no-fail-on-empty-changeset
+ else
+ # Create new stack
+ aws cloudformation create-stack \
+ --stack-name $STACK_NAME \
+ --template-body file://infra/aws-cloudformation/preview.yml \
+ --parameters $STACK_PARAMS \
+ --capabilities CAPABILITY_IAM
+
+ # Wait for stack creation to complete
+ aws cloudformation wait stack-create-complete --stack-name $STACK_NAME
+ fi
+
+ - name: Get Stack Outputs
+ id: stack-output
+ run: |
+ DOMAIN=$(aws cloudformation describe-stacks \
+ --stack-name $STACK_NAME \
+ --query 'Stacks[0].Outputs[?OutputKey==`DomainRecord`].OutputValue' \
+ --output text)
+ echo "PREVIEW_URL=https://$DOMAIN" >> $GITHUB_OUTPUT
+
+ - name: Comment Preview URL
+ uses: actions/github-script@v7
+ with:
+ script: |
+ const previewUrl = '${{ steps.stack-output.outputs.PREVIEW_URL }}';
+ const message = `Preview URL: ${previewUrl}`;
+ github.rest.issues.createComment({
+ issue_number: context.issue.number,
+ owner: context.repo.owner,
+ repo: context.repo.name,
+ body: message
+ });
diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-publish-images.yml
similarity index 96%
rename from .github/workflows/preview-branches-publish-images.yml
rename to .github/workflows/preview-publish-images.yml
index 772001b137..a07fac0fa0 100644
--- a/.github/workflows/preview-branches-publish-images.yml
+++ b/.github/workflows/preview-publish-images.yml
@@ -5,7 +5,7 @@ on:
types: [opened, synchronize, reopened]
concurrency:
- group: ${{ github.head_ref }}-build-images
+ group: pr-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
@@ -92,6 +92,7 @@ jobs:
BUILDKIT_INLINE_CACHE=1
BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }}
GIT_SHA=${{ github.sha }}
+ BRANCH_NAME=${{ steps.build-metadata.outputs.BRANCH_NAME }}
PR_NUMBER=${{ github.event.pull_request.number }}
labels: |
org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}
diff --git a/.vscode/settings.json b/.vscode/settings.json
index f0d8987b2b..9b548d8a97 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -2,19 +2,23 @@
"editor.formatOnSave": true,
"cSpell.words": [
"actix",
+ "awscliv",
"ayush",
"bigdecimal",
"bincode",
"bindgen",
"buildkit",
"Buildx",
+ "containerd",
"CRPXNLSKVLJFHH",
"dashmap",
"dbgjs",
"dcell",
"ddimaria",
+ "dearmor",
"dgraph",
"dotenv",
+ "dpkg",
"endregion",
"finitize",
"Fuzzysort",
@@ -31,6 +35,7 @@
"indicies",
"itertools",
"jwks",
+ "keyrings",
"MDSL",
"micropip",
"minmax",
@@ -51,6 +56,8 @@
"relcells",
"reqwest",
"scrollend",
+ "selfhost",
+ "selfhosted",
"shadcn",
"Signin",
"smallpop",
@@ -63,6 +70,7 @@
"undoable",
"unspill",
"useblacksmith",
+ "usermod",
"vals",
"vcpu",
"websockets",
diff --git a/infra/aws-cloudformation/preview.yml b/infra/aws-cloudformation/preview.yml
new file mode 100644
index 0000000000..c7195e4bb9
--- /dev/null
+++ b/infra/aws-cloudformation/preview.yml
@@ -0,0 +1,233 @@
+AWSTemplateFormatVersion: "2010-09-09"
+Description: "Quadratic Selfhost Cloudformation Template"
+
+Parameters:
+ LicenseKey:
+ Type: String
+ Description: "Your license key for Quadratic. Get one here https://selfhost.quadratichq.com/"
+ ImageTag:
+ Type: String
+ Description: "Image tag to use for all services"
+ Default: "latest"
+
+Mappings:
+ EnvironmentConfig:
+ Preview:
+ DomainName: "quadratic-preview.com"
+ RegionMap:
+ us-west-2:
+ AMI: "ami-05134c8ef96964280" # Example AMI for us-west-2 (Ubuntu 20.04)
+ ap-south-1:
+ AMI: "ami-0522ab6e1ddcc7055" # Example AMI for ap-south-1 (Ubuntu 24.04 LTS)
+
+Resources:
+ EC2Role:
+ Type: AWS::IAM::Role
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ Properties:
+ AssumeRolePolicyDocument:
+ Version: "2012-10-17"
+ Statement:
+ - Effect: Allow
+ Principal:
+ Service: ec2.amazonaws.com
+ Action: sts:AssumeRole
+ ManagedPolicyArns:
+ - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
+ - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
+ Policies:
+ - PolicyName: ECRAccess
+ PolicyDocument:
+ Version: "2012-10-17"
+ Statement:
+ - Effect: Allow
+ Action:
+ - ecr:GetAuthorizationToken
+ - ecr:BatchCheckLayerAvailability
+ - ecr:GetDownloadUrlForLayer
+ - ecr:BatchGetImage
+ - ecr:DescribeImages
+ - ecr:ListImages
+ Resource: "*"
+
+ EC2InstanceProfile:
+ Type: AWS::IAM::InstanceProfile
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ Properties:
+ Roles:
+ - !Ref EC2Role
+
+ OpenSecurityGroup:
+ Type: "AWS::EC2::SecurityGroup"
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ Properties:
+ GroupDescription: !Sub "${ImageTag}"
+ SecurityGroupIngress:
+ - IpProtocol: "tcp"
+ FromPort: "80"
+ ToPort: "80"
+ CidrIp: "0.0.0.0/0"
+ - IpProtocol: "tcp"
+ FromPort: "443"
+ ToPort: "443"
+ CidrIp: "0.0.0.0/0"
+
+ EC2Instance:
+ Type: "AWS::EC2::Instance"
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ Properties:
+ InstanceType: "m6a.large"
+ ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI]
+ IamInstanceProfile: !Ref EC2InstanceProfile
+ SecurityGroups:
+ - !Ref OpenSecurityGroup
+ BlockDeviceMappings:
+ - DeviceName: "/dev/sda1"
+ Ebs:
+ VolumeSize: "25"
+ VolumeType: "gp3"
+ UserData:
+ Fn::Base64: !Sub |
+ #!/bin/bash
+
+ # Update and install dependencies
+ sudo apt-get update
+ sudo apt-get install -y apt-transport-https ca-certificates curl software-properties-common unzip jq
+
+ # Install AWS CLI v2
+ curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+ unzip awscliv2.zip
+ sudo ./aws/install
+ rm -rf aws awscliv2.zip
+
+ # Install Docker
+ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+ echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+ sudo apt-get update
+ sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+
+ # Install Docker Compose
+ sudo curl -L "https://github.com/docker/compose/releases/download/v2.21.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+ sudo chmod +x /usr/local/bin/docker-compose
+
+ # Configure Docker
+ sudo systemctl enable docker
+ sudo systemctl start docker
+ sudo usermod -aG docker ubuntu
+
+ # Export environment variables for docker-compose
+ echo 'export ECR_URL="${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com"' >> /home/ubuntu/.profile
+ echo 'export IMAGE_TAG="${ImageTag}"' >> /home/ubuntu/.profile
+
+ # Source the new environment variables
+ source /home/ubuntu/.profile
+
+ # Create login.sh script
+ echo '#!/bin/bash
+ aws ecr get-login-password --region ${AWS::Region} | docker login --username AWS --password-stdin ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com' > /quadratic-selfhost/login.sh
+ chmod +x /quadratic-selfhost/login.sh
+
+ # Run Quadratic initialization
+ curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic-selfhost/main/init.sh -o init.sh && bash -i init.sh ${LicenseKey} ${ImageTag}.${!FindInMap [EnvironmentConfig, Preview, DomainName]}
+
+ DnsRecords:
+ Type: AWS::Route53::RecordSet
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ Properties:
+ HostedZoneName: !Sub "${!FindInMap [EnvironmentConfig, Preview, DomainName]}."
+ Name: !Sub "${ImageTag}.${!FindInMap [EnvironmentConfig, Preview, DomainName]}"
+ Type: A
+ TTL: 300
+ ResourceRecords:
+ - !GetAtt EC2Instance.PublicIp
+
+ WildcardDnsRecord:
+ Type: AWS::Route53::RecordSet
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ Properties:
+ HostedZoneName: !Sub "${!FindInMap [EnvironmentConfig, Preview, DomainName]}."
+ Name: !Sub "*.${ImageTag}.${!FindInMap [EnvironmentConfig, Preview, DomainName]}"
+ Type: A
+ TTL: 300
+ ResourceRecords:
+ - !GetAtt EC2Instance.PublicIp
+
+ EventBridgeRole:
+ Type: AWS::IAM::Role
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ Properties:
+ AssumeRolePolicyDocument:
+ Version: "2012-10-17"
+ Statement:
+ - Effect: Allow
+ Principal:
+ Service: events.amazonaws.com
+ Action: sts:AssumeRole
+ Policies:
+ - PolicyName: InvokeSsmAutomation
+ PolicyDocument:
+ Version: "2012-10-17"
+ Statement:
+ - Effect: Allow
+ Action:
+ - ssm:SendCommand
+ Resource: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:*"
+
+ EcrPushRule:
+ Type: AWS::Events::Rule
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ DependsOn:
+ - EC2Instance
+ - EventBridgeRole
+ Properties:
+ Description: "Rule to detect ECR image pushes"
+ EventPattern:
+ source:
+ - aws.ecr
+ detail-type:
+ - "ECR Image Action"
+ detail:
+ action-type:
+ - "PUSH"
+ repository-name:
+ - "quadratic-client-development"
+ - "quadratic-api-development"
+ - "quadratic-multiplayer-development"
+ - "quadratic-files-development"
+ - "quadratic-connection-development"
+ image-tag:
+ - !Ref ImageTag
+ State: "ENABLED"
+ Targets:
+ - Arn: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:automation-definition/AWS-RunShellScript"
+ Id: "TriggerScriptOnInstance"
+ RoleArn: !GetAtt EventBridgeRole.Arn
+ InputTransformer:
+ InputPathsMap:
+ "imageTag": "$.detail.image-tag"
+ "repository": "$.detail.repository-name"
+ InputTemplate: |
+ {
+ "InstanceIds": ["${EC2Instance.InstanceId}"],
+ "DocumentName": "AWS-RunShellScript",
+ "Parameters": {
+ "commands": [
+ "cd /quadratic-selfhost",
+ "./login.sh",
+ "./start.sh",
+ ]
+ }
+ }
+
+Outputs:
+ DomainRecord:
+ Description: "Url of the selfhosted instance"
+ Value: !Sub "${ImageTag}.${!FindInMap [EnvironmentConfig, Preview, DomainName]}"
\ No newline at end of file
From 0384d50a13cb8e25878a033ca710611c3cad1ba1 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 22 Dec 2024 16:37:52 +0530
Subject: [PATCH 058/155] trigger CI
---
.github/workflows/preview-cloudformation-deploy.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml
index 37e0c0e0aa..3cbe2edf7c 100644
--- a/.github/workflows/preview-cloudformation-deploy.yml
+++ b/.github/workflows/preview-cloudformation-deploy.yml
@@ -1,4 +1,4 @@
-name: Build Docker Images for Preview Branches
+name: Deploy CloudFormation Stack for Preview Branches
on:
pull_request:
From deaa27586e8ee92fa44d91b88a2e32166c013428 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 22 Dec 2024 16:41:37 +0530
Subject: [PATCH 059/155] concurrency group
---
.github/workflows/preview-cloudformation-deploy.yml | 4 ++--
.github/workflows/preview-publish-images.yml | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml
index 3cbe2edf7c..064c6de0f3 100644
--- a/.github/workflows/preview-cloudformation-deploy.yml
+++ b/.github/workflows/preview-cloudformation-deploy.yml
@@ -1,11 +1,11 @@
-name: Deploy CloudFormation Stack for Preview Branches
+name: Deploy CloudFormation Stack for Preview Branch
on:
pull_request:
types: [opened, reopened]
concurrency:
- group: pr-${{ github.event.pull_request.number }}
+ group: pr-${{ github.event.pull_request.number }}-deploy-cloudformation
jobs:
deploy_cloudformation:
diff --git a/.github/workflows/preview-publish-images.yml b/.github/workflows/preview-publish-images.yml
index a07fac0fa0..7b49f79b4c 100644
--- a/.github/workflows/preview-publish-images.yml
+++ b/.github/workflows/preview-publish-images.yml
@@ -5,7 +5,7 @@ on:
types: [opened, synchronize, reopened]
concurrency:
- group: pr-${{ github.event.pull_request.number }}
+ group: pr-${{ github.event.pull_request.number }}-build-images
cancel-in-progress: true
jobs:
From 0a995ccefbd7e910424566e2c17169245ea520c9 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 22 Dec 2024 17:13:56 +0530
Subject: [PATCH 060/155] fix HostedZoneName
---
.../workflows/preview-cloudformation-deploy.yml | 4 ++--
.github/workflows/preview-publish-images.yml | 4 ++--
infra/aws-cloudformation/preview.yml | 15 ++++++---------
3 files changed, 10 insertions(+), 13 deletions(-)
diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml
index 064c6de0f3..bff4f28e2e 100644
--- a/.github/workflows/preview-cloudformation-deploy.yml
+++ b/.github/workflows/preview-cloudformation-deploy.yml
@@ -2,7 +2,7 @@ name: Deploy CloudFormation Stack for Preview Branch
on:
pull_request:
- types: [opened, reopened]
+ types: [opened, synchronize, reopened]
concurrency:
group: pr-${{ github.event.pull_request.number }}-deploy-cloudformation
@@ -16,7 +16,7 @@ jobs:
runs-on: blacksmith-2vcpu-ubuntu-2204
env:
- STACK_NAME: preview-pr-${{ github.event.pull_request.number }}
+ STACK_NAME: pr-${{ github.event.pull_request.number }}
STACK_PARAMS: >-
ParameterKey=LicenseKey,ParameterValue=5a32bd8a-409e-4733-8846-1868c568a813
ParameterKey=ImageTag,ParameterValue=pr-${{ github.event.pull_request.number }}
diff --git a/.github/workflows/preview-publish-images.yml b/.github/workflows/preview-publish-images.yml
index 7b49f79b4c..4c4c099224 100644
--- a/.github/workflows/preview-publish-images.yml
+++ b/.github/workflows/preview-publish-images.yml
@@ -1,4 +1,4 @@
-name: Build Docker Images for Preview Branches
+name: Build and Publish Images to ECR - Preview
on:
pull_request:
@@ -9,7 +9,7 @@ concurrency:
cancel-in-progress: true
jobs:
- build_images:
+ publish_images:
permissions:
contents: read
diff --git a/infra/aws-cloudformation/preview.yml b/infra/aws-cloudformation/preview.yml
index c7195e4bb9..32e1b142e2 100644
--- a/infra/aws-cloudformation/preview.yml
+++ b/infra/aws-cloudformation/preview.yml
@@ -11,9 +11,6 @@ Parameters:
Default: "latest"
Mappings:
- EnvironmentConfig:
- Preview:
- DomainName: "quadratic-preview.com"
RegionMap:
us-west-2:
AMI: "ami-05134c8ef96964280" # Example AMI for us-west-2 (Ubuntu 20.04)
@@ -132,15 +129,15 @@ Resources:
chmod +x /quadratic-selfhost/login.sh
# Run Quadratic initialization
- curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic-selfhost/main/init.sh -o init.sh && bash -i init.sh ${LicenseKey} ${ImageTag}.${!FindInMap [EnvironmentConfig, Preview, DomainName]}
+ curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic-selfhost/main/init.sh -o init.sh && bash -i init.sh ${LicenseKey} ${ImageTag}.quadratic-preview.com
DnsRecords:
Type: AWS::Route53::RecordSet
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
Properties:
- HostedZoneName: !Sub "${!FindInMap [EnvironmentConfig, Preview, DomainName]}."
- Name: !Sub "${ImageTag}.${!FindInMap [EnvironmentConfig, Preview, DomainName]}"
+ HostedZoneName: "quadratic-preview.com."
+ Name: !Sub "${ImageTag}.quadratic-preview.com"
Type: A
TTL: 300
ResourceRecords:
@@ -151,8 +148,8 @@ Resources:
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
Properties:
- HostedZoneName: !Sub "${!FindInMap [EnvironmentConfig, Preview, DomainName]}."
- Name: !Sub "*.${ImageTag}.${!FindInMap [EnvironmentConfig, Preview, DomainName]}"
+ HostedZoneName: "quadratic-preview.com."
+ Name: !Sub "*.${ImageTag}.quadratic-preview.com"
Type: A
TTL: 300
ResourceRecords:
@@ -230,4 +227,4 @@ Resources:
Outputs:
DomainRecord:
Description: "Url of the selfhosted instance"
- Value: !Sub "${ImageTag}.${!FindInMap [EnvironmentConfig, Preview, DomainName]}"
\ No newline at end of file
+ Value: !Sub "${ImageTag}.quadratic-preview.com"
From 55c770ba9dcad7559a72bacafc719b3a0e3b78ff Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 22 Dec 2024 17:20:17 +0530
Subject: [PATCH 061/155] fix InputTemplate
---
infra/aws-cloudformation/preview.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/infra/aws-cloudformation/preview.yml b/infra/aws-cloudformation/preview.yml
index 32e1b142e2..74331749f9 100644
--- a/infra/aws-cloudformation/preview.yml
+++ b/infra/aws-cloudformation/preview.yml
@@ -219,7 +219,7 @@ Resources:
"commands": [
"cd /quadratic-selfhost",
"./login.sh",
- "./start.sh",
+ "./start.sh"
]
}
}
From 315520948da32f2aa31d9bcfd07e220c6f1e6e91 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Mon, 23 Dec 2024 02:33:28 +0530
Subject: [PATCH 062/155] fix login script
---
.github/workflows/preview-cloudformation-deploy.yml | 2 +-
infra/aws-cloudformation/preview.yml | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml
index bff4f28e2e..4f26dfc3ca 100644
--- a/.github/workflows/preview-cloudformation-deploy.yml
+++ b/.github/workflows/preview-cloudformation-deploy.yml
@@ -1,4 +1,4 @@
-name: Deploy CloudFormation Stack for Preview Branch
+name: Deploy CloudFormation Stack - Preview
on:
pull_request:
diff --git a/infra/aws-cloudformation/preview.yml b/infra/aws-cloudformation/preview.yml
index 74331749f9..8e5fb026ad 100644
--- a/infra/aws-cloudformation/preview.yml
+++ b/infra/aws-cloudformation/preview.yml
@@ -123,14 +123,14 @@ Resources:
# Source the new environment variables
source /home/ubuntu/.profile
+ # Run Quadratic initialization
+ curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic-selfhost/main/init.sh -o init.sh && bash -i init.sh ${LicenseKey} ${ImageTag}.quadratic-preview.com
+
# Create login.sh script
echo '#!/bin/bash
aws ecr get-login-password --region ${AWS::Region} | docker login --username AWS --password-stdin ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com' > /quadratic-selfhost/login.sh
chmod +x /quadratic-selfhost/login.sh
- # Run Quadratic initialization
- curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic-selfhost/main/init.sh -o init.sh && bash -i init.sh ${LicenseKey} ${ImageTag}.quadratic-preview.com
-
DnsRecords:
Type: AWS::Route53::RecordSet
DeletionPolicy: Delete
From e17e26741b05e34fcfa1fd4e7f753b5c8b62c557 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Mon, 23 Dec 2024 03:27:39 +0530
Subject: [PATCH 063/155] fix github comment
---
.../preview-cloudformation-deploy.yml | 4 ++--
.github/workflows/preview-publish-images.yml | 8 ++++----
infra/aws-cloudformation/preview.yml | 19 ++++++++++++++++++-
3 files changed, 24 insertions(+), 7 deletions(-)
diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml
index 4f26dfc3ca..3e4946b625 100644
--- a/.github/workflows/preview-cloudformation-deploy.yml
+++ b/.github/workflows/preview-cloudformation-deploy.yml
@@ -69,9 +69,9 @@ jobs:
script: |
const previewUrl = '${{ steps.stack-output.outputs.PREVIEW_URL }}';
const message = `Preview URL: ${previewUrl}`;
- github.rest.issues.createComment({
- issue_number: context.issue.number,
+ await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.name,
+ issue_number: context.issue.number,
body: message
});
diff --git a/.github/workflows/preview-publish-images.yml b/.github/workflows/preview-publish-images.yml
index 4c4c099224..8bbc2ddbf4 100644
--- a/.github/workflows/preview-publish-images.yml
+++ b/.github/workflows/preview-publish-images.yml
@@ -5,7 +5,7 @@ on:
types: [opened, synchronize, reopened]
concurrency:
- group: pr-${{ github.event.pull_request.number }}-build-images
+ group: pr-${{ github.event.pull_request.number }}-publish-images
cancel-in-progress: true
jobs:
@@ -87,13 +87,13 @@ jobs:
context: .
file: quadratic-${{ matrix.service }}/Dockerfile
push: true
- tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }}
+ tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }}-${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
build-args: |
BUILDKIT_INLINE_CACHE=1
BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }}
- GIT_SHA=${{ github.sha }}
+ GIT_SHA_SHORT=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
BRANCH_NAME=${{ steps.build-metadata.outputs.BRANCH_NAME }}
PR_NUMBER=${{ github.event.pull_request.number }}
labels: |
org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}
- org.opencontainers.image.revision=${{ github.sha }}
+ org.opencontainers.image.revision=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
diff --git a/infra/aws-cloudformation/preview.yml b/infra/aws-cloudformation/preview.yml
index 8e5fb026ad..b6c50c94e9 100644
--- a/infra/aws-cloudformation/preview.yml
+++ b/infra/aws-cloudformation/preview.yml
@@ -8,7 +8,6 @@ Parameters:
ImageTag:
Type: String
Description: "Image tag to use for all services"
- Default: "latest"
Mappings:
RegionMap:
@@ -23,6 +22,9 @@ Resources:
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
Properties:
+ Tags:
+ - Key: Name
+ Value: !Sub "pr-${ImageTag}"
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
@@ -53,6 +55,9 @@ Resources:
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
Properties:
+ Tags:
+ - Key: Name
+ Value: !Sub "pr-${ImageTag}"
Roles:
- !Ref EC2Role
@@ -61,6 +66,9 @@ Resources:
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
Properties:
+ Tags:
+ - Key: Name
+ Value: !Sub "pr-${ImageTag}"
GroupDescription: !Sub "${ImageTag}"
SecurityGroupIngress:
- IpProtocol: "tcp"
@@ -77,6 +85,9 @@ Resources:
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
Properties:
+ Tags:
+ - Key: Name
+ Value: !Sub "pr-${ImageTag}"
InstanceType: "m6a.large"
ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI]
IamInstanceProfile: !Ref EC2InstanceProfile
@@ -160,6 +171,9 @@ Resources:
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
Properties:
+ Tags:
+ - Key: Name
+ Value: !Sub "pr-${ImageTag}"
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
@@ -185,6 +199,9 @@ Resources:
- EC2Instance
- EventBridgeRole
Properties:
+ Tags:
+ - Key: Name
+ Value: !Sub "pr-${ImageTag}"
Description: "Rule to detect ECR image pushes"
EventPattern:
source:
From ad15f68dfbdb63a99747fae2d747e54fdf192826 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Mon, 23 Dec 2024 03:30:54 +0530
Subject: [PATCH 064/155] remove tag on ec2 profile
---
infra/aws-cloudformation/preview.yml | 3 ---
1 file changed, 3 deletions(-)
diff --git a/infra/aws-cloudformation/preview.yml b/infra/aws-cloudformation/preview.yml
index b6c50c94e9..a7a71962c1 100644
--- a/infra/aws-cloudformation/preview.yml
+++ b/infra/aws-cloudformation/preview.yml
@@ -55,9 +55,6 @@ Resources:
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
Properties:
- Tags:
- - Key: Name
- Value: !Sub "pr-${ImageTag}"
Roles:
- !Ref EC2Role
From 6555269fd2f89578463a717352667d7f17de654b Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Mon, 23 Dec 2024 03:56:11 +0530
Subject: [PATCH 065/155] use aws-cloudformation-github-deploy action
---
.../preview-cloudformation-deploy.yml | 75 ++++++++-----------
infra/aws-cloudformation/preview.yml | 13 +++-
2 files changed, 39 insertions(+), 49 deletions(-)
diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml
index 3e4946b625..579669ef47 100644
--- a/.github/workflows/preview-cloudformation-deploy.yml
+++ b/.github/workflows/preview-cloudformation-deploy.yml
@@ -1,4 +1,4 @@
-name: Deploy CloudFormation Stack - Preview
+name: Deploy to AWS CloudFormation - Preview
on:
pull_request:
@@ -17,9 +17,6 @@ jobs:
env:
STACK_NAME: pr-${{ github.event.pull_request.number }}
- STACK_PARAMS: >-
- ParameterKey=LicenseKey,ParameterValue=5a32bd8a-409e-4733-8846-1868c568a813
- ParameterKey=ImageTag,ParameterValue=pr-${{ github.event.pull_request.number }}
steps:
- uses: actions/checkout@v4
@@ -32,46 +29,34 @@ jobs:
aws-region: ${{ secrets.AWS_REGION }}
- name: Deploy CloudFormation Stack
- run: |
- # Check if stack exists
- if aws cloudformation describe-stacks --stack-name $STACK_NAME 2>/dev/null; then
- # Update existing stack
- aws cloudformation update-stack \
- --stack-name $STACK_NAME \
- --template-body file://infra/aws-cloudformation/preview.yml \
- --parameters $STACK_PARAMS \
- --capabilities CAPABILITY_IAM \
- --no-fail-on-empty-changeset
- else
- # Create new stack
- aws cloudformation create-stack \
- --stack-name $STACK_NAME \
- --template-body file://infra/aws-cloudformation/preview.yml \
- --parameters $STACK_PARAMS \
- --capabilities CAPABILITY_IAM
-
- # Wait for stack creation to complete
- aws cloudformation wait stack-create-complete --stack-name $STACK_NAME
- fi
-
- - name: Get Stack Outputs
- id: stack-output
- run: |
- DOMAIN=$(aws cloudformation describe-stacks \
- --stack-name $STACK_NAME \
- --query 'Stacks[0].Outputs[?OutputKey==`DomainRecord`].OutputValue' \
- --output text)
- echo "PREVIEW_URL=https://$DOMAIN" >> $GITHUB_OUTPUT
+ id: deploy-stack
+ uses: aws-actions/aws-cloudformation-github-deploy@v1
+ with:
+ name: ${{ env.STACK_NAME }}
+ template: infra/aws-cloudformation/preview.yml
+ parameter-overrides: >-
+ LicenseKey=5a32bd8a-409e-4733-8846-1868c568a813,
+ ImageTag=pr-${{ github.event.pull_request.number }}
+ capabilities: CAPABILITY_IAM
+ no-fail-on-empty-changeset: "1"
+ disable-rollback: false
+
+ - name: Find Comment
+ uses: peter-evans/find-comment@v3
+ id: preview-comment
+ with:
+ issue-number: ${{ github.event.pull_request.number }}
+ comment-author: 'github-actions[bot]'
+ body-includes: 'Preview Deployment'
- - name: Comment Preview URL
- uses: actions/github-script@v7
+ - name: Create or update comment
+ if: steps.deploy-stack.outputs.DnsRecord != ''
+ uses: peter-evans/create-or-update-comment@v3
with:
- script: |
- const previewUrl = '${{ steps.stack-output.outputs.PREVIEW_URL }}';
- const message = `Preview URL: ${previewUrl}`;
- await github.rest.issues.createComment({
- owner: context.repo.owner,
- repo: context.repo.name,
- issue_number: context.issue.number,
- body: message
- });
+ comment-id: ${{ steps.preview-comment.outputs.comment-id }}
+ issue-number: ${{ github.event.pull_request.number }}
+ body: |
+ ## Preview Deployment
+ Preview URL: https://${{ steps.deploy-stack.outputs.DnsRecord }}
+ edit-mode: replace
+
diff --git a/infra/aws-cloudformation/preview.yml b/infra/aws-cloudformation/preview.yml
index a7a71962c1..68211d8f38 100644
--- a/infra/aws-cloudformation/preview.yml
+++ b/infra/aws-cloudformation/preview.yml
@@ -16,6 +16,14 @@ Mappings:
ap-south-1:
AMI: "ami-0522ab6e1ddcc7055" # Example AMI for ap-south-1 (Ubuntu 24.04 LTS)
+RollbackConfiguration:
+ MonitoringTimeInMinutes: 0
+ RollbackTriggers: []
+
+OnFailure: ROLLBACK
+DeletionPolicy: Delete
+UpdateReplacePolicy: Delete
+
Resources:
EC2Role:
Type: AWS::IAM::Role
@@ -196,9 +204,6 @@ Resources:
- EC2Instance
- EventBridgeRole
Properties:
- Tags:
- - Key: Name
- Value: !Sub "pr-${ImageTag}"
Description: "Rule to detect ECR image pushes"
EventPattern:
source:
@@ -239,6 +244,6 @@ Resources:
}
Outputs:
- DomainRecord:
+ DnsRecord:
Description: "Url of the selfhosted instance"
Value: !Sub "${ImageTag}.quadratic-preview.com"
From 74ec7ce38178e60f3ee11e3f355548f61f6d0184 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Mon, 23 Dec 2024 04:03:45 +0530
Subject: [PATCH 066/155] remove rollback and delete attributes
---
.github/workflows/preview-cloudformation-deploy.yml | 1 +
infra/aws-cloudformation/preview.yml | 8 --------
2 files changed, 1 insertion(+), 8 deletions(-)
diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml
index 579669ef47..10912d404b 100644
--- a/.github/workflows/preview-cloudformation-deploy.yml
+++ b/.github/workflows/preview-cloudformation-deploy.yml
@@ -40,6 +40,7 @@ jobs:
capabilities: CAPABILITY_IAM
no-fail-on-empty-changeset: "1"
disable-rollback: false
+ termination-protection: false
- name: Find Comment
uses: peter-evans/find-comment@v3
diff --git a/infra/aws-cloudformation/preview.yml b/infra/aws-cloudformation/preview.yml
index 68211d8f38..23bc166c9a 100644
--- a/infra/aws-cloudformation/preview.yml
+++ b/infra/aws-cloudformation/preview.yml
@@ -16,14 +16,6 @@ Mappings:
ap-south-1:
AMI: "ami-0522ab6e1ddcc7055" # Example AMI for ap-south-1 (Ubuntu 24.04 LTS)
-RollbackConfiguration:
- MonitoringTimeInMinutes: 0
- RollbackTriggers: []
-
-OnFailure: ROLLBACK
-DeletionPolicy: Delete
-UpdateReplacePolicy: Delete
-
Resources:
EC2Role:
Type: AWS::IAM::Role
From 8bafe875c66a82d5a0d4638986b880f209e52c84 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Mon, 23 Dec 2024 04:14:15 +0530
Subject: [PATCH 067/155] add write permission for comment
---
.github/workflows/preview-cloudformation-deploy.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml
index 10912d404b..7c579d1514 100644
--- a/.github/workflows/preview-cloudformation-deploy.yml
+++ b/.github/workflows/preview-cloudformation-deploy.yml
@@ -12,6 +12,7 @@ jobs:
permissions:
contents: read
id-token: write
+ pull-requests: write
runs-on: blacksmith-2vcpu-ubuntu-2204
From 62e476df0f91d11c363755ff539d3e6544c1afe2 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Mon, 23 Dec 2024 21:04:22 +0530
Subject: [PATCH 068/155] simple selfhosting deployment
---
.../preview-cloudformation-deploy.yml | 8 +-
infra/aws-cloudformation/ami.sh | 28 ++
.../preview-deployment-template.yml | 227 +++++++++++++++++
infra/aws-cloudformation/preview.yml | 241 ------------------
4 files changed, 262 insertions(+), 242 deletions(-)
create mode 100644 infra/aws-cloudformation/ami.sh
create mode 100644 infra/aws-cloudformation/preview-deployment-template.yml
delete mode 100644 infra/aws-cloudformation/preview.yml
diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml
index 7c579d1514..36d4cc6097 100644
--- a/.github/workflows/preview-cloudformation-deploy.yml
+++ b/.github/workflows/preview-cloudformation-deploy.yml
@@ -22,6 +22,11 @@ jobs:
steps:
- uses: actions/checkout@v4
+ - name: Generate Build Metadata
+ id: build-metadata
+ run: |
+ echo "GIT_SHA_SHORT=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
@@ -34,10 +39,11 @@ jobs:
uses: aws-actions/aws-cloudformation-github-deploy@v1
with:
name: ${{ env.STACK_NAME }}
- template: infra/aws-cloudformation/preview.yml
+ template: infra/aws-cloudformation/preview-deployment-template.yml
parameter-overrides: >-
LicenseKey=5a32bd8a-409e-4733-8846-1868c568a813,
ImageTag=pr-${{ github.event.pull_request.number }}
+ GitSHA=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
capabilities: CAPABILITY_IAM
no-fail-on-empty-changeset: "1"
disable-rollback: false
diff --git a/infra/aws-cloudformation/ami.sh b/infra/aws-cloudformation/ami.sh
new file mode 100644
index 0000000000..968860d9c0
--- /dev/null
+++ b/infra/aws-cloudformation/ami.sh
@@ -0,0 +1,28 @@
+# script used to build quadratic-preview-ami-base
+
+#!/bin/bash
+
+# Update and install dependencies
+sudo apt-get update
+sudo apt-get install -y apt-transport-https ca-certificates curl software-properties-common unzip jq
+
+# Install AWS CLI v2
+curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+unzip awscliv2.zip
+sudo ./aws/install
+rm -rf aws awscliv2.zip
+
+# Install Docker
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+sudo apt-get update
+sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+
+# Install Docker Compose
+sudo curl -L "https://github.com/docker/compose/releases/download/v2.21.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+sudo chmod +x /usr/local/bin/docker-compose
+
+# Configure Docker
+sudo systemctl enable docker
+sudo systemctl start docker
+sudo usermod -aG docker ubuntu
\ No newline at end of file
diff --git a/infra/aws-cloudformation/preview-deployment-template.yml b/infra/aws-cloudformation/preview-deployment-template.yml
new file mode 100644
index 0000000000..c71d69bd0d
--- /dev/null
+++ b/infra/aws-cloudformation/preview-deployment-template.yml
@@ -0,0 +1,227 @@
+AWSTemplateFormatVersion: 2010-09-09
+Description: Quadratic Preview - Deployment Template
+
+Parameters:
+ LicenseKey:
+ Type: String
+ Description: Your license key for Quadratic. Get one here https://selfhost.quadratichq.com/
+ ImageTag:
+ Type: String
+ Description: Image tag to use for all services
+ GitSHA:
+ Type: String
+ Description: Git short SHA to use for all services
+
+Resources:
+ SecurityGroup:
+ Type: AWS::EC2::SecurityGroup
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ Properties:
+ Tags:
+ - Key: Name
+ Value: !Ref ImageTag
+ GroupDescription: !Ref ImageTag
+ SecurityGroupIngress:
+ - IpProtocol: tcp
+ FromPort: 80
+ ToPort: 80
+ CidrIp: 0.0.0.0/0
+ - IpProtocol: tcp
+ FromPort: 443
+ ToPort: 443
+ CidrIp: 0.0.0.0/0
+
+ EC2Role:
+ Type: AWS::IAM::Role
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ Properties:
+ Tags:
+ - Key: Name
+ Value: !Ref ImageTag
+ AssumeRolePolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ - Effect: Allow
+ Principal:
+ Service: ec2.amazonaws.com
+ Action: sts:AssumeRole
+ ManagedPolicyArns:
+ - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
+ - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
+ Policies:
+ - PolicyName: ECRAccess
+ PolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ - Effect: Allow
+ Action:
+ - ecr:GetAuthorizationToken
+ - ecr:BatchCheckLayerAvailability
+ - ecr:GetDownloadUrlForLayer
+ - ecr:BatchGetImage
+ - ecr:DescribeImages
+ - ecr:ListImages
+ Resource: "*"
+
+ EC2InstanceProfile:
+ Type: AWS::IAM::InstanceProfile
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ DependsOn:
+ - EC2Role
+ Properties:
+ Roles:
+ - !Ref EC2Role
+
+ EC2Instance:
+ Type: AWS::EC2::Instance
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ DependsOn:
+ - EC2InstanceProfile
+ - SecurityGroup
+ Properties:
+ Tags:
+ - Key: Name
+ Value: !Ref ImageTag
+ InstanceType: m6a.large
+ ImageId: ami-0fd0cf59e80d3f545 # quadratic-preview-ami-base
+ IamInstanceProfile: !Ref EC2InstanceProfile
+ SecurityGroups:
+ - !Ref SecurityGroup
+ BlockDeviceMappings:
+ - DeviceName: /dev/sda1
+ Ebs:
+ VolumeSize: 30
+ VolumeType: gp3
+ UserData:
+ Fn::Base64: !Sub |
+ #!/bin/bash
+
+ # Export environment variables for docker-compose
+ echo "export ECR_URL=${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com" >> /home/ubuntu/.profile
+ echo "export IMAGE_TAG=${ImageTag}" >> /home/ubuntu/.profile
+ echo "export GIT_SHA=${GitSHA}" >> /home/ubuntu/.profile
+
+ # Source the new environment variables
+ source /home/ubuntu/.profile
+
+ # Create login.sh script and run it
+ cat << 'EOF' > /home/ubuntu/quadratic-selfhost/login.sh
+ #!/bin/bash
+ aws ecr get-login-password --region ${AWS::Region} | docker login --username AWS --password-stdin ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com
+ EOF
+
+ chmod +x /home/ubuntu/quadratic-selfhost/login.sh
+ cd /home/ubuntu/quadratic-selfhost
+ ./login.sh
+
+ # Run Quadratic initialization script
+ ./init.sh "${LicenseKey}" "${ImageTag}-${GitSHA}.quadratic-preview.com"
+
+ EventBridgeRole:
+ Type: AWS::IAM::Role
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ Properties:
+ Tags:
+ - Key: Name
+ Value: !Ref ImageTag
+ AssumeRolePolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ - Effect: Allow
+ Principal:
+ Service: events.amazonaws.com
+ Action: sts:AssumeRole
+ Policies:
+ - PolicyName: InvokeSsmAutomation
+ PolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ - Effect: Allow
+ Action:
+ - ssm:SendCommand
+ Resource: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:*"
+
+ EcrPushRule:
+ Type: AWS::Events::Rule
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ DependsOn:
+ - EC2Instance
+ - EventBridgeRole
+ Properties:
+ Description: Rule to detect ECR image pushes
+ EventPattern:
+ source:
+ - aws.ecr
+ detail-type:
+ - ECR Image Action
+ detail:
+ action-type:
+ - PUSH
+ repository-name:
+ - quadratic-client-development
+ - quadratic-api-development
+ - quadratic-multiplayer-development
+ - quadratic-files-development
+ - quadratic-connection-development
+ image-tag:
+ - !Ref ImageTag
+ State: ENABLED
+ Targets:
+ - Arn: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:automation-definition/AWS-RunShellScript"
+ Id: TriggerScriptOnInstance
+ RoleArn: !GetAtt EventBridgeRole.Arn
+ InputTransformer:
+ InputPathsMap:
+ imageTag: $.detail.image-tag
+ repository: $.detail.repository-name
+ InputTemplate: |
+ {
+ "InstanceIds": ["${EC2Instance.InstanceId}"],
+ "DocumentName": "AWS-RunShellScript",
+ "Parameters": {
+ "commands": [
+ "cd /home/ubuntu/quadratic-selfhost",
+ "./login.sh",
+ "./start.sh"
+ ]
+ }
+ }
+
+ DnsRecords:
+ Type: AWS::Route53::RecordSet
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ DependsOn:
+ - EC2Instance
+ Properties:
+ HostedZoneId: Z0126430TJ1UYIMO3SYX
+ Name: !Sub "${ImageTag}-${GitSHA}.quadratic-preview.com"
+ Type: A
+ TTL: 300
+ ResourceRecords:
+ - !GetAtt EC2Instance.PublicIp
+
+ WildcardDnsRecord:
+ Type: AWS::Route53::RecordSet
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ DependsOn:
+ - EC2Instance
+ Properties:
+ HostedZoneId: Z0126430TJ1UYIMO3SYX
+ Name: !Sub "*.${ImageTag}-${GitSHA}.quadratic-preview.com"
+ Type: A
+ TTL: 300
+ ResourceRecords:
+ - !GetAtt EC2Instance.PublicIp
+
+Outputs:
+ DnsRecord:
+ Description: Url of the preview instance
+ Value: !Sub "${ImageTag}-${GitSHA}.quadratic-preview.com"
diff --git a/infra/aws-cloudformation/preview.yml b/infra/aws-cloudformation/preview.yml
deleted file mode 100644
index 23bc166c9a..0000000000
--- a/infra/aws-cloudformation/preview.yml
+++ /dev/null
@@ -1,241 +0,0 @@
-AWSTemplateFormatVersion: "2010-09-09"
-Description: "Quadratic Selfhost Cloudformation Template"
-
-Parameters:
- LicenseKey:
- Type: String
- Description: "Your license key for Quadratic. Get one here https://selfhost.quadratichq.com/"
- ImageTag:
- Type: String
- Description: "Image tag to use for all services"
-
-Mappings:
- RegionMap:
- us-west-2:
- AMI: "ami-05134c8ef96964280" # Example AMI for us-west-2 (Ubuntu 20.04)
- ap-south-1:
- AMI: "ami-0522ab6e1ddcc7055" # Example AMI for ap-south-1 (Ubuntu 24.04 LTS)
-
-Resources:
- EC2Role:
- Type: AWS::IAM::Role
- DeletionPolicy: Delete
- UpdateReplacePolicy: Delete
- Properties:
- Tags:
- - Key: Name
- Value: !Sub "pr-${ImageTag}"
- AssumeRolePolicyDocument:
- Version: "2012-10-17"
- Statement:
- - Effect: Allow
- Principal:
- Service: ec2.amazonaws.com
- Action: sts:AssumeRole
- ManagedPolicyArns:
- - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
- - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
- Policies:
- - PolicyName: ECRAccess
- PolicyDocument:
- Version: "2012-10-17"
- Statement:
- - Effect: Allow
- Action:
- - ecr:GetAuthorizationToken
- - ecr:BatchCheckLayerAvailability
- - ecr:GetDownloadUrlForLayer
- - ecr:BatchGetImage
- - ecr:DescribeImages
- - ecr:ListImages
- Resource: "*"
-
- EC2InstanceProfile:
- Type: AWS::IAM::InstanceProfile
- DeletionPolicy: Delete
- UpdateReplacePolicy: Delete
- Properties:
- Roles:
- - !Ref EC2Role
-
- OpenSecurityGroup:
- Type: "AWS::EC2::SecurityGroup"
- DeletionPolicy: Delete
- UpdateReplacePolicy: Delete
- Properties:
- Tags:
- - Key: Name
- Value: !Sub "pr-${ImageTag}"
- GroupDescription: !Sub "${ImageTag}"
- SecurityGroupIngress:
- - IpProtocol: "tcp"
- FromPort: "80"
- ToPort: "80"
- CidrIp: "0.0.0.0/0"
- - IpProtocol: "tcp"
- FromPort: "443"
- ToPort: "443"
- CidrIp: "0.0.0.0/0"
-
- EC2Instance:
- Type: "AWS::EC2::Instance"
- DeletionPolicy: Delete
- UpdateReplacePolicy: Delete
- Properties:
- Tags:
- - Key: Name
- Value: !Sub "pr-${ImageTag}"
- InstanceType: "m6a.large"
- ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI]
- IamInstanceProfile: !Ref EC2InstanceProfile
- SecurityGroups:
- - !Ref OpenSecurityGroup
- BlockDeviceMappings:
- - DeviceName: "/dev/sda1"
- Ebs:
- VolumeSize: "25"
- VolumeType: "gp3"
- UserData:
- Fn::Base64: !Sub |
- #!/bin/bash
-
- # Update and install dependencies
- sudo apt-get update
- sudo apt-get install -y apt-transport-https ca-certificates curl software-properties-common unzip jq
-
- # Install AWS CLI v2
- curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
- unzip awscliv2.zip
- sudo ./aws/install
- rm -rf aws awscliv2.zip
-
- # Install Docker
- curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
- echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
- sudo apt-get update
- sudo apt-get install -y docker-ce docker-ce-cli containerd.io
-
- # Install Docker Compose
- sudo curl -L "https://github.com/docker/compose/releases/download/v2.21.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
- sudo chmod +x /usr/local/bin/docker-compose
-
- # Configure Docker
- sudo systemctl enable docker
- sudo systemctl start docker
- sudo usermod -aG docker ubuntu
-
- # Export environment variables for docker-compose
- echo 'export ECR_URL="${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com"' >> /home/ubuntu/.profile
- echo 'export IMAGE_TAG="${ImageTag}"' >> /home/ubuntu/.profile
-
- # Source the new environment variables
- source /home/ubuntu/.profile
-
- # Run Quadratic initialization
- curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic-selfhost/main/init.sh -o init.sh && bash -i init.sh ${LicenseKey} ${ImageTag}.quadratic-preview.com
-
- # Create login.sh script
- echo '#!/bin/bash
- aws ecr get-login-password --region ${AWS::Region} | docker login --username AWS --password-stdin ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com' > /quadratic-selfhost/login.sh
- chmod +x /quadratic-selfhost/login.sh
-
- DnsRecords:
- Type: AWS::Route53::RecordSet
- DeletionPolicy: Delete
- UpdateReplacePolicy: Delete
- Properties:
- HostedZoneName: "quadratic-preview.com."
- Name: !Sub "${ImageTag}.quadratic-preview.com"
- Type: A
- TTL: 300
- ResourceRecords:
- - !GetAtt EC2Instance.PublicIp
-
- WildcardDnsRecord:
- Type: AWS::Route53::RecordSet
- DeletionPolicy: Delete
- UpdateReplacePolicy: Delete
- Properties:
- HostedZoneName: "quadratic-preview.com."
- Name: !Sub "*.${ImageTag}.quadratic-preview.com"
- Type: A
- TTL: 300
- ResourceRecords:
- - !GetAtt EC2Instance.PublicIp
-
- EventBridgeRole:
- Type: AWS::IAM::Role
- DeletionPolicy: Delete
- UpdateReplacePolicy: Delete
- Properties:
- Tags:
- - Key: Name
- Value: !Sub "pr-${ImageTag}"
- AssumeRolePolicyDocument:
- Version: "2012-10-17"
- Statement:
- - Effect: Allow
- Principal:
- Service: events.amazonaws.com
- Action: sts:AssumeRole
- Policies:
- - PolicyName: InvokeSsmAutomation
- PolicyDocument:
- Version: "2012-10-17"
- Statement:
- - Effect: Allow
- Action:
- - ssm:SendCommand
- Resource: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:*"
-
- EcrPushRule:
- Type: AWS::Events::Rule
- DeletionPolicy: Delete
- UpdateReplacePolicy: Delete
- DependsOn:
- - EC2Instance
- - EventBridgeRole
- Properties:
- Description: "Rule to detect ECR image pushes"
- EventPattern:
- source:
- - aws.ecr
- detail-type:
- - "ECR Image Action"
- detail:
- action-type:
- - "PUSH"
- repository-name:
- - "quadratic-client-development"
- - "quadratic-api-development"
- - "quadratic-multiplayer-development"
- - "quadratic-files-development"
- - "quadratic-connection-development"
- image-tag:
- - !Ref ImageTag
- State: "ENABLED"
- Targets:
- - Arn: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:automation-definition/AWS-RunShellScript"
- Id: "TriggerScriptOnInstance"
- RoleArn: !GetAtt EventBridgeRole.Arn
- InputTransformer:
- InputPathsMap:
- "imageTag": "$.detail.image-tag"
- "repository": "$.detail.repository-name"
- InputTemplate: |
- {
- "InstanceIds": ["${EC2Instance.InstanceId}"],
- "DocumentName": "AWS-RunShellScript",
- "Parameters": {
- "commands": [
- "cd /quadratic-selfhost",
- "./login.sh",
- "./start.sh"
- ]
- }
- }
-
-Outputs:
- DnsRecord:
- Description: "Url of the selfhosted instance"
- Value: !Sub "${ImageTag}.quadratic-preview.com"
From e903e739047c1ec0fec78205dd0afe0cbb47ecd4 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Mon, 23 Dec 2024 21:07:30 +0530
Subject: [PATCH 069/155] comma
---
.github/workflows/preview-cloudformation-deploy.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml
index 36d4cc6097..0bc7d69a79 100644
--- a/.github/workflows/preview-cloudformation-deploy.yml
+++ b/.github/workflows/preview-cloudformation-deploy.yml
@@ -42,7 +42,7 @@ jobs:
template: infra/aws-cloudformation/preview-deployment-template.yml
parameter-overrides: >-
LicenseKey=5a32bd8a-409e-4733-8846-1868c568a813,
- ImageTag=pr-${{ github.event.pull_request.number }}
+ ImageTag=pr-${{ github.event.pull_request.number }},
GitSHA=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
capabilities: CAPABILITY_IAM
no-fail-on-empty-changeset: "1"
From c7518827b23d8d54320d8e651d35b975c69b1709 Mon Sep 17 00:00:00 2001
From: Jim Nielsen
Date: Mon, 23 Dec 2024 09:32:04 -0700
Subject: [PATCH 070/155] Update EmptyGridMessage.tsx
---
quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx b/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
index b779dc9ae3..bc91b026b8 100644
--- a/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
+++ b/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
@@ -22,7 +22,7 @@ export function EmptyGridMessage() {
team: { uuid: teamUuid },
} = useFileRouteLoaderData();
const canEdit = filePermissions.includes('FILE_EDIT');
- const [open, setOpen] = useState(false);
+ const [open, setOpen] = useState(fileHasData() ? false : true);
const showConnectionsMenu = useSetRecoilState(editorInteractionStateShowConnectionsMenuAtom);
const showCellTypeMenu = useSetRecoilState(editorInteractionStateShowCellTypeMenuAtom);
const { data } = useConnectionsFetcher();
From 0cceb805b25c2f1cb6dba59f4b8a36d5f1dbf46d Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Mon, 23 Dec 2024 22:25:26 +0530
Subject: [PATCH 071/155] license key
---
.github/workflows/preview-cloudformation-deploy.yml | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml
index 0bc7d69a79..a0514566e2 100644
--- a/.github/workflows/preview-cloudformation-deploy.yml
+++ b/.github/workflows/preview-cloudformation-deploy.yml
@@ -13,7 +13,7 @@ jobs:
contents: read
id-token: write
pull-requests: write
-
+
runs-on: blacksmith-2vcpu-ubuntu-2204
env:
@@ -41,7 +41,7 @@ jobs:
name: ${{ env.STACK_NAME }}
template: infra/aws-cloudformation/preview-deployment-template.yml
parameter-overrides: >-
- LicenseKey=5a32bd8a-409e-4733-8846-1868c568a813,
+ LicenseKey=${{ secrets.QUADRATIC_LICENSE_KEY }},
ImageTag=pr-${{ github.event.pull_request.number }},
GitSHA=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
capabilities: CAPABILITY_IAM
@@ -54,8 +54,8 @@ jobs:
id: preview-comment
with:
issue-number: ${{ github.event.pull_request.number }}
- comment-author: 'github-actions[bot]'
- body-includes: 'Preview Deployment'
+ comment-author: "github-actions[bot]"
+ body-includes: "Preview Deployment"
- name: Create or update comment
if: steps.deploy-stack.outputs.DnsRecord != ''
@@ -67,4 +67,3 @@ jobs:
## Preview Deployment
Preview URL: https://${{ steps.deploy-stack.outputs.DnsRecord }}
edit-mode: replace
-
From c0c776d102a02bc32a637be7e97935af5d4cea6f Mon Sep 17 00:00:00 2001
From: Jim Nielsen
Date: Mon, 23 Dec 2024 10:33:50 -0700
Subject: [PATCH 072/155] Update ci.yml
---
.github/workflows/ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b53a5372d4..57b930abb8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -195,7 +195,7 @@ jobs:
lint:
runs-on: ubuntu-latest-8-cores
- timeout-minutes: 10
+ timeout-minutes: 15
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
From 005196003a8af42cd321ebdac73e17b73cab8538 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Mon, 23 Dec 2024 23:44:29 +0530
Subject: [PATCH 073/155] clippy
---
quadratic-core/src/grid/sheet/cell_array.rs | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/quadratic-core/src/grid/sheet/cell_array.rs b/quadratic-core/src/grid/sheet/cell_array.rs
index df6a704fa8..a6e087db1b 100644
--- a/quadratic-core/src/grid/sheet/cell_array.rs
+++ b/quadratic-core/src/grid/sheet/cell_array.rs
@@ -73,11 +73,11 @@ impl Sheet {
.x_range()
.map(|x| {
let pos = Pos { x, y };
- let cell_value = self.cell_value(pos).unwrap_or_else(|| CellValue::Blank);
+ let cell_value = self.cell_value(pos).unwrap_or(CellValue::Blank);
match (include_code, &cell_value) {
(true, CellValue::Code(_)) => cell_value,
- (_, _) => self.display_value(pos).unwrap_or_else(|| CellValue::Blank),
+ (_, _) => self.display_value(pos).unwrap_or(CellValue::Blank),
}
})
.collect::>()
From fb67c8f7c9f34b89e0a3e65981251bb1a2108639 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 00:27:12 +0530
Subject: [PATCH 074/155] use global accelerator
---
.../preview-deployment-template.yml | 63 ++++++++++++++++---
1 file changed, 53 insertions(+), 10 deletions(-)
diff --git a/infra/aws-cloudformation/preview-deployment-template.yml b/infra/aws-cloudformation/preview-deployment-template.yml
index c71d69bd0d..91197dcc7b 100644
--- a/infra/aws-cloudformation/preview-deployment-template.yml
+++ b/infra/aws-cloudformation/preview-deployment-template.yml
@@ -104,7 +104,7 @@ Resources:
echo "export ECR_URL=${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com" >> /home/ubuntu/.profile
echo "export IMAGE_TAG=${ImageTag}" >> /home/ubuntu/.profile
echo "export GIT_SHA=${GitSHA}" >> /home/ubuntu/.profile
-
+
# Source the new environment variables
source /home/ubuntu/.profile
@@ -113,7 +113,7 @@ Resources:
#!/bin/bash
aws ecr get-login-password --region ${AWS::Region} | docker login --username AWS --password-stdin ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com
EOF
-
+
chmod +x /home/ubuntu/quadratic-selfhost/login.sh
cd /home/ubuntu/quadratic-selfhost
./login.sh
@@ -193,33 +193,76 @@ Resources:
}
}
+ GlobalAccelerator:
+ Type: AWS::GlobalAccelerator::Accelerator
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ Properties:
+ Name: !Sub "${ImageTag}-${GitSHA}"
+ Enabled: true
+ Tags:
+ - Key: Name
+ Value: !Ref ImageTag
+
+ GlobalAcceleratorListener:
+ Type: AWS::GlobalAccelerator::Listener
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ DependsOn:
+ - GlobalAccelerator
+ Properties:
+ AcceleratorArn: !Ref GlobalAccelerator
+ Protocol: TCP
+ PortRanges:
+ - FromPort: 80
+ ToPort: 80
+ - FromPort: 443
+ ToPort: 443
+
+ GlobalAcceleratorEndpointGroup:
+ Type: AWS::GlobalAccelerator::EndpointGroup
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ DependsOn:
+ - GlobalAcceleratorListener
+ - EC2Instance
+ Properties:
+ ListenerArn: !Ref GlobalAcceleratorListener
+ EndpointGroupRegion: !Ref AWS::Region
+ EndpointConfigurations:
+ - EndpointId: !Ref EC2Instance
+ Weight: 100
+ ClientIPPreservationEnabled: true
+
DnsRecords:
Type: AWS::Route53::RecordSet
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
DependsOn:
- - EC2Instance
+ - GlobalAccelerator
Properties:
HostedZoneId: Z0126430TJ1UYIMO3SYX
Name: !Sub "${ImageTag}-${GitSHA}.quadratic-preview.com"
Type: A
- TTL: 300
- ResourceRecords:
- - !GetAtt EC2Instance.PublicIp
+ AliasTarget:
+ DNSName: !GetAtt GlobalAccelerator.DnsName
+ HostedZoneId: Z2BJ6XQ5FK7U4H
+ EvaluateTargetHealth: true
WildcardDnsRecord:
Type: AWS::Route53::RecordSet
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
DependsOn:
- - EC2Instance
+ - GlobalAccelerator
Properties:
HostedZoneId: Z0126430TJ1UYIMO3SYX
Name: !Sub "*.${ImageTag}-${GitSHA}.quadratic-preview.com"
Type: A
- TTL: 300
- ResourceRecords:
- - !GetAtt EC2Instance.PublicIp
+ AliasTarget:
+ DNSName: !GetAtt GlobalAccelerator.DnsName
+ HostedZoneId: Z2BJ6XQ5FK7U4H
+ EvaluateTargetHealth: true
Outputs:
DnsRecord:
From 5b3bd0b8339b4fa5bd56fddaa829a707f51b0d06 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 00:46:27 +0530
Subject: [PATCH 075/155] handle stack rollback
---
.../preview-cloudformation-deploy.yml | 49 ++++++++++++++++++-
1 file changed, 47 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml
index a0514566e2..a2525abee7 100644
--- a/.github/workflows/preview-cloudformation-deploy.yml
+++ b/.github/workflows/preview-cloudformation-deploy.yml
@@ -34,6 +34,28 @@ jobs:
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }}
aws-region: ${{ secrets.AWS_REGION }}
+ - name: Check and Delete Stack if in ROLLBACK_COMPLETE
+ id: check-stack
+ run: |
+ if aws cloudformation describe-stacks --stack-name ${{ env.STACK_NAME }} 2>/dev/null; then
+ STACK_STATUS=$(aws cloudformation describe-stacks --stack-name ${{ env.STACK_NAME }} --query 'Stacks[0].StackStatus' --output text)
+ echo "Current stack status: $STACK_STATUS"
+
+ if [ "$STACK_STATUS" = "ROLLBACK_COMPLETE" ] || [ "$STACK_STATUS" = "ROLLBACK_FAILED" ]; then
+ echo "Stack is in $STACK_STATUS state. Deleting..."
+ aws cloudformation delete-stack --stack-name ${{ env.STACK_NAME }}
+ echo "Waiting for stack deletion to complete..."
+ aws cloudformation wait stack-delete-complete --stack-name ${{ env.STACK_NAME }}
+ echo "deleted=true" >> $GITHUB_OUTPUT
+ else
+ echo "Stack exists but is not in ROLLBACK state"
+ echo "deleted=false" >> $GITHUB_OUTPUT
+ fi
+ else
+ echo "Stack does not exist"
+ echo "deleted=false" >> $GITHUB_OUTPUT
+ fi
+
- name: Deploy CloudFormation Stack
id: deploy-stack
uses: aws-actions/aws-cloudformation-github-deploy@v1
@@ -49,6 +71,16 @@ jobs:
disable-rollback: false
termination-protection: false
+ - name: Verify Stack Deployment
+ if: success()
+ run: |
+ STACK_STATUS=$(aws cloudformation describe-stacks --stack-name ${{ env.STACK_NAME }} --query 'Stacks[0].StackStatus' --output text)
+ echo "Final stack status: $STACK_STATUS"
+ if [ "$STACK_STATUS" != "CREATE_COMPLETE" ] && [ "$STACK_STATUS" != "UPDATE_COMPLETE" ]; then
+ echo "Stack deployment did not complete successfully. Status: $STACK_STATUS"
+ exit 1
+ fi
+
- name: Find Comment
uses: peter-evans/find-comment@v3
id: preview-comment
@@ -57,8 +89,8 @@ jobs:
comment-author: "github-actions[bot]"
body-includes: "Preview Deployment"
- - name: Create or update comment
- if: steps.deploy-stack.outputs.DnsRecord != ''
+ - name: Create or update comment on build success
+ if: success() && steps.deploy-stack.outputs.DnsRecord != ''
uses: peter-evans/create-or-update-comment@v3
with:
comment-id: ${{ steps.preview-comment.outputs.comment-id }}
@@ -66,4 +98,17 @@ jobs:
body: |
## Preview Deployment
Preview URL: https://${{ steps.deploy-stack.outputs.DnsRecord }}
+ Stack Status: ${{ steps.deploy-stack.outputs.stack-status }}
+ Deployment Time: ${{ steps.deploy-stack.outputs.deployment-time }}
+ edit-mode: replace
+
+ - name: Create or update comment on build failure
+ if: failure()
+ uses: peter-evans/create-or-update-comment@v3
+ with:
+ comment-id: ${{ steps.preview-comment.outputs.comment-id }}
+ issue-number: ${{ github.event.pull_request.number }}
+ body: |
+ ## Preview Deployment
+ The preview deployment encountered an error. Please check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
edit-mode: replace
From a6bdcacc089ec262af3be30ae57e0965e5346192 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 01:57:18 +0530
Subject: [PATCH 076/155] remove depends on, sort in order of dependency
---
.../preview-deployment-template.yml | 153 ++++++++----------
1 file changed, 68 insertions(+), 85 deletions(-)
diff --git a/infra/aws-cloudformation/preview-deployment-template.yml b/infra/aws-cloudformation/preview-deployment-template.yml
index 91197dcc7b..05582e9723 100644
--- a/infra/aws-cloudformation/preview-deployment-template.yml
+++ b/infra/aws-cloudformation/preview-deployment-template.yml
@@ -65,12 +65,35 @@ Resources:
- ecr:ListImages
Resource: "*"
+ EventBridgeRole:
+ Type: AWS::IAM::Role
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ Properties:
+ Tags:
+ - Key: Name
+ Value: !Ref ImageTag
+ AssumeRolePolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ - Effect: Allow
+ Principal:
+ Service: events.amazonaws.com
+ Action: sts:AssumeRole
+ Policies:
+ - PolicyName: InvokeSsmAutomation
+ PolicyDocument:
+ Version: 2012-10-17
+ Statement:
+ - Effect: Allow
+ Action:
+ - ssm:SendCommand
+ Resource: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:*"
+
EC2InstanceProfile:
Type: AWS::IAM::InstanceProfile
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
- DependsOn:
- - EC2Role
Properties:
Roles:
- !Ref EC2Role
@@ -79,9 +102,6 @@ Resources:
Type: AWS::EC2::Instance
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
- DependsOn:
- - EC2InstanceProfile
- - SecurityGroup
Properties:
Tags:
- Key: Name
@@ -121,38 +141,60 @@ Resources:
# Run Quadratic initialization script
./init.sh "${LicenseKey}" "${ImageTag}-${GitSHA}.quadratic-preview.com"
- EventBridgeRole:
- Type: AWS::IAM::Role
+ GlobalAccelerator:
+ Type: AWS::GlobalAccelerator::Accelerator
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
Properties:
+ Name: !Sub "${ImageTag}-${GitSHA}"
+ Enabled: true
Tags:
- Key: Name
Value: !Ref ImageTag
- AssumeRolePolicyDocument:
- Version: 2012-10-17
- Statement:
- - Effect: Allow
- Principal:
- Service: events.amazonaws.com
- Action: sts:AssumeRole
- Policies:
- - PolicyName: InvokeSsmAutomation
- PolicyDocument:
- Version: 2012-10-17
- Statement:
- - Effect: Allow
- Action:
- - ssm:SendCommand
- Resource: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:*"
+
+ DnsRecords:
+ Type: AWS::Route53::RecordSet
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ Properties:
+ HostedZoneId: Z0126430TJ1UYIMO3SYX
+ Name: !Sub "${ImageTag}-${GitSHA}.quadratic-preview.com"
+ Type: A
+ AliasTarget:
+ DNSName: !GetAtt GlobalAccelerator.DnsName
+ HostedZoneId: Z2BJ6XQ5FK7U4H
+ EvaluateTargetHealth: true
+
+ WildcardDnsRecord:
+ Type: AWS::Route53::RecordSet
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ Properties:
+ HostedZoneId: Z0126430TJ1UYIMO3SYX
+ Name: !Sub "*.${ImageTag}-${GitSHA}.quadratic-preview.com"
+ Type: A
+ AliasTarget:
+ DNSName: !GetAtt GlobalAccelerator.DnsName
+ HostedZoneId: Z2BJ6XQ5FK7U4H
+ EvaluateTargetHealth: true
+
+ GlobalAcceleratorListener:
+ Type: AWS::GlobalAccelerator::Listener
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ Properties:
+ AcceleratorArn: !Ref GlobalAccelerator
+ Protocol: TCP
+ PortRanges:
+ - FromPort: 80
+ ToPort: 80
+ - FromPort: 443
+ ToPort: 443
EcrPushRule:
Type: AWS::Events::Rule
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
- DependsOn:
- - EC2Instance
- - EventBridgeRole
Properties:
Description: Rule to detect ECR image pushes
EventPattern:
@@ -193,39 +235,10 @@ Resources:
}
}
- GlobalAccelerator:
- Type: AWS::GlobalAccelerator::Accelerator
- DeletionPolicy: Delete
- UpdateReplacePolicy: Delete
- Properties:
- Name: !Sub "${ImageTag}-${GitSHA}"
- Enabled: true
- Tags:
- - Key: Name
- Value: !Ref ImageTag
-
- GlobalAcceleratorListener:
- Type: AWS::GlobalAccelerator::Listener
- DeletionPolicy: Delete
- UpdateReplacePolicy: Delete
- DependsOn:
- - GlobalAccelerator
- Properties:
- AcceleratorArn: !Ref GlobalAccelerator
- Protocol: TCP
- PortRanges:
- - FromPort: 80
- ToPort: 80
- - FromPort: 443
- ToPort: 443
-
GlobalAcceleratorEndpointGroup:
Type: AWS::GlobalAccelerator::EndpointGroup
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
- DependsOn:
- - GlobalAcceleratorListener
- - EC2Instance
Properties:
ListenerArn: !Ref GlobalAcceleratorListener
EndpointGroupRegion: !Ref AWS::Region
@@ -234,36 +247,6 @@ Resources:
Weight: 100
ClientIPPreservationEnabled: true
- DnsRecords:
- Type: AWS::Route53::RecordSet
- DeletionPolicy: Delete
- UpdateReplacePolicy: Delete
- DependsOn:
- - GlobalAccelerator
- Properties:
- HostedZoneId: Z0126430TJ1UYIMO3SYX
- Name: !Sub "${ImageTag}-${GitSHA}.quadratic-preview.com"
- Type: A
- AliasTarget:
- DNSName: !GetAtt GlobalAccelerator.DnsName
- HostedZoneId: Z2BJ6XQ5FK7U4H
- EvaluateTargetHealth: true
-
- WildcardDnsRecord:
- Type: AWS::Route53::RecordSet
- DeletionPolicy: Delete
- UpdateReplacePolicy: Delete
- DependsOn:
- - GlobalAccelerator
- Properties:
- HostedZoneId: Z0126430TJ1UYIMO3SYX
- Name: !Sub "*.${ImageTag}-${GitSHA}.quadratic-preview.com"
- Type: A
- AliasTarget:
- DNSName: !GetAtt GlobalAccelerator.DnsName
- HostedZoneId: Z2BJ6XQ5FK7U4H
- EvaluateTargetHealth: true
-
Outputs:
DnsRecord:
Description: Url of the preview instance
From 88d27bdb775fdc0381167c1f2537e072062f2536 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 02:09:07 +0530
Subject: [PATCH 077/155] try depot
---
.github/workflows/preview-publish-images.yml | 25 ++++++++++++++++++--
1 file changed, 23 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/preview-publish-images.yml b/.github/workflows/preview-publish-images.yml
index 8bbc2ddbf4..fe99b5fb9b 100644
--- a/.github/workflows/preview-publish-images.yml
+++ b/.github/workflows/preview-publish-images.yml
@@ -22,8 +22,10 @@ jobs:
include:
- service: api
runner: blacksmith-2vcpu-ubuntu-2204
+ # - service: client
+ # runner: blacksmith-4vcpu-ubuntu-2204 # no speed benefit of 4vcpu, 2vcpu runs out of memory
- service: client
- runner: blacksmith-4vcpu-ubuntu-2204 # no speed benefit of 4vcpu, 2vcpu runs out of memory
+ runner: depot-ubuntu-24.04-64
- service: connection
runner: blacksmith-2vcpu-ubuntu-2204
- service: files
@@ -82,8 +84,10 @@ jobs:
network=host
- name: Build and push
- uses: useblacksmith/build-push-action@v1
+ uses: depot/build-push-action@v1
with:
+ project: 473fbzf7w6
+ token: ${{ secrets.DEPOT_PROJECT_TOKEN }}
context: .
file: quadratic-${{ matrix.service }}/Dockerfile
push: true
@@ -97,3 +101,20 @@ jobs:
labels: |
org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}
org.opencontainers.image.revision=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
+
+ # - name: Build and push
+ # uses: useblacksmith/build-push-action@v1
+ # with:
+ # context: .
+ # file: quadratic-${{ matrix.service }}/Dockerfile
+ # push: true
+ # tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }}-${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
+ # build-args: |
+ # BUILDKIT_INLINE_CACHE=1
+ # BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }}
+ # GIT_SHA_SHORT=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
+ # BRANCH_NAME=${{ steps.build-metadata.outputs.BRANCH_NAME }}
+ # PR_NUMBER=${{ github.event.pull_request.number }}
+ # labels: |
+ # org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}
+ # org.opencontainers.image.revision=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
From bd07d6e44f4d9fde1ace781461a69e2fffe8ebb9 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 02:14:18 +0530
Subject: [PATCH 078/155] use setup action
---
.github/workflows/preview-publish-images.yml | 28 +++++++++++++++-----
1 file changed, 21 insertions(+), 7 deletions(-)
diff --git a/.github/workflows/preview-publish-images.yml b/.github/workflows/preview-publish-images.yml
index fe99b5fb9b..010d63f7c8 100644
--- a/.github/workflows/preview-publish-images.yml
+++ b/.github/workflows/preview-publish-images.yml
@@ -21,21 +21,35 @@ jobs:
matrix:
include:
- service: api
- runner: blacksmith-2vcpu-ubuntu-2204
- # - service: client
- # runner: blacksmith-4vcpu-ubuntu-2204 # no speed benefit of 4vcpu, 2vcpu runs out of memory
+ runner: depot-ubuntu-22.04
- service: client
runner: depot-ubuntu-24.04-64
- service: connection
- runner: blacksmith-2vcpu-ubuntu-2204
+ runner: depot-ubuntu-22.04
- service: files
- runner: blacksmith-2vcpu-ubuntu-2204
+ runner: depot-ubuntu-22.04
- service: multiplayer
- runner: blacksmith-2vcpu-ubuntu-2204
+ runner: depot-ubuntu-22.04
+ # - service: api
+ # runner: blacksmith-2vcpu-ubuntu-2204
+ # - service: client
+ # runner: blacksmith-4vcpu-ubuntu-2204 # no speed benefit of 4vcpu, 2vcpu runs out of memory
+ # - service: client
+ # runner: depot-ubuntu-24.04-64
+ # - service: connection
+ # runner: blacksmith-2vcpu-ubuntu-2204
+ # - service: files
+ # runner: blacksmith-2vcpu-ubuntu-2204
+ # - service: multiplayer
+ # runner: blacksmith-2vcpu-ubuntu-2204
fail-fast: true
steps:
- - uses: actions/checkout@v4
+ - name: Checkout repo
+ uses: actions/checkout@v4
+
+ - name: Set up Depot CLI
+ uses: depot/setup-action@v1
- name: Generate Build Metadata
id: build-metadata
From 8c0ceeebe877584a73561c991ae1545b8ad11cc9 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 02:24:41 +0530
Subject: [PATCH 079/155] allow token
---
.github/workflows/preview-publish-images.yml | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/.github/workflows/preview-publish-images.yml b/.github/workflows/preview-publish-images.yml
index 010d63f7c8..cccbc4a957 100644
--- a/.github/workflows/preview-publish-images.yml
+++ b/.github/workflows/preview-publish-images.yml
@@ -12,6 +12,7 @@ jobs:
publish_images:
permissions:
contents: read
+ id-token: write
timeout-minutes: 30
@@ -34,8 +35,6 @@ jobs:
# runner: blacksmith-2vcpu-ubuntu-2204
# - service: client
# runner: blacksmith-4vcpu-ubuntu-2204 # no speed benefit of 4vcpu, 2vcpu runs out of memory
- # - service: client
- # runner: depot-ubuntu-24.04-64
# - service: connection
# runner: blacksmith-2vcpu-ubuntu-2204
# - service: files
From 064b78bcf24f16256f6a4cc02438af87f3f086a7 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 02:34:08 +0530
Subject: [PATCH 080/155] change project id
---
.github/workflows/preview-publish-images.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/preview-publish-images.yml b/.github/workflows/preview-publish-images.yml
index cccbc4a957..a94e78f751 100644
--- a/.github/workflows/preview-publish-images.yml
+++ b/.github/workflows/preview-publish-images.yml
@@ -99,7 +99,7 @@ jobs:
- name: Build and push
uses: depot/build-push-action@v1
with:
- project: 473fbzf7w6
+ project: bt0vcnzjf4
token: ${{ secrets.DEPOT_PROJECT_TOKEN }}
context: .
file: quadratic-${{ matrix.service }}/Dockerfile
From bb869fa5bd6c1f3de23de45ecd6abdf57d9dab34 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 02:53:41 +0530
Subject: [PATCH 081/155] revert to blacksmith, try 32 core for lint
---
.github/workflows/ci.yml | 68 ++++++++++----------
.github/workflows/preview-publish-images.yml | 47 ++------------
2 files changed, 41 insertions(+), 74 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 57b930abb8..6f63d1f89d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -194,7 +194,7 @@ jobs:
cargo clippy -- -D warnings
lint:
- runs-on: ubuntu-latest-8-cores
+ runs-on: blacksmith-32vcpu-ubuntu-2204
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
@@ -229,42 +229,42 @@ jobs:
# If we are merging into main, but not pushed on main
if: github.base_ref == 'main' && github.ref != 'refs/heads/main'
steps:
- - name: Checkout current branch
- uses: actions/checkout@v3
- with:
- fetch-depth: 0
-
- - name: Get current VERSION
- id: current_version
- run: echo "CURRENT_VERSION=$(cat VERSION)" >> $GITHUB_OUTPUT
-
- - name: Checkout main branch
- uses: actions/checkout@v3
- with:
- ref: main
- fetch-depth: 1
-
- - name: Get main VERSION
- id: main_version
- run: echo "MAIN_VERSION=$(cat VERSION)" >> $GITHUB_OUTPUT
-
- - name: Compare versions to main, verify this version is higher
- run: |
- current_version="${{ steps.current_version.outputs.CURRENT_VERSION }}"
- main_version="${{ steps.main_version.outputs.MAIN_VERSION }}"
- if [ "$(printf '%s\n' "$main_version" "$current_version" | sort -V | tail -n1)" != "$current_version" ]; then
- echo "Error: VERSION in the current branch ($current_version) is not greater than VERSION in main ($main_version)"
- exit 1
- else
- echo "VERSION check passed: Current branch ($current_version) > main ($main_version)"
- fi
+ - name: Checkout current branch
+ uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+
+ - name: Get current VERSION
+ id: current_version
+ run: echo "CURRENT_VERSION=$(cat VERSION)" >> $GITHUB_OUTPUT
+
+ - name: Checkout main branch
+ uses: actions/checkout@v3
+ with:
+ ref: main
+ fetch-depth: 1
+
+ - name: Get main VERSION
+ id: main_version
+ run: echo "MAIN_VERSION=$(cat VERSION)" >> $GITHUB_OUTPUT
+
+ - name: Compare versions to main, verify this version is higher
+ run: |
+ current_version="${{ steps.current_version.outputs.CURRENT_VERSION }}"
+ main_version="${{ steps.main_version.outputs.MAIN_VERSION }}"
+ if [ "$(printf '%s\n' "$main_version" "$current_version" | sort -V | tail -n1)" != "$current_version" ]; then
+ echo "Error: VERSION in the current branch ($current_version) is not greater than VERSION in main ($main_version)"
+ exit 1
+ else
+ echo "VERSION check passed: Current branch ($current_version) > main ($main_version)"
+ fi
check-versions-match:
runs-on: ubuntu-latest
steps:
- - name: Checkout current branch
- uses: actions/checkout@v3
+ - name: Checkout current branch
+ uses: actions/checkout@v3
- - name: Verify that all versions match
- run: ./bump.sh verify
+ - name: Verify that all versions match
+ run: ./bump.sh verify
diff --git a/.github/workflows/preview-publish-images.yml b/.github/workflows/preview-publish-images.yml
index a94e78f751..606f8a43b0 100644
--- a/.github/workflows/preview-publish-images.yml
+++ b/.github/workflows/preview-publish-images.yml
@@ -12,7 +12,6 @@ jobs:
publish_images:
permissions:
contents: read
- id-token: write
timeout-minutes: 30
@@ -22,34 +21,21 @@ jobs:
matrix:
include:
- service: api
- runner: depot-ubuntu-22.04
+ runner: blacksmith-2vcpu-ubuntu-2204
- service: client
- runner: depot-ubuntu-24.04-64
+ runner: blacksmith-4vcpu-ubuntu-2204 # no speed benefit of 4vcpu, 2vcpu runs out of memory
- service: connection
- runner: depot-ubuntu-22.04
+ runner: blacksmith-2vcpu-ubuntu-2204
- service: files
- runner: depot-ubuntu-22.04
+ runner: blacksmith-2vcpu-ubuntu-2204
- service: multiplayer
- runner: depot-ubuntu-22.04
- # - service: api
- # runner: blacksmith-2vcpu-ubuntu-2204
- # - service: client
- # runner: blacksmith-4vcpu-ubuntu-2204 # no speed benefit of 4vcpu, 2vcpu runs out of memory
- # - service: connection
- # runner: blacksmith-2vcpu-ubuntu-2204
- # - service: files
- # runner: blacksmith-2vcpu-ubuntu-2204
- # - service: multiplayer
- # runner: blacksmith-2vcpu-ubuntu-2204
+ runner: blacksmith-2vcpu-ubuntu-2204
fail-fast: true
steps:
- - name: Checkout repo
+ - name: Checkout code
uses: actions/checkout@v4
- - name: Set up Depot CLI
- uses: depot/setup-action@v1
-
- name: Generate Build Metadata
id: build-metadata
run: |
@@ -97,10 +83,8 @@ jobs:
network=host
- name: Build and push
- uses: depot/build-push-action@v1
+ uses: useblacksmith/build-push-action@v1
with:
- project: bt0vcnzjf4
- token: ${{ secrets.DEPOT_PROJECT_TOKEN }}
context: .
file: quadratic-${{ matrix.service }}/Dockerfile
push: true
@@ -114,20 +98,3 @@ jobs:
labels: |
org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}
org.opencontainers.image.revision=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
-
- # - name: Build and push
- # uses: useblacksmith/build-push-action@v1
- # with:
- # context: .
- # file: quadratic-${{ matrix.service }}/Dockerfile
- # push: true
- # tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }}-${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
- # build-args: |
- # BUILDKIT_INLINE_CACHE=1
- # BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }}
- # GIT_SHA_SHORT=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
- # BRANCH_NAME=${{ steps.build-metadata.outputs.BRANCH_NAME }}
- # PR_NUMBER=${{ github.event.pull_request.number }}
- # labels: |
- # org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}
- # org.opencontainers.image.revision=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
From 5829ff7cf250b0f65ce31f6061acb5ab942f4783 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 03:07:39 +0530
Subject: [PATCH 082/155] change all non prod runners to blacksmith
---
.github/workflows/ci.yml | 16 ++++++++--------
.github/workflows/qa-wolf-notify.yml | 2 +-
2 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6f63d1f89d..73a6872bbf 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -12,7 +12,7 @@ on:
jobs:
test_rust:
- runs-on: ubuntu-latest-8-cores
+ runs-on: blacksmith-32vcpu-ubuntu-2204
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
@@ -109,7 +109,7 @@ jobs:
npm run test:docker:ci
test_unit:
- runs-on: ubuntu-latest-8-cores
+ runs-on: blacksmith-4vcpu-ubuntu-2204
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
@@ -145,7 +145,7 @@ jobs:
npm run test:ts
test_python:
- runs-on: ubuntu-latest-8-cores
+ runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
@@ -163,7 +163,7 @@ jobs:
npm run test
test_api:
- runs-on: ubuntu-latest-8-cores
+ runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
@@ -178,7 +178,7 @@ jobs:
npm run docker:test:ci
lint_rust:
- runs-on: ubuntu-latest-8-cores
+ runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
@@ -194,7 +194,7 @@ jobs:
cargo clippy -- -D warnings
lint:
- runs-on: blacksmith-32vcpu-ubuntu-2204
+ runs-on: blacksmith-4vcpu-ubuntu-2204
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
@@ -225,7 +225,7 @@ jobs:
npm run lint:ts
check-version-increment:
- runs-on: ubuntu-latest
+ runs-on: blacksmith-2vcpu-ubuntu-2204
# If we are merging into main, but not pushed on main
if: github.base_ref == 'main' && github.ref != 'refs/heads/main'
steps:
@@ -260,7 +260,7 @@ jobs:
fi
check-versions-match:
- runs-on: ubuntu-latest
+ runs-on: blacksmith-2vcpu-ubuntu-2204
steps:
- name: Checkout current branch
diff --git a/.github/workflows/qa-wolf-notify.yml b/.github/workflows/qa-wolf-notify.yml
index 0790fb3b87..4db6ff0c3c 100644
--- a/.github/workflows/qa-wolf-notify.yml
+++ b/.github/workflows/qa-wolf-notify.yml
@@ -7,7 +7,7 @@ on:
jobs:
qa-wolf-notify:
- runs-on: ubuntu-latest
+ runs-on: blacksmith-2vcpu-ubuntu-2204
concurrency:
group: qa-wolf-notify
cancel-in-progress: true
From e6de92a129577245a5c81fbfa6b4c7af764650aa Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 03:18:09 +0530
Subject: [PATCH 083/155] change
---
.github/workflows/ci.yml | 10 +++++++---
.github/workflows/production-publish-images.yml | 1 +
.github/workflows/qa-wolf-notify.yml | 3 +++
3 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 73a6872bbf..d0f00ef83f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -10,9 +10,13 @@ on:
- main
pull_request:
+concurrency:
+ group: ci
+ cancel-in-progress: true
+
jobs:
test_rust:
- runs-on: blacksmith-32vcpu-ubuntu-2204
+ runs-on: blacksmith-8vcpu-ubuntu-2204
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
@@ -109,7 +113,7 @@ jobs:
npm run test:docker:ci
test_unit:
- runs-on: blacksmith-4vcpu-ubuntu-2204
+ runs-on: blacksmith-8vcpu-ubuntu-2204
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
@@ -194,7 +198,7 @@ jobs:
cargo clippy -- -D warnings
lint:
- runs-on: blacksmith-4vcpu-ubuntu-2204
+ runs-on: blacksmith-8vcpu-ubuntu-2204
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
diff --git a/.github/workflows/production-publish-images.yml b/.github/workflows/production-publish-images.yml
index 79c20b0abd..4d9801d17a 100644
--- a/.github/workflows/production-publish-images.yml
+++ b/.github/workflows/production-publish-images.yml
@@ -7,6 +7,7 @@ on:
concurrency:
group: production-publish-images
+ cancel-in-progress: true
jobs:
publish_images:
diff --git a/.github/workflows/qa-wolf-notify.yml b/.github/workflows/qa-wolf-notify.yml
index 4db6ff0c3c..b03148cc0a 100644
--- a/.github/workflows/qa-wolf-notify.yml
+++ b/.github/workflows/qa-wolf-notify.yml
@@ -5,6 +5,9 @@ on:
branches:
- qa
+concurrency:
+ group: qa-wolf-notify
+
jobs:
qa-wolf-notify:
runs-on: blacksmith-2vcpu-ubuntu-2204
From c587ff3acd20b04e686871269b5c764b945d2575 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 03:28:46 +0530
Subject: [PATCH 084/155] revert to 4vcpu, better than 8vcpu
---
.github/workflows/ci.yml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d0f00ef83f..3b9709083d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -16,7 +16,7 @@ concurrency:
jobs:
test_rust:
- runs-on: blacksmith-8vcpu-ubuntu-2204
+ runs-on: blacksmith-4vcpu-ubuntu-2204
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
@@ -113,7 +113,7 @@ jobs:
npm run test:docker:ci
test_unit:
- runs-on: blacksmith-8vcpu-ubuntu-2204
+ runs-on: blacksmith-4vcpu-ubuntu-2204
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
@@ -198,7 +198,7 @@ jobs:
cargo clippy -- -D warnings
lint:
- runs-on: blacksmith-8vcpu-ubuntu-2204
+ runs-on: blacksmith-4vcpu-ubuntu-2204
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
From ead4fe18172c142848abd92952234e0a92f6a945 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 03:46:08 +0530
Subject: [PATCH 085/155] try caching in ci workflow
---
.github/workflows/ci.yml | 32 +++++++++++++++++++++++++++++---
1 file changed, 29 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3b9709083d..a55a0a6815 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -23,12 +23,21 @@ jobs:
- uses: actions/setup-node@v4
with:
node-version: 18
+ cache: "npm"
+ cache-dependency-path: "**/package-lock.json"
- name: Set up Rust
uses: moonrepo/setup-rust@v1
with:
components: clippy, llvm-tools-preview
- cache: false
+ cache: true
+ cache-target: true
+
+ - name: Cache grcov
+ uses: actions/cache@v3
+ with:
+ path: ~/.cargo/bin/grcov
+ key: ${{ runner.os }}-grcov
- name: Install grcov
run: if ! which grcov; then cargo install grcov; fi
@@ -120,14 +129,20 @@ jobs:
- uses: actions/setup-node@v4
with:
node-version: 18
+ cache: "npm"
+ cache-dependency-path: "**/package-lock.json"
+
- uses: actions/setup-python@v5
with:
python-version: "3.11.3"
+ cache: "pip"
- name: Set up Rust
uses: moonrepo/setup-rust@v1
with:
channel: "nightly"
+ cache: true
+ cache-target: true
- uses: jetli/wasm-pack-action@v0.4.0
@@ -156,6 +171,9 @@ jobs:
- uses: actions/setup-node@v4
with:
node-version: 18
+ cache: "npm"
+ cache-dependency-path: "**/package-lock.json"
+
- uses: actions/setup-python@v5
with:
python-version: "3.11.3"
@@ -174,6 +192,8 @@ jobs:
- uses: actions/setup-node@v4
with:
node-version: 18
+ cache: "npm"
+ cache-dependency-path: "**/package-lock.json"
- name: Run npm test:ci in quadratic-api
run: |
@@ -190,7 +210,8 @@ jobs:
uses: moonrepo/setup-rust@v1
with:
components: clippy
- cache: false
+ cache: true
+ cache-target: true
- name: Run cargo clippy in quadratic-core
run: |
@@ -205,10 +226,15 @@ jobs:
- uses: actions/setup-node@v4
with:
node-version: 18
+ cache: "npm"
+ cache-dependency-path: "**/package-lock.json"
+
- name: Set up Rust
uses: moonrepo/setup-rust@v1
with:
- cache: false
+ cache: true
+ cache-target: true
+
- uses: jetli/wasm-pack-action@v0.4.0
with:
version: "latest"
From d720329bb9c68943b3d436659587eaf60db7135d Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 04:14:22 +0530
Subject: [PATCH 086/155] Revert "try caching in ci workflow"
This reverts commit ead4fe18172c142848abd92952234e0a92f6a945.
---
.github/workflows/ci.yml | 32 +++-----------------------------
1 file changed, 3 insertions(+), 29 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a55a0a6815..3b9709083d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -23,21 +23,12 @@ jobs:
- uses: actions/setup-node@v4
with:
node-version: 18
- cache: "npm"
- cache-dependency-path: "**/package-lock.json"
- name: Set up Rust
uses: moonrepo/setup-rust@v1
with:
components: clippy, llvm-tools-preview
- cache: true
- cache-target: true
-
- - name: Cache grcov
- uses: actions/cache@v3
- with:
- path: ~/.cargo/bin/grcov
- key: ${{ runner.os }}-grcov
+ cache: false
- name: Install grcov
run: if ! which grcov; then cargo install grcov; fi
@@ -129,20 +120,14 @@ jobs:
- uses: actions/setup-node@v4
with:
node-version: 18
- cache: "npm"
- cache-dependency-path: "**/package-lock.json"
-
- uses: actions/setup-python@v5
with:
python-version: "3.11.3"
- cache: "pip"
- name: Set up Rust
uses: moonrepo/setup-rust@v1
with:
channel: "nightly"
- cache: true
- cache-target: true
- uses: jetli/wasm-pack-action@v0.4.0
@@ -171,9 +156,6 @@ jobs:
- uses: actions/setup-node@v4
with:
node-version: 18
- cache: "npm"
- cache-dependency-path: "**/package-lock.json"
-
- uses: actions/setup-python@v5
with:
python-version: "3.11.3"
@@ -192,8 +174,6 @@ jobs:
- uses: actions/setup-node@v4
with:
node-version: 18
- cache: "npm"
- cache-dependency-path: "**/package-lock.json"
- name: Run npm test:ci in quadratic-api
run: |
@@ -210,8 +190,7 @@ jobs:
uses: moonrepo/setup-rust@v1
with:
components: clippy
- cache: true
- cache-target: true
+ cache: false
- name: Run cargo clippy in quadratic-core
run: |
@@ -226,15 +205,10 @@ jobs:
- uses: actions/setup-node@v4
with:
node-version: 18
- cache: "npm"
- cache-dependency-path: "**/package-lock.json"
-
- name: Set up Rust
uses: moonrepo/setup-rust@v1
with:
- cache: true
- cache-target: true
-
+ cache: false
- uses: jetli/wasm-pack-action@v0.4.0
with:
version: "latest"
From 5de031268a2d098c2d1743510c30ba8d4702de6a Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 04:26:12 +0530
Subject: [PATCH 087/155] try dev build for test
---
package.json | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/package.json b/package.json
index 180cfea028..9407784c95 100644
--- a/package.json
+++ b/package.json
@@ -34,7 +34,8 @@
"watch:wasm:perf:javascript": "cd quadratic-core && cargo watch -s 'wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs'",
"build:wasm:perf:javascript": "cd quadratic-core && wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs",
"watch:rust-client": "cd quadratic-rust-client && cargo watch -s 'wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs'",
- "build:rust-client": "cd quadratic-rust-client && wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs",
+ "build:rust-client": "cd quadratic-rust-client && wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs",
+ "build:rust-client:perf": "cd quadratic-rust-client && wasm-pack build--target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs",
"watch:python": "cd quadratic-kernels/python-wasm && npm run dev",
"build:python": "./quadratic-kernels/python-wasm/package.sh",
"build:file:blank": "cd quadratic-core && cargo run --bin generate_blank_current_file",
From 3bfb2c17b0a3a1adc6edd22e4481e44c44eb6066 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 04:33:34 +0530
Subject: [PATCH 088/155] try 2vCPU
---
.github/workflows/ci.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3b9709083d..215970bfa6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -113,7 +113,7 @@ jobs:
npm run test:docker:ci
test_unit:
- runs-on: blacksmith-4vcpu-ubuntu-2204
+ runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
@@ -198,7 +198,7 @@ jobs:
cargo clippy -- -D warnings
lint:
- runs-on: blacksmith-4vcpu-ubuntu-2204
+ runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
From 9cd0fb47f05aab62e673f44f3528b84388b87d34 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 04:35:10 +0530
Subject: [PATCH 089/155] try parallel flah
---
.github/workflows/ci.yml | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 215970bfa6..a8c29bb69e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -45,11 +45,12 @@ jobs:
- name: Test quadratic-core
env:
LLVM_PROFILE_FILE: grcov-%p-%m.profraw
- RUSTFLAGS: -Cinstrument-coverage
+ RUSTFLAGS: "-Cinstrument-coverage -j 4"
RUSTC_BOOTSTRAP: 1
+ CARGO_BUILD_JOBS: 4
run: |
cd quadratic-core
- cargo test
+ cargo test -- --test-threads=4
- name: Generate coverage for quadratic-core
env:
From a1d847ad24999602ba1db2e4daffd90df12f66e8 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 04:48:33 +0530
Subject: [PATCH 090/155] break rust tests into 3 tests
---
.github/workflows/ci.yml | 87 +++++++++++++++++++++++++++++++++++++---
1 file changed, 82 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a8c29bb69e..d4670aa9db 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -15,8 +15,8 @@ concurrency:
cancel-in-progress: true
jobs:
- test_rust:
- runs-on: blacksmith-4vcpu-ubuntu-2204
+ test_core:
+ runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
@@ -45,7 +45,7 @@ jobs:
- name: Test quadratic-core
env:
LLVM_PROFILE_FILE: grcov-%p-%m.profraw
- RUSTFLAGS: "-Cinstrument-coverage -j 4"
+ RUSTFLAGS: -Cinstrument-coverage
RUSTC_BOOTSTRAP: 1
CARGO_BUILD_JOBS: 4
run: |
@@ -68,21 +68,49 @@ jobs:
--ignore "./docker/*" \
-o lcov.info
- - name: Upload coverage reports to Codecov
+ - name: Upload coverage reports to Codecov quadratic-core
uses: codecov/codecov-action@v3
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+ test_multiplayer:
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 15
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 18
+
+ - name: Set up Rust
+ uses: moonrepo/setup-rust@v1
+ with:
+ components: clippy, llvm-tools-preview
+ cache: false
+
+ - name: Install grcov
+ run: if ! which grcov; then cargo install grcov; fi
+
+ - name: Install llvm-tools-preview
+ run: if ! which llvm-tools-preview; then rustup component add llvm-tools-preview; fi
+
+ - name: Install pkg-config
+ if: github.runner.isHosted == true
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y pkg-config
+
- name: Test quadratic-multiplayer
env:
LLVM_PROFILE_FILE: grcov-%p-%m.profraw
RUSTFLAGS: -Cinstrument-coverage
RUSTC_BOOTSTRAP: 1
+ CARGO_BUILD_JOBS: 4
run: |
cd quadratic-multiplayer
npm run docker:test
- - name: Generate coverage quadratic-multiplayer
+ - name: Generate coverage for quadratic-multiplayer
env:
RUSTC_BOOTSTRAP: 1
run: |
@@ -104,15 +132,64 @@ jobs:
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+ test_connection:
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 15
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 18
+
+ - name: Set up Rust
+ uses: moonrepo/setup-rust@v1
+ with:
+ components: clippy, llvm-tools-preview
+ cache: false
+
+ - name: Install grcov
+ run: if ! which grcov; then cargo install grcov; fi
+
+ - name: Install llvm-tools-preview
+ run: if ! which llvm-tools-preview; then rustup component add llvm-tools-preview; fi
+
+ - name: Install pkg-config
+ if: github.runner.isHosted == true
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y pkg-config
+
- name: Test quadratic-connection
env:
LLVM_PROFILE_FILE: grcov-%p-%m.profraw
RUSTFLAGS: -Cinstrument-coverage
RUSTC_BOOTSTRAP: 1
+ CARGO_BUILD_JOBS: 4
run: |
cd quadratic-connection
npm run test:docker:ci
+ - name: Generate coverage for quadratic-connection
+ env:
+ RUSTC_BOOTSTRAP: 1
+ run: |
+ grcov $(find . -name "grcov-*.profraw" -print) \
+ --branch \
+ --ignore-not-existing \
+ --binary-path ./target/debug/ \
+ -s . \
+ -t lcov \
+ --ignore "/*" \
+ --ignore "./src/wasm_bindings/*" \
+ --ignore "./src/bin/*" \
+ --ignore "./docker/*" \
+ -o lcov.info
+
+ - name: Upload coverage reports to Codecov quadratic-connection
+ uses: codecov/codecov-action@v3
+ env:
+ CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
test_unit:
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 10
From b9980fbb6df7ec3873c93d033a8ce7861904225a Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 04:56:21 +0530
Subject: [PATCH 091/155] change
---
.github/workflows/ci.yml | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d4670aa9db..ba221c39d2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -16,7 +16,7 @@ concurrency:
jobs:
test_core:
- runs-on: blacksmith-2vcpu-ubuntu-2204
+ runs-on: blacksmith-4vcpu-ubuntu-2204
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
@@ -74,7 +74,7 @@ jobs:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
test_multiplayer:
- runs-on: blacksmith-2vcpu-ubuntu-2204
+ runs-on: blacksmith-4vcpu-ubuntu-2204
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
@@ -133,7 +133,7 @@ jobs:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
test_connection:
- runs-on: blacksmith-2vcpu-ubuntu-2204
+ runs-on: blacksmith-4vcpu-ubuntu-2204
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
@@ -191,7 +191,7 @@ jobs:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
test_unit:
- runs-on: blacksmith-2vcpu-ubuntu-2204
+ runs-on: blacksmith-4vcpu-ubuntu-2204
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
@@ -276,7 +276,7 @@ jobs:
cargo clippy -- -D warnings
lint:
- runs-on: blacksmith-2vcpu-ubuntu-2204
+ runs-on: blacksmith-4vcpu-ubuntu-2204
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
From 0b44a5ed9cf28abb7638dc4c4ec7a430dc6a8940 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 04:58:37 +0530
Subject: [PATCH 092/155] connections codecov
---
.github/workflows/ci.yml | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ba221c39d2..3dd327a019 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -173,7 +173,14 @@ jobs:
env:
RUSTC_BOOTSTRAP: 1
run: |
- grcov $(find . -name "grcov-*.profraw" -print) \
+ mkdir -p coverage
+ chmod -R 755 coverage
+
+ # Find profraw files in current project directory only
+ find quadratic-*/target -name "grcov-*.profraw" -type f -print > profraw_files.txt
+
+ # Run grcov with explicit file list
+ grcov $(cat profraw_files.txt) \
--branch \
--ignore-not-existing \
--binary-path ./target/debug/ \
@@ -183,7 +190,8 @@ jobs:
--ignore "./src/wasm_bindings/*" \
--ignore "./src/bin/*" \
--ignore "./docker/*" \
- -o lcov.info
+ --ignore "**/target/*" \
+ -o coverage/lcov.info
- name: Upload coverage reports to Codecov quadratic-connection
uses: codecov/codecov-action@v3
From 8756a50e9d5d7d9d0bc02e5355808a4f88c375e2 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 05:06:59 +0530
Subject: [PATCH 093/155] try
---
.github/workflows/ci.yml | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3dd327a019..a354825945 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -176,8 +176,7 @@ jobs:
mkdir -p coverage
chmod -R 755 coverage
- # Find profraw files in current project directory only
- find quadratic-*/target -name "grcov-*.profraw" -type f -print > profraw_files.txt
+ find . -name "grcov-*.profraw" -type f -print > profraw_files.txt
# Run grcov with explicit file list
grcov $(cat profraw_files.txt) \
From b348c374076f5b045cb39b849c7df8576f06ce0b Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 05:14:25 +0530
Subject: [PATCH 094/155] fix test
---
.github/workflows/ci.yml | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a354825945..a8108dec62 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -176,9 +176,11 @@ jobs:
mkdir -p coverage
chmod -R 755 coverage
- find . -name "grcov-*.profraw" -type f -print > profraw_files.txt
+ find . -name "grcov-*.profraw" -type f \
+ ! -path "./docker/mysql-connection/data/*" \
+ ! -path "./docker/postgres-connection/data/*" \
+ -print > profraw_files.txt
- # Run grcov with explicit file list
grcov $(cat profraw_files.txt) \
--branch \
--ignore-not-existing \
From c037d74c8e52f3ecc4a3dd061392a2d17dbb10ec Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 24 Dec 2024 05:20:46 +0530
Subject: [PATCH 095/155] revert connections codecov
---
.github/workflows/ci.yml | 30 ------------------------------
1 file changed, 30 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a8108dec62..c5665604af 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -169,36 +169,6 @@ jobs:
cd quadratic-connection
npm run test:docker:ci
- - name: Generate coverage for quadratic-connection
- env:
- RUSTC_BOOTSTRAP: 1
- run: |
- mkdir -p coverage
- chmod -R 755 coverage
-
- find . -name "grcov-*.profraw" -type f \
- ! -path "./docker/mysql-connection/data/*" \
- ! -path "./docker/postgres-connection/data/*" \
- -print > profraw_files.txt
-
- grcov $(cat profraw_files.txt) \
- --branch \
- --ignore-not-existing \
- --binary-path ./target/debug/ \
- -s . \
- -t lcov \
- --ignore "/*" \
- --ignore "./src/wasm_bindings/*" \
- --ignore "./src/bin/*" \
- --ignore "./docker/*" \
- --ignore "**/target/*" \
- -o coverage/lcov.info
-
- - name: Upload coverage reports to Codecov quadratic-connection
- uses: codecov/codecov-action@v3
- env:
- CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-
test_unit:
runs-on: blacksmith-4vcpu-ubuntu-2204
timeout-minutes: 10
From 4fc9e81bb81a9aed23e09da82cfc534bb4db371b Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Wed, 25 Dec 2024 00:15:26 +0530
Subject: [PATCH 096/155] add wasm profiles in rust client
---
package-lock.json | 14 ++++++--------
quadratic-rust-client/Cargo.toml | 24 ++++++++++++++++++++++++
2 files changed, 30 insertions(+), 8 deletions(-)
diff --git a/package-lock.json b/package-lock.json
index ce031cc916..7fad79bc60 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -14,12 +14,12 @@
"quadratic-files",
"quadratic-multiplayer",
"quadratic-rust-client",
+ "quadratic-core",
"quadratic-client",
"quadratic-kernels/python-wasm"
],
"dependencies": {
"@ory/kratos-client": "^1.2.1",
- "tsc": "^2.0.4",
"vitest": "^1.5.0",
"zod": "^3.23.8"
},
@@ -27138,6 +27138,10 @@
"resolved": "quadratic-client",
"link": true
},
+ "node_modules/quadratic-core": {
+ "resolved": "quadratic-core",
+ "link": true
+ },
"node_modules/quadratic-files": {
"resolved": "quadratic-files",
"link": true
@@ -29795,13 +29799,6 @@
"devOptional": true,
"license": "MIT"
},
- "node_modules/tsc": {
- "version": "2.0.4",
- "license": "MIT",
- "bin": {
- "tsc": "bin/tsc"
- }
- },
"node_modules/tsconfck": {
"version": "3.0.2",
"dev": true,
@@ -32182,6 +32179,7 @@
"version": "4.0.0",
"license": "ISC"
},
+ "quadratic-core": {},
"quadratic-files": {
"devDependencies": {}
},
diff --git a/quadratic-rust-client/Cargo.toml b/quadratic-rust-client/Cargo.toml
index a2c534dd9a..04efc1204a 100644
--- a/quadratic-rust-client/Cargo.toml
+++ b/quadratic-rust-client/Cargo.toml
@@ -22,3 +22,27 @@ chrono = "0.4.38"
[lib]
crate-type = ["cdylib", "rlib"]
+
+[package.metadata.wasm-pack.profile.release]
+wasm-opt = ['-Os', '-g'] # TODO: -g seems to fix the name mangling problem
+
+[package.metadata.wasm-pack.profile.dev]
+wasm-opt = false
+
+[package.metadata.wasm-pack.profile.dev.wasm-bindgen]
+debug-js-glue = true
+demangle-name-section = true
+dwarf-debug-info = true
+
+[package.metadata.wasm-pack.profile.profiling]
+wasm-opt = ['-O', '-g']
+
+[package.metadata.wasm-pack.profile.profiling.wasm-bindgen]
+debug-js-glue = false
+demangle-name-section = true
+dwarf-debug-info = true
+
+[package.metadata.wasm-pack.profile.release.wasm-bindgen]
+debug-js-glue = false
+demangle-name-section = true
+dwarf-debug-info = false
From c78c42006d9abf3854f3969e8ce8be041ed7a219 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Wed, 25 Dec 2024 00:55:37 +0530
Subject: [PATCH 097/155] fix deploy template
---
.github/workflows/preview-cloudformation-deploy.yml | 2 +-
infra/aws-cloudformation/preview-deployment-template.yml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml
index a2525abee7..938794846f 100644
--- a/.github/workflows/preview-cloudformation-deploy.yml
+++ b/.github/workflows/preview-cloudformation-deploy.yml
@@ -34,7 +34,7 @@ jobs:
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }}
aws-region: ${{ secrets.AWS_REGION }}
- - name: Check and Delete Stack if in ROLLBACK_COMPLETE
+ - name: Check and Delete Stack if in ROLLBACK
id: check-stack
run: |
if aws cloudformation describe-stacks --stack-name ${{ env.STACK_NAME }} 2>/dev/null; then
diff --git a/infra/aws-cloudformation/preview-deployment-template.yml b/infra/aws-cloudformation/preview-deployment-template.yml
index 05582e9723..82e75806f6 100644
--- a/infra/aws-cloudformation/preview-deployment-template.yml
+++ b/infra/aws-cloudformation/preview-deployment-template.yml
@@ -222,7 +222,7 @@ Resources:
InputPathsMap:
imageTag: $.detail.image-tag
repository: $.detail.repository-name
- InputTemplate: |
+ InputTemplate: !Sub |
{
"InstanceIds": ["${EC2Instance.InstanceId}"],
"DocumentName": "AWS-RunShellScript",
From 6a7cbe367a6cd03ea95cb1fee55e8eefc07ed287 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Wed, 25 Dec 2024 02:24:23 +0530
Subject: [PATCH 098/155] fix files env
---
infra/aws-cloudformation/preview-deployment-template.yml | 4 +++-
quadratic-client/Dockerfile | 1 +
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/infra/aws-cloudformation/preview-deployment-template.yml b/infra/aws-cloudformation/preview-deployment-template.yml
index 82e75806f6..377fb3e5db 100644
--- a/infra/aws-cloudformation/preview-deployment-template.yml
+++ b/infra/aws-cloudformation/preview-deployment-template.yml
@@ -126,7 +126,7 @@ Resources:
echo "export GIT_SHA=${GitSHA}" >> /home/ubuntu/.profile
# Source the new environment variables
- source /home/ubuntu/.profile
+ sudo -u ubuntu bash -c 'source /home/ubuntu/.profile'
# Create login.sh script and run it
cat << 'EOF' > /home/ubuntu/quadratic-selfhost/login.sh
@@ -195,6 +195,8 @@ Resources:
Type: AWS::Events::Rule
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
+ DependsOn:
+ - EC2Instance
Properties:
Description: Rule to detect ECR image pushes
EventPattern:
diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile
index 1a4a0d2f3e..6280a1f31a 100644
--- a/quadratic-client/Dockerfile
+++ b/quadratic-client/Dockerfile
@@ -58,6 +58,7 @@ ENV VITE_QUADRATIC_API_URL=VITE_QUADRATIC_API_URL_VAL
ENV VITE_QUADRATIC_MULTIPLAYER_URL=VITE_QUADRATIC_MULTIPLAYER_URL_VAL
ENV VITE_QUADRATIC_CONNECTION_URL=VITE_QUADRATIC_CONNECTION_URL_VAL
ENV VITE_AUTH_TYPE=VITE_AUTH_TYPE_VAL
+ENV VITE_STORAGE_TYPE=VITE_STORAGE_TYPE_VAL
ENV VITE_ORY_HOST=VITE_ORY_HOST_VAL
RUN echo 'Building front-end...' && npm run build --workspace=quadratic-client
From 3f7a2c232f10c3c4110abf8d3a35a9821bd94078 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Wed, 25 Dec 2024 03:04:02 +0530
Subject: [PATCH 099/155] fix env in ec2
---
.github/workflows/ci.yml | 10 +++++-----
.github/workflows/preview-cloudformation-deploy.yml | 2 --
.../preview-deployment-template.yml | 13 ++++++-------
3 files changed, 11 insertions(+), 14 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c5665604af..85afcb214f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -16,7 +16,7 @@ concurrency:
jobs:
test_core:
- runs-on: blacksmith-4vcpu-ubuntu-2204
+ runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
@@ -74,7 +74,7 @@ jobs:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
test_multiplayer:
- runs-on: blacksmith-4vcpu-ubuntu-2204
+ runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
@@ -133,7 +133,7 @@ jobs:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
test_connection:
- runs-on: blacksmith-4vcpu-ubuntu-2204
+ runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
@@ -170,7 +170,7 @@ jobs:
npm run test:docker:ci
test_unit:
- runs-on: blacksmith-4vcpu-ubuntu-2204
+ runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
@@ -255,7 +255,7 @@ jobs:
cargo clippy -- -D warnings
lint:
- runs-on: blacksmith-4vcpu-ubuntu-2204
+ runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml
index 938794846f..df037ffe9b 100644
--- a/.github/workflows/preview-cloudformation-deploy.yml
+++ b/.github/workflows/preview-cloudformation-deploy.yml
@@ -98,8 +98,6 @@ jobs:
body: |
## Preview Deployment
Preview URL: https://${{ steps.deploy-stack.outputs.DnsRecord }}
- Stack Status: ${{ steps.deploy-stack.outputs.stack-status }}
- Deployment Time: ${{ steps.deploy-stack.outputs.deployment-time }}
edit-mode: replace
- name: Create or update comment on build failure
diff --git a/infra/aws-cloudformation/preview-deployment-template.yml b/infra/aws-cloudformation/preview-deployment-template.yml
index 377fb3e5db..6acc0cd24c 100644
--- a/infra/aws-cloudformation/preview-deployment-template.yml
+++ b/infra/aws-cloudformation/preview-deployment-template.yml
@@ -120,13 +120,12 @@ Resources:
Fn::Base64: !Sub |
#!/bin/bash
- # Export environment variables for docker-compose
- echo "export ECR_URL=${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com" >> /home/ubuntu/.profile
- echo "export IMAGE_TAG=${ImageTag}" >> /home/ubuntu/.profile
- echo "export GIT_SHA=${GitSHA}" >> /home/ubuntu/.profile
-
- # Source the new environment variables
- sudo -u ubuntu bash -c 'source /home/ubuntu/.profile'
+ # Export environment variables to .env file
+ cat << EOF > /home/ubuntu/quadratic-selfhost/.env
+ ECR_URL=${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com
+ IMAGE_TAG=${ImageTag}
+ GIT_SHA=${GitSHA}
+ EOF
# Create login.sh script and run it
cat << 'EOF' > /home/ubuntu/quadratic-selfhost/login.sh
From e7ac1041b8a98f91ef68cbc4491145722f91c6a3 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Wed, 25 Dec 2024 03:31:40 +0530
Subject: [PATCH 100/155] fix env append
---
infra/aws-cloudformation/preview-deployment-template.yml | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/infra/aws-cloudformation/preview-deployment-template.yml b/infra/aws-cloudformation/preview-deployment-template.yml
index 6acc0cd24c..e88d6a6141 100644
--- a/infra/aws-cloudformation/preview-deployment-template.yml
+++ b/infra/aws-cloudformation/preview-deployment-template.yml
@@ -120,8 +120,10 @@ Resources:
Fn::Base64: !Sub |
#!/bin/bash
- # Export environment variables to .env file
- cat << EOF > /home/ubuntu/quadratic-selfhost/.env
+ # Append environment variables to .env file
+ cat << EOF >> /home/ubuntu/quadratic-selfhost/.env
+
+ #AWS
ECR_URL=${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com
IMAGE_TAG=${ImageTag}
GIT_SHA=${GitSHA}
@@ -194,8 +196,6 @@ Resources:
Type: AWS::Events::Rule
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
- DependsOn:
- - EC2Instance
Properties:
Description: Rule to detect ECR image pushes
EventPattern:
From 4089cac96bbeadf31d826bf3220a810f55b026d8 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Wed, 25 Dec 2024 11:33:37 +0530
Subject: [PATCH 101/155] use github build action, cache not invalidating
---
.../preview-cloudformation-deploy.yml | 5 +-
.github/workflows/preview-publish-images.yml | 3 +-
.../preview-deployment-template.yml | 256 +++++++++---------
3 files changed, 135 insertions(+), 129 deletions(-)
diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml
index df037ffe9b..d94b782a65 100644
--- a/.github/workflows/preview-cloudformation-deploy.yml
+++ b/.github/workflows/preview-cloudformation-deploy.yml
@@ -65,7 +65,6 @@ jobs:
parameter-overrides: >-
LicenseKey=${{ secrets.QUADRATIC_LICENSE_KEY }},
ImageTag=pr-${{ github.event.pull_request.number }},
- GitSHA=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
capabilities: CAPABILITY_IAM
no-fail-on-empty-changeset: "1"
disable-rollback: false
@@ -90,14 +89,14 @@ jobs:
body-includes: "Preview Deployment"
- name: Create or update comment on build success
- if: success() && steps.deploy-stack.outputs.DnsRecord != ''
+ if: success() && steps.deploy-stack.outputs.WebsiteURL != ''
uses: peter-evans/create-or-update-comment@v3
with:
comment-id: ${{ steps.preview-comment.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: |
## Preview Deployment
- Preview URL: https://${{ steps.deploy-stack.outputs.DnsRecord }}
+ Preview URL: https://${{ steps.deploy-stack.outputs.WebsiteURL }}
edit-mode: replace
- name: Create or update comment on build failure
diff --git a/.github/workflows/preview-publish-images.yml b/.github/workflows/preview-publish-images.yml
index 606f8a43b0..b8bd8ea9fb 100644
--- a/.github/workflows/preview-publish-images.yml
+++ b/.github/workflows/preview-publish-images.yml
@@ -83,7 +83,8 @@ jobs:
network=host
- name: Build and push
- uses: useblacksmith/build-push-action@v1
+ # uses: useblacksmith/build-push-action@v1
+ uses: docker/build-push-action@v6
with:
context: .
file: quadratic-${{ matrix.service }}/Dockerfile
diff --git a/infra/aws-cloudformation/preview-deployment-template.yml b/infra/aws-cloudformation/preview-deployment-template.yml
index e88d6a6141..a6e6b74361 100644
--- a/infra/aws-cloudformation/preview-deployment-template.yml
+++ b/infra/aws-cloudformation/preview-deployment-template.yml
@@ -8,11 +8,24 @@ Parameters:
ImageTag:
Type: String
Description: Image tag to use for all services
- GitSHA:
- Type: String
- Description: Git short SHA to use for all services
Resources:
+ # AMI Lookup for quadratic-preview-base-ami (custom AMI for preview deployments)
+ AMIInfo:
+ Type: AWS::EC2::Image
+ Properties:
+ Filters:
+ - Name: name
+ Values:
+ - "quadratic-preview-base-ami"
+ - Name: state
+ Values:
+ - available
+ Owners:
+ - !Ref AWS::AccountId
+ MostRecent: true
+
+ # Security Group for the EC2 instance
SecurityGroup:
Type: AWS::EC2::SecurityGroup
DeletionPolicy: Delete
@@ -32,6 +45,7 @@ Resources:
ToPort: 443
CidrIp: 0.0.0.0/0
+ # IAM Role for the EC2 instance
EC2Role:
Type: AWS::IAM::Role
DeletionPolicy: Delete
@@ -49,47 +63,8 @@ Resources:
Action: sts:AssumeRole
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
- - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
- Policies:
- - PolicyName: ECRAccess
- PolicyDocument:
- Version: 2012-10-17
- Statement:
- - Effect: Allow
- Action:
- - ecr:GetAuthorizationToken
- - ecr:BatchCheckLayerAvailability
- - ecr:GetDownloadUrlForLayer
- - ecr:BatchGetImage
- - ecr:DescribeImages
- - ecr:ListImages
- Resource: "*"
-
- EventBridgeRole:
- Type: AWS::IAM::Role
- DeletionPolicy: Delete
- UpdateReplacePolicy: Delete
- Properties:
- Tags:
- - Key: Name
- Value: !Ref ImageTag
- AssumeRolePolicyDocument:
- Version: 2012-10-17
- Statement:
- - Effect: Allow
- Principal:
- Service: events.amazonaws.com
- Action: sts:AssumeRole
- Policies:
- - PolicyName: InvokeSsmAutomation
- PolicyDocument:
- Version: 2012-10-17
- Statement:
- - Effect: Allow
- Action:
- - ssm:SendCommand
- Resource: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:*"
+ # IAM Instance Profile for the EC2 instance
EC2InstanceProfile:
Type: AWS::IAM::InstanceProfile
DeletionPolicy: Delete
@@ -98,6 +73,7 @@ Resources:
Roles:
- !Ref EC2Role
+ # EC2 instance
EC2Instance:
Type: AWS::EC2::Instance
DeletionPolicy: Delete
@@ -105,9 +81,9 @@ Resources:
Properties:
Tags:
- Key: Name
- Value: !Ref ImageTag
+ Value: !Sub "${ImageTag}"
InstanceType: m6a.large
- ImageId: ami-0fd0cf59e80d3f545 # quadratic-preview-ami-base
+ ImageId: !GetAtt AMIInfo.Id
IamInstanceProfile: !Ref EC2InstanceProfile
SecurityGroups:
- !Ref SecurityGroup
@@ -121,64 +97,109 @@ Resources:
#!/bin/bash
# Append environment variables to .env file
- cat << EOF >> /home/ubuntu/quadratic-selfhost/.env
+ cat << EOF >> /quadratic-selfhost/.env
#AWS
ECR_URL=${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com
IMAGE_TAG=${ImageTag}
- GIT_SHA=${GitSHA}
EOF
# Create login.sh script and run it
- cat << 'EOF' > /home/ubuntu/quadratic-selfhost/login.sh
+ cat << 'EOF' > /quadratic-selfhost/login.sh
#!/bin/bash
aws ecr get-login-password --region ${AWS::Region} | docker login --username AWS --password-stdin ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com
EOF
- chmod +x /home/ubuntu/quadratic-selfhost/login.sh
- cd /home/ubuntu/quadratic-selfhost
+ chmod +x /quadratic-selfhost/login.sh
+ cd /quadratic-selfhost
./login.sh
# Run Quadratic initialization script
- ./init.sh "${LicenseKey}" "${ImageTag}-${GitSHA}.quadratic-preview.com"
+ ./init.sh "${LicenseKey}" "${ImageTag}.quadratic-preview.com"
+ # # IAM Role for the EventBridge rule
+ # EventBridgeRole:
+ # Type: AWS::IAM::Role
+ # DeletionPolicy: Delete
+ # UpdateReplacePolicy: Delete
+ # Properties:
+ # Tags:
+ # - Key: Name
+ # Value: !Ref ImageTag
+ # AssumeRolePolicyDocument:
+ # Version: 2012-10-17
+ # Statement:
+ # - Effect: Allow
+ # Principal:
+ # Service: events.amazonaws.com
+ # Action: sts:AssumeRole
+ # Policies:
+ # - PolicyName: InvokeSsmAutomation
+ # PolicyDocument:
+ # Version: 2012-10-17
+ # Statement:
+ # - Effect: Allow
+ # Action:
+ # - ssm:SendCommand
+ # Resource: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:*"
+
+ # EcrPushRule:
+ # Type: AWS::Events::Rule
+ # DeletionPolicy: Delete
+ # UpdateReplacePolicy: Delete
+ # Properties:
+ # Description: Rule to detect ECR image pushes
+ # EventPattern:
+ # source:
+ # - aws.ecr
+ # detail-type:
+ # - ECR Image Action
+ # detail:
+ # action-type:
+ # - PUSH
+ # repository-name:
+ # - quadratic-client-development
+ # - quadratic-api-development
+ # - quadratic-multiplayer-development
+ # - quadratic-files-development
+ # - quadratic-connection-development
+ # image-tag:
+ # - !Ref ImageTag
+ # State: ENABLED
+ # Targets:
+ # - Arn: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:automation-definition/AWS-RunShellScript"
+ # Id: TriggerScriptOnInstance
+ # RoleArn: !GetAtt EventBridgeRole.Arn
+ # InputTransformer:
+ # InputPathsMap:
+ # imageTag: $.detail.image-tag
+ # repository: $.detail.repository-name
+ # InputTemplate: !Sub |
+ # {
+ # "InstanceIds": ["${EC2Instance.InstanceId}"],
+ # "DocumentName": "AWS-RunShellScript",
+ # "Parameters": {
+ # "commands": [
+ # "cd /quadratic-selfhost",
+ # "./login.sh",
+ # "./start.sh"
+ # ]
+ # }
+ # }
+
+ # Global Accelerator
GlobalAccelerator:
Type: AWS::GlobalAccelerator::Accelerator
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
Properties:
- Name: !Sub "${ImageTag}-${GitSHA}"
+ Name: !Sub "${ImageTag}"
Enabled: true
Tags:
- Key: Name
Value: !Ref ImageTag
- DnsRecords:
- Type: AWS::Route53::RecordSet
- DeletionPolicy: Delete
- UpdateReplacePolicy: Delete
- Properties:
- HostedZoneId: Z0126430TJ1UYIMO3SYX
- Name: !Sub "${ImageTag}-${GitSHA}.quadratic-preview.com"
- Type: A
- AliasTarget:
- DNSName: !GetAtt GlobalAccelerator.DnsName
- HostedZoneId: Z2BJ6XQ5FK7U4H
- EvaluateTargetHealth: true
-
- WildcardDnsRecord:
- Type: AWS::Route53::RecordSet
- DeletionPolicy: Delete
- UpdateReplacePolicy: Delete
- Properties:
- HostedZoneId: Z0126430TJ1UYIMO3SYX
- Name: !Sub "*.${ImageTag}-${GitSHA}.quadratic-preview.com"
- Type: A
- AliasTarget:
- DNSName: !GetAtt GlobalAccelerator.DnsName
- HostedZoneId: Z2BJ6XQ5FK7U4H
- EvaluateTargetHealth: true
-
+ # Global Accelerator Listener
GlobalAcceleratorListener:
Type: AWS::GlobalAccelerator::Listener
DeletionPolicy: Delete
@@ -192,50 +213,7 @@ Resources:
- FromPort: 443
ToPort: 443
- EcrPushRule:
- Type: AWS::Events::Rule
- DeletionPolicy: Delete
- UpdateReplacePolicy: Delete
- Properties:
- Description: Rule to detect ECR image pushes
- EventPattern:
- source:
- - aws.ecr
- detail-type:
- - ECR Image Action
- detail:
- action-type:
- - PUSH
- repository-name:
- - quadratic-client-development
- - quadratic-api-development
- - quadratic-multiplayer-development
- - quadratic-files-development
- - quadratic-connection-development
- image-tag:
- - !Ref ImageTag
- State: ENABLED
- Targets:
- - Arn: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:automation-definition/AWS-RunShellScript"
- Id: TriggerScriptOnInstance
- RoleArn: !GetAtt EventBridgeRole.Arn
- InputTransformer:
- InputPathsMap:
- imageTag: $.detail.image-tag
- repository: $.detail.repository-name
- InputTemplate: !Sub |
- {
- "InstanceIds": ["${EC2Instance.InstanceId}"],
- "DocumentName": "AWS-RunShellScript",
- "Parameters": {
- "commands": [
- "cd /home/ubuntu/quadratic-selfhost",
- "./login.sh",
- "./start.sh"
- ]
- }
- }
-
+ # Global Accelerator Endpoint Group
GlobalAcceleratorEndpointGroup:
Type: AWS::GlobalAccelerator::EndpointGroup
DeletionPolicy: Delete
@@ -248,7 +226,35 @@ Resources:
Weight: 100
ClientIPPreservationEnabled: true
+ # DNS record for the preview deployment for the main application
+ DNSRecord:
+ Type: AWS::Route53::RecordSet
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ Properties:
+ HostedZoneId: Z0126430TJ1UYIMO3SYX
+ Name: !Sub "${ImageTag}.quadratic-preview.com"
+ Type: A
+ AliasTarget:
+ DNSName: !GetAtt GlobalAccelerator.DnsName
+ HostedZoneId: Z2BJ6XQ5FK7U4H
+ EvaluateTargetHealth: true
+
+ # Wildcard DNS record for the preview deployment for all subdomains
+ WildcardDNSRecord:
+ Type: AWS::Route53::RecordSet
+ DeletionPolicy: Delete
+ UpdateReplacePolicy: Delete
+ Properties:
+ HostedZoneId: Z0126430TJ1UYIMO3SYX
+ Name: !Sub "*.${ImageTag}.quadratic-preview.com"
+ Type: A
+ AliasTarget:
+ DNSName: !GetAtt GlobalAccelerator.DnsName
+ HostedZoneId: Z2BJ6XQ5FK7U4H
+ EvaluateTargetHealth: true
+
Outputs:
- DnsRecord:
- Description: Url of the preview instance
- Value: !Sub "${ImageTag}-${GitSHA}.quadratic-preview.com"
+ WebsiteURL:
+ Description: Website URL
+ Value: !Sub "https://${ImageTag}.quadratic-preview.com"
From 7f49585ee0de1deff9beccb37130eb5e8f844243 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Wed, 25 Dec 2024 14:49:37 +0530
Subject: [PATCH 102/155] more bugs
---
.github/workflows/preview-cloudformation-deploy.yml | 9 +++++----
.github/workflows/preview-publish-images.yml | 4 ++--
client.Dockerfile | 1 +
docker-compose.yml | 1 +
4 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml
index d94b782a65..a47525b12c 100644
--- a/.github/workflows/preview-cloudformation-deploy.yml
+++ b/.github/workflows/preview-cloudformation-deploy.yml
@@ -34,21 +34,22 @@ jobs:
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }}
aws-region: ${{ secrets.AWS_REGION }}
- - name: Check and Delete Stack if in ROLLBACK
+ - name: Check and Delete Stack if in ROLLBACK or DELETE
id: check-stack
run: |
if aws cloudformation describe-stacks --stack-name ${{ env.STACK_NAME }} 2>/dev/null; then
STACK_STATUS=$(aws cloudformation describe-stacks --stack-name ${{ env.STACK_NAME }} --query 'Stacks[0].StackStatus' --output text)
echo "Current stack status: $STACK_STATUS"
- if [ "$STACK_STATUS" = "ROLLBACK_COMPLETE" ] || [ "$STACK_STATUS" = "ROLLBACK_FAILED" ]; then
+ if [ "$STACK_STATUS" = "ROLLBACK_COMPLETE" ] || [ "$STACK_STATUS" = "ROLLBACK_FAILED" ] || \
+ [ "$STACK_STATUS" = "DELETE_FAILED" ] || [ "$STACK_STATUS" = "DELETE_IN_PROGRESS" ]; then
echo "Stack is in $STACK_STATUS state. Deleting..."
aws cloudformation delete-stack --stack-name ${{ env.STACK_NAME }}
echo "Waiting for stack deletion to complete..."
aws cloudformation wait stack-delete-complete --stack-name ${{ env.STACK_NAME }}
echo "deleted=true" >> $GITHUB_OUTPUT
else
- echo "Stack exists but is not in ROLLBACK state"
+ echo "Stack exists but is not in ROLLBACK or DELETE state"
echo "deleted=false" >> $GITHUB_OUTPUT
fi
else
@@ -64,7 +65,7 @@ jobs:
template: infra/aws-cloudformation/preview-deployment-template.yml
parameter-overrides: >-
LicenseKey=${{ secrets.QUADRATIC_LICENSE_KEY }},
- ImageTag=pr-${{ github.event.pull_request.number }},
+ ImageTag=pr-${{ github.event.pull_request.number }}
capabilities: CAPABILITY_IAM
no-fail-on-empty-changeset: "1"
disable-rollback: false
diff --git a/.github/workflows/preview-publish-images.yml b/.github/workflows/preview-publish-images.yml
index b8bd8ea9fb..260cbc01aa 100644
--- a/.github/workflows/preview-publish-images.yml
+++ b/.github/workflows/preview-publish-images.yml
@@ -2,7 +2,7 @@ name: Build and Publish Images to ECR - Preview
on:
pull_request:
- types: [opened, synchronize, reopened]
+ types: [opened, reopened]
concurrency:
group: pr-${{ github.event.pull_request.number }}-publish-images
@@ -89,7 +89,7 @@ jobs:
context: .
file: quadratic-${{ matrix.service }}/Dockerfile
push: true
- tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }}-${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
+ tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }}
build-args: |
BUILDKIT_INLINE_CACHE=1
BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }}
diff --git a/client.Dockerfile b/client.Dockerfile
index ea1366e10e..79caeca4a4 100644
--- a/client.Dockerfile
+++ b/client.Dockerfile
@@ -67,6 +67,7 @@ ENV VITE_QUADRATIC_API_URL=VITE_QUADRATIC_API_URL_VAL
ENV VITE_QUADRATIC_MULTIPLAYER_URL=VITE_QUADRATIC_MULTIPLAYER_URL_VAL
ENV VITE_QUADRATIC_CONNECTION_URL=VITE_QUADRATIC_CONNECTION_URL_VAL
ENV VITE_AUTH_TYPE=VITE_AUTH_TYPE_VAL
+ENV VITE_STORAGE_TYPE=VITE_STORAGE_TYPE_VAL
ENV VITE_ORY_HOST=VITE_ORY_HOST_VAL
RUN npm run build --workspace=quadratic-client
diff --git a/docker-compose.yml b/docker-compose.yml
index db29d07208..7a1d918fa6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -92,6 +92,7 @@ services:
VITE_QUADRATIC_MULTIPLAYER_URL: ws://localhost:3001/ws
VITE_QUADRATIC_CONNECTION_URL: http://localhost:3003
VITE_AUTH_TYPE: ory
+ VITE_STORAGE_TYPE: file-system
VITE_ORY_HOST: http://localhost:4433
restart: "always"
ports:
From 63b420f4228fe963a3bb833f45702e8fec1a2719 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Wed, 25 Dec 2024 14:51:59 +0530
Subject: [PATCH 103/155] fix
---
.github/workflows/preview-cloudformation-deploy.yml | 2 +-
.github/workflows/preview-publish-images.yml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml
index a47525b12c..b0ff2b6796 100644
--- a/.github/workflows/preview-cloudformation-deploy.yml
+++ b/.github/workflows/preview-cloudformation-deploy.yml
@@ -2,7 +2,7 @@ name: Deploy to AWS CloudFormation - Preview
on:
pull_request:
- types: [opened, synchronize, reopened]
+ types: [opened, reopened]
concurrency:
group: pr-${{ github.event.pull_request.number }}-deploy-cloudformation
diff --git a/.github/workflows/preview-publish-images.yml b/.github/workflows/preview-publish-images.yml
index 260cbc01aa..75e2b2242e 100644
--- a/.github/workflows/preview-publish-images.yml
+++ b/.github/workflows/preview-publish-images.yml
@@ -2,7 +2,7 @@ name: Build and Publish Images to ECR - Preview
on:
pull_request:
- types: [opened, reopened]
+ types: [opened, synchronize, reopened]
concurrency:
group: pr-${{ github.event.pull_request.number }}-publish-images
From 1bcab8d8aee66e15fc7ddef34e71a30e165507fa Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Wed, 25 Dec 2024 15:48:40 +0530
Subject: [PATCH 104/155] fix python build
---
quadratic-client/Dockerfile | 30 ++++++++++++++++++------------
1 file changed, 18 insertions(+), 12 deletions(-)
diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile
index 6280a1f31a..599723c836 100644
--- a/quadratic-client/Dockerfile
+++ b/quadratic-client/Dockerfile
@@ -21,14 +21,20 @@ RUN apt-get update && apt-get install -y --no-install-recommends python-is-pytho
WORKDIR /quadratic
-# Copy root dependencies
-# Any changes to these files will trigger a full rebuild (version change)
+# Copy updateAlertVersion.json
COPY updateAlertVersion.json .
+
+# Copy all package.json files
COPY package.json .
+COPY package-lock.json .
+COPY ./quadratic-kernels/python-wasm/package*.json ./quadratic-kernels/python-wasm/
+COPY ./quadratic-core/package*.json ./quadratic-core/
+COPY ./quadratic-rust-client/package*.json ./quadratic-rust-client/
+COPY ./quadratic-shared/package*.json ./quadratic-shared/
+COPY ./quadratic-client/package*.json ./quadratic-client/
-# Run the packaging script for quadratic_py
-COPY ./quadratic-kernels/python-wasm/. ./quadratic-kernels/python-wasm/
-RUN ./quadratic-kernels/python-wasm/package.sh --no-poetry
+# Install npm dependencies
+RUN npm install --no-audit --no-fund
# Build wasm and export TS/Rust types
COPY ./quadratic-core/. ./quadratic-core/
@@ -41,18 +47,18 @@ ARG GIT_COMMIT
ENV GIT_COMMIT=$GIT_COMMIT
RUN echo 'Building quadratic-rust-client...' && npm run build --workspace=quadratic-rust-client
-# Install npm dependencies
-COPY package-lock.json .
-COPY ./quadratic-shared/package*.json ./quadratic-shared/
-COPY ./quadratic-client/package*.json ./quadratic-client/
-RUN npm install --no-audit --no-fund
+# Copy rest of the files
+COPY ./quadratic-kernels/python-wasm/. ./quadratic-kernels/python-wasm/
+COPY ./quadratic-shared/. ./quadratic-shared/
+COPY ./quadratic-client/. ./quadratic-client/
+
+# Run the packaging script for quadratic_py
+RUN ./quadratic-kernels/python-wasm/package.sh --no-poetry
# Build the quadratic-shared
-COPY ./quadratic-shared/. ./quadratic-shared/
RUN echo 'Building quadratic-shared...' && npm run compile --workspace=quadratic-shared
# Build the front-end
-COPY ./quadratic-client/. ./quadratic-client/
ENV VITE_DEBUG=VITE_DEBUG_VAL
ENV VITE_QUADRATIC_API_URL=VITE_QUADRATIC_API_URL_VAL
ENV VITE_QUADRATIC_MULTIPLAYER_URL=VITE_QUADRATIC_MULTIPLAYER_URL_VAL
From eb8b885858370ae3a351c75f6ca9918b7c107875 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Thu, 26 Dec 2024 05:14:11 +0530
Subject: [PATCH 105/155] run locally in docker - build or pull from ecr
---
.env.docker | 104 ++++++
.env.example | 6 -
.vscode/settings.json | 16 +
client.Dockerfile | 89 -----
dev/compile.sh | 2 +-
dev/control.js | 8 +-
dev/control.ts | 8 +-
docker-compose.base.yml | 53 ---
docker-compose.build.yml | 35 ++
docker-compose.ecr.yml | 19 +
docker-compose.yml | 337 ++++++++++--------
docker/client/config/default.conf | 24 ++
docker/client/scripts/replace_env_vars.sh | 34 ++
docker/file-storage/.gitignore | 0
docker/ory-auth/config/kratos.yml | 8 +-
docker/postgres/.gitignore | 0
docker/redis/.gitignore | 0
docker/static/certs/example.com+1-key.pem | 28 --
docker/static/certs/example.com+1.pem | 26 --
docker/static/certs/example.com+2-key.pem | 28 --
docker/static/certs/example.com+2.pem | 27 --
docker/static/conf/example.com+1.conf | 40 ---
docker/static/default.conf | 13 -
infra/aws-cloudformation/{ => ami}/ami.sh | 2 +-
.../ami/quadratic-selfhost/.env.docker | 95 +++++
.../ami/quadratic-selfhost/docker-compose.yml | 297 +++++++++++++++
.../docker/admin/config/openapi.yaml | 48 +++
.../docker/caddy/config/Caddyfile | 27 ++
.../docker/client/config/default.conf | 24 ++
.../docker/client/scripts/replace_env_vars.sh | 34 ++
.../ory-auth/config/identity.schema.json | 47 +++
.../docker/ory-auth/config/kratos.yml | 138 +++++++
.../docker/postgres/scripts/init.sh | 18 +
.../ami/quadratic-selfhost/init.sh | 135 +++++++
.../ami/quadratic-selfhost/start.sh | 12 +
.../ami/quadratic-selfhost/stop.sh | 8 +
.../preview-deployment-template.yml | 33 +-
package.json | 4 +-
quadratic-api/.env.docker | 20 --
quadratic-api/.env.example | 4 +-
quadratic-api/.env.test | 4 +-
quadratic-client/.env.docker | 9 -
quadratic-client/Dockerfile | 3 -
quadratic-connection/.env.docker | 9 -
quadratic-files/.env.docker | 30 --
quadratic-files/.env.example | 4 +-
quadratic-files/.env.test | 4 +-
quadratic-multiplayer/.env.docker | 15 -
48 files changed, 1345 insertions(+), 584 deletions(-)
create mode 100644 .env.docker
delete mode 100644 .env.example
delete mode 100644 client.Dockerfile
delete mode 100644 docker-compose.base.yml
create mode 100644 docker-compose.build.yml
create mode 100644 docker-compose.ecr.yml
create mode 100644 docker/client/config/default.conf
create mode 100755 docker/client/scripts/replace_env_vars.sh
delete mode 100644 docker/file-storage/.gitignore
delete mode 100644 docker/postgres/.gitignore
delete mode 100644 docker/redis/.gitignore
delete mode 100644 docker/static/certs/example.com+1-key.pem
delete mode 100644 docker/static/certs/example.com+1.pem
delete mode 100644 docker/static/certs/example.com+2-key.pem
delete mode 100644 docker/static/certs/example.com+2.pem
delete mode 100644 docker/static/conf/example.com+1.conf
delete mode 100644 docker/static/default.conf
rename infra/aws-cloudformation/{ => ami}/ami.sh (92%)
create mode 100644 infra/aws-cloudformation/ami/quadratic-selfhost/.env.docker
create mode 100644 infra/aws-cloudformation/ami/quadratic-selfhost/docker-compose.yml
create mode 100644 infra/aws-cloudformation/ami/quadratic-selfhost/docker/admin/config/openapi.yaml
create mode 100644 infra/aws-cloudformation/ami/quadratic-selfhost/docker/caddy/config/Caddyfile
create mode 100644 infra/aws-cloudformation/ami/quadratic-selfhost/docker/client/config/default.conf
create mode 100755 infra/aws-cloudformation/ami/quadratic-selfhost/docker/client/scripts/replace_env_vars.sh
create mode 100644 infra/aws-cloudformation/ami/quadratic-selfhost/docker/ory-auth/config/identity.schema.json
create mode 100644 infra/aws-cloudformation/ami/quadratic-selfhost/docker/ory-auth/config/kratos.yml
create mode 100644 infra/aws-cloudformation/ami/quadratic-selfhost/docker/postgres/scripts/init.sh
create mode 100755 infra/aws-cloudformation/ami/quadratic-selfhost/init.sh
create mode 100755 infra/aws-cloudformation/ami/quadratic-selfhost/start.sh
create mode 100755 infra/aws-cloudformation/ami/quadratic-selfhost/stop.sh
delete mode 100644 quadratic-api/.env.docker
delete mode 100644 quadratic-client/.env.docker
delete mode 100644 quadratic-connection/.env.docker
delete mode 100644 quadratic-files/.env.docker
delete mode 100644 quadratic-multiplayer/.env.docker
diff --git a/.env.docker b/.env.docker
new file mode 100644
index 0000000000..1d3c9653dd
--- /dev/null
+++ b/.env.docker
@@ -0,0 +1,104 @@
+# global
+ENVIRONMENT=docker
+
+# Your license key for Quadratic. Get one here https://selfhost.quadratichq.com/
+LICENSE_KEY=LICENSE_KEY
+
+# use image from ECR or build locally
+ECR_OR_BUILD=ecr
+
+ECR_URL=public.ecr.aws/z7e3d4w1
+IMAGE_TAG=latest
+
+# postgres database
+DATABASE_IN_DOCKER_COMPOSE=true
+DATABASE_DSN=postgresql://postgres:postgres@host.docker.internal:5432/postgres
+
+# pubsub
+PUBSUB_IN_DOCKER_COMPOSE=true
+PUBSUB_HOST=host.docker.internal
+PUBSUB_PORT=6379
+PUBSUB_PASSWORD=""
+PUBSUB_ACTIVE_CHANNELS=active_channels
+PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL=processed_transactions
+
+# auth: ory or auth0
+AUTH_TYPE=ory
+JWKS_URI=http://host.docker.internal:3000/.well-known/jwks.json
+M2M_AUTH_TOKEN=M2M_AUTH_TOKEN
+ENCRYPTION_KEY=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc
+
+# auth=ory
+ORY_IN_DOCKER_COMPOSE=true
+ORY_DSN=postgresql://postgres:postgres@host.docker.internal:5432/kratos?sslmode=disable
+ORY_LOG_LEVEL=trace
+ORY_ADMIN_HOST=http://host.docker.internal:4434
+KRATOS_URL_INTERNAL=http://host.docker.internal:4433/
+KRATOS_URL_EXTERNAL=http://localhost:4433/
+KRATOS_NODE_PORT=4455
+KRATOS_COOKIE_SECRET=changeme
+KRATOS_CSRF_COOKIE_NAME=__HOST-localhost-x-csrf-token
+KRATOS_CSRF_COOKIE_SECRET=changeme
+
+# storage - s3 or file-system
+STORAGE_TYPE=file-system
+
+# storage=s3
+AWS_S3_REGION=
+AWS_S3_BUCKET_NAME=quadratic-api-docker
+AWS_S3_ACCESS_KEY_ID=
+AWS_S3_SECRET_ACCESS_KEY=
+
+# storage=file-system
+STORAGE_DIR=/file-storage
+
+# caddy
+CADDY_IN_DOCKER_COMPOSE=true
+
+# client
+QUADRATIC_CLIENT_IN_DOCKER_COMPOSE=true
+
+# api
+QUADRATIC_API_IN_DOCKER_COMPOSE=true
+QUADRATIC_API_URL_EXTERNAL=http://localhost:8000
+QUADRATIC_API_URL_INTERNAL=http://host.docker.internal:8000
+
+# multiplayer
+QUADRATIC_MULTIPLAYER_IN_DOCKER_COMPOSE=true
+QUADRATIC_MULTIPLAYER_RUST_LOG=info
+QUADRATIC_MULTIPLAYER_HOST=0.0.0.0
+QUADRATIC_MULTIPLAYER_PORT=3001
+QUADRATIC_MULTIPLAYER_HEARTBEAT_CHECK_S=3
+QUADRATIC_MULTIPLAYER_HEARTBEAT_TIMEOUT_S=600
+QUADRATIC_MULTIPLAYER_URL_EXTERNAL=ws://localhost:3001/ws
+QUADRATIC_MULTIPLAYER_URL_INTERNAL=ws://host.docker.internal:3001
+
+# files
+QUADRATIC_FILES_IN_DOCKER_COMPOSE=true
+QUADRATIC_FILES_RUST_LOG=info
+QUADRATIC_FILES_HOST=0.0.0.0
+QUADRATIC_FILES_PORT=3002
+QUADRATIC_FILES_FILE_CHECK_S=5
+QUADRATIC_FILES_FILES_PER_CHECK=1000
+QUADRATIC_FILES_TRUNCATE_FILE_CHECK_S=60
+QUADRATIC_FILES_TRUNCATE_TRANSACTION_AGE_DAYS=5
+QUADRATIC_FILES_URL_EXTERNAL=http://localhost:3002
+QUADRATIC_FILES_URL_INTERNAL=http://host.docker.internal:3002
+
+# connection
+QUADRATIC_CONNECTION_IN_DOCKER_COMPOSE=true
+QUADRATIC_CONNECTION_RUST_LOG=info
+QUADRATIC_CONNECTION_HOST=0.0.0.0
+QUADRATIC_CONNECTION_PORT=3003
+QUADRATIC_CONNECTION_URL_EXTERNAL=http://localhost:3003
+QUADRATIC_CONNECTION_URL_INTERNAL=http://host.docker.internal:3003
+QUADRATIC_CONNECTION_MAX_RESPONSE_BYTES=15728640 # 15MB
+QUADRATIC_CONNECTION_STATIC_IPS=0.0.0.0,127.0.0.1
+
+# stripe
+STRIPE_SECRET_KEY=STRIPE_SECRET_KEY
+STRIPE_WEBHOOK_SECRET=STRIPE_WEBHOOK_SECRET
+
+# ai
+OPENAI_API_KEY=
+ANTHROPIC_API_KEY=
diff --git a/.env.example b/.env.example
deleted file mode 100644
index c1fe3a6a24..0000000000
--- a/.env.example
+++ /dev/null
@@ -1,6 +0,0 @@
-VITE_AUTH0_ISSUER=https://quadratic-community.us.auth0.com/
-VITE_AUTH0_DOMAIN=quadratic-community.us.auth0.com
-VITE_AUTH0_CLIENT_ID=DCPCvqyU5Q0bJD8Q3QmJEoV48x1zLH7W
-VITE_AUTH0_AUDIENCE=community-quadratic
-
-VITE_QUADRATIC_API_URL=http://localhost:8000
\ No newline at end of file
diff --git a/.vscode/settings.json b/.vscode/settings.json
index 9b548d8a97..d7dffa6a46 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -33,19 +33,26 @@
"htmlescape",
"indexmap",
"indicies",
+ "initdb",
+ "isready",
"itertools",
"jwks",
"keyrings",
+ "localstack",
+ "mailslurper",
"MDSL",
"micropip",
"minmax",
"moby",
"msdf",
+ "mysqladmin",
"nonblank",
"Northbridge",
"openai",
"opensans",
+ "oryd",
"peekable",
+ "PGUSER",
"pixi",
"pixiapp",
"Plotly",
@@ -58,10 +65,15 @@
"scrollend",
"selfhost",
"selfhosted",
+ "selfservice",
"shadcn",
"Signin",
"smallpop",
+ "smtps",
"Southborough",
+ "sqlcmd",
+ "sqlservr",
+ "sslmode",
"Strftime",
"szhsin",
"thiserror",
@@ -73,6 +85,7 @@
"usermod",
"vals",
"vcpu",
+ "VITE",
"websockets",
"Westborough",
"zstd"
@@ -120,5 +133,8 @@
},
"[html]": {
"editor.defaultFormatter": "esbenp.prettier-vscode"
+ },
+ "[markdown]": {
+ "editor.defaultFormatter": "esbenp.prettier-vscode"
}
}
diff --git a/client.Dockerfile b/client.Dockerfile
deleted file mode 100644
index 79caeca4a4..0000000000
--- a/client.Dockerfile
+++ /dev/null
@@ -1,89 +0,0 @@
-# Use an official node image as a parent image
-FROM node:18 AS build
-
-# Install rustup
-RUN echo 'Installing rustup...' && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-ENV PATH="/root/.cargo/bin:${PATH}"
-
-# Install wasm-pack
-RUN echo 'Installing wasm-pack...' && curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
-
-# Install wasm32-unknown-unknown target
-RUN rustup target add wasm32-unknown-unknown
-
-# Install python, binaryen & clean up
-RUN apt-get update && apt-get install -y python-is-python3 python3-pip binaryen && apt-get clean && rm -rf /var/lib/apt/lists/*
-
-# Install npm dependencies
-WORKDIR /app
-COPY package.json .
-COPY package-lock.json .
-COPY ./quadratic-kernels/python-wasm/package*.json ./quadratic-kernels/python-wasm/
-COPY ./quadratic-core/package*.json ./quadratic-core/
-COPY ./quadratic-rust-client/package*.json ./quadratic-rust-client/
-COPY ./quadratic-shared/package*.json ./quadratic-shared/
-COPY ./quadratic-client/package*.json ./quadratic-client/
-RUN npm install
-
-# Install typescript
-RUN npm install -D typescript
-
-# Copy the rest of the application
-WORKDIR /app
-COPY updateAlertVersion.json .
-COPY ./quadratic-kernels/python-wasm/. ./quadratic-kernels/python-wasm/
-COPY ./quadratic-core/. ./quadratic-core/
-COPY ./quadratic-rust-client/. ./quadratic-rust-client/
-COPY ./quadratic-shared/. ./quadratic-shared/
-COPY ./quadratic-client/. ./quadratic-client/
-
-# Run the packaging script for quadratic_py
-WORKDIR /app
-RUN ./quadratic-kernels/python-wasm/package.sh --no-poetry
-
-# Build wasm
-WORKDIR /app/quadratic-core
-RUN echo 'Building wasm...' && wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs
-
-# Export TS/Rust types
-WORKDIR /app/quadratic-core
-RUN echo 'Exporting TS/Rust types...' && cargo run --bin export_types
-
-# Build the quadratic-rust-client
-WORKDIR /app
-ARG GIT_COMMIT
-ENV GIT_COMMIT=$GIT_COMMIT
-RUN echo 'Building quadratic-rust-client...' && npm run build --workspace=quadratic-rust-client
-
-# Build the quadratic-shared
-WORKDIR /app
-RUN echo 'Building quadratic-shared...' && npm run compile --workspace=quadratic-shared
-
-# Build the front-end
-WORKDIR /app
-RUN echo 'Building front-end...'
-ENV VITE_DEBUG=VITE_DEBUG_VAL
-ENV VITE_QUADRATIC_API_URL=VITE_QUADRATIC_API_URL_VAL
-ENV VITE_QUADRATIC_MULTIPLAYER_URL=VITE_QUADRATIC_MULTIPLAYER_URL_VAL
-ENV VITE_QUADRATIC_CONNECTION_URL=VITE_QUADRATIC_CONNECTION_URL_VAL
-ENV VITE_AUTH_TYPE=VITE_AUTH_TYPE_VAL
-ENV VITE_STORAGE_TYPE=VITE_STORAGE_TYPE_VAL
-ENV VITE_ORY_HOST=VITE_ORY_HOST_VAL
-RUN npm run build --workspace=quadratic-client
-
-# The default command to run the application
-# CMD ["npm", "run", "start:production"]
-
-FROM nginx:stable-alpine
-COPY --from=build /app/build /usr/share/nginx/html
-
-EXPOSE 80 443 3000
-
-CMD ["nginx", "-g", "daemon off;"]
-
-
-
-
-
-
-
diff --git a/dev/compile.sh b/dev/compile.sh
index 44fc06dc96..37f948c500 100755
--- a/dev/compile.sh
+++ b/dev/compile.sh
@@ -1,4 +1,4 @@
#!/bin/bash
-tsc --module nodenext --moduleResolution nodenext dev/index.ts
+npx tsc --module nodenext --moduleResolution nodenext --skipLibCheck true dev/index.ts
node dev/index.js
\ No newline at end of file
diff --git a/dev/control.js b/dev/control.js
index 17395f08d9..1dd5175b20 100644
--- a/dev/control.js
+++ b/dev/control.js
@@ -446,7 +446,11 @@ export class Control {
this.signals.rustClient = new AbortController();
this.rustClient = spawn("npm", [
"run",
- this.cli.options.rustClient ? (this.cli.options.perf ? "dev:perf" : "dev") : "build",
+ this.cli.options.rustClient
+ ? this.cli.options.perf
+ ? "dev:perf"
+ : "dev"
+ : "build",
"--workspace=quadratic-rust-client",
], { signal: this.signals.rustClient.signal });
this.ui.printOutput("rustClient", (data) => this.handleResponse("rustClient", data, {
@@ -524,7 +528,7 @@ export class Control {
const servicesLocal = this.cli.options.servicesLocal;
const redis = servicesLocal
? spawn("redis-cli", ["ping"])
- : spawn("docker", ["exec", "quadratic-redis-1", "redis-cli", "ping"]);
+ : spawn("docker", ["exec", "redis", "redis-cli", "ping"]);
redis.on("error", (e) => {
if (e.code === "ENOENT") {
resolve("not found");
diff --git a/dev/control.ts b/dev/control.ts
index 3f520ada5a..5af7e840e0 100644
--- a/dev/control.ts
+++ b/dev/control.ts
@@ -531,7 +531,11 @@ export class Control {
"npm",
[
"run",
- this.cli.options.rustClient ? (this.cli.options.perf ? "dev:perf" :"dev") : "build",
+ this.cli.options.rustClient
+ ? this.cli.options.perf
+ ? "dev:perf"
+ : "dev"
+ : "build",
"--workspace=quadratic-rust-client",
],
{ signal: this.signals.rustClient.signal }
@@ -611,7 +615,7 @@ export class Control {
const servicesLocal = this.cli.options.servicesLocal;
const redis = servicesLocal
? spawn("redis-cli", ["ping"])
- : spawn("docker", ["exec", "quadratic-redis-1", "redis-cli", "ping"]);
+ : spawn("docker", ["exec", "redis", "redis-cli", "ping"]);
redis.on("error", (e: any) => {
if (e.code === "ENOENT") {
resolve("not found");
diff --git a/docker-compose.base.yml b/docker-compose.base.yml
deleted file mode 100644
index 278c217a08..0000000000
--- a/docker-compose.base.yml
+++ /dev/null
@@ -1,53 +0,0 @@
-version: "3.8"
-
-services:
- redis:
- image: redis/redis-stack:latest
- restart: always
- ports:
- - "6379:6379"
- - "8001:8001"
- healthcheck:
- test: ["CMD", "redis-cli", "ping"]
- interval: "5s"
- networks:
- - host
-
- postgres:
- image: postgres:15
- restart: always
- container_name: postgres
- ports:
- - "5432:5432"
- environment:
- POSTGRES_USER: postgres
- PGUSER: postgres
- POSTGRES_PASSWORD: postgres
- healthcheck:
- test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"]
- interval: 10s
- timeout: 5s
- retries: 5
-
- localstack:
- container_name: "${LOCALSTACK_DOCKER_NAME:-localstack}"
- image: localstack/localstack:latest
- ports:
- - "127.0.0.1:4566:4566" # LocalStack Gateway
- - "127.0.0.1:4510-4559:4510-4559"
- environment:
- # LocalStack configuration: https://docs.localstack.cloud/references/configuration/
- - LOCALSTACK_AUTH_TOKEN=${LOCALSTACK_AUTH_TOKEN:-}
- - DEBUG=${DEBUG:-0}
- - SERVICES=s3:4566
- - HOSTNAME=localstack
- - HOSTNAME_EXTERNAL=localstack
- - DEFAULT_REGION=us-east-2
- - DISABLE_CUSTOM_CORS_S3=true
- - DISABLE_CORS_CHECKS=true
- - EXTRA_CORS_ALLOWED_ORIGINS=*
- networks:
- - host
-
-networks:
- host:
diff --git a/docker-compose.build.yml b/docker-compose.build.yml
new file mode 100644
index 0000000000..975a6ae339
--- /dev/null
+++ b/docker-compose.build.yml
@@ -0,0 +1,35 @@
+# used to build the images locally
+
+services:
+ # quadratic services - client, api, multiplayer, files, connection
+
+ quadratic-client:
+ build:
+ context: .
+ dockerfile: quadratic-client/Dockerfile
+
+ quadratic-api:
+ build:
+ context: .
+ dockerfile: quadratic-api/Dockerfile
+
+ quadratic-multiplayer:
+ build:
+ context: .
+ dockerfile: quadratic-multiplayer/Dockerfile
+ args:
+ - binary=quadratic-multiplayer
+
+ quadratic-files:
+ build:
+ context: .
+ dockerfile: quadratic-files/Dockerfile
+ args:
+ - binary=quadratic-files
+
+ quadratic-connection:
+ build:
+ context: .
+ dockerfile: quadratic-connection/Dockerfile
+ args:
+ - binary=quadratic-connection
diff --git a/docker-compose.ecr.yml b/docker-compose.ecr.yml
new file mode 100644
index 0000000000..0050c8f72c
--- /dev/null
+++ b/docker-compose.ecr.yml
@@ -0,0 +1,19 @@
+# used to pull production images from ECR
+
+services:
+ # quadratic services - client, api, multiplayer, files, connection
+
+ quadratic-client:
+ image: ${ECR_URL}/quadratic-client:${IMAGE_TAG}
+
+ quadratic-api:
+ image: ${ECR_URL}/quadratic-api:${IMAGE_TAG}
+
+ quadratic-multiplayer:
+ image: ${ECR_URL}/quadratic-multiplayer:${IMAGE_TAG}
+
+ quadratic-files:
+ image: ${ECR_URL}/quadratic-files:${IMAGE_TAG}
+
+ quadratic-connection:
+ image: ${ECR_URL}/quadratic-connection:${IMAGE_TAG}
diff --git a/docker-compose.yml b/docker-compose.yml
index 7a1d918fa6..1f4679d7f0 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,223 +1,258 @@
-version: "3.8"
-
services:
+ # base services - redis, postgres
+
redis:
- extends:
- file: docker-compose.base.yml
- service: redis
+ image: redis/redis-stack:latest
+ restart: always
+ container_name: redis
+ ports:
+ - "6379:6379"
+ - "8001:8001"
+ healthcheck:
+ test: ["CMD", "redis-cli", "ping"]
+ interval: "5s"
volumes:
- ./docker/redis/data:/data
profiles:
- base
+ - all
postgres:
- extends:
- file: docker-compose.base.yml
- service: postgres
+ image: postgres:15
+ restart: always
+ container_name: postgres
+ ports:
+ - "5432:5432"
environment:
+ POSTGRES_USER: postgres
+ PGUSER: postgres
+ POSTGRES_PASSWORD: postgres
ADDITIONAL_DATABASES: kratos
+ healthcheck:
+ test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"]
+ interval: 10s
+ timeout: 5s
+ retries: 5
volumes:
- ./docker/postgres/data:/var/lib/postgresql/data
- ./docker/postgres/scripts:/docker-entrypoint-initdb.d
profiles:
- base
+ - all
+
+ # files service - local aws alternative, use this or file-storage
localstack:
- extends:
- file: docker-compose.base.yml
- service: localstack
+ container_name: "${LOCALSTACK_DOCKER_NAME:-localstack}"
+ image: localstack/localstack:latest
+ ports:
+ - "127.0.0.1:4566:4566" # LocalStack Gateway
+ - "127.0.0.1:4510-4559:4510-4559"
+ environment:
+ # LocalStack configuration: https://docs.localstack.cloud/references/configuration/
+ - LOCALSTACK_AUTH_TOKEN=${LOCALSTACK_AUTH_TOKEN:-}
+ - DEBUG=${DEBUG:-0}
+ - SERVICES=s3:4566
+ - HOSTNAME=localstack
+ - HOSTNAME_EXTERNAL=localstack
+ - DEFAULT_REGION=us-east-2
+ - DISABLE_CUSTOM_CORS_S3=true
+ - DISABLE_CORS_CHECKS=true
+ - EXTRA_CORS_ALLOWED_ORIGINS=*
volumes:
- "./docker/localstack/scripts/init-aws.sh:/etc/localstack/init/ready.d/init-aws.sh" # ready hook
- "./docker/localstack/data:/var/lib/localstack"
- "/var/run/docker.sock:/var/run/docker.sock"
profiles:
- base
+ networks:
+ - host
- # nginx:
- # restart: always
- # image: nginx:1-alpine
- # ports:
- # - 80:80
- # - 443:443
- # - 3000:80
- # volumes:
- # - ./docker/static/html:/usr/share/nginx/html
- # - ./docker/static/conf/:/etc/nginx/conf.d/:ro
- # - ./docker/static/certs:/etc/nginx/ssl
- # # - ./docker/static/default.conf:/etc/nginx/conf.d/default.conf
- # depends_on:
- # - quadratic-api
- # # networks:
- # # - host
-
- # https-portal:
- # image: steveltn/https-portal:1
- # ports:
- # - "80:80"
- # - "443:443"
- # restart: always
- # env_file:
- # - quadratic-client/.env.local
- # - quadratic-client/.env.docker
- # # override env vars here
- # environment:
- # DOMAINS: >
- # quadratic.lvh.me,
- # quadratic-api.lvh.me -> http://quadratic-api:8000,
- # quadratic-multiplayer.lvh.me -> http://quadratic-multiplayer:3001/ws,
- # quadratic-localstack.lvh.me -> http://localstack:4566
- # STAGE: "local" # Use 'production' to use a LetsEncrypt signed SSL cert
- # FORCE_RENEW: "false"
- # WEBSOCKET: "true"
-
- # VITE_QUADRATIC_API_URL: http://0.0.0.0:8000
- # VITE_QUADRATIC_MULTIPLAYER_URL: ws://0.0.0.0:3001
- # VITE_QUADRATIC_CONNECTION_URL: http://0.0.0.0:3003
- # depends_on:
- # # - quadratic-client
- # - quadratic-api
- # volumes:
- # - ./docker/https-portal/data:/var/lib/https-portal
- # - ./docker/https-portal/vhosts:/var/www/vhosts
- # - ./docker/https-portal/quadratic.lvh.me.conf.erb:/var/lib/nginx-conf/quadratic.lvh.me.conf.erb:ro
- # - ./docker/https-portal/quadratic.lvh.me.ssl.conf.erb:/var/lib/nginx-conf/quadratic.lvh.me.ssl.conf.erb:ro
+ # quadratic services - client, api, multiplayer, files, connection
quadratic-client:
- build:
- context: .
- dockerfile: client.Dockerfile
+ extends:
+ file: docker-compose.${ECR_OR_BUILD}.yml
+ service: quadratic-client
environment:
VITE_DEBUG: 1
- VITE_QUADRATIC_API_URL: http://localhost:8000
- VITE_QUADRATIC_MULTIPLAYER_URL: ws://localhost:3001/ws
- VITE_QUADRATIC_CONNECTION_URL: http://localhost:3003
- VITE_AUTH_TYPE: ory
- VITE_STORAGE_TYPE: file-system
- VITE_ORY_HOST: http://localhost:4433
- restart: "always"
+ VITE_QUADRATIC_API_URL: ${QUADRATIC_API_URL_EXTERNAL}
+ VITE_QUADRATIC_MULTIPLAYER_URL: ${QUADRATIC_MULTIPLAYER_URL_EXTERNAL}
+ VITE_QUADRATIC_CONNECTION_URL: ${QUADRATIC_CONNECTION_URL_EXTERNAL}
+ VITE_AUTH_TYPE: ${AUTH_TYPE}
+ VITE_STORAGE_TYPE: ${STORAGE_TYPE}
+ VITE_ORY_HOST: ${KRATOS_URL_EXTERNAL}
ports:
- # - "3000:3000"
- - 80:80
- - 443:443
- 3000:80
- # command: "npm run start:production --workspace=quadratic-client"
+ command: >
+ sh -c "/client/scripts/replace_env_vars.sh &&
+ nginx -g \"daemon off;\""
+ healthcheck:
+ test: ["CMD-SHELL", "curl -f http://host.docker.internal:3000/ || exit 1"]
+ interval: 10s
+ timeout: 5s
+ restart: "always"
depends_on:
- postgres:
- condition: service_healthy
+ quadratic-api:
+ condition: service_started
+ volumes:
+ - ./docker/client:/client
+ - ./docker/client/config/default.conf:/etc/nginx/conf.d/default.conf
profiles:
- - client
+ - quadratic-client
- frontend
- volumes:
- # - ./docker/static/html:/usr/share/nginx/html
- - ./docker/static/conf/:/etc/nginx/conf.d/:ro
- - ./docker/static/certs:/etc/nginx/ssl
- # - ./docker/static/default.conf:/etc/nginx/conf.d/default.conf
- # networks:
- # - host
+ - all
+ networks:
+ - host
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
quadratic-api:
- build:
- context: .
- dockerfile: quadratic-api/Dockerfile
- env_file:
- - quadratic-api/.env
- - quadratic-api/.env.docker
- # override env vars here
+ extends:
+ file: docker-compose.${ECR_OR_BUILD}.yml
+ service: quadratic-api
environment:
- AWS_S3_ENDPOINT: https://localhost/localstack
+ CORS: "*"
+ DATABASE_URL: ${DATABASE_DSN}
+ ENVIRONMENT: ${ENVIRONMENT}
+ STRIPE_SECRET_KEY: ${STRIPE_SECRET_KEY}
+ STRIPE_WEBHOOK_SECRET: ${STRIPE_WEBHOOK_SECRET}
+ OPENAI_API_KEY: ${OPENAI_API_KEY}
+ M2M_AUTH_TOKEN: ${M2M_AUTH_TOKEN}
+ ENCRYPTION_KEY: ${ENCRYPTION_KEY}
+ AUTH_TYPE: ${AUTH_TYPE}
+ ORY_JWKS_URI: ${JWKS_URI}
+ ORY_ADMIN_HOST: ${ORY_ADMIN_HOST}
+ STORAGE_TYPE: ${STORAGE_TYPE}
+ QUADRATIC_FILE_URI: ${QUADRATIC_FILES_URL_INTERNAL}
+ QUADRATIC_FILE_URI_PUBLIC: ${QUADRATIC_FILES_URL_EXTERNAL}
+ LICENSE_KEY: ${LICENSE_KEY}
restart: "always"
ports:
- "8000:8000"
- command: "npm run start:prod --workspace=quadratic-api"
+ command: bash -c "npx prisma migrate deploy --schema quadratic-api/prisma/schema.prisma && npm run start:prod --workspace=quadratic-api"
depends_on:
postgres:
condition: service_healthy
profiles:
- api
- - frontend
+ - backend
+ - all
+ networks:
+ - host
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
quadratic-multiplayer:
- build:
- context: .
- dockerfile: quadratic-multiplayer/Dockerfile
- args:
- - binary=quadratic-multiplayer
- env_file:
- - quadratic-multiplayer/.env.docker
- # override env vars here
+ extends:
+ file: docker-compose.${ECR_OR_BUILD}.yml
+ service: quadratic-multiplayer
environment:
- QUADRATIC_API_URI: http://host.docker.internal:8000
RUST_LOG: info
+ HOST: 0.0.0.0
+ PORT: 3001
+ HEARTBEAT_CHECK_S: 3
+ HEARTBEAT_TIMEOUT_S: 600
+ QUADRATIC_API_URI: ${QUADRATIC_API_URL_INTERNAL}
+ M2M_AUTH_TOKEN: ${M2M_AUTH_TOKEN}
+ ENVIRONMENT: ${ENVIRONMENT}
+ PUBSUB_HOST: ${PUBSUB_HOST}
+ PUBSUB_PORT: ${PUBSUB_PORT}
+ PUBSUB_PASSWORD: ${PUBSUB_PASSWORD}
+ PUBSUB_ACTIVE_CHANNELS: ${PUBSUB_ACTIVE_CHANNELS}
+ AUTH0_JWKS_URI: ${JWKS_URI}
+ AUTHENTICATE_JWT: true
restart: "always"
ports:
- "3001:3001"
depends_on:
- postgres:
- condition: service_healthy
redis:
condition: service_healthy
quadratic-api:
condition: service_started
profiles:
- - backend
- quadratic-multiplayer
+ - backend
+ - all
networks:
- host
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
quadratic-files:
- build:
- context: .
- dockerfile: quadratic-files/Dockerfile
- args:
- - binary=quadratic-files
- env_file:
- - quadratic-files/.env.docker
- # override env vars here
+ extends:
+ file: docker-compose.${ECR_OR_BUILD}.yml
+ service: quadratic-files
environment:
- QUADRATIC_API_URI: http://host.docker.internal:8000
- RUST_LOG: info
+ RUST_LOG: ${QUADRATIC_FILES_RUST_LOG}
+ HOST: ${QUADRATIC_FILES_HOST}
+ PORT: ${QUADRATIC_FILES_PORT}
+ FILE_CHECK_S: ${QUADRATIC_FILES_FILE_CHECK_S}
+ FILES_PER_CHECK: ${QUADRATIC_FILES_FILES_PER_CHECK}
+ TRUNCATE_FILE_CHECK_S: ${QUADRATIC_FILES_TRUNCATE_FILE_CHECK_S}
+ TRUNCATE_TRANSACTION_AGE_DAYS: ${QUADRATIC_FILES_TRUNCATE_TRANSACTION_AGE_DAYS}
+ ENVIRONMENT: ${ENVIRONMENT}
+ AUTH0_JWKS_URI: ${JWKS_URI}
+ QUADRATIC_API_URI: ${QUADRATIC_API_URL_INTERNAL}
+ M2M_AUTH_TOKEN: ${M2M_AUTH_TOKEN}
+ PUBSUB_HOST: ${PUBSUB_HOST}
+ PUBSUB_PORT: ${PUBSUB_PORT}
+ PUBSUB_PASSWORD: ${PUBSUB_PASSWORD}
+ PUBSUB_ACTIVE_CHANNELS: ${PUBSUB_ACTIVE_CHANNELS}
+ PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL: ${PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL}
+ STORAGE_TYPE: ${STORAGE_TYPE}
+ AWS_S3_REGION: ${AWS_S3_REGION}
+ AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+ AWS_S3_ACCESS_KEY_ID: ${AWS_S3_ACCESS_KEY_ID}
+ AWS_S3_SECRET_ACCESS_KEY: ${AWS_S3_SECRET_ACCESS_KEY}
+ STORAGE_DIR: ${STORAGE_DIR}
+ STORAGE_ENCRYPTION_KEYS: ${ENCRYPTION_KEY}
restart: "always"
ports:
- "3002:3002"
depends_on:
- postgres:
- condition: service_healthy
redis:
condition: service_healthy
quadratic-api:
condition: service_started
+ volumes:
+ - ./docker/file-storage:/file-storage
profiles:
- - backend
- quadratic-files
+ - backend
+ - all
networks:
- host
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
quadratic-connection:
- build:
- context: .
- dockerfile: quadratic-connection/Dockerfile
- args:
- - binary=quadratic-connection
- env_file:
- - quadratic-connection/.env.docker
- # override env vars here
+ extends:
+ file: docker-compose.${ECR_OR_BUILD}.yml
+ service: quadratic-connection
environment:
- QUADRATIC_API_URI: http://host.docker.internal:8000
- RUST_LOG: info
+ RUST_LOG: ${QUADRATIC_CONNECTION_RUST_LOG}
+ HOST: ${QUADRATIC_CONNECTION_HOST}
+ PORT: ${QUADRATIC_CONNECTION_PORT}
+ ENVIRONMENT: ${ENVIRONMENT}
+ AUTH0_JWKS_URI: ${JWKS_URI}
+ QUADRATIC_API_URI: ${QUADRATIC_API_URL_INTERNAL}
+ M2M_AUTH_TOKEN: ${M2M_AUTH_TOKEN}
+ MAX_RESPONSE_BYTES: ${QUADRATIC_CONNECTION_MAX_RESPONSE_BYTES}
+ STATIC_IPS: ${QUADRATIC_CONNECTION_STATIC_IPS}
restart: "always"
ports:
- "3003:3003"
- # depends_on:
- # postgres:
- # condition: service_healthy
- # quadratic-api:
- # condition: service_started
-
profiles:
- - backend
- quadratic-connection
+ - backend
+ - all
+ networks:
+ - host
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
- # Auth Providers
+ # auth service - ory
ory-auth:
image: oryd/kratos:v1.2.0
@@ -228,8 +263,8 @@ services:
volumes:
- ./docker/ory-auth/config:/etc/config/kratos
environment:
- DSN: postgresql://postgres:postgres@host.docker.internal:5432/kratos?sslmode=disable
- LOG_LEVEL: trace
+ DSN: ${ORY_DSN}
+ LOG_LEVEL: ${ORY_LOG_LEVEL}
restart: unless-stopped
depends_on:
- postgres
@@ -239,6 +274,8 @@ services:
- all
networks:
- host
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
ory-auth-migrate:
image: oryd/kratos:v1.2.0
@@ -246,7 +283,7 @@ services:
volumes:
- ./docker/ory-auth/config:/etc/config/kratos
environment:
- DSN: postgresql://postgres:postgres@host.docker.internal:5432/kratos?sslmode=disable
+ DSN: ${ORY_DSN}
restart: on-failure
depends_on:
- postgres
@@ -255,25 +292,28 @@ services:
- all
networks:
- host
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
ory-auth-node:
image: oryd/kratos-selfservice-ui-node:v1.2.0
ports:
- "4455:4455"
environment:
- PORT: 4455
- SECURITY_MODE:
- KRATOS_PUBLIC_URL: http://host.docker.internal:4433/
- KRATOS_BROWSER_URL: http://localhost:4433/
- COOKIE_SECRET: changeme
- CSRF_COOKIE_NAME: ory_csrf_ui
- CSRF_COOKIE_SECRET: changeme
+ PORT: ${KRATOS_NODE_PORT}
+ KRATOS_PUBLIC_URL: ${KRATOS_URL_INTERNAL}
+ KRATOS_BROWSER_URL: ${KRATOS_URL_EXTERNAL}
+ COOKIE_SECRET: ${KRATOS_COOKIE_SECRET}
+ CSRF_COOKIE_NAME: ${KRATOS_CSRF_COOKIE_NAME}
+ CSRF_COOKIE_SECRET: ${KRATOS_CSRF_COOKIE_SECRET}
restart: on-failure
profiles:
- ory
- all
networks:
- host
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
ory-auth-mail:
image: oryd/mailslurper:latest-smtps
@@ -287,8 +327,11 @@ services:
- all
networks:
- host
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
- # Databases to be used for testing by the connection service
+ # databases to be used for testing by the connection service
+ # postgres, mysql, mssql
postgres-connection:
image: postgres:15
diff --git a/docker/client/config/default.conf b/docker/client/config/default.conf
new file mode 100644
index 0000000000..a58529d83b
--- /dev/null
+++ b/docker/client/config/default.conf
@@ -0,0 +1,24 @@
+server {
+ listen 80;
+ server_name localhost;
+
+ root /usr/share/nginx/html;
+ index index.html;
+
+ location / {
+ try_files $uri $uri/ /index.html =404;
+ }
+
+ location ~* \.(?:css|js|json|gif|png|jpg|jpeg|svg|ico)$ {
+ expires 1y;
+ access_log off;
+ add_header Cache-Control "public, no-transform";
+
+ add_header Cross-Origin-Opener-Policy "same-origin";
+ add_header Cross-Origin-Embedder-Policy "require-corp";
+ }
+
+ # Add CORS headers to all requests
+ add_header Cross-Origin-Opener-Policy "same-origin";
+ add_header Cross-Origin-Embedder-Policy "require-corp";
+}
diff --git a/docker/client/scripts/replace_env_vars.sh b/docker/client/scripts/replace_env_vars.sh
new file mode 100755
index 0000000000..60160a8ccf
--- /dev/null
+++ b/docker/client/scripts/replace_env_vars.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+
+escape_for_sed() {
+ input="$1"
+ printf '%s\n' "$input" | sed -e 's/[\/&]/\\&/g'
+}
+
+replace_env_vars() {
+ vite_vars=""
+
+ for env_var in $(env); do
+ case "$env_var" in
+ VITE_*)
+ vite_vars="$vite_vars $env_var"
+ ;;
+ esac
+ done
+
+ find "/usr/share/nginx/html/assets" -type f -name "*.js" | xargs grep -l "VITE_" | while read file; do
+
+ for env_var in $vite_vars; do
+ var="$(echo "$env_var" | cut -d'=' -f1)"
+ val="$(echo "$env_var" | cut -d'=' -f2-)"
+ appended_var="${var}_VAL"
+ escaped_val=$(escape_for_sed "$val")
+
+ # echo "Replacing $appended_var with $escaped_val in $file"
+ sed -i "s/${appended_var}/${escaped_val}/g" "$file"
+ done
+ done
+}
+
+echo "Replacing .env values in $ENV_PATH"
+replace_env_vars
diff --git a/docker/file-storage/.gitignore b/docker/file-storage/.gitignore
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/docker/ory-auth/config/kratos.yml b/docker/ory-auth/config/kratos.yml
index c815b5713f..60381666f0 100644
--- a/docker/ory-auth/config/kratos.yml
+++ b/docker/ory-auth/config/kratos.yml
@@ -70,7 +70,7 @@ selfservice:
ui_url: http://localhost:4455/verification
use: link
after:
- default_browser_return_url: http://localhost:3000
+ default_browser_return_url: http://localhost:3000/login-result
logout:
after:
@@ -99,6 +99,10 @@ session:
jwks_url: http://host.docker.internal:3000/.well-known/jwks.json
# claims_mapper_url: base64://... # A JsonNet template for modifying the claims
ttl: 24h # 24 hours (defaults to 10 minutes)
+cookies:
+ domain: "localhost"
+ path: /
+ same_site: Lax
log:
level: debug
@@ -130,4 +134,4 @@ courier:
connection_uri: smtps://test:test@host.docker.internal:1025/?skip_ssl_verify=true
feature_flags:
- use_continue_with_transitions: true
\ No newline at end of file
+ use_continue_with_transitions: true
diff --git a/docker/postgres/.gitignore b/docker/postgres/.gitignore
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/docker/redis/.gitignore b/docker/redis/.gitignore
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/docker/static/certs/example.com+1-key.pem b/docker/static/certs/example.com+1-key.pem
deleted file mode 100644
index d0abec41d1..0000000000
--- a/docker/static/certs/example.com+1-key.pem
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDtw3Bzjnl5uSTf
-F4Wb8iRanOx9PhfAKY6M1CsssVeHm3tLiy41LLM2pI8Aq1aXqnrCZAEJQj/hdXuJ
-7WXcN1VOur8xQP+N23oPSjwzRIkfwwRAd/VS0VPsnp+xajq268bI5BlVt8M3XwLG
-Etgh46m1ezuapyN+cbqqRFsDUtDSKNK4Hv6HJa1XyPi8ogC4deVst2QIfuasYcJv
-ZytMRNmGr/otdPuenjpGsa/HszERWidueVpQ84s+XSQBXW+j0yBMZXx9a2FxRlQ1
-Gk/Vyzm1GQk8kU1ngVG+jgKFnoi6qIn0v2bFPEOaiULrcqDQ1gUXL4vHcHlXeiF3
-fZEbWnC3AgMBAAECggEAE6cI33RSVB5ajtoZ4Bb2rEq2PW/pdKe6sadD3lDWRE40
-tbzOV/TW51hYvZxr7uNXfEPZ1hMUxqT4TiFCPx6PvY8wCHkv5mDSyPrA2Rf8IZT0
-AAQqUesdfbxqxLZcHgyFBMvd0Dj9ONFwoECkfsCboXWLKrBP/b6WyEYYI8Evo381
-hIN0xTujmakVhJ33wOWTiSoJPX7gZlFh9HnU/A3sGcfuqrmRvwKZWwlLkW5GMeEJ
-HC4bSh25H6UUq8/F0fywPjqCWloTOYxFqgvlXvW2BAi+sq3XNj9Ec0/CV9UAfbyT
-HUcQRZeDaIDfxUqyzrT8VFaTkeaL0cDmanjRDiUR8QKBgQDuZFXDWY8YPgW9RZCq
-n9T9IvyEs2ANtAtfcALlCohbXs62cEAH4mwBUyQshHBpuMePgDOvDoZG1p9JAfYi
-3pMAZYV8f+3ndmhut+N2YGX/FB7KAb5wFWZVT89bsWK529VR+5IzYEwuWxSf5ZWM
-BXRzFOlo5VxSungznUrLlm8ByQKBgQD/UzheuzP2J9xRdFncXUediYCJP9k9kxKO
-xAuAP50zQ87oNx95cx42+9bskWMG2rZGNdvVr+yqRwdsFjVdarnmM/ctLQ8x4Y9X
-srtXyadYUbxyA0Tzt5gkJlWoyTTAU71kZ0fgPEIfvQ5C/EuRNQ+rXxgob+V4+I+O
-ItLn+PsefwKBgHoICo7xbXqvZSi6T4/IObNLEZCsceMR4zB7mj+84IhFQ/PICj7+
-/OLAuKfBM/oqiJ1AtzRJbxscCnXI90JqRT3Suj49Dn+J8XOb1mhmeE/W8NvfgKjH
-i5boP/FkIHGbwtswuGpsRRMFtM0VLTR8JlwyvDjiEByZL6bcQcltvG1hAoGBAM95
-yZm5F51UgMSz/n2CUzqhzJA7EQXnKDJY/luF1fEdjdnHSU1AjXHyrZBpCAY+3dUp
-2OzI21D3DQH4/f5eRpfY7GeKcQmAmCGUfIX2uISdTrt7CqHdM8VUXVEdxz7uDT20
-a4S8kqMF1rv6FlH1wzjnulLJsrfdi5HdnHKiMTVzAoGBAK2SJ47mXybh/9FYM0VD
-7gLs+xKCvKN4XgXyO8W3nT0JXBoB5uxNUqZvfaFKJj1J7JcrNP7txdjenYX7kGgy
-BKC6zKX+Ayw96FDrH/6Fv2l3JcSJS0HkiaJBOUivehWL3aYTyZvq1fYdKnX3o963
-TmV8WP6TIFJUl4BbikDoF2NN
------END PRIVATE KEY-----
diff --git a/docker/static/certs/example.com+1.pem b/docker/static/certs/example.com+1.pem
deleted file mode 100644
index 9021dcee1c..0000000000
--- a/docker/static/certs/example.com+1.pem
+++ /dev/null
@@ -1,26 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEdDCCAtygAwIBAgIRAKCFg40tG4cXo0xziiChOQ0wDQYJKoZIhvcNAQELBQAw
-gZkxHjAcBgNVBAoTFW1rY2VydCBkZXZlbG9wbWVudCBDQTE3MDUGA1UECwwuZGF2
-aWRkaW1hcmlhQE1hY0Jvb2stUHJvLmxvY2FsIChEYXZpZCBEaU1hcmlhKTE+MDwG
-A1UEAww1bWtjZXJ0IGRhdmlkZGltYXJpYUBNYWNCb29rLVByby5sb2NhbCAoRGF2
-aWQgRGlNYXJpYSkwHhcNMjQwNTI0MTk0ODQ0WhcNMjYwODI0MTk0ODQ0WjBkMScw
-JQYDVQQKEx5ta2NlcnQgZGV2ZWxvcG1lbnQgY2VydGlmaWNhdGUxOTA3BgNVBAsM
-MGRhdmlkZGltYXJpYUBNYWNCb29rLVByby0zLmxvY2FsIChEYXZpZCBEaU1hcmlh
-KTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAO3DcHOOeXm5JN8XhZvy
-JFqc7H0+F8ApjozUKyyxV4ebe0uLLjUsszakjwCrVpeqesJkAQlCP+F1e4ntZdw3
-VU66vzFA/43beg9KPDNEiR/DBEB39VLRU+yen7FqOrbrxsjkGVW3wzdfAsYS2CHj
-qbV7O5qnI35xuqpEWwNS0NIo0rge/oclrVfI+LyiALh15Wy3ZAh+5qxhwm9nK0xE
-2Yav+i10+56eOkaxr8ezMRFaJ255WlDziz5dJAFdb6PTIExlfH1rYXFGVDUaT9XL
-ObUZCTyRTWeBUb6OAoWeiLqoifS/ZsU8Q5qJQutyoNDWBRcvi8dweVd6IXd9kRta
-cLcCAwEAAaNrMGkwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMB
-MB8GA1UdIwQYMBaAFGDgmbnR2urMnwPJ/36wuw1Kit+IMCEGA1UdEQQaMBiCC2V4
-YW1wbGUuY29tgglsb2NhbGhvc3QwDQYJKoZIhvcNAQELBQADggGBANQQ8M4lmJIP
-5R10rJl1i8v0KJQpMXQwTVH902YIKQUXgy79uxJ2L7W9dmkNXtw6R6CLXy+KAE03
-fswt28E1cbTO8ObcdW2gIaey0RCzTf034yhyDt9PdKKs+kG5MiIUu9v75JhyUlIK
-mqPKH8F0ad+D7gxCK+bzNMCRmk1uWh7LnNrDmkHhv6gtCSmWzIeQS+6qlh39mePn
-raiiXMLymX0tR0B34gjSi4ozp//P+8XbeznJXXsBiTMyTapfZhG1y67mqI9bcx28
-MzvREMlFauubeiSUNMuPciQInA8MUWN291hfYlSDOq1aSe99h5J+evEr2Tmp54yb
-jbqoDSHDPvX0KvFBQOEmLu3tHogiYUKvsAk/92Vjat2I/pKP+oJR9cO7MUxPXYgv
-bzuobMaGH8JEYyUBn7VoheZBnBEimo5fKLJCkR0JfifvGVOZKN0S37N7qQrgcJUj
-td0vqF0RRbx9unraQoyXJlKHsOc+J8FlBRhX7jngi48o1GRq2oWNCg==
------END CERTIFICATE-----
diff --git a/docker/static/certs/example.com+2-key.pem b/docker/static/certs/example.com+2-key.pem
deleted file mode 100644
index 2fa836d6c7..0000000000
--- a/docker/static/certs/example.com+2-key.pem
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDbAlyE39O83JE+
-p0EBrySzVxpGBrVKtwUa4iVnF7OCvHiHEQ6p04wrvT73Aeob1zxRiwDYKa4rIgfk
-Fkm8rjYWvcUsPzb+JVf3WGdF41eBxuMOvZ2GozcDLUe9qVZYLArrjots2W2p0l7r
-5JPxdUEYjqRdcG4G3Hxq0Xeze8zyi9rQQxYXKxt18ZOdrsoW1Tblsgwvzoom0Wne
-89klOq/nmpinocX2nEJaZOr/M7GJJAjGrc2F5b+EqOwiZ1QshVEMEETJKRcaXKrp
-I+iKCKKlYnN72hkpfLxDPC3x2BxIUjn8Lyg+x6BCpEwKCFI5cNpQVJMjSrzPUcNd
-3apkvZOpAgMBAAECggEAM3kjm4srrai9bldK/QVX/9qJ70lmVJAdYAsktU0pwKFh
-RoVox9tTh3gsE0vc5Pw6TgP4h9WL/NE7v4zJedIMHl4tuE+HMkY+nYCmjRN4nqTK
-+szv+BPmOxXqVZY2F/UKAAD5nfSgRJiP/Ks1ZjujuAzWbqudAHnTuBtRIdsH+T0c
-AHC9YixwT7wUN7OrrQ8hItsQda7kQu+0FW97wc69jkLihhr3wGkwVfl5SuQxB1dt
-PwFPdZpcumkF2GtCocDLn6uyyDhYbei/LoOj5ePMA4OFcxCPRdhA5zA1P5wHlSFm
-Hy4iwL7VGc9XRhAuZzs1gmTF7e+Vn8e2WWfN9wbdMQKBgQDd+L151UQBiFC8Z0TP
-2sJoSIi6oX91g/cO1mxKRw5K7hRxXTaQ0e9t3q0dPnRqR7gohymKKyi8IwUH1SHB
-jur6p8qZsP+BWe/qcPhrMOyjM37MSs0R4RYq0gONyKF73hWMEQzlEaylyI2w3WKY
-Q24go0V4vhnpsLp0Qd9CvFX8/QKBgQD8lVyHQEKjDuDyAgM1eSwdxqAUsnfvD5Ud
-ot8PVjIfUdYM0KKFlBS1yH1UYvfjSw2yTYjotauKOXCykSnMs+qoBaX5ee/E8tju
-SWuesiYFeQstKCRwzB+Fb2hieFZVq+uNYhz2MZVFn1wkt0j2L31R20qhjhnYqiDO
-JtgbH/kHHQKBgQCfMbfiWtuNJdCRbpbhY4kt9WdqQk0BYQWdNJcxpkhP4PP0Yd+y
-eX34FE2fvZ0MZCdlmZpnJ5DtbUg8V9T/1pob7p7VHyYABRqVzzO63Lm6SkJUDgmJ
-Gx8k0r4Nv6hhB+P4MnpHBygFNhK4l+4QObwP2EkI0X4QJdlza5LNb/lTmQKBgEYy
-pKKaQ2rZ6b6YvJeR86ba2walixuPsxundmLmy8tUjS7GlUSWoSLcc6iOUOKEq2vQ
-jKpQQzqJOD8IhRt1LVRBLZ2mO/L6ozumgBh83oBK4cZND8Ohl2kYS2SCmUv6Gd8T
-U4VAxoGxBoTVw5tYG3Yygg1gVuKWdcOVnB39xtIpAoGBANPNiiD9Pn2VZyac0QjL
-FWhbFWCI85c/cOfP4Ws4Z6+tD0Xh53VM6ea4tp87PYdN0j+MFqW/evTo1/ybfzRP
-du3JVrYFLnwp/NnPfbakhbCY1IMVmFs8Om9hcQRTEAbFDxf06PHrhSkbmqrAWIf/
-T1lYjEM7fOdoalKmEr5CtCR6
------END PRIVATE KEY-----
diff --git a/docker/static/certs/example.com+2.pem b/docker/static/certs/example.com+2.pem
deleted file mode 100644
index 89ec8c8997..0000000000
--- a/docker/static/certs/example.com+2.pem
+++ /dev/null
@@ -1,27 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEkjCCAvqgAwIBAgIQGtOcyR+KllSAwkLKWUv91TANBgkqhkiG9w0BAQsFADCB
-mTEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMTcwNQYDVQQLDC5kYXZp
-ZGRpbWFyaWFATWFjQm9vay1Qcm8ubG9jYWwgKERhdmlkIERpTWFyaWEpMT4wPAYD
-VQQDDDVta2NlcnQgZGF2aWRkaW1hcmlhQE1hY0Jvb2stUHJvLmxvY2FsIChEYXZp
-ZCBEaU1hcmlhKTAeFw0yNDA1MjQxOTQ3MjJaFw0yNjA4MjQxOTQ3MjJaMGQxJzAl
-BgNVBAoTHm1rY2VydCBkZXZlbG9wbWVudCBjZXJ0aWZpY2F0ZTE5MDcGA1UECwww
-ZGF2aWRkaW1hcmlhQE1hY0Jvb2stUHJvLTMubG9jYWwgKERhdmlkIERpTWFyaWEp
-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2wJchN/TvNyRPqdBAa8k
-s1caRga1SrcFGuIlZxezgrx4hxEOqdOMK70+9wHqG9c8UYsA2CmuKyIH5BZJvK42
-Fr3FLD82/iVX91hnReNXgcbjDr2dhqM3Ay1HvalWWCwK646LbNltqdJe6+ST8XVB
-GI6kXXBuBtx8atF3s3vM8ova0EMWFysbdfGTna7KFtU25bIML86KJtFp3vPZJTqv
-55qYp6HF9pxCWmTq/zOxiSQIxq3NheW/hKjsImdULIVRDBBEySkXGlyq6SPoigii
-pWJze9oZKXy8Qzwt8dgcSFI5/C8oPsegQqRMCghSOXDaUFSTI0q8z1HDXd2qZL2T
-qQIDAQABo4GJMIGGMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcD
-ATAfBgNVHSMEGDAWgBRg4Jm50drqzJ8Dyf9+sLsNSorfiDA+BgNVHREENzA1ggtl
-eGFtcGxlLmNvbYIbZXhhbXBsZS1zdGFnaW5nLmFwcHNwb3QuY29tgglsb2NhbGhv
-c3QwDQYJKoZIhvcNAQELBQADggGBALMDi2IFk0GeWk1x6Ulsp3BDOhEyIR2tBjDk
-3m+B1emyhT2gXnfSLh/yGN4Sac8W+KtRVjvcBDHWvxPrqwChyuRgaev+jsgIGCVn
-dMweTf2C75NSq8l5MlitzF6i74/i2vQT3CxJcblFlxL+j7fOq1ikTuuTGUz7f4PZ
-XMl+g7dmLgDwLirU+PvEVRhztOGav7tl3hfnVPvDsEkL3G8xomNALfj6/6Qy3e2l
-bNZKx6+jrLUidzPNrTKMOPnPugfDRamAi1vcl1B4N1WcwmoKWbY+HMbie3e7SqwL
-tebyYrJACklFbal0UidcQJtweTkt1gMPCbkOhWk2IOHrcF7fiIi6fy8jWTpHB4Oq
-Nv0HcOVrvEjAtT87v6X/fNqvcRwAMuP0oaWQz1ZTY0RQP5BgxEX0WzuRt0Nn1KAs
-+auoqim5stKYZ4MSA+H52LVepA+f63e/is6rxvFJliVZFJrlkxNG3kETDy69/oik
-KgiL6PePzuopMkRnZgiFzL52BzXfpw==
------END CERTIFICATE-----
diff --git a/docker/static/conf/example.com+1.conf b/docker/static/conf/example.com+1.conf
deleted file mode 100644
index 414c9c28b2..0000000000
--- a/docker/static/conf/example.com+1.conf
+++ /dev/null
@@ -1,40 +0,0 @@
-server {
- listen 80;
- listen [::]:80;
- server_name localhost;
-
- location / {
- proxy_set_header X-Forwarded-For $remote_addr;
- proxy_set_header X-Forwarded-Host $host;
- proxy_set_header X-Forwarded-Proto $scheme;
-
- root /usr/share/nginx/html;
- try_files $uri $uri/ /index.html =404;
- }
-
- location = /localstack/ {
- proxy_pass http://localhost:4566/;
- }
-}
-server {
- listen 443 ssl;
- listen [::]:443 ssl;
- http2 on;
- server_name localhost;
-
- ssl_certificate /etc/nginx/ssl/example.com+1.pem;
- ssl_certificate_key /etc/nginx/ssl/example.com+1-key.pem;
-
- location / {
- proxy_set_header X-Forwarded-For $remote_addr;
- proxy_set_header X-Forwarded-Host $host;
- proxy_set_header X-Forwarded-Proto $scheme;
-
- root /usr/share/nginx/html;
- try_files $uri $uri/ /index.html =404;
- }
-
- location = /localstack/ {
- proxy_pass http://localhost:4566/;
- }
-}
\ No newline at end of file
diff --git a/docker/static/default.conf b/docker/static/default.conf
deleted file mode 100644
index 0d3076d98e..0000000000
--- a/docker/static/default.conf
+++ /dev/null
@@ -1,13 +0,0 @@
-server {
- listen 443 ssl;
-
- server_name localhost;
- ssl_certificate /etc/nginx/certs/nginx-selfsigned.crt;
- ssl_certificate_key /etc/nginx/certs/nginx-selfsigned.key;
- proxy_set_header Host $host;
- proxy_set_header X-Forwarded-For $remote_addr;
-
- location / {
- try_files $uri $uri/ /index.html =404;
- }
-}
\ No newline at end of file
diff --git a/infra/aws-cloudformation/ami.sh b/infra/aws-cloudformation/ami/ami.sh
similarity index 92%
rename from infra/aws-cloudformation/ami.sh
rename to infra/aws-cloudformation/ami/ami.sh
index 968860d9c0..eb5b52bda9 100644
--- a/infra/aws-cloudformation/ami.sh
+++ b/infra/aws-cloudformation/ami/ami.sh
@@ -1,4 +1,4 @@
-# script used to build quadratic-preview-ami-base
+# script used to build quadratic-preview-ami-base, all ami images have this script pre-run
#!/bin/bash
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/.env.docker b/infra/aws-cloudformation/ami/quadratic-selfhost/.env.docker
new file mode 100644
index 0000000000..a80254118b
--- /dev/null
+++ b/infra/aws-cloudformation/ami/quadratic-selfhost/.env.docker
@@ -0,0 +1,95 @@
+# global
+LICENSE_KEY="#LICENSE_KEY#"
+ENVIRONMENT=docker
+
+# postgres database
+DATABASE_IN_DOCKER_COMPOSE=true
+DATABASE_DSN=postgresql://postgres:postgres@host.docker.internal:5432/postgres
+
+# pubsub
+PUBSUB_IN_DOCKER_COMPOSE=true
+PUBSUB_HOST=host.docker.internal
+PUBSUB_PORT=6379
+PUBSUB_PASSWORD=""
+PUBSUB_ACTIVE_CHANNELS=active_channels
+PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL=processed_transactions
+
+# auth: ory or auth0
+AUTH_TYPE=ory
+JWKS_URI=http://host.docker.internal:3000/.well-known/jwks.json
+M2M_AUTH_TOKEN=M2M_AUTH_TOKEN
+ENCRYPTION_KEY=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc
+
+# auth=ory
+ORY_IN_DOCKER_COMPOSE=true
+ORY_DSN=postgresql://postgres:postgres@host.docker.internal:5432/kratos?sslmode=disable
+ORY_LOG_LEVEL=trace
+ORY_ADMIN_HOST=http://host.docker.internal:4434
+KRATOS_URL_INTERNAL=http://host.docker.internal:4433/
+KRATOS_URL_EXTERNAL=https://ory.#HOST#/
+KRATOS_NODE_PORT=4455
+KRATOS_COOKIE_SECRET=changeme
+KRATOS_CSRF_COOKIE_NAME=__HOST-#HOST#-x-csrf-token
+KRATOS_CSRF_COOKIE_SECRET=changeme
+
+# storage: s3 or file-system
+STORAGE_TYPE=file-system
+
+# storage=s3
+AWS_S3_REGION=
+AWS_S3_BUCKET_NAME=quadratic-api-docker
+AWS_S3_ACCESS_KEY_ID=
+AWS_S3_SECRET_ACCESS_KEY=
+
+# storage=file-system
+STORAGE_DIR=/file-storage
+
+# caddy
+CADDY_IN_DOCKER_COMPOSE=true
+
+# client
+QUADRATIC_CLIENT_IN_DOCKER_COMPOSE=true
+
+# api
+QUADRATIC_API_IN_DOCKER_COMPOSE=true
+QUADRATIC_API_URL_EXTERNAL=https://api.#HOST#
+QUADRATIC_API_URL_INTERNAL=http://host.docker.internal:8000
+
+# multiplayer
+QUADRATIC_MULTIPLAYER_IN_DOCKER_COMPOSE=true
+QUADRATIC_MULTIPLAYER_RUST_LOG=info
+QUADRATIC_MULTIPLAYER_HOST=0.0.0.0
+QUADRATIC_MULTIPLAYER_PORT=3001
+QUADRATIC_MULTIPLAYER_HEARTBEAT_CHECK_S=3
+QUADRATIC_MULTIPLAYER_HEARTBEAT_TIMEOUT_S=600
+QUADRATIC_MULTIPLAYER_URL_EXTERNAL=wss://multiplayer.#HOST#/ws
+QUADRATIC_MULTIPLAYER_URL_INTERNAL=ws://host.docker.internal:3001
+
+# files
+QUADRATIC_FILES_IN_DOCKER_COMPOSE=true
+QUADRATIC_FILES_RUST_LOG=info
+QUADRATIC_FILES_HOST=0.0.0.0
+QUADRATIC_FILES_PORT=3002
+QUADRATIC_FILES_FILE_CHECK_S=5
+QUADRATIC_FILES_FILES_PER_CHECK=1000
+QUADRATIC_FILES_TRUNCATE_FILE_CHECK_S=60
+QUADRATIC_FILES_TRUNCATE_TRANSACTION_AGE_DAYS=5
+QUADRATIC_FILES_URL_EXTERNAL=https://files.#HOST#
+QUADRATIC_FILES_URL_INTERNAL=http://host.docker.internal:3002
+
+# connection
+QUADRATIC_CONNECTION_IN_DOCKER_COMPOSE=true
+QUADRATIC_CONNECTION_RUST_LOG=info
+QUADRATIC_CONNECTION_HOST=0.0.0.0
+QUADRATIC_CONNECTION_PORT=3003
+QUADRATIC_CONNECTION_URL_EXTERNAL=https://connection.#HOST#
+QUADRATIC_CONNECTION_URL_INTERNAL=http://host.docker.internal:3003
+QUADRATIC_CONNECTION_MAX_RESPONSE_BYTES=15728640 # 15MB
+QUADRATIC_CONNECTION_STATIC_IPS=0.0.0.0,127.0.0.1
+
+# stripe
+STRIPE_SECRET_KEY=STRIPE_SECRET_KEY
+STRIPE_WEBHOOK_SECRET=STRIPE_WEBHOOK_SECRET
+
+# ai
+OPENAI_API_KEY=
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/docker-compose.yml b/infra/aws-cloudformation/ami/quadratic-selfhost/docker-compose.yml
new file mode 100644
index 0000000000..a60ac02fff
--- /dev/null
+++ b/infra/aws-cloudformation/ami/quadratic-selfhost/docker-compose.yml
@@ -0,0 +1,297 @@
+services:
+ # base services - redis, postgres
+
+ redis:
+ image: redis/redis-stack:latest
+ restart: always
+ ports:
+ - "6379:6379"
+ - "8001:8001"
+ healthcheck:
+ test: ["CMD", "redis-cli", "ping"]
+ interval: "5s"
+ volumes:
+ - ./docker/redis/data:/data
+ profiles:
+ - pubsub
+
+ postgres:
+ image: postgres:15
+ restart: always
+ ports:
+ - "5432:5432"
+ environment:
+ POSTGRES_USER: postgres
+ PGUSER: postgres
+ POSTGRES_PASSWORD: postgres
+ ADDITIONAL_DATABASES: kratos
+ healthcheck:
+ test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"]
+ interval: 10s
+ timeout: 5s
+ retries: 5
+ volumes:
+ - ./docker/postgres/data:/var/lib/postgresql/data
+ - ./docker/postgres/scripts:/docker-entrypoint-initdb.d
+ profiles:
+ - database
+
+ caddy:
+ image: caddy:latest
+ ports:
+ - "80:80"
+ - "443:443"
+ volumes:
+ - ./docker/caddy/config/Caddyfile:/etc/caddy/Caddyfile
+ - ./docker/caddy/data:/data/caddy
+ profiles:
+ - caddy
+ networks:
+ - host
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+
+ # quadratic services - client, api, multiplayer, files, connection
+
+ quadratic-client:
+ image: ${ECR_URL}/quadratic-client-development:${IMAGE_TAG}
+ environment:
+ VITE_DEBUG: 1
+ VITE_QUADRATIC_API_URL: ${QUADRATIC_API_URL_EXTERNAL}
+ VITE_QUADRATIC_MULTIPLAYER_URL: ${QUADRATIC_MULTIPLAYER_URL_EXTERNAL}
+ VITE_QUADRATIC_CONNECTION_URL: ${QUADRATIC_CONNECTION_URL_EXTERNAL}
+ VITE_AUTH_TYPE: ${AUTH_TYPE}
+ VITE_STORAGE_TYPE: ${STORAGE_TYPE}
+ VITE_ORY_HOST: ${KRATOS_URL_EXTERNAL}
+ ports:
+ - "3000:80"
+ command: >
+ sh -c "/client/scripts/replace_env_vars.sh &&
+ nginx -g \"daemon off;\""
+ healthcheck:
+ test: ["CMD-SHELL", "curl -f http://host.docker.internal:3000/ || exit 1"]
+ interval: 10s
+ timeout: 5s
+ restart: "always"
+ depends_on:
+ quadratic-api:
+ condition: service_started
+ volumes:
+ - ./docker/client:/client
+ - ./docker/client/config/default.conf:/etc/nginx/conf.d/default.conf
+ profiles:
+ - quadratic_client
+ networks:
+ - host
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+
+ quadratic-api:
+ image: ${ECR_URL}/quadratic-api-development:${IMAGE_TAG}
+ environment:
+ CORS: "*"
+ DATABASE_URL: ${DATABASE_DSN}
+ ENVIRONMENT: ${ENVIRONMENT}
+ STRIPE_SECRET_KEY: ${STRIPE_SECRET_KEY}
+ STRIPE_WEBHOOK_SECRET: ${STRIPE_WEBHOOK_SECRET}
+ OPENAI_API_KEY: ${OPENAI_API_KEY}
+ M2M_AUTH_TOKEN: ${M2M_AUTH_TOKEN}
+ ENCRYPTION_KEY: ${ENCRYPTION_KEY}
+ AUTH_TYPE: ${AUTH_TYPE}
+ ORY_JWKS_URI: ${JWKS_URI}
+ ORY_ADMIN_HOST: ${ORY_ADMIN_HOST}
+ STORAGE_TYPE: ${STORAGE_TYPE}
+ QUADRATIC_FILE_URI: ${QUADRATIC_FILES_URL_INTERNAL}
+ QUADRATIC_FILE_URI_PUBLIC: ${QUADRATIC_FILES_URL_EXTERNAL}
+ LICENSE_KEY: ${LICENSE_KEY}
+ restart: "always"
+ ports:
+ - "8000:8000"
+ command: bash -c "npx prisma migrate deploy --schema quadratic-api/prisma/schema.prisma && npm run start:prod --workspace=quadratic-api"
+ depends_on:
+ postgres:
+ condition: service_healthy
+ profiles:
+ - quadratic_api
+ networks:
+ - host
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+
+ quadratic-multiplayer:
+ image: ${ECR_URL}/quadratic-multiplayer-development:${IMAGE_TAG}
+ environment:
+ RUST_LOG: info
+ HOST: 0.0.0.0
+ PORT: 3001
+ HEARTBEAT_CHECK_S: 3
+ HEARTBEAT_TIMEOUT_S: 600
+ QUADRATIC_API_URI: ${QUADRATIC_API_URL_INTERNAL}
+ M2M_AUTH_TOKEN: ${M2M_AUTH_TOKEN}
+ ENVIRONMENT: ${ENVIRONMENT}
+ PUBSUB_HOST: ${PUBSUB_HOST}
+ PUBSUB_PORT: ${PUBSUB_PORT}
+ PUBSUB_PASSWORD: ${PUBSUB_PASSWORD}
+ PUBSUB_ACTIVE_CHANNELS: ${PUBSUB_ACTIVE_CHANNELS}
+ AUTH0_JWKS_URI: ${JWKS_URI}
+ AUTHENTICATE_JWT: true
+ restart: "always"
+ ports:
+ - "3001:3001"
+ depends_on:
+ redis:
+ condition: service_healthy
+ quadratic-api:
+ condition: service_started
+ profiles:
+ - quadratic_multiplayer
+ networks:
+ - host
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+
+ quadratic-files:
+ image: ${ECR_URL}/quadratic-files-development:${IMAGE_TAG}
+ environment:
+ RUST_LOG: ${QUADRATIC_FILES_RUST_LOG}
+ HOST: ${QUADRATIC_FILES_HOST}
+ PORT: ${QUADRATIC_FILES_PORT}
+ FILE_CHECK_S: ${QUADRATIC_FILES_FILE_CHECK_S}
+ FILES_PER_CHECK: ${QUADRATIC_FILES_FILES_PER_CHECK}
+ TRUNCATE_FILE_CHECK_S: ${QUADRATIC_FILES_TRUNCATE_FILE_CHECK_S}
+ TRUNCATE_TRANSACTION_AGE_DAYS: ${QUADRATIC_FILES_TRUNCATE_TRANSACTION_AGE_DAYS}
+ ENVIRONMENT: ${ENVIRONMENT}
+ AUTH0_JWKS_URI: ${JWKS_URI}
+ QUADRATIC_API_URI: ${QUADRATIC_API_URL_INTERNAL}
+ M2M_AUTH_TOKEN: ${M2M_AUTH_TOKEN}
+ PUBSUB_HOST: ${PUBSUB_HOST}
+ PUBSUB_PORT: ${PUBSUB_PORT}
+ PUBSUB_PASSWORD: ${PUBSUB_PASSWORD}
+ PUBSUB_ACTIVE_CHANNELS: ${PUBSUB_ACTIVE_CHANNELS}
+ PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL: ${PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL}
+ STORAGE_TYPE: ${STORAGE_TYPE}
+ AWS_S3_REGION: ${AWS_S3_REGION}
+ AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+ AWS_S3_ACCESS_KEY_ID: ${AWS_S3_ACCESS_KEY_ID}
+ AWS_S3_SECRET_ACCESS_KEY: ${AWS_S3_SECRET_ACCESS_KEY}
+ STORAGE_DIR: ${STORAGE_DIR}
+ STORAGE_ENCRYPTION_KEYS: ${ENCRYPTION_KEY}
+ restart: "always"
+ ports:
+ - "3002:3002"
+ depends_on:
+ redis:
+ condition: service_healthy
+ quadratic-api:
+ condition: service_started
+ volumes:
+ - ./docker/file-storage:/file-storage
+ profiles:
+ - quadratic_files
+ networks:
+ - host
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+
+ quadratic-connection:
+ image: ${ECR_URL}/quadratic-connection-development:${IMAGE_TAG}
+ environment:
+ RUST_LOG: ${QUADRATIC_CONNECTION_RUST_LOG}
+ HOST: ${QUADRATIC_CONNECTION_HOST}
+ PORT: ${QUADRATIC_CONNECTION_PORT}
+ ENVIRONMENT: ${ENVIRONMENT}
+ AUTH0_JWKS_URI: ${JWKS_URI}
+ QUADRATIC_API_URI: ${QUADRATIC_API_URL_INTERNAL}
+ M2M_AUTH_TOKEN: ${M2M_AUTH_TOKEN}
+ MAX_RESPONSE_BYTES: ${QUADRATIC_CONNECTION_MAX_RESPONSE_BYTES}
+ STATIC_IPS: ${QUADRATIC_CONNECTION_STATIC_IPS}
+ restart: "always"
+ ports:
+ - "3003:3003"
+ profiles:
+ - quadratic_connection
+ networks:
+ - host
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+
+ # auth service - ory
+
+ ory-auth:
+ image: oryd/kratos:v1.2.0
+ ports:
+ - "4433:4433" # public
+ - "4434:4434" # admin
+ command: serve -c /etc/config/kratos/kratos.yml --dev --watch-courier
+ volumes:
+ - ./docker/ory-auth/config:/etc/config/kratos
+ environment:
+ DSN: ${ORY_DSN}
+ LOG_LEVEL: ${ORY_LOG_LEVEL}
+ restart: unless-stopped
+ depends_on:
+ - postgres
+ - ory-auth-migrate
+ profiles:
+ - ory
+ networks:
+ - host
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+
+ ory-auth-migrate:
+ image: oryd/kratos:v1.2.0
+ command: migrate -c /etc/config/kratos/kratos.yml sql -e --yes
+ volumes:
+ - ./docker/ory-auth/config:/etc/config/kratos
+ environment:
+ DSN: ${ORY_DSN}
+ restart: on-failure
+ depends_on:
+ - postgres
+ profiles:
+ - ory
+ networks:
+ - host
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+
+ ory-auth-node:
+ image: oryd/kratos-selfservice-ui-node:v1.2.0
+ ports:
+ - "4455:4455"
+ environment:
+ PORT: ${KRATOS_NODE_PORT}
+ KRATOS_PUBLIC_URL: ${KRATOS_URL_INTERNAL}
+ KRATOS_BROWSER_URL: ${KRATOS_URL_EXTERNAL}
+ COOKIE_SECRET: ${KRATOS_COOKIE_SECRET}
+ CSRF_COOKIE_NAME: ${KRATOS_CSRF_COOKIE_NAME}
+ CSRF_COOKIE_SECRET: ${KRATOS_CSRF_COOKIE_SECRET}
+ restart: on-failure
+ profiles:
+ - ory
+ networks:
+ - host
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+
+ ory-auth-mail:
+ image: oryd/mailslurper:latest-smtps
+ ports:
+ - "1025:1025"
+ - "4436:4436"
+ - "4437:4437"
+ - "8080:8080"
+ profiles:
+ - ory
+ networks:
+ - host
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+
+volumes:
+ docker:
+ name: docker
+
+networks:
+ host:
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/admin/config/openapi.yaml b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/admin/config/openapi.yaml
new file mode 100644
index 0000000000..ff6fd4fe92
--- /dev/null
+++ b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/admin/config/openapi.yaml
@@ -0,0 +1,48 @@
+openapi: 3.0.3
+info:
+ title: My API
+ version: '1.0'
+ x-logo:
+ url: ''
+paths:
+ /license/{licenseKey}:
+ post:
+ tags: []
+ operationId: license
+ parameters:
+ - name: licenseKey
+ in: path
+ required: true
+ deprecated: false
+ example: sdaf
+ schema:
+ type: string
+ x-last-modified: 1724376657454
+ responses:
+ '200':
+ content:
+ application/json:
+ schema:
+ type: object
+ properties:
+ limits:
+ type: object
+ properties:
+ seats:
+ type: number
+ example: 5
+ x-last-modified: 1724376729775
+ description: ''
+ headers: {}
+ links: {}
+ x-last-modified: 1724376538356
+components:
+ securitySchemes: {}
+ schemas: {}
+ headers: {}
+ responses: {}
+ parameters: {}
+tags: []
+servers:
+ - url: https://api.example.io
+security: []
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/caddy/config/Caddyfile b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/caddy/config/Caddyfile
new file mode 100644
index 0000000000..3d44781eb6
--- /dev/null
+++ b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/caddy/config/Caddyfile
@@ -0,0 +1,27 @@
+#HOST# {
+ reverse_proxy http://host.docker.internal:3000
+}
+
+api.#HOST# {
+ reverse_proxy http://host.docker.internal:8000
+}
+
+multiplayer.#HOST# {
+ reverse_proxy http://host.docker.internal:3001
+}
+
+files.#HOST# {
+ reverse_proxy http://host.docker.internal:3002
+}
+
+connection.#HOST# {
+ reverse_proxy http://host.docker.internal:3003
+}
+
+ory.#HOST# {
+ reverse_proxy http://host.docker.internal:4433
+}
+
+ory-node.#HOST# {
+ reverse_proxy http://host.docker.internal:4455
+}
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/client/config/default.conf b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/client/config/default.conf
new file mode 100644
index 0000000000..a58529d83b
--- /dev/null
+++ b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/client/config/default.conf
@@ -0,0 +1,24 @@
+server {
+ listen 80;
+ server_name localhost;
+
+ root /usr/share/nginx/html;
+ index index.html;
+
+ location / {
+ try_files $uri $uri/ /index.html =404;
+ }
+
+ location ~* \.(?:css|js|json|gif|png|jpg|jpeg|svg|ico)$ {
+ expires 1y;
+ access_log off;
+ add_header Cache-Control "public, no-transform";
+
+ add_header Cross-Origin-Opener-Policy "same-origin";
+ add_header Cross-Origin-Embedder-Policy "require-corp";
+ }
+
+ # Add CORS headers to all requests
+ add_header Cross-Origin-Opener-Policy "same-origin";
+ add_header Cross-Origin-Embedder-Policy "require-corp";
+}
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/client/scripts/replace_env_vars.sh b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/client/scripts/replace_env_vars.sh
new file mode 100755
index 0000000000..60160a8ccf
--- /dev/null
+++ b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/client/scripts/replace_env_vars.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+
+escape_for_sed() {
+ input="$1"
+ printf '%s\n' "$input" | sed -e 's/[\/&]/\\&/g'
+}
+
+replace_env_vars() {
+ vite_vars=""
+
+ for env_var in $(env); do
+ case "$env_var" in
+ VITE_*)
+ vite_vars="$vite_vars $env_var"
+ ;;
+ esac
+ done
+
+ find "/usr/share/nginx/html/assets" -type f -name "*.js" | xargs grep -l "VITE_" | while read file; do
+
+ for env_var in $vite_vars; do
+ var="$(echo "$env_var" | cut -d'=' -f1)"
+ val="$(echo "$env_var" | cut -d'=' -f2-)"
+ appended_var="${var}_VAL"
+ escaped_val=$(escape_for_sed "$val")
+
+ # echo "Replacing $appended_var with $escaped_val in $file"
+ sed -i "s/${appended_var}/${escaped_val}/g" "$file"
+ done
+ done
+}
+
+echo "Replacing .env values in $ENV_PATH"
+replace_env_vars
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/ory-auth/config/identity.schema.json b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/ory-auth/config/identity.schema.json
new file mode 100644
index 0000000000..a953fc68ec
--- /dev/null
+++ b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/ory-auth/config/identity.schema.json
@@ -0,0 +1,47 @@
+{
+ "$id": "https://schemas.ory.sh/presets/kratos/quickstart/email-password/identity.schema.json",
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "Person",
+ "type": "object",
+ "properties": {
+ "traits": {
+ "type": "object",
+ "properties": {
+ "email": {
+ "type": "string",
+ "format": "email",
+ "title": "E-Mail",
+ "minLength": 3,
+ "ory.sh/kratos": {
+ "credentials": {
+ "password": {
+ "identifier": true
+ }
+ },
+ "verification": {
+ "via": "email"
+ },
+ "recovery": {
+ "via": "email"
+ }
+ }
+ },
+ "name": {
+ "type": "object",
+ "properties": {
+ "first": {
+ "title": "First Name",
+ "type": "string"
+ },
+ "last": {
+ "title": "Last Name",
+ "type": "string"
+ }
+ }
+ }
+ },
+ "required": ["email"],
+ "additionalProperties": false
+ }
+ }
+}
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/ory-auth/config/kratos.yml b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/ory-auth/config/kratos.yml
new file mode 100644
index 0000000000..810ec81a24
--- /dev/null
+++ b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/ory-auth/config/kratos.yml
@@ -0,0 +1,138 @@
+# https://raw.githubusercontent.com/ory/kratos/v1.2.0/.schemastore/config.schema.json
+version: v1.2.0
+
+dsn: memory
+
+serve:
+ public:
+ base_url: https://ory.#HOST#/
+ cors:
+ enabled: true
+ allowed_origins:
+ - https://#HOST#
+ allowed_methods:
+ - POST
+ - GET
+ - PUT
+ - PATCH
+ - DELETE
+ allowed_headers:
+ - Authorization
+ - Access-Control-Allow-Origin
+ - Cookie
+ - Content-Type
+ exposed_headers:
+ - Content-Type
+ - Set-Cookie
+ admin:
+ base_url: http://kratos:4434/
+
+selfservice:
+ default_browser_return_url: https://#HOST#
+ allowed_return_urls:
+ - https://#HOST#
+ - https://ory-node.#HOST#
+ - https://#HOST#:3000
+ - https://#HOST#:19006/Callback
+ - exp://localhost:8081/--/Callback
+
+ methods:
+ password:
+ enabled: true
+ totp:
+ config:
+ issuer: Kratos
+ enabled: true
+ lookup_secret:
+ enabled: true
+ link:
+ enabled: true
+ code:
+ enabled: true
+
+ flows:
+ error:
+ ui_url: https://ory-node.#HOST#/error
+
+ settings:
+ ui_url: https://ory-node.#HOST#/settings
+ privileged_session_max_age: 15m
+ required_aal: highest_available
+
+ recovery:
+ enabled: true
+ ui_url: https://ory-node.#HOST#/recovery
+ use: code
+
+ verification:
+ # we disable verification for self-hosting
+ enabled: false
+ ui_url: https://ory-node.#HOST#/verification
+ use: code
+ after:
+ default_browser_return_url: https://#HOST#/login-result
+
+ logout:
+ after:
+ default_browser_return_url: https://ory-node.#HOST#/login
+
+ login:
+ ui_url: https://ory-node.#HOST#/login
+ lifespan: 10m
+
+ registration:
+ lifespan: 10m
+ ui_url: https://ory-node.#HOST#/registration
+ after:
+ password:
+ default_browser_return_url: https://#HOST#/login-result
+ hooks:
+ - hook: session
+ - hook: show_verification_ui
+ default_browser_return_url: https://#HOST#/login-result
+
+session:
+ whoami:
+ tokenizer:
+ templates:
+ jwt_template:
+ jwks_url: http://host.docker.internal:3000/.well-known/jwks.json
+ # claims_mapper_url: base64://... # A JsonNet template for modifying the claims
+ ttl: 24h # 24 hours (defaults to 10 minutes)
+cookies:
+ domain: "#HOST#"
+ path: /
+ same_site: Lax
+
+log:
+ level: warning
+ format: json
+ redaction_text: ""
+ leak_sensitive_values: false
+
+secrets:
+ cookie:
+ - PLEASE-CHANGE-ME-I-AM-VERY-INSECURE
+ cipher:
+ - 32-LONG-SECRET-NOT-SECURE-AT-ALL
+
+ciphers:
+ algorithm: xchacha20-poly1305
+
+hashers:
+ algorithm: bcrypt
+ bcrypt:
+ cost: 8
+
+identity:
+ default_schema_id: default
+ schemas:
+ - id: default
+ url: file:///etc/config/kratos/identity.schema.json
+
+courier:
+ smtp:
+ connection_uri: smtps://test:test@host.docker.internal:1025/?skip_ssl_verify=true
+
+feature_flags:
+ use_continue_with_transitions: true
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/postgres/scripts/init.sh b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/postgres/scripts/init.sh
new file mode 100644
index 0000000000..5e5b12df77
--- /dev/null
+++ b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/postgres/scripts/init.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+set -u
+
+function create_user_and_database() {
+ local database=$1
+ echo "Creating database '$database' with user '$POSTGRES_USER'"
+ psql -c "CREATE DATABASE $database;" || { echo "Failed to create database '$database'"; exit 1; }
+ echo "Database '$database' created"
+}
+
+if [ -n "$ADDITIONAL_DATABASES" ]; then
+ for i in ${ADDITIONAL_DATABASES//,/ }
+ do
+ create_user_and_database $i
+ done
+fi
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/init.sh b/infra/aws-cloudformation/ami/quadratic-selfhost/init.sh
new file mode 100755
index 0000000000..8c19cb0334
--- /dev/null
+++ b/infra/aws-cloudformation/ami/quadratic-selfhost/init.sh
@@ -0,0 +1,135 @@
+#!/bin/bash
+
+# Self-Hosting Initialization
+#
+# Usage:
+#
+# ./init.sh 83f0ebdf-eafb-4c8d-bd7b-04ea07d61b7f localhost
+#
+#
+# Flow:
+#
+# First, check to see if there is a VERSION file, if so, use that version.
+# If not, then check for the first command line argument, if so, use that version.
+# Else, prompt the user.
+#
+# First, check to see if there is a HOST file, if so, use that host.
+# If not, then check for the first command line argument, if so, use that host.
+# Else, prompt the user.
+
+REPO="https://github.com/quadratichq/quadratic-selfhost.git"
+SELF_HOSTING_URI="https://selfhost.quadratichq.com/"
+INVALID_LICENSE_KEY="Invalid license key."
+PROFILE=""
+LICENSE_KEY=""
+HOST=""
+
+get_license_key() {
+ read -p "Enter your license key (Get one for free instantly at $SELF_HOSTING_URI): " user_input
+
+ if [[ $user_input =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then
+ echo $user_input
+ else
+ echo $INVALID_LICENSE_KEY
+ return 1
+ fi
+}
+
+get_host() {
+ read -p "What public host name or public IP address are you using for this setup (e.g. localhost, app.quadratic.com, or other): " user_input
+
+ # TODO: validate host
+ echo $user_input
+}
+
+parse_profile() {
+ # automatically export all variables
+ set -a
+ [[ -f ".env" ]] && source .env
+ # disable auto export
+ set +a
+
+ values=()
+ variables=(
+ "DATABASE_IN_DOCKER_COMPOSE"
+ "PUBSUB_IN_DOCKER_COMPOSE"
+ "CADDY_IN_DOCKER_COMPOSE"
+ "ORY_IN_DOCKER_COMPOSE"
+ "QUADRATIC_CLIENT_IN_DOCKER_COMPOSE"
+ "QUADRATIC_API_IN_DOCKER_COMPOSE"
+ "QUADRATIC_MULTIPLAYER_IN_DOCKER_COMPOSE"
+ "QUADRATIC_FILES_IN_DOCKER_COMPOSE"
+ "QUADRATIC_FILES_URL_INTERNAL"
+ "QUADRATIC_FILES_URL_EXTERNAL"
+ "QUADRATIC_CONNECTION_IN_DOCKER_COMPOSE"
+ )
+
+ for var_name in "${variables[@]}"; do
+ local var_value=$(eval echo \$$var_name)
+
+ if [ "$var_value" == "true" ]; then
+ # store the lowercase variable name
+ var_name_stripped=$(echo "$var_name" | sed 's/_IN_DOCKER_COMPOSE//g')
+ var_name_lower=$(echo "$var_name_stripped" | awk '{print tolower($0)}')
+ values+=("--profile ${var_name_lower}")
+ fi
+ done
+
+ echo "${values[@]}"
+}
+
+checkout() {
+ git clone $REPO
+ cd quadratic-selfhost
+ git checkout
+}
+
+
+if [ -f "quadratic-selfhost/LICENSE_KEY" ]; then
+ LICENSE_KEY=$( LICENSE_KEY
+
+# write docker compose profile to PROFILE file
+PROFILE=$(parse_profile)
+touch PROFILE
+echo $PROFILE > PROFILE
+
+# write host to HOST file
+touch HOST
+echo $HOST > HOST
+
+# remove the init.sh script
+# rm ../init.sh
+
+# adding .bak for compatibility with both GNU (Linux) and BSD (MacOS) sed
+sed -i.bak "s/#LICENSE_KEY#/$LICENSE_KEY/g" ".env"
+sed -i.bak "s/#HOST#/$HOST/g" ".env"
+sed -i.bak "s/#HOST#/$HOST/g" "docker/ory-auth/config/kratos.yml"
+sed -i.bak "s/#HOST#/$HOST/g" "docker/caddy/config/Caddyfile"
+sed -i.bak "s/#HOST#/$HOST/g" "docker/nginx/conf.d/default.conf"
+
+rm .env.bak
+rm docker/ory-auth/config/kratos.yml.bak
+rm docker/caddy/config/Caddyfile.bak
+rm docker/nginx/conf.d/default.conf.bak
+
+sh start.sh
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/start.sh b/infra/aws-cloudformation/ami/quadratic-selfhost/start.sh
new file mode 100755
index 0000000000..e0d7b82226
--- /dev/null
+++ b/infra/aws-cloudformation/ami/quadratic-selfhost/start.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+# read the value of PROFILE from the file
+PROFILE=$(cat PROFILE)
+
+start() {
+ docker compose $PROFILE down -v --remove-orphans
+ docker system prune -af
+ docker compose $PROFILE up -d
+}
+
+start
\ No newline at end of file
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/stop.sh b/infra/aws-cloudformation/ami/quadratic-selfhost/stop.sh
new file mode 100755
index 0000000000..7e8a39a665
--- /dev/null
+++ b/infra/aws-cloudformation/ami/quadratic-selfhost/stop.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+stop() {
+ docker compose --profile "*" down -v --remove-orphans
+ docker system prune -af
+}
+
+stop
\ No newline at end of file
diff --git a/infra/aws-cloudformation/preview-deployment-template.yml b/infra/aws-cloudformation/preview-deployment-template.yml
index a6e6b74361..e0d5d3f988 100644
--- a/infra/aws-cloudformation/preview-deployment-template.yml
+++ b/infra/aws-cloudformation/preview-deployment-template.yml
@@ -10,21 +10,6 @@ Parameters:
Description: Image tag to use for all services
Resources:
- # AMI Lookup for quadratic-preview-base-ami (custom AMI for preview deployments)
- AMIInfo:
- Type: AWS::EC2::Image
- Properties:
- Filters:
- - Name: name
- Values:
- - "quadratic-preview-base-ami"
- - Name: state
- Values:
- - available
- Owners:
- - !Ref AWS::AccountId
- MostRecent: true
-
# Security Group for the EC2 instance
SecurityGroup:
Type: AWS::EC2::SecurityGroup
@@ -83,7 +68,7 @@ Resources:
- Key: Name
Value: !Sub "${ImageTag}"
InstanceType: m6a.large
- ImageId: !GetAtt AMIInfo.Id
+ ImageId: ami-0aa830d419bb978e9 # quadratic-preview-base-ami
IamInstanceProfile: !Ref EC2InstanceProfile
SecurityGroups:
- !Ref SecurityGroup
@@ -115,7 +100,7 @@ Resources:
./login.sh
# Run Quadratic initialization script
- ./init.sh "${LicenseKey}" "${ImageTag}.quadratic-preview.com"
+ ./init.sh "${LicenseKey}" "${ImageTag}.quadratic-selfhost.com"
# # IAM Role for the EventBridge rule
# EventBridgeRole:
@@ -226,28 +211,28 @@ Resources:
Weight: 100
ClientIPPreservationEnabled: true
- # DNS record for the preview deployment for the main application
+ # DNS record for the main application
DNSRecord:
Type: AWS::Route53::RecordSet
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
Properties:
- HostedZoneId: Z0126430TJ1UYIMO3SYX
- Name: !Sub "${ImageTag}.quadratic-preview.com"
+ HostedZoneId: Z08322143IM80L9CZZQV5
+ Name: !Sub "${ImageTag}.quadratic-selfhost.com."
Type: A
AliasTarget:
DNSName: !GetAtt GlobalAccelerator.DnsName
HostedZoneId: Z2BJ6XQ5FK7U4H
EvaluateTargetHealth: true
- # Wildcard DNS record for the preview deployment for all subdomains
+ # Wildcard DNS record for all services on subdomains
WildcardDNSRecord:
Type: AWS::Route53::RecordSet
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
Properties:
- HostedZoneId: Z0126430TJ1UYIMO3SYX
- Name: !Sub "*.${ImageTag}.quadratic-preview.com"
+ HostedZoneId: Z08322143IM80L9CZZQV5
+ Name: !Sub "*.${ImageTag}.quadratic-selfhost.com."
Type: A
AliasTarget:
DNSName: !GetAtt GlobalAccelerator.DnsName
@@ -257,4 +242,4 @@ Resources:
Outputs:
WebsiteURL:
Description: Website URL
- Value: !Sub "https://${ImageTag}.quadratic-preview.com"
+ Value: !Sub "https://${ImageTag}.quadratic-selfhost.com"
diff --git a/package.json b/package.json
index 9407784c95..d87766426b 100644
--- a/package.json
+++ b/package.json
@@ -61,7 +61,9 @@
"docker:down:kill-volumes": "docker compose down -v",
"copy:esbuild": "cp -f node_modules/esbuild-wasm/esbuild.wasm quadratic-client/public/",
"gen:pyright:initialization": "npm run gen:pyright:worker --workspace=quadratic-kernels/python-wasm",
- "gen:pyright:worker": "npm run gen:pyright:worker --workspace=quadratic-kernels/python-wasm"
+ "gen:pyright:worker": "npm run gen:pyright:worker --workspace=quadratic-kernels/python-wasm",
+ "docker": "ECR_OR_BUILD=ecr docker compose --profile all --env-file .env.docker up",
+ "docker:build": "ECR_OR_BUILD=build docker compose --profile all --env-file .env.docker up"
},
"dependencies": {
"@ory/kratos-client": "^1.2.1",
diff --git a/quadratic-api/.env.docker b/quadratic-api/.env.docker
deleted file mode 100644
index f4f8144b51..0000000000
--- a/quadratic-api/.env.docker
+++ /dev/null
@@ -1,20 +0,0 @@
-CORS="*"
-DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/postgres"
-ENVIRONMENT=docker
-STRIPE_SECRET_KEY=STRIPE_SECRET_KEY
-STRIPE_WEBHOOK_SECRET=STRIPE_WEBHOOK_SECRET
-OPENAI_API_KEY=
-M2M_AUTH_TOKEN=M2M_AUTH_TOKEN
-
-# Hex string to be used as the key for enctyption, use npm run key:generate
-ENCRYPTION_KEY=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc
-
-# Auth
-AUTH_TYPE=auth0
-AWS_S3_ENDPOINT=http://localstack:4566
-
-# Storage
-STORAGE_TYPE=s3
-
-# Admin
-LICENSE_KEY=LICENSE_KEY
diff --git a/quadratic-api/.env.example b/quadratic-api/.env.example
index 466ea24bf8..9b772338c0 100644
--- a/quadratic-api/.env.example
+++ b/quadratic-api/.env.example
@@ -27,8 +27,8 @@ AUTH0_AUDIENCE=community-quadratic
ORY_JWKS_URI='http://host.docker.internal:3000/.well-known/jwks.json'
ORY_ADMIN_HOST=http://0.0.0.0:4434
-# Storage
-STORAGE_TYPE=s3 # s3 or file-system
+# Storage - s3 or file-system
+STORAGE_TYPE=s3
QUADRATIC_FILE_URI=http://localhost:3002
QUADRATIC_FILE_URI_PUBLIC=http://localhost:3002
AWS_S3_REGION=us-east-2
diff --git a/quadratic-api/.env.test b/quadratic-api/.env.test
index 27d0e2cfca..0c043da96d 100644
--- a/quadratic-api/.env.test
+++ b/quadratic-api/.env.test
@@ -18,8 +18,8 @@ AUTH0_DOMAIN="AUTH0_DOMAIN"
ORY_JWKS_URI='http://host.docker.internal:3000/.well-known/jwks.json'
ORY_ADMIN_HOST=http://0.0.0.0:4434
-# Storage
-STORAGE_TYPE=s3 # s3 or file-system
+# Storage - s3 or file-system
+STORAGE_TYPE=s3
QUADRATIC_FILE_URI=http://localhost:3002
QUADRATIC_FILE_URI_PUBLIC=http://localhost:3002
AWS_S3_REGION=us-west-2
diff --git a/quadratic-client/.env.docker b/quadratic-client/.env.docker
deleted file mode 100644
index ee0c206b50..0000000000
--- a/quadratic-client/.env.docker
+++ /dev/null
@@ -1,9 +0,0 @@
-VITE_DEBUG=1 // use =1 to enable debug flags
-VITE_QUADRATIC_API_URL=http://localhost:8000
-VITE_QUADRATIC_MULTIPLAYER_URL=ws://localhost:3001/ws
-VITE_QUADRATIC_CONNECTION_URL=http://localhost:3003
-
-# Auth
-VITE_AUTH_TYPE=ory
-VITE_STORAGE_TYPE=file-system
-VITE_ORY_HOST=http://localhost:4433
diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile
index 599723c836..0e11342a44 100644
--- a/quadratic-client/Dockerfile
+++ b/quadratic-client/Dockerfile
@@ -68,9 +68,6 @@ ENV VITE_STORAGE_TYPE=VITE_STORAGE_TYPE_VAL
ENV VITE_ORY_HOST=VITE_ORY_HOST_VAL
RUN echo 'Building front-end...' && npm run build --workspace=quadratic-client
-# The default command to run the application
-# CMD ["npm", "run", "start:production"]
-
FROM nginx:stable-alpine
COPY --from=build /quadratic/build /usr/share/nginx/html
diff --git a/quadratic-connection/.env.docker b/quadratic-connection/.env.docker
deleted file mode 100644
index 522914b032..0000000000
--- a/quadratic-connection/.env.docker
+++ /dev/null
@@ -1,9 +0,0 @@
-HOST=0.0.0.0
-PORT=3003
-ENVIRONMENT=docker
-
-AUTH0_JWKS_URI=http://host.docker.internal:3000/.well-known/jwks.json
-QUADRATIC_API_URI=http://host.docker.internal:8000
-M2M_AUTH_TOKEN=M2M_AUTH_TOKEN
-MAX_RESPONSE_BYTES=15728640 # 15MB
-STATIC_IPS=0.0.0.0,127.0.0.1
diff --git a/quadratic-files/.env.docker b/quadratic-files/.env.docker
deleted file mode 100644
index 1691e5bfd9..0000000000
--- a/quadratic-files/.env.docker
+++ /dev/null
@@ -1,30 +0,0 @@
-HOST=0.0.0.0
-PORT=3002
-FILE_CHECK_S=5
-FILES_PER_CHECK=1000
-TRUNCATE_FILE_CHECK_S=60
-TRUNCATE_TRANSACTION_AGE_DAYS=5 #
-ENVIRONMENT=docker
-
-AUTH0_JWKS_URI=http://host.docker.internal:3000/.well-known/jwks.json
-QUADRATIC_API_URI=http://host.docker.internal:8000
-M2M_AUTH_TOKEN=M2M_AUTH_TOKEN
-
-PUBSUB_HOST=host.docker.internal
-PUBSUB_PORT=6379
-PUBSUB_PASSWORD=
-PUBSUB_ACTIVE_CHANNELS=active_channels
-PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL=processed_transactions
-
-# Storage
-STORAGE_TYPE=file-system # s3 or file-system
-
-# Storage: s3
-AWS_S3_REGION=
-AWS_S3_BUCKET_NAME=quadratic-api-docker
-AWS_S3_ACCESS_KEY_ID=
-AWS_S3_SECRET_ACCESS_KEY=
-
-# Storage: file-system
-STORAGE_DIR=/file-storage
-STORAGE_ENCRYPTION_KEYS=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc
diff --git a/quadratic-files/.env.example b/quadratic-files/.env.example
index 8d18ad5335..860c4cb29d 100644
--- a/quadratic-files/.env.example
+++ b/quadratic-files/.env.example
@@ -16,8 +16,8 @@ PUBSUB_PASSWORD=
PUBSUB_ACTIVE_CHANNELS=active_channels
PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL=processed_transactions
-# Storage
-STORAGE_TYPE=s3 # s3 or file-system
+# Storage - s3 or file-system
+STORAGE_TYPE=s3
# Storage: s3
AWS_S3_REGION=us-east-2
diff --git a/quadratic-files/.env.test b/quadratic-files/.env.test
index fd3e72165d..b4e4bc85f2 100644
--- a/quadratic-files/.env.test
+++ b/quadratic-files/.env.test
@@ -16,8 +16,8 @@ PUBSUB_PASSWORD=
PUBSUB_ACTIVE_CHANNELS=active_channels
PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL=processed_transactions
-# Storage
-STORAGE_TYPE=s3 # s3 or file-system
+# Storage - s3 or file-system
+STORAGE_TYPE=s3
# Storage: s3
AWS_S3_REGION=
diff --git a/quadratic-multiplayer/.env.docker b/quadratic-multiplayer/.env.docker
deleted file mode 100644
index a6dfb7fa0f..0000000000
--- a/quadratic-multiplayer/.env.docker
+++ /dev/null
@@ -1,15 +0,0 @@
-HOST=0.0.0.0
-PORT=3001
-HEARTBEAT_CHECK_S=3
-HEARTBEAT_TIMEOUT_S=600
-QUADRATIC_API_URI=http://host.docker.internal:8000
-M2M_AUTH_TOKEN=M2M_AUTH_TOKEN
-ENVIRONMENT=docker
-
-PUBSUB_HOST=host.docker.internal
-PUBSUB_PORT=6379
-PUBSUB_PASSWORD=
-PUBSUB_ACTIVE_CHANNELS=active_channels
-
-AUTH0_JWKS_URI=http://host.docker.internal:3000/.well-known/jwks.json
-AUTHENTICATE_JWT=true
\ No newline at end of file
From 8c9816988e7a6d347ed9e87b53f9e1f94569a9e1 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Thu, 26 Dec 2024 06:01:33 +0530
Subject: [PATCH 106/155] fix docker tests
---
.env.docker | 2 +-
docker-compose.yml | 23 ++++++++++-------------
quadratic-connection/package.json | 4 ++--
quadratic-multiplayer/package.json | 6 +++---
4 files changed, 16 insertions(+), 19 deletions(-)
diff --git a/.env.docker b/.env.docker
index 1d3c9653dd..615aff28b8 100644
--- a/.env.docker
+++ b/.env.docker
@@ -5,7 +5,7 @@ ENVIRONMENT=docker
LICENSE_KEY=LICENSE_KEY
# use image from ECR or build locally
-ECR_OR_BUILD=ecr
+ECR_OR_BUILD=build
ECR_URL=public.ecr.aws/z7e3d4w1
IMAGE_TAG=latest
diff --git a/docker-compose.yml b/docker-compose.yml
index 1f4679d7f0..ab2fdc3dca 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -361,6 +361,7 @@ services:
image: mysql:8
restart: always
container_name: mysql-connection
+ user: "999:999"
ports:
- 3306:3306
environment:
@@ -374,7 +375,7 @@ services:
timeout: 5s
retries: 5
volumes:
- - ./docker/mysql-connection/data:/var/lib/mysql
+ - ./docker/mysql-connection/data:/var/lib/mysql:rw
- ./docker/mysql-connection/scripts:/docker-entrypoint-initdb.d/
profiles:
- quadratic-connection
@@ -385,19 +386,16 @@ services:
image: mcr.microsoft.com/mssql/server:2022-latest
restart: always
container_name: mssql-connection
- user: root
ports:
- 1433:1433
environment:
ACCEPT_EULA: Y
- MSSQL_SA_PASSWORD: yourStrong(!)Password
+ MSSQL_SA_PASSWORD: YourStrong@Passw0rd
MSSQL_PID: Express
+ MSSQL_TCP_PORT: 1433
healthcheck:
test:
- [
- "CMD-SHELL",
- "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P '${MSSQL_SA_PASSWORD}' -Q 'SELECT 1' || exit 1",
- ]
+ ["CMD-SHELL", '/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P "YourStrong@Passw0rd" -Q "SELECT 1" || exit 1']
interval: 10s
timeout: 5s
retries: 5
@@ -405,12 +403,11 @@ services:
- ./docker/mssql-connection/data:/var/opt/mssql
- ./docker/mssql-connection/scripts:/docker-entrypoint-initdb.d/
command: >
- bash -c "
- /opt/mssql/bin/sqlservr &
- /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P \"$$MSSQL_SA_PASSWORD\" -i /docker-entrypoint-initdb.d/create_db.sql -C &&
- /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P \"$$MSSQL_SA_PASSWORD\" -i /docker-entrypoint-initdb.d/seed_db.sql -C &&
- tail -f /dev/null
- "
+ bash -c "/opt/mssql/bin/sqlservr &
+ sleep 30 &&
+ /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P YourStrong@Passw0rd -i /docker-entrypoint-initdb.d/create_db.sql -C &&
+ /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P YourStrong@Passw0rd -i /docker-entrypoint-initdb.d/seed_db.sql -C &&
+ tail -f /dev/null"
profiles:
- quadratic-connection
- quadratic-connection-db
diff --git a/quadratic-connection/package.json b/quadratic-connection/package.json
index 062ad84fdb..e172777da7 100644
--- a/quadratic-connection/package.json
+++ b/quadratic-connection/package.json
@@ -15,7 +15,7 @@
"coverage:gen": "CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='coverage/cargo-test-%p-%m.profraw' cargo test",
"coverage:html": "grcov . --binary-path ./target/debug/deps/ -s . -t html --branch --ignore-not-existing --ignore '../*' --ignore '/*' -o coverage/html",
"coverage:view": "open coverage/html/index.html",
- "docker:up": "docker compose --profile quadratic-connection-db up -d && sleep 3",
- "docker:down": "docker compose --profile quadratic-connection-db down"
+ "docker:up": "ECR_OR_BUILD=build docker compose -f ../docker-compose.yml --profile quadratic-connection-db --env-file ../.env.docker up -d && sleep 3",
+ "docker:down": "ECR_OR_BUILD=build docker compose -f ../docker-compose.yml --profile quadratic-connection-db --env-file ../.env.docker down"
}
}
diff --git a/quadratic-multiplayer/package.json b/quadratic-multiplayer/package.json
index f877bf4f8e..8a98dbe325 100644
--- a/quadratic-multiplayer/package.json
+++ b/quadratic-multiplayer/package.json
@@ -14,8 +14,8 @@
"coverage:gen": "CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='coverage/cargo-test-%p-%m.profraw' cargo test",
"coverage:html": "grcov . --binary-path ./target/debug/deps/ -s . -t html --branch --ignore-not-existing --ignore '../*' --ignore '/*' -o coverage/html",
"coverage:view": "open coverage/html/index.html",
- "docker:up": "docker compose -f ../docker-compose.base.yml up -d --wait",
- "docker:down": "docker compose down -v",
- "docker:test": "docker compose kill && npm run docker:up && npm run test && npm run docker:down"
+ "docker:up": "ECR_OR_BUILD=build docker compose -f ../docker-compose.yml --profile base --env-file ../.env.docker up -d --wait",
+ "docker:down": "ECR_OR_BUILD=build docker compose -f ../docker-compose.yml --profile base --env-file ../.env.docker down -v",
+ "docker:test": "ECR_OR_BUILD=build docker compose kill && npm run docker:up && npm run test && npm run docker:down"
}
}
From 74f5baadfb4150008c25c94ee5440c744e184e2d Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Thu, 26 Dec 2024 06:29:47 +0530
Subject: [PATCH 107/155] fix mssql docker and tests
---
docker-compose.yml | 37 ++++++++++++++++++++++---------
quadratic-connection/package.json | 2 +-
2 files changed, 28 insertions(+), 11 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index ab2fdc3dca..73f538dfed 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -386,27 +386,44 @@ services:
image: mcr.microsoft.com/mssql/server:2022-latest
restart: always
container_name: mssql-connection
+ user: mssql
ports:
- 1433:1433
environment:
ACCEPT_EULA: Y
- MSSQL_SA_PASSWORD: YourStrong@Passw0rd
+ MSSQL_SA_PASSWORD: yourStrong(!)Password
MSSQL_PID: Express
- MSSQL_TCP_PORT: 1433
+ volumes:
+ - ./docker/mssql-connection/data:/var/opt/mssql
+ - ./docker/mssql-connection/scripts:/docker-entrypoint-initdb.d/
healthcheck:
test:
- ["CMD-SHELL", '/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P "YourStrong@Passw0rd" -Q "SELECT 1" || exit 1']
+ [
+ "CMD",
+ "/opt/mssql-tools18/bin/sqlcmd",
+ "-S",
+ "localhost",
+ "-U",
+ "sa",
+ "-P",
+ "yourStrong(!)Password",
+ "-Q",
+ "SELECT 1",
+ "-C",
+ "-N",
+ "-t",
+ "30",
+ ]
interval: 10s
timeout: 5s
+ start_period: 30s
retries: 5
- volumes:
- - ./docker/mssql-connection/data:/var/opt/mssql
- - ./docker/mssql-connection/scripts:/docker-entrypoint-initdb.d/
command: >
- bash -c "/opt/mssql/bin/sqlservr &
- sleep 30 &&
- /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P YourStrong@Passw0rd -i /docker-entrypoint-initdb.d/create_db.sql -C &&
- /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P YourStrong@Passw0rd -i /docker-entrypoint-initdb.d/seed_db.sql -C &&
+ bash -c "
+ /opt/mssql/bin/sqlservr &
+ sleep 10 &&
+ /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'yourStrong(!)Password' -i /docker-entrypoint-initdb.d/create_db.sql -C -N -t 30 &&
+ /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'yourStrong(!)Password' -i /docker-entrypoint-initdb.d/seed_db.sql -C -N -t 30 &&
tail -f /dev/null"
profiles:
- quadratic-connection
diff --git a/quadratic-connection/package.json b/quadratic-connection/package.json
index e172777da7..9b0d0459f6 100644
--- a/quadratic-connection/package.json
+++ b/quadratic-connection/package.json
@@ -15,7 +15,7 @@
"coverage:gen": "CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='coverage/cargo-test-%p-%m.profraw' cargo test",
"coverage:html": "grcov . --binary-path ./target/debug/deps/ -s . -t html --branch --ignore-not-existing --ignore '../*' --ignore '/*' -o coverage/html",
"coverage:view": "open coverage/html/index.html",
- "docker:up": "ECR_OR_BUILD=build docker compose -f ../docker-compose.yml --profile quadratic-connection-db --env-file ../.env.docker up -d && sleep 3",
+ "docker:up": "ECR_OR_BUILD=build docker compose -f ../docker-compose.yml --profile quadratic-connection-db --env-file ../.env.docker up -d --wait && sleep 10",
"docker:down": "ECR_OR_BUILD=build docker compose -f ../docker-compose.yml --profile quadratic-connection-db --env-file ../.env.docker down"
}
}
From 8c9a1b07cbe947e590e4dd084c18e456c541e066 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Thu, 26 Dec 2024 13:15:57 +0530
Subject: [PATCH 108/155] build client using multiple builders, builds in half
time without cache
---
quadratic-client/Dockerfile | 112 +++++++++++++++++++++++++++++++-----
quadratic-core/package.json | 2 +-
2 files changed, 100 insertions(+), 14 deletions(-)
diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile
index 0e11342a44..29fb324300 100644
--- a/quadratic-client/Dockerfile
+++ b/quadratic-client/Dockerfile
@@ -1,5 +1,5 @@
-# Use an official node image as a parent image
-FROM node:18 AS build
+## Builder 1 - Build wasm (core)
+FROM node:18 AS wasm-builder
# Install rustup
RUN echo 'Installing rustup...' && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
@@ -16,6 +16,93 @@ RUN echo 'wasm-pack version:' && wasm-pack --version
# Install wasm32-unknown-unknown target
RUN rustup target add wasm32-unknown-unknown
+WORKDIR /quadratic
+
+# trigger rebuild if updateAlertVersion.json changes
+COPY updateAlertVersion.json .
+
+# Copy required files for building wasm
+COPY package.json .
+COPY ./quadratic-core/. ./quadratic-core/
+COPY ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts
+
+# Build wasm
+RUN echo 'Building wasm...' && npm run build --workspace=quadratic-core
+
+
+
+## Builder 2 - Build TS/Rust types
+FROM node:18 AS ts-rust-types-builder
+
+# Install rustup
+RUN echo 'Installing rustup...' && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+ENV CARGO_TARGET_DIR=/quadratic/target
+ENV CARGO_HOME=/quadratic/.cargo
+ENV CARGO_BUILD_JOBS=64
+ENV RUSTFLAGS='-C codegen-units=64'
+
+# Install wasm-pack
+RUN echo 'Installing wasm-pack...' && curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
+RUN echo 'wasm-pack version:' && wasm-pack --version
+
+# Install wasm32-unknown-unknown target
+RUN rustup target add wasm32-unknown-unknown
+
+WORKDIR /quadratic
+
+# trigger rebuild if updateAlertVersion.json changes
+COPY updateAlertVersion.json .
+
+# Copy required files for building ts/rust types
+COPY package.json .
+COPY ./quadratic-core/. ./quadratic-core/
+COPY ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts
+
+# Build TS/Rust types
+RUN echo 'Building TS/Rust types...' && npm run export_types --workspace=quadratic-core
+
+
+
+## Builder 3 - Build rust client
+FROM node:18 AS rust-client-builder
+
+# Install rustup
+RUN echo 'Installing rustup...' && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+ENV CARGO_TARGET_DIR=/quadratic/target
+ENV CARGO_HOME=/quadratic/.cargo
+ENV CARGO_BUILD_JOBS=64
+ENV RUSTFLAGS='-C codegen-units=64'
+
+# Install wasm-pack
+RUN echo 'Installing wasm-pack...' && curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
+RUN echo 'wasm-pack version:' && wasm-pack --version
+
+# Install wasm32-unknown-unknown target
+RUN rustup target add wasm32-unknown-unknown
+
+WORKDIR /quadratic
+
+# trigger rebuild if updateAlertVersion.json changes
+COPY updateAlertVersion.json .
+
+# Copy required files for building rust client
+COPY package.json .
+COPY ./quadratic-core/. ./quadratic-core/
+COPY ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts
+COPY ./quadratic-rust-client/. ./quadratic-rust-client/
+
+# Build the rust-client
+ARG GIT_COMMIT
+ENV GIT_COMMIT=$GIT_COMMIT
+RUN echo 'Building rust-client...' && npm run build --workspace=quadratic-rust-client
+
+
+
+## Builder 4 - Combine all builder files and build the client
+FROM node:18 AS final-build
+
# Install python
RUN apt-get update && apt-get install -y --no-install-recommends python-is-python3 python3-pip
@@ -36,22 +123,18 @@ COPY ./quadratic-client/package*.json ./quadratic-client/
# Install npm dependencies
RUN npm install --no-audit --no-fund
-# Build wasm and export TS/Rust types
+# Copy rest of the files
COPY ./quadratic-core/. ./quadratic-core/
-COPY ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts
-RUN echo 'Building wasm...' && npm run build --workspace=quadratic-core
-
-# Build the quadratic-rust-client
COPY ./quadratic-rust-client/. ./quadratic-rust-client/
-ARG GIT_COMMIT
-ENV GIT_COMMIT=$GIT_COMMIT
-RUN echo 'Building quadratic-rust-client...' && npm run build --workspace=quadratic-rust-client
-
-# Copy rest of the files
COPY ./quadratic-kernels/python-wasm/. ./quadratic-kernels/python-wasm/
COPY ./quadratic-shared/. ./quadratic-shared/
COPY ./quadratic-client/. ./quadratic-client/
+# Copy files from wasm, ts/rust types, and rust client builders
+COPY --from=wasm-builder /quadratic/quadratic-client/src/app/quadratic-core/. ./quadratic-client/src/app/quadratic-core
+COPY --from=ts-rust-types-builder /quadratic/quadratic-client/src/app/quadratic-core-types/. ./quadratic-client/src/app/quadratic-core-types
+COPY --from=rust-client-builder /quadratic/quadratic-client/src/app/quadratic-rust-client/. ./quadratic-client/src/app/quadratic-rust-client
+
# Run the packaging script for quadratic_py
RUN ./quadratic-kernels/python-wasm/package.sh --no-poetry
@@ -68,8 +151,11 @@ ENV VITE_STORAGE_TYPE=VITE_STORAGE_TYPE_VAL
ENV VITE_ORY_HOST=VITE_ORY_HOST_VAL
RUN echo 'Building front-end...' && npm run build --workspace=quadratic-client
+
+
+## Runner - Serve the client with nginx
FROM nginx:stable-alpine
-COPY --from=build /quadratic/build /usr/share/nginx/html
+COPY --from=final-build /quadratic/build /usr/share/nginx/html
EXPOSE 80 443 3000
diff --git a/quadratic-core/package.json b/quadratic-core/package.json
index 0af92d0167..1f5028f447 100644
--- a/quadratic-core/package.json
+++ b/quadratic-core/package.json
@@ -6,7 +6,7 @@
"start": "cargo watch -s 'wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs'",
"performance": "cargo watch -s 'wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs'",
"build": "wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs",
- "postbuild": "cargo run --bin export_types --features js",
+ "export_types": "cargo run --bin export_types --features js",
"coverage": "npm run coverage:clean && npm run coverage:wasm:gen && npm run coverage:wasm:html && npm run coverage:wasm:view",
"coverage:wasm:gen": "CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='coverage/cargo-test-%p-%m.profraw' cargo test",
"coverage:wasm:html": "grcov . --binary-path ../target/debug/deps/ -s src -t html --branch --ignore-not-existing --ignore 'src/wasm_bindings/*' --ignore 'src/bin/*' --ignore '../*' --ignore '/*' -o coverage/html",
From 703a8d11df245b56372e88a15519114b73bb3437 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Thu, 26 Dec 2024 13:26:30 +0530
Subject: [PATCH 109/155] try more cores
---
.github/workflows/preview-publish-images.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/preview-publish-images.yml b/.github/workflows/preview-publish-images.yml
index 75e2b2242e..286f9335d0 100644
--- a/.github/workflows/preview-publish-images.yml
+++ b/.github/workflows/preview-publish-images.yml
@@ -23,7 +23,7 @@ jobs:
- service: api
runner: blacksmith-2vcpu-ubuntu-2204
- service: client
- runner: blacksmith-4vcpu-ubuntu-2204 # no speed benefit of 4vcpu, 2vcpu runs out of memory
+ runner: blacksmith-8vcpu-ubuntu-2204 # no speed benefit of 4vcpu, 2vcpu runs out of memory
- service: connection
runner: blacksmith-2vcpu-ubuntu-2204
- service: files
From 37aa830bec540bfe1f9a9095e52704ce6b645e98 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Thu, 26 Dec 2024 13:36:33 +0530
Subject: [PATCH 110/155] try 16
---
.github/workflows/preview-publish-images.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/preview-publish-images.yml b/.github/workflows/preview-publish-images.yml
index 286f9335d0..97b5224b3c 100644
--- a/.github/workflows/preview-publish-images.yml
+++ b/.github/workflows/preview-publish-images.yml
@@ -23,7 +23,7 @@ jobs:
- service: api
runner: blacksmith-2vcpu-ubuntu-2204
- service: client
- runner: blacksmith-8vcpu-ubuntu-2204 # no speed benefit of 4vcpu, 2vcpu runs out of memory
+ runner: blacksmith-16vcpu-ubuntu-2204
- service: connection
runner: blacksmith-2vcpu-ubuntu-2204
- service: files
From c39c4949b2fe8a3d8191cd21ac970b8f7736d54e Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Thu, 26 Dec 2024 14:10:53 +0530
Subject: [PATCH 111/155] node dev in docker
---
.env.docker | 4 +-
.github/workflows/preview-publish-images.yml | 2 +-
.gitignore | 3 +-
.vscode/settings.json | 3 +
Dockerfile | 28 +++++
docker-compose.yml | 101 ++++++++++++++++--
.../ami/quadratic-selfhost/.env.docker | 8 +-
.../ami/quadratic-selfhost/docker-compose.yml | 14 +--
.../ami/quadratic-selfhost/start.sh | 4 +-
package.json | 11 +-
quadratic-client/vite.config.js | 1 +
11 files changed, 147 insertions(+), 32 deletions(-)
create mode 100644 Dockerfile
diff --git a/.env.docker b/.env.docker
index 615aff28b8..6d69401a0a 100644
--- a/.env.docker
+++ b/.env.docker
@@ -1,5 +1,6 @@
# global
ENVIRONMENT=docker
+RUST_LOG=info
# Your license key for Quadratic. Get one here https://selfhost.quadratichq.com/
LICENSE_KEY=LICENSE_KEY
@@ -65,7 +66,6 @@ QUADRATIC_API_URL_INTERNAL=http://host.docker.internal:8000
# multiplayer
QUADRATIC_MULTIPLAYER_IN_DOCKER_COMPOSE=true
-QUADRATIC_MULTIPLAYER_RUST_LOG=info
QUADRATIC_MULTIPLAYER_HOST=0.0.0.0
QUADRATIC_MULTIPLAYER_PORT=3001
QUADRATIC_MULTIPLAYER_HEARTBEAT_CHECK_S=3
@@ -75,7 +75,6 @@ QUADRATIC_MULTIPLAYER_URL_INTERNAL=ws://host.docker.internal:3001
# files
QUADRATIC_FILES_IN_DOCKER_COMPOSE=true
-QUADRATIC_FILES_RUST_LOG=info
QUADRATIC_FILES_HOST=0.0.0.0
QUADRATIC_FILES_PORT=3002
QUADRATIC_FILES_FILE_CHECK_S=5
@@ -87,7 +86,6 @@ QUADRATIC_FILES_URL_INTERNAL=http://host.docker.internal:3002
# connection
QUADRATIC_CONNECTION_IN_DOCKER_COMPOSE=true
-QUADRATIC_CONNECTION_RUST_LOG=info
QUADRATIC_CONNECTION_HOST=0.0.0.0
QUADRATIC_CONNECTION_PORT=3003
QUADRATIC_CONNECTION_URL_EXTERNAL=http://localhost:3003
diff --git a/.github/workflows/preview-publish-images.yml b/.github/workflows/preview-publish-images.yml
index 97b5224b3c..75e2b2242e 100644
--- a/.github/workflows/preview-publish-images.yml
+++ b/.github/workflows/preview-publish-images.yml
@@ -23,7 +23,7 @@ jobs:
- service: api
runner: blacksmith-2vcpu-ubuntu-2204
- service: client
- runner: blacksmith-16vcpu-ubuntu-2204
+ runner: blacksmith-4vcpu-ubuntu-2204 # no speed benefit of 4vcpu, 2vcpu runs out of memory
- service: connection
runner: blacksmith-2vcpu-ubuntu-2204
- service: files
diff --git a/.gitignore b/.gitignore
index 513b8e7b81..da9a09843f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,7 +5,8 @@ node_modules
**/node_modules
/.pnp
.pnp.js
-
+.cache
+.cargo
# testing
/coverage
diff --git a/.vscode/settings.json b/.vscode/settings.json
index d7dffa6a46..7d95b148ea 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -2,6 +2,7 @@
"editor.formatOnSave": true,
"cSpell.words": [
"actix",
+ "autoclean",
"awscliv",
"ayush",
"bigdecimal",
@@ -9,6 +10,7 @@
"bindgen",
"buildkit",
"Buildx",
+ "codegen",
"containerd",
"CRPXNLSKVLJFHH",
"dashmap",
@@ -62,6 +64,7 @@
"RELCELL",
"relcells",
"reqwest",
+ "rustup",
"scrollend",
"selfhost",
"selfhosted",
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000000..e5aeace60f
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,28 @@
+FROM node:18
+
+SHELL ["/bin/bash", "-c"]
+
+# Install build-essential, curl, python and python3-pip
+RUN echo 'Installing build-essential, curl, python and python3-pip...' && apt-get update && \
+ apt-get install -y \
+ build-essential \
+ curl \
+ python-is-python3 \
+ python3-pip
+
+# Install rustup
+RUN echo 'Installing rustup...' && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install wasm-pack
+RUN echo 'Installing wasm-pack...' && curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
+
+# Install wasm32-unknown-unknown target
+RUN echo 'Installing wasm32-unknown-unknown target...' && rustup target add wasm32-unknown-unknown
+
+# Install cargo-watch
+RUN echo 'Installing cargo-watch...' && cargo install cargo-watch
+
+WORKDIR /quadratic
+
+CMD ["bash", "-c", "source ~/.bashrc && npm install --no-audit --no-fund && npm run compile --workspace=quadratic-shared && npm run dev"]
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
index 73f538dfed..8836d1a00b 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -15,6 +15,7 @@ services:
- ./docker/redis/data:/data
profiles:
- base
+ - dev
- all
postgres:
@@ -38,6 +39,7 @@ services:
- ./docker/postgres/scripts:/docker-entrypoint-initdb.d
profiles:
- base
+ - dev
- all
# files service - local aws alternative, use this or file-storage
@@ -148,11 +150,11 @@ services:
file: docker-compose.${ECR_OR_BUILD}.yml
service: quadratic-multiplayer
environment:
- RUST_LOG: info
- HOST: 0.0.0.0
- PORT: 3001
- HEARTBEAT_CHECK_S: 3
- HEARTBEAT_TIMEOUT_S: 600
+ RUST_LOG: ${RUST_LOG}
+ HOST: ${QUADRATIC_MULTIPLAYER_HOST}
+ PORT: ${QUADRATIC_MULTIPLAYER_PORT}
+ HEARTBEAT_CHECK_S: ${QUADRATIC_MULTIPLAYER_HEARTBEAT_CHECK_S}
+ HEARTBEAT_TIMEOUT_S: ${QUADRATIC_MULTIPLAYER_HEARTBEAT_TIMEOUT_S}
QUADRATIC_API_URI: ${QUADRATIC_API_URL_INTERNAL}
M2M_AUTH_TOKEN: ${M2M_AUTH_TOKEN}
ENVIRONMENT: ${ENVIRONMENT}
@@ -184,7 +186,7 @@ services:
file: docker-compose.${ECR_OR_BUILD}.yml
service: quadratic-files
environment:
- RUST_LOG: ${QUADRATIC_FILES_RUST_LOG}
+ RUST_LOG: ${RUST_LOG}
HOST: ${QUADRATIC_FILES_HOST}
PORT: ${QUADRATIC_FILES_PORT}
FILE_CHECK_S: ${QUADRATIC_FILES_FILE_CHECK_S}
@@ -231,7 +233,7 @@ services:
file: docker-compose.${ECR_OR_BUILD}.yml
service: quadratic-connection
environment:
- RUST_LOG: ${QUADRATIC_CONNECTION_RUST_LOG}
+ RUST_LOG: ${RUST_LOG}
HOST: ${QUADRATIC_CONNECTION_HOST}
PORT: ${QUADRATIC_CONNECTION_PORT}
ENVIRONMENT: ${ENVIRONMENT}
@@ -252,6 +254,84 @@ services:
extra_hosts:
- "host.docker.internal:host-gateway"
+ quadratic-dev:
+ build:
+ context: .
+ dockerfile: Dockerfile
+ container_name: quadratic-dev
+ environment:
+ # common
+ ENVIRONMENT: ${ENVIRONMENT}
+ RUST_LOG: ${RUST_LOG}
+ STORAGE_TYPE: ${STORAGE_TYPE}
+ QUADRATIC_API_URI: ${QUADRATIC_API_URL_INTERNAL}
+ AUTH0_JWKS_URI: ${JWKS_URI}
+ M2M_AUTH_TOKEN: ${M2M_AUTH_TOKEN}
+ PUBSUB_HOST: ${PUBSUB_HOST}
+ PUBSUB_PORT: ${PUBSUB_PORT}
+ PUBSUB_PASSWORD: ${PUBSUB_PASSWORD}
+ PUBSUB_ACTIVE_CHANNELS: ${PUBSUB_ACTIVE_CHANNELS}
+ HOST: 0.0.0.0
+ # client
+ VITE_DEBUG: 1
+ VITE_QUADRATIC_API_URL: ${QUADRATIC_API_URL_EXTERNAL}
+ VITE_QUADRATIC_MULTIPLAYER_URL: ${QUADRATIC_MULTIPLAYER_URL_EXTERNAL}
+ VITE_QUADRATIC_CONNECTION_URL: ${QUADRATIC_CONNECTION_URL_EXTERNAL}
+ VITE_AUTH_TYPE: ${AUTH_TYPE}
+ VITE_STORAGE_TYPE: ${STORAGE_TYPE}
+ VITE_ORY_HOST: ${KRATOS_URL_EXTERNAL}
+ # api
+ CORS: "*"
+ DATABASE_URL: ${DATABASE_DSN}
+ STRIPE_SECRET_KEY: ${STRIPE_SECRET_KEY}
+ STRIPE_WEBHOOK_SECRET: ${STRIPE_WEBHOOK_SECRET}
+ OPENAI_API_KEY: ${OPENAI_API_KEY}
+ ENCRYPTION_KEY: ${ENCRYPTION_KEY}
+ AUTH_TYPE: ${AUTH_TYPE}
+ ORY_JWKS_URI: ${JWKS_URI}
+ ORY_ADMIN_HOST: ${ORY_ADMIN_HOST}
+ QUADRATIC_FILE_URI: ${QUADRATIC_FILES_URL_INTERNAL}
+ QUADRATIC_FILE_URI_PUBLIC: ${QUADRATIC_FILES_URL_EXTERNAL}
+ LICENSE_KEY: ${LICENSE_KEY}
+ # multiplayer
+ # PORT: ${QUADRATIC_MULTIPLAYER_PORT}
+ HEARTBEAT_CHECK_S: ${QUADRATIC_MULTIPLAYER_HEARTBEAT_CHECK_S}
+ HEARTBEAT_TIMEOUT_S: ${QUADRATIC_MULTIPLAYER_HEARTBEAT_TIMEOUT_S}
+ AUTHENTICATE_JWT: true
+ # files
+ # PORT: ${QUADRATIC_FILES_PORT}
+ FILE_CHECK_S: ${QUADRATIC_FILES_FILE_CHECK_S}
+ FILES_PER_CHECK: ${QUADRATIC_FILES_FILES_PER_CHECK}
+ TRUNCATE_FILE_CHECK_S: ${QUADRATIC_FILES_TRUNCATE_FILE_CHECK_S}
+ TRUNCATE_TRANSACTION_AGE_DAYS: ${QUADRATIC_FILES_TRUNCATE_TRANSACTION_AGE_DAYS}
+ PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL: ${PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL}
+ STORAGE_DIR: ${STORAGE_DIR}
+ STORAGE_ENCRYPTION_KEYS: ${ENCRYPTION_KEY}
+ # connection
+ # PORT: ${QUADRATIC_CONNECTION_PORT}
+ MAX_RESPONSE_BYTES: ${QUADRATIC_CONNECTION_MAX_RESPONSE_BYTES}
+ STATIC_IPS: ${QUADRATIC_CONNECTION_STATIC_IPS}
+ ports:
+ - "3000:3000"
+ - "8000:8000"
+ - "3001:3001"
+ - "3002:3002"
+ - "3003:3003"
+ volumes:
+ - ./:/quadratic
+ depends_on:
+ - postgres
+ - redis
+ profiles:
+ - quadratic-dev
+ - dev
+ networks:
+ - host
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+ tty: true
+ stdin_open: true
+
# auth service - ory
ory-auth:
@@ -271,6 +351,7 @@ services:
- ory-auth-migrate
profiles:
- ory
+ - dev
- all
networks:
- host
@@ -289,6 +370,7 @@ services:
- postgres
profiles:
- ory
+ - dev
- all
networks:
- host
@@ -309,6 +391,7 @@ services:
restart: on-failure
profiles:
- ory
+ - dev
- all
networks:
- host
@@ -324,14 +407,14 @@ services:
- "8080:8080"
profiles:
- ory
+ - dev
- all
networks:
- host
extra_hosts:
- "host.docker.internal:host-gateway"
- # databases to be used for testing by the connection service
- # postgres, mysql, mssql
+ # databases to be used for testing by the connection service - postgres, mysql, mssql
postgres-connection:
image: postgres:15
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/.env.docker b/infra/aws-cloudformation/ami/quadratic-selfhost/.env.docker
index a80254118b..89dceba3ee 100644
--- a/infra/aws-cloudformation/ami/quadratic-selfhost/.env.docker
+++ b/infra/aws-cloudformation/ami/quadratic-selfhost/.env.docker
@@ -1,6 +1,9 @@
# global
-LICENSE_KEY="#LICENSE_KEY#"
ENVIRONMENT=docker
+RUST_LOG=info
+
+# Your license key for Quadratic. Get one here https://selfhost.quadratichq.com/
+LICENSE_KEY="#LICENSE_KEY#"
# postgres database
DATABASE_IN_DOCKER_COMPOSE=true
@@ -57,7 +60,6 @@ QUADRATIC_API_URL_INTERNAL=http://host.docker.internal:8000
# multiplayer
QUADRATIC_MULTIPLAYER_IN_DOCKER_COMPOSE=true
-QUADRATIC_MULTIPLAYER_RUST_LOG=info
QUADRATIC_MULTIPLAYER_HOST=0.0.0.0
QUADRATIC_MULTIPLAYER_PORT=3001
QUADRATIC_MULTIPLAYER_HEARTBEAT_CHECK_S=3
@@ -67,7 +69,6 @@ QUADRATIC_MULTIPLAYER_URL_INTERNAL=ws://host.docker.internal:3001
# files
QUADRATIC_FILES_IN_DOCKER_COMPOSE=true
-QUADRATIC_FILES_RUST_LOG=info
QUADRATIC_FILES_HOST=0.0.0.0
QUADRATIC_FILES_PORT=3002
QUADRATIC_FILES_FILE_CHECK_S=5
@@ -79,7 +80,6 @@ QUADRATIC_FILES_URL_INTERNAL=http://host.docker.internal:3002
# connection
QUADRATIC_CONNECTION_IN_DOCKER_COMPOSE=true
-QUADRATIC_CONNECTION_RUST_LOG=info
QUADRATIC_CONNECTION_HOST=0.0.0.0
QUADRATIC_CONNECTION_PORT=3003
QUADRATIC_CONNECTION_URL_EXTERNAL=https://connection.#HOST#
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/docker-compose.yml b/infra/aws-cloudformation/ami/quadratic-selfhost/docker-compose.yml
index a60ac02fff..8f9391c4ac 100644
--- a/infra/aws-cloudformation/ami/quadratic-selfhost/docker-compose.yml
+++ b/infra/aws-cloudformation/ami/quadratic-selfhost/docker-compose.yml
@@ -121,11 +121,11 @@ services:
quadratic-multiplayer:
image: ${ECR_URL}/quadratic-multiplayer-development:${IMAGE_TAG}
environment:
- RUST_LOG: info
- HOST: 0.0.0.0
- PORT: 3001
- HEARTBEAT_CHECK_S: 3
- HEARTBEAT_TIMEOUT_S: 600
+ RUST_LOG: ${RUST_LOG}
+ HOST: ${QUADRATIC_MULTIPLAYER_HOST}
+ PORT: ${QUADRATIC_MULTIPLAYER_PORT}
+ HEARTBEAT_CHECK_S: ${QUADRATIC_MULTIPLAYER_HEARTBEAT_CHECK_S}
+ HEARTBEAT_TIMEOUT_S: ${QUADRATIC_MULTIPLAYER_HEARTBEAT_TIMEOUT_S}
QUADRATIC_API_URI: ${QUADRATIC_API_URL_INTERNAL}
M2M_AUTH_TOKEN: ${M2M_AUTH_TOKEN}
ENVIRONMENT: ${ENVIRONMENT}
@@ -153,7 +153,7 @@ services:
quadratic-files:
image: ${ECR_URL}/quadratic-files-development:${IMAGE_TAG}
environment:
- RUST_LOG: ${QUADRATIC_FILES_RUST_LOG}
+ RUST_LOG: ${RUST_LOG}
HOST: ${QUADRATIC_FILES_HOST}
PORT: ${QUADRATIC_FILES_PORT}
FILE_CHECK_S: ${QUADRATIC_FILES_FILE_CHECK_S}
@@ -196,7 +196,7 @@ services:
quadratic-connection:
image: ${ECR_URL}/quadratic-connection-development:${IMAGE_TAG}
environment:
- RUST_LOG: ${QUADRATIC_CONNECTION_RUST_LOG}
+ RUST_LOG: ${RUST_LOG}
HOST: ${QUADRATIC_CONNECTION_HOST}
PORT: ${QUADRATIC_CONNECTION_PORT}
ENVIRONMENT: ${ENVIRONMENT}
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/start.sh b/infra/aws-cloudformation/ami/quadratic-selfhost/start.sh
index e0d7b82226..07c6db13a5 100755
--- a/infra/aws-cloudformation/ami/quadratic-selfhost/start.sh
+++ b/infra/aws-cloudformation/ami/quadratic-selfhost/start.sh
@@ -5,8 +5,8 @@ PROFILE=$(cat PROFILE)
start() {
docker compose $PROFILE down -v --remove-orphans
- docker system prune -af
- docker compose $PROFILE up -d
+ docker compose $PROFILE up -d
+ docker system prune -f
}
start
\ No newline at end of file
diff --git a/package.json b/package.json
index d87766426b..a92627d20a 100644
--- a/package.json
+++ b/package.json
@@ -56,14 +56,15 @@
"heroku-postbuild": "npm run build --workspace=quadratic-api",
"kill": "kill-port 3000 && kill-port 3001 && kill-port 8001 && kill-port 8000",
"prisma:migrate": "cd quadratic-api && npm run prisma:migrate",
- "docker:up": "docker compose up -d --wait && npm run prisma:migrate --workspace=quadratic-api",
- "docker:down": "docker compose down",
- "docker:down:kill-volumes": "docker compose down -v",
"copy:esbuild": "cp -f node_modules/esbuild-wasm/esbuild.wasm quadratic-client/public/",
"gen:pyright:initialization": "npm run gen:pyright:worker --workspace=quadratic-kernels/python-wasm",
"gen:pyright:worker": "npm run gen:pyright:worker --workspace=quadratic-kernels/python-wasm",
- "docker": "ECR_OR_BUILD=ecr docker compose --profile all --env-file .env.docker up",
- "docker:build": "ECR_OR_BUILD=build docker compose --profile all --env-file .env.docker up"
+ "docker:base": "ECR_OR_BUILD=build docker compose --profile base --env-file .env.docker up",
+ "docker:dev": "ECR_OR_BUILD=build docker compose --profile dev --env-file .env.docker up -d && docker attach quadratic-dev",
+ "docker:build": "ECR_OR_BUILD=build docker compose --profile all --env-file .env.docker up",
+ "docker:ecr": "ECR_OR_BUILD=ecr docker compose --profile all --env-file .env.docker up",
+ "docker:down": "ECR_OR_BUILD=build docker compose --profile all --env-file .env.docker down",
+ "docker:clean": "npm run docker:down && docker system prune -af"
},
"dependencies": {
"@ory/kratos-client": "^1.2.1",
diff --git a/quadratic-client/vite.config.js b/quadratic-client/vite.config.js
index 0da34f0f54..42189311d0 100644
--- a/quadratic-client/vite.config.js
+++ b/quadratic-client/vite.config.js
@@ -46,6 +46,7 @@ export default defineConfig(() => {
publicDir: './public',
assetsInclude: ['**/*.py'],
server: {
+ host: '0.0.0.0',
port: 3000,
},
resolve: {
From 8a62ba0ef7cd93ef265a5955350f639203b84cab Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Fri, 27 Dec 2024 15:08:01 +0530
Subject: [PATCH 112/155] finish ci workflow
---
.env.docker | 39 ++++----
.github/workflows/ci.yml | 10 +-
...> staging-build-publish-deploy-images.yml} | 52 ++++++++--
...on-deploy.yml => staging-create-stack.yml} | 67 ++++++-------
.github/workflows/staging-delete-stack.yml | 62 ++++++++++++
.vscode/settings.json | 1 +
docker-compose.yml | 6 ++
.../ami/quadratic-selfhost/.env.docker | 22 ++---
.../ami/quadratic-selfhost/docker-compose.yml | 6 ++
.../ami/quadratic-selfhost/init.sh | 10 +-
.../ami/quadratic-selfhost/start.sh | 4 +-
.../ami/quadratic-selfhost/stop.sh | 2 +-
...nt-template.yml => quadratic-selfhost.yml} | 94 ++++---------------
quadratic-client/Dockerfile | 31 +++---
14 files changed, 224 insertions(+), 182 deletions(-)
rename .github/workflows/{preview-publish-images.yml => staging-build-publish-deploy-images.yml} (69%)
rename .github/workflows/{preview-cloudformation-deploy.yml => staging-create-stack.yml} (60%)
create mode 100644 .github/workflows/staging-delete-stack.yml
rename infra/aws-cloudformation/{preview-deployment-template.yml => quadratic-selfhost.yml} (63%)
diff --git a/.env.docker b/.env.docker
index 6d69401a0a..aeeeb8f308 100644
--- a/.env.docker
+++ b/.env.docker
@@ -3,13 +3,7 @@ ENVIRONMENT=docker
RUST_LOG=info
# Your license key for Quadratic. Get one here https://selfhost.quadratichq.com/
-LICENSE_KEY=LICENSE_KEY
-
-# use image from ECR or build locally
-ECR_OR_BUILD=build
-
-ECR_URL=public.ecr.aws/z7e3d4w1
-IMAGE_TAG=latest
+LICENSE_KEY="#LICENSE_KEY#"
# postgres database
DATABASE_IN_DOCKER_COMPOSE=true
@@ -41,18 +35,6 @@ KRATOS_COOKIE_SECRET=changeme
KRATOS_CSRF_COOKIE_NAME=__HOST-localhost-x-csrf-token
KRATOS_CSRF_COOKIE_SECRET=changeme
-# storage - s3 or file-system
-STORAGE_TYPE=file-system
-
-# storage=s3
-AWS_S3_REGION=
-AWS_S3_BUCKET_NAME=quadratic-api-docker
-AWS_S3_ACCESS_KEY_ID=
-AWS_S3_SECRET_ACCESS_KEY=
-
-# storage=file-system
-STORAGE_DIR=/file-storage
-
# caddy
CADDY_IN_DOCKER_COMPOSE=true
@@ -97,6 +79,25 @@ QUADRATIC_CONNECTION_STATIC_IPS=0.0.0.0,127.0.0.1
STRIPE_SECRET_KEY=STRIPE_SECRET_KEY
STRIPE_WEBHOOK_SECRET=STRIPE_WEBHOOK_SECRET
+# storage - s3 or file-system
+STORAGE_TYPE=file-system
+
+# storage=file-system
+STORAGE_DIR=/file-storage
+
+# storage=s3
+AWS_S3_BUCKET_NAME=quadratic-api-docker
+AWS_S3_REGION=
+AWS_S3_ACCESS_KEY_ID=
+AWS_S3_SECRET_ACCESS_KEY=
+
# ai
OPENAI_API_KEY=
ANTHROPIC_API_KEY=
+EXA_API_KEY=
+
+# use image from ECR or build locally
+ECR_OR_BUILD=build
+
+ECR_URL=public.ecr.aws/z7e3d4w1
+IMAGE_TAG=latest
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 85afcb214f..306bd0770c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -17,7 +17,7 @@ concurrency:
jobs:
test_core:
runs-on: blacksmith-2vcpu-ubuntu-2204
- timeout-minutes: 15
+ timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
@@ -75,7 +75,7 @@ jobs:
test_multiplayer:
runs-on: blacksmith-2vcpu-ubuntu-2204
- timeout-minutes: 15
+ timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
@@ -134,7 +134,7 @@ jobs:
test_connection:
runs-on: blacksmith-2vcpu-ubuntu-2204
- timeout-minutes: 15
+ timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
@@ -256,7 +256,7 @@ jobs:
lint:
runs-on: blacksmith-2vcpu-ubuntu-2204
- timeout-minutes: 15
+ timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
@@ -287,6 +287,7 @@ jobs:
check-version-increment:
runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 10
# If we are merging into main, but not pushed on main
if: github.base_ref == 'main' && github.ref != 'refs/heads/main'
steps:
@@ -322,6 +323,7 @@ jobs:
check-versions-match:
runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 10
steps:
- name: Checkout current branch
diff --git a/.github/workflows/preview-publish-images.yml b/.github/workflows/staging-build-publish-deploy-images.yml
similarity index 69%
rename from .github/workflows/preview-publish-images.yml
rename to .github/workflows/staging-build-publish-deploy-images.yml
index 75e2b2242e..47ca28680f 100644
--- a/.github/workflows/preview-publish-images.yml
+++ b/.github/workflows/staging-build-publish-deploy-images.yml
@@ -1,29 +1,26 @@
-name: Build and Publish Images to ECR - Preview
+name: Build, Publish & Deploy Images - Staging
on:
pull_request:
types: [opened, synchronize, reopened]
concurrency:
- group: pr-${{ github.event.pull_request.number }}-publish-images
+ group: pr-${{ github.event.pull_request.number }}-build-images
cancel-in-progress: true
jobs:
- publish_images:
+ build_images:
permissions:
contents: read
-
- timeout-minutes: 30
-
runs-on: ${{ matrix.runner }}
-
+ timeout-minutes: 30
strategy:
matrix:
include:
- - service: api
- runner: blacksmith-2vcpu-ubuntu-2204
- service: client
runner: blacksmith-4vcpu-ubuntu-2204 # no speed benefit of 4vcpu, 2vcpu runs out of memory
+ - service: api
+ runner: blacksmith-2vcpu-ubuntu-2204
- service: connection
runner: blacksmith-2vcpu-ubuntu-2204
- service: files
@@ -31,7 +28,6 @@ jobs:
- service: multiplayer
runner: blacksmith-2vcpu-ubuntu-2204
fail-fast: true
-
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -99,3 +95,39 @@ jobs:
labels: |
org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}
org.opencontainers.image.revision=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
+
+ deploy_images:
+ needs: build_images
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ env:
+ STACK_NAME: pr-${{ github.event.pull_request.number }}
+ steps:
+ - name: Configure AWS Credentials
+ uses: aws-actions/configure-aws-credentials@v4
+ with:
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEVELOPMENT }}
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }}
+ aws-region: ${{ secrets.AWS_REGION }}
+
+ - name: Get EC2 Instance ID
+ id: get-instance
+ run: |
+ INSTANCE_ID=$(aws cloudformation describe-stack-resources \
+ --stack-name ${{ env.STACK_NAME }} \
+ --logical-resource-id EC2Instance \
+ --query 'StackResources[0].PhysicalResourceId' \
+ --output text)
+ echo "instance_id=$INSTANCE_ID" >> $GITHUB_OUTPUT
+
+ - name: Wait for instance to be ready
+ run: |
+ aws ec2 wait instance-status-ok \
+ --instance-ids ${{ steps.get-instance.outputs.instance_id }}
+
+ - name: Run deployment script on EC2
+ run: |
+ aws ssm send-command \
+ --instance-ids ${{ steps.get-instance.outputs.instance_id }} \
+ --document-name "AWS-RunShellScript" \
+ --parameters commands=["cd /quadratic-selfhost && ./login.sh && ./start.sh"] \
+ --comment "Deploying new images after build"
diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/staging-create-stack.yml
similarity index 60%
rename from .github/workflows/preview-cloudformation-deploy.yml
rename to .github/workflows/staging-create-stack.yml
index b0ff2b6796..ba91875a65 100644
--- a/.github/workflows/preview-cloudformation-deploy.yml
+++ b/.github/workflows/staging-create-stack.yml
@@ -1,32 +1,25 @@
-name: Deploy to AWS CloudFormation - Preview
+name: Create Stack - Staging
on:
pull_request:
types: [opened, reopened]
concurrency:
- group: pr-${{ github.event.pull_request.number }}-deploy-cloudformation
+ group: pr-${{ github.event.pull_request.number }}-stack
jobs:
- deploy_cloudformation:
+ create_stack:
+ name: Create Stack - Staging
permissions:
contents: read
- id-token: write
pull-requests: write
-
runs-on: blacksmith-2vcpu-ubuntu-2204
-
+ timeout-minutes: 30
env:
STACK_NAME: pr-${{ github.event.pull_request.number }}
-
steps:
- uses: actions/checkout@v4
- - name: Generate Build Metadata
- id: build-metadata
- run: |
- echo "GIT_SHA_SHORT=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
@@ -34,79 +27,81 @@ jobs:
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }}
aws-region: ${{ secrets.AWS_REGION }}
- - name: Check and Delete Stack if in ROLLBACK or DELETE
+ - name: Delete Stack if Exists
id: check-stack
run: |
if aws cloudformation describe-stacks --stack-name ${{ env.STACK_NAME }} 2>/dev/null; then
STACK_STATUS=$(aws cloudformation describe-stacks --stack-name ${{ env.STACK_NAME }} --query 'Stacks[0].StackStatus' --output text)
echo "Current stack status: $STACK_STATUS"
- if [ "$STACK_STATUS" = "ROLLBACK_COMPLETE" ] || [ "$STACK_STATUS" = "ROLLBACK_FAILED" ] || \
- [ "$STACK_STATUS" = "DELETE_FAILED" ] || [ "$STACK_STATUS" = "DELETE_IN_PROGRESS" ]; then
- echo "Stack is in $STACK_STATUS state. Deleting..."
- aws cloudformation delete-stack --stack-name ${{ env.STACK_NAME }}
+ echo "Stack exists. Attempting deletion..."
+ if aws cloudformation delete-stack --stack-name ${{ env.STACK_NAME }}; then
echo "Waiting for stack deletion to complete..."
- aws cloudformation wait stack-delete-complete --stack-name ${{ env.STACK_NAME }}
- echo "deleted=true" >> $GITHUB_OUTPUT
+ if aws cloudformation wait stack-delete-complete --stack-name ${{ env.STACK_NAME }}; then
+ echo "Stack deleted successfully"
+ echo "deleted=true" >> $GITHUB_OUTPUT
+ else
+ echo "::error::Stack deletion wait timed out"
+ exit 1
+ fi
else
- echo "Stack exists but is not in ROLLBACK or DELETE state"
- echo "deleted=false" >> $GITHUB_OUTPUT
+ echo "::error::Failed to initiate stack deletion"
+ exit 1
fi
else
echo "Stack does not exist"
echo "deleted=false" >> $GITHUB_OUTPUT
fi
- - name: Deploy CloudFormation Stack
- id: deploy-stack
+ - name: Create Stack
+ id: create-stack
uses: aws-actions/aws-cloudformation-github-deploy@v1
with:
name: ${{ env.STACK_NAME }}
- template: infra/aws-cloudformation/preview-deployment-template.yml
+ template: infra/aws-cloudformation/quadratic-selfhost.yml
parameter-overrides: >-
- LicenseKey=${{ secrets.QUADRATIC_LICENSE_KEY }},
ImageTag=pr-${{ github.event.pull_request.number }}
capabilities: CAPABILITY_IAM
no-fail-on-empty-changeset: "1"
disable-rollback: false
termination-protection: false
- - name: Verify Stack Deployment
+ - name: Verify Stack Creation
if: success()
run: |
STACK_STATUS=$(aws cloudformation describe-stacks --stack-name ${{ env.STACK_NAME }} --query 'Stacks[0].StackStatus' --output text)
echo "Final stack status: $STACK_STATUS"
if [ "$STACK_STATUS" != "CREATE_COMPLETE" ] && [ "$STACK_STATUS" != "UPDATE_COMPLETE" ]; then
- echo "Stack deployment did not complete successfully. Status: $STACK_STATUS"
+ echo "Stack creation did not complete successfully. Status: $STACK_STATUS"
exit 1
fi
- name: Find Comment
uses: peter-evans/find-comment@v3
- id: preview-comment
+ id: staging-comment
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: "github-actions[bot]"
- body-includes: "Preview Deployment"
+ body-includes: "Staging Deployment"
- name: Create or update comment on build success
- if: success() && steps.deploy-stack.outputs.WebsiteURL != ''
+ if: success() && steps.create-stack.outputs.WebsiteURL != ''
uses: peter-evans/create-or-update-comment@v3
with:
- comment-id: ${{ steps.preview-comment.outputs.comment-id }}
+ comment-id: ${{ steps.staging-comment.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: |
- ## Preview Deployment
- Preview URL: https://${{ steps.deploy-stack.outputs.WebsiteURL }}
+ ## Staging Deployment
+ Staging URL: https://${{ steps.create-stack.outputs.WebsiteURL }}
edit-mode: replace
- name: Create or update comment on build failure
if: failure()
uses: peter-evans/create-or-update-comment@v3
with:
- comment-id: ${{ steps.preview-comment.outputs.comment-id }}
+ comment-id: ${{ steps.staging-comment.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: |
- ## Preview Deployment
- The preview deployment encountered an error. Please check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
+ ## Staging Deployment
+ The staging deployment encountered an error. Please check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
edit-mode: replace
diff --git a/.github/workflows/staging-delete-stack.yml b/.github/workflows/staging-delete-stack.yml
new file mode 100644
index 0000000000..9512219725
--- /dev/null
+++ b/.github/workflows/staging-delete-stack.yml
@@ -0,0 +1,62 @@
+name: Delete Stack - Staging
+
+on:
+ pull_request:
+ types: [closed]
+
+concurrency:
+ group: pr-${{ github.event.pull_request.number }}-stack
+
+jobs:
+ delete_stack:
+ name: Delete Stack - Staging
+ permissions:
+ contents: read
+ pull-requests: write
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 30
+ env:
+ STACK_NAME: pr-${{ github.event.pull_request.number }}
+ steps:
+ - name: Configure AWS Credentials
+ uses: aws-actions/configure-aws-credentials@v4
+ with:
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEVELOPMENT }}
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }}
+ aws-region: ${{ secrets.AWS_REGION }}
+
+ - name: Delete CloudFormation Stack
+ run: |
+ if aws cloudformation describe-stacks --stack-name ${{ env.STACK_NAME }} 2>/dev/null; then
+ echo "Deleting stack ${{ env.STACK_NAME }}..."
+ aws cloudformation delete-stack --stack-name ${{ env.STACK_NAME }}
+
+ echo "Waiting for stack deletion to complete..."
+ if aws cloudformation wait stack-delete-complete --stack-name ${{ env.STACK_NAME }}; then
+ echo "Stack deleted successfully"
+ else
+ echo "::error::Stack deletion wait timed out"
+ exit 1
+ fi
+ else
+ echo "Stack ${{ env.STACK_NAME }} does not exist"
+ fi
+
+ - name: Find Comment
+ uses: peter-evans/find-comment@v3
+ id: staging-comment
+ with:
+ issue-number: ${{ github.event.pull_request.number }}
+ comment-author: "github-actions[bot]"
+ body-includes: "Staging Deployment"
+
+ - name: Update comment
+ if: steps.staging-comment.outputs.comment-id != ''
+ uses: peter-evans/create-or-update-comment@v3
+ with:
+ comment-id: ${{ steps.staging-comment.outputs.comment-id }}
+ issue-number: ${{ github.event.pull_request.number }}
+ body: |
+ ## Staging Deployment
+ Stack deleted.
+ edit-mode: replace
diff --git a/.vscode/settings.json b/.vscode/settings.json
index 7d95b148ea..22d0b2c32c 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -61,6 +61,7 @@
"pulumi",
"pyimport",
"rects",
+ "Referer",
"RELCELL",
"relcells",
"reqwest",
diff --git a/docker-compose.yml b/docker-compose.yml
index 8836d1a00b..0ada6dcdd2 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -120,6 +120,12 @@ services:
STRIPE_SECRET_KEY: ${STRIPE_SECRET_KEY}
STRIPE_WEBHOOK_SECRET: ${STRIPE_WEBHOOK_SECRET}
OPENAI_API_KEY: ${OPENAI_API_KEY}
+ ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY}
+ EXA_API_KEY: ${EXA_API_KEY}
+ AWS_S3_REGION: ${AWS_S3_REGION}
+ AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+ AWS_S3_ACCESS_KEY_ID: ${AWS_S3_ACCESS_KEY_ID}
+ AWS_S3_SECRET_ACCESS_KEY: ${AWS_S3_SECRET_ACCESS_KEY}
M2M_AUTH_TOKEN: ${M2M_AUTH_TOKEN}
ENCRYPTION_KEY: ${ENCRYPTION_KEY}
AUTH_TYPE: ${AUTH_TYPE}
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/.env.docker b/infra/aws-cloudformation/ami/quadratic-selfhost/.env.docker
index 89dceba3ee..a5922e79e0 100644
--- a/infra/aws-cloudformation/ami/quadratic-selfhost/.env.docker
+++ b/infra/aws-cloudformation/ami/quadratic-selfhost/.env.docker
@@ -35,18 +35,6 @@ KRATOS_COOKIE_SECRET=changeme
KRATOS_CSRF_COOKIE_NAME=__HOST-#HOST#-x-csrf-token
KRATOS_CSRF_COOKIE_SECRET=changeme
-# storage: s3 or file-system
-STORAGE_TYPE=file-system
-
-# storage=s3
-AWS_S3_REGION=
-AWS_S3_BUCKET_NAME=quadratic-api-docker
-AWS_S3_ACCESS_KEY_ID=
-AWS_S3_SECRET_ACCESS_KEY=
-
-# storage=file-system
-STORAGE_DIR=/file-storage
-
# caddy
CADDY_IN_DOCKER_COMPOSE=true
@@ -91,5 +79,11 @@ QUADRATIC_CONNECTION_STATIC_IPS=0.0.0.0,127.0.0.1
STRIPE_SECRET_KEY=STRIPE_SECRET_KEY
STRIPE_WEBHOOK_SECRET=STRIPE_WEBHOOK_SECRET
-# ai
-OPENAI_API_KEY=
+# storage: s3 or file-system
+STORAGE_TYPE=file-system
+
+# storage=file-system
+STORAGE_DIR=/file-storage
+
+# storage=s3
+AWS_S3_BUCKET_NAME=quadratic-api-docker
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/docker-compose.yml b/infra/aws-cloudformation/ami/quadratic-selfhost/docker-compose.yml
index 8f9391c4ac..7fc0e95117 100644
--- a/infra/aws-cloudformation/ami/quadratic-selfhost/docker-compose.yml
+++ b/infra/aws-cloudformation/ami/quadratic-selfhost/docker-compose.yml
@@ -95,6 +95,12 @@ services:
STRIPE_SECRET_KEY: ${STRIPE_SECRET_KEY}
STRIPE_WEBHOOK_SECRET: ${STRIPE_WEBHOOK_SECRET}
OPENAI_API_KEY: ${OPENAI_API_KEY}
+ ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY}
+ EXA_API_KEY: ${EXA_API_KEY}
+ AWS_S3_REGION: ${AWS_S3_REGION}
+ AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+ AWS_S3_ACCESS_KEY_ID: ${AWS_S3_ACCESS_KEY_ID}
+ AWS_S3_SECRET_ACCESS_KEY: ${AWS_S3_SECRET_ACCESS_KEY}
M2M_AUTH_TOKEN: ${M2M_AUTH_TOKEN}
ENCRYPTION_KEY: ${ENCRYPTION_KEY}
AUTH_TYPE: ${AUTH_TYPE}
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/init.sh b/infra/aws-cloudformation/ami/quadratic-selfhost/init.sh
index 8c19cb0334..d6b70c69cf 100755
--- a/infra/aws-cloudformation/ami/quadratic-selfhost/init.sh
+++ b/infra/aws-cloudformation/ami/quadratic-selfhost/init.sh
@@ -45,7 +45,7 @@ get_host() {
parse_profile() {
# automatically export all variables
set -a
- [[ -f ".env" ]] && source .env
+ [[ -f ".env.docker" ]] && source .env.docker
# disable auto export
set +a
@@ -121,15 +121,13 @@ echo $HOST > HOST
# rm ../init.sh
# adding .bak for compatibility with both GNU (Linux) and BSD (MacOS) sed
-sed -i.bak "s/#LICENSE_KEY#/$LICENSE_KEY/g" ".env"
-sed -i.bak "s/#HOST#/$HOST/g" ".env"
+sed -i.bak "s/#LICENSE_KEY#/$LICENSE_KEY/g" ".env.docker"
+sed -i.bak "s/#HOST#/$HOST/g" ".env.docker"
sed -i.bak "s/#HOST#/$HOST/g" "docker/ory-auth/config/kratos.yml"
sed -i.bak "s/#HOST#/$HOST/g" "docker/caddy/config/Caddyfile"
-sed -i.bak "s/#HOST#/$HOST/g" "docker/nginx/conf.d/default.conf"
-rm .env.bak
+rm .env.docker.bak
rm docker/ory-auth/config/kratos.yml.bak
rm docker/caddy/config/Caddyfile.bak
-rm docker/nginx/conf.d/default.conf.bak
sh start.sh
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/start.sh b/infra/aws-cloudformation/ami/quadratic-selfhost/start.sh
index 07c6db13a5..68ea70654e 100755
--- a/infra/aws-cloudformation/ami/quadratic-selfhost/start.sh
+++ b/infra/aws-cloudformation/ami/quadratic-selfhost/start.sh
@@ -4,8 +4,8 @@
PROFILE=$(cat PROFILE)
start() {
- docker compose $PROFILE down -v --remove-orphans
- docker compose $PROFILE up -d
+ docker compose $PROFILE --env-file .env.docker down -v --remove-orphans
+ docker compose $PROFILE --env-file .env.docker up -d
docker system prune -f
}
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/stop.sh b/infra/aws-cloudformation/ami/quadratic-selfhost/stop.sh
index 7e8a39a665..2403c44b1b 100755
--- a/infra/aws-cloudformation/ami/quadratic-selfhost/stop.sh
+++ b/infra/aws-cloudformation/ami/quadratic-selfhost/stop.sh
@@ -1,7 +1,7 @@
#!/bin/sh
stop() {
- docker compose --profile "*" down -v --remove-orphans
+ docker compose --profile "*" --env-file .env.docker down -v --remove-orphans
docker system prune -af
}
diff --git a/infra/aws-cloudformation/preview-deployment-template.yml b/infra/aws-cloudformation/quadratic-selfhost.yml
similarity index 63%
rename from infra/aws-cloudformation/preview-deployment-template.yml
rename to infra/aws-cloudformation/quadratic-selfhost.yml
index e0d5d3f988..cd3520c050 100644
--- a/infra/aws-cloudformation/preview-deployment-template.yml
+++ b/infra/aws-cloudformation/quadratic-selfhost.yml
@@ -1,10 +1,7 @@
AWSTemplateFormatVersion: 2010-09-09
-Description: Quadratic Preview - Deployment Template
+Description: Quadratic Preview - DockerDeployment Template
Parameters:
- LicenseKey:
- Type: String
- Description: Your license key for Quadratic. Get one here https://selfhost.quadratichq.com/
ImageTag:
Type: String
Description: Image tag to use for all services
@@ -47,7 +44,9 @@ Resources:
Service: ec2.amazonaws.com
Action: sts:AssumeRole
ManagedPolicyArns:
+ - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
- arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
+ - arn:aws:iam::aws:policy/AmazonSSMReadOnlyAccess
# IAM Instance Profile for the EC2 instance
EC2InstanceProfile:
@@ -68,7 +67,7 @@ Resources:
- Key: Name
Value: !Sub "${ImageTag}"
InstanceType: m6a.large
- ImageId: ami-0aa830d419bb978e9 # quadratic-preview-base-ami
+ ImageId: ami-094c4f62c0fffae5e # quadratic-selfhost-base-ami
IamInstanceProfile: !Ref EC2InstanceProfile
SecurityGroups:
- !Ref SecurityGroup
@@ -82,9 +81,18 @@ Resources:
#!/bin/bash
# Append environment variables to .env file
- cat << EOF >> /quadratic-selfhost/.env
+ cat << EOF >> /quadratic-selfhost/.env.docker
+
+ AWS_S3_REGION=$(aws ssm get-parameter --name "/quadratic-development/AWS_S3_REGION" --with-decryption --query "Parameter.Value" --output text)
+ AWS_S3_ACCESS_KEY_ID=$(aws ssm get-parameter --name "/quadratic-development/AWS_S3_ACCESS_KEY_ID" --with-decryption --query "Parameter.Value" --output text)
+ AWS_S3_SECRET_ACCESS_KEY=$(aws ssm get-parameter --name "/quadratic-development/AWS_S3_SECRET_ACCESS_KEY" --with-decryption --query "Parameter.Value" --output text)
+
+ # AI API Keys
+ OPENAI_API_KEY=$(aws ssm get-parameter --name "/quadratic-development/OPENAI_API_KEY" --with-decryption --query "Parameter.Value" --output text)
+ ANTHROPIC_API_KEY=$(aws ssm get-parameter --name "/quadratic-development/ANTHROPIC_API_KEY" --with-decryption --query "Parameter.Value" --output text)
+ EXA_API_KEY=$(aws ssm get-parameter --name "/quadratic-development/EXA_API_KEY" --with-decryption --query "Parameter.Value" --output text)
- #AWS
+ # aws ecr
ECR_URL=${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com
IMAGE_TAG=${ImageTag}
EOF
@@ -100,77 +108,7 @@ Resources:
./login.sh
# Run Quadratic initialization script
- ./init.sh "${LicenseKey}" "${ImageTag}.quadratic-selfhost.com"
-
- # # IAM Role for the EventBridge rule
- # EventBridgeRole:
- # Type: AWS::IAM::Role
- # DeletionPolicy: Delete
- # UpdateReplacePolicy: Delete
- # Properties:
- # Tags:
- # - Key: Name
- # Value: !Ref ImageTag
- # AssumeRolePolicyDocument:
- # Version: 2012-10-17
- # Statement:
- # - Effect: Allow
- # Principal:
- # Service: events.amazonaws.com
- # Action: sts:AssumeRole
- # Policies:
- # - PolicyName: InvokeSsmAutomation
- # PolicyDocument:
- # Version: 2012-10-17
- # Statement:
- # - Effect: Allow
- # Action:
- # - ssm:SendCommand
- # Resource: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:*"
-
- # EcrPushRule:
- # Type: AWS::Events::Rule
- # DeletionPolicy: Delete
- # UpdateReplacePolicy: Delete
- # Properties:
- # Description: Rule to detect ECR image pushes
- # EventPattern:
- # source:
- # - aws.ecr
- # detail-type:
- # - ECR Image Action
- # detail:
- # action-type:
- # - PUSH
- # repository-name:
- # - quadratic-client-development
- # - quadratic-api-development
- # - quadratic-multiplayer-development
- # - quadratic-files-development
- # - quadratic-connection-development
- # image-tag:
- # - !Ref ImageTag
- # State: ENABLED
- # Targets:
- # - Arn: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:automation-definition/AWS-RunShellScript"
- # Id: TriggerScriptOnInstance
- # RoleArn: !GetAtt EventBridgeRole.Arn
- # InputTransformer:
- # InputPathsMap:
- # imageTag: $.detail.image-tag
- # repository: $.detail.repository-name
- # InputTemplate: !Sub |
- # {
- # "InstanceIds": ["${EC2Instance.InstanceId}"],
- # "DocumentName": "AWS-RunShellScript",
- # "Parameters": {
- # "commands": [
- # "cd /quadratic-selfhost",
- # "./login.sh",
- # "./start.sh"
- # ]
- # }
- # }
+ ./init.sh "$(aws ssm get-parameter --name "/quadratic-development/QUADRATIC_LICENSE_KEY" --with-decryption --query "Parameter.Value" --output text)" "${ImageTag}.quadratic-selfhost.com"
# Global Accelerator
GlobalAccelerator:
diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile
index 29fb324300..d5d53f910f 100644
--- a/quadratic-client/Dockerfile
+++ b/quadratic-client/Dockerfile
@@ -1,4 +1,6 @@
-## Builder 1 - Build wasm (core)
+###########################################################################################
+# Builder 1 - Build wasm (core)
+###########################################################################################
FROM node:18 AS wasm-builder
# Install rustup
@@ -30,8 +32,9 @@ COPY ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.t
RUN echo 'Building wasm...' && npm run build --workspace=quadratic-core
-
-## Builder 2 - Build TS/Rust types
+###########################################################################################
+# Builder 2 - Build TS/Rust types
+###########################################################################################
FROM node:18 AS ts-rust-types-builder
# Install rustup
@@ -63,8 +66,9 @@ COPY ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.t
RUN echo 'Building TS/Rust types...' && npm run export_types --workspace=quadratic-core
-
-## Builder 3 - Build rust client
+###########################################################################################
+# Builder 3 - Build rust client
+###########################################################################################
FROM node:18 AS rust-client-builder
# Install rustup
@@ -99,9 +103,10 @@ ENV GIT_COMMIT=$GIT_COMMIT
RUN echo 'Building rust-client...' && npm run build --workspace=quadratic-rust-client
-
-## Builder 4 - Combine all builder files and build the client
-FROM node:18 AS final-build
+###########################################################################################
+# Builder 4 - Combine all builder files and build the client
+###########################################################################################
+FROM node:18 AS vite-build
# Install python
RUN apt-get update && apt-get install -y --no-install-recommends python-is-python3 python3-pip
@@ -152,11 +157,13 @@ ENV VITE_ORY_HOST=VITE_ORY_HOST_VAL
RUN echo 'Building front-end...' && npm run build --workspace=quadratic-client
-
-## Runner - Serve the client with nginx
+###########################################################################################
+# Runner - Serve the client with nginx
+###########################################################################################
FROM nginx:stable-alpine
-COPY --from=final-build /quadratic/build /usr/share/nginx/html
+
+COPY --from=vite-build /quadratic/build /usr/share/nginx/html
EXPOSE 80 443 3000
-CMD ["nginx", "-g", "daemon off;"]
+CMD ["nginx", "-g", "daemon off;"]
\ No newline at end of file
From 586397eab340d9b630694dbb038196161405ba33 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 28 Dec 2024 00:39:57 +0530
Subject: [PATCH 113/155] add error handling in deploy images
---
...es.yml => staging-build-deploy-images.yml} | 65 ++++++++++++++++++-
.github/workflows/staging-create-stack.yml | 2 +-
.github/workflows/staging-delete-stack.yml | 2 +-
3 files changed, 64 insertions(+), 5 deletions(-)
rename .github/workflows/{staging-build-publish-deploy-images.yml => staging-build-deploy-images.yml} (67%)
diff --git a/.github/workflows/staging-build-publish-deploy-images.yml b/.github/workflows/staging-build-deploy-images.yml
similarity index 67%
rename from .github/workflows/staging-build-publish-deploy-images.yml
rename to .github/workflows/staging-build-deploy-images.yml
index 47ca28680f..7b6f2f6809 100644
--- a/.github/workflows/staging-build-publish-deploy-images.yml
+++ b/.github/workflows/staging-build-deploy-images.yml
@@ -1,4 +1,4 @@
-name: Build, Publish & Deploy Images - Staging
+name: Build & Deploy Images
on:
pull_request:
@@ -99,8 +99,10 @@ jobs:
deploy_images:
needs: build_images
runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 30
env:
STACK_NAME: pr-${{ github.event.pull_request.number }}
+ MAX_ATTEMPTS: 20
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
@@ -109,6 +111,50 @@ jobs:
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }}
aws-region: ${{ secrets.AWS_REGION }}
+ - name: Wait for stack deployment
+ id: check-stack
+ run: |
+ ATTEMPTS=0
+ while [ $ATTEMPTS -lt ${{ env.MAX_ATTEMPTS }} ]; do
+ if ! STATUS=$(aws cloudformation describe-stacks \
+ --stack-name ${{ env.STACK_NAME }} \
+ --query 'Stacks[0].StackStatus' \
+ --output text 2>&1); then
+ echo "Error getting stack status: $STATUS"
+ echo "Stack might not exist yet. Waiting..."
+ sleep 30
+ ATTEMPTS=$((ATTEMPTS + 1))
+ continue
+ fi
+
+ echo "Current stack status: $STATUS"
+
+ # Fail if stack is in a failed or rollback state
+ if [[ $STATUS == *FAILED* ]] || [[ $STATUS == *ROLLBACK* ]]; then
+ echo "::error::Stack is in a failed or rollback state: $STATUS"
+ exit 1
+ fi
+
+ # Continue if stack is ready
+ if [[ $STATUS == "CREATE_COMPLETE" ]] || [[ $STATUS == "UPDATE_COMPLETE" ]]; then
+ echo "::notice::Stack is ready with status: $STATUS"
+ break
+ fi
+
+ # Wait and check again if stack is still being created/updated
+ if [[ $STATUS == *IN_PROGRESS* ]]; then
+ echo "Stack operation in progress. Waiting 30 seconds..."
+ sleep 30
+ ATTEMPTS=$((ATTEMPTS + 1))
+ continue
+ fi
+ done
+
+ if [ $ATTEMPTS -eq ${{ env.MAX_ATTEMPTS }} ]; then
+ echo "::error::Timeout waiting for stack to be ready"
+ exit 1
+ fi
+
- name: Get EC2 Instance ID
id: get-instance
run: |
@@ -117,6 +163,10 @@ jobs:
--logical-resource-id EC2Instance \
--query 'StackResources[0].PhysicalResourceId' \
--output text)
+ if [ -z "$INSTANCE_ID" ]; then
+ echo "::error::Failed to get EC2 instance ID"
+ exit 1
+ fi
echo "instance_id=$INSTANCE_ID" >> $GITHUB_OUTPUT
- name: Wait for instance to be ready
@@ -125,9 +175,18 @@ jobs:
--instance-ids ${{ steps.get-instance.outputs.instance_id }}
- name: Run deployment script on EC2
+ id: deploy
run: |
- aws ssm send-command \
+ COMMAND_ID=$(aws ssm send-command \
--instance-ids ${{ steps.get-instance.outputs.instance_id }} \
--document-name "AWS-RunShellScript" \
--parameters commands=["cd /quadratic-selfhost && ./login.sh && ./start.sh"] \
- --comment "Deploying new images after build"
+ --comment "Deploying new images after build" \
+ --query 'Command.CommandId' \
+ --output text)
+
+ # Wait for command completion
+ echo "Waiting for deployment command to complete..."
+ aws ssm wait command-executed \
+ --command-id "$COMMAND_ID" \
+ --instance-id ${{ steps.get-instance.outputs.instance_id }}
diff --git a/.github/workflows/staging-create-stack.yml b/.github/workflows/staging-create-stack.yml
index ba91875a65..a78e6dfa14 100644
--- a/.github/workflows/staging-create-stack.yml
+++ b/.github/workflows/staging-create-stack.yml
@@ -1,4 +1,4 @@
-name: Create Stack - Staging
+name: Create Stack
on:
pull_request:
diff --git a/.github/workflows/staging-delete-stack.yml b/.github/workflows/staging-delete-stack.yml
index 9512219725..1ce6993807 100644
--- a/.github/workflows/staging-delete-stack.yml
+++ b/.github/workflows/staging-delete-stack.yml
@@ -1,4 +1,4 @@
-name: Delete Stack - Staging
+name: Delete Stack
on:
pull_request:
From 5efc196a0c457550412d93badf12d03632317ba8 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 28 Dec 2024 06:44:18 +0530
Subject: [PATCH 114/155] add client dev build option
---
.env.docker | 17 +++--
.../workflows/staging-build-deploy-images.yml | 26 +++----
.vscode/settings.json | 2 +
docker-compose.build.yml | 2 +
docker-compose.yml | 32 ++++++---
package.json | 13 ++--
quadratic-client/Dockerfile | 72 ++++++++++++-------
quadratic-connection/package.json | 4 +-
quadratic-core/Cargo.toml | 12 ++--
quadratic-core/package.json | 1 +
quadratic-multiplayer/package.json | 6 +-
quadratic-rust-client/Cargo.toml | 12 ++--
quadratic-rust-client/package.json | 1 +
13 files changed, 125 insertions(+), 75 deletions(-)
diff --git a/.env.docker b/.env.docker
index aeeeb8f308..7edccc20bf 100644
--- a/.env.docker
+++ b/.env.docker
@@ -87,17 +87,20 @@ STORAGE_DIR=/file-storage
# storage=s3
AWS_S3_BUCKET_NAME=quadratic-api-docker
-AWS_S3_REGION=
-AWS_S3_ACCESS_KEY_ID=
-AWS_S3_SECRET_ACCESS_KEY=
+AWS_S3_REGION=us-west-2
+AWS_S3_ACCESS_KEY_ID=AWS_S3_ACCESS_KEY_ID
+AWS_S3_SECRET_ACCESS_KEY=AWS_S3_SECRET_ACCESS_KEY
# ai
-OPENAI_API_KEY=
-ANTHROPIC_API_KEY=
-EXA_API_KEY=
+OPENAI_API_KEY=OPENAI_API_KEY
+ANTHROPIC_API_KEY=ANTHROPIC_API_KEY
+EXA_API_KEY=EXA_API_KEY
# use image from ECR or build locally
ECR_OR_BUILD=build
+# build client in without wasm-opt
+DEV=false
+
ECR_URL=public.ecr.aws/z7e3d4w1
-IMAGE_TAG=latest
+IMAGE_TAG=latest
\ No newline at end of file
diff --git a/.github/workflows/staging-build-deploy-images.yml b/.github/workflows/staging-build-deploy-images.yml
index 7b6f2f6809..201bc6bae7 100644
--- a/.github/workflows/staging-build-deploy-images.yml
+++ b/.github/workflows/staging-build-deploy-images.yml
@@ -2,7 +2,7 @@ name: Build & Deploy Images
on:
pull_request:
- types: [opened, synchronize, reopened]
+ types: [opened, synchronize, reopened, labeled, unlabeled]
concurrency:
group: pr-${{ github.event.pull_request.number }}-build-images
@@ -12,26 +12,25 @@ jobs:
build_images:
permissions:
contents: read
- runs-on: ${{ matrix.runner }}
+ runs-on: blacksmith-4vcpu-ubuntu-2204
timeout-minutes: 30
strategy:
matrix:
- include:
- - service: client
- runner: blacksmith-4vcpu-ubuntu-2204 # no speed benefit of 4vcpu, 2vcpu runs out of memory
- - service: api
- runner: blacksmith-2vcpu-ubuntu-2204
- - service: connection
- runner: blacksmith-2vcpu-ubuntu-2204
- - service: files
- runner: blacksmith-2vcpu-ubuntu-2204
- - service: multiplayer
- runner: blacksmith-2vcpu-ubuntu-2204
+ service: [client, api, multiplayer, files, connection]
fail-fast: true
steps:
- name: Checkout code
uses: actions/checkout@v4
+ - name: Check for build:dev label
+ id: check-dev
+ run: |
+ if [[ ${{ contains(github.event.pull_request.labels.*.name, 'build:dev') }} == true ]]; then
+ echo "DEV=true" >> $GITHUB_OUTPUT
+ else
+ echo "DEV=false" >> $GITHUB_OUTPUT
+ fi
+
- name: Generate Build Metadata
id: build-metadata
run: |
@@ -92,6 +91,7 @@ jobs:
GIT_SHA_SHORT=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
BRANCH_NAME=${{ steps.build-metadata.outputs.BRANCH_NAME }}
PR_NUMBER=${{ github.event.pull_request.number }}
+ DEV=${{ steps.check-dev.outputs.DEV }}
labels: |
org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}
org.opencontainers.image.revision=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
diff --git a/.vscode/settings.json b/.vscode/settings.json
index 22d0b2c32c..40c306c509 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -18,6 +18,7 @@
"dcell",
"ddimaria",
"dearmor",
+ "demangle",
"dgraph",
"dotenv",
"dpkg",
@@ -70,6 +71,7 @@
"selfhost",
"selfhosted",
"selfservice",
+ "serde",
"shadcn",
"Signin",
"smallpop",
diff --git a/docker-compose.build.yml b/docker-compose.build.yml
index 975a6ae339..55877651d1 100644
--- a/docker-compose.build.yml
+++ b/docker-compose.build.yml
@@ -7,6 +7,8 @@ services:
build:
context: .
dockerfile: quadratic-client/Dockerfile
+ args:
+ DEV: ${DEV}
quadratic-api:
build:
diff --git a/docker-compose.yml b/docker-compose.yml
index 0ada6dcdd2..bcc56c948d 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -3,8 +3,8 @@ services:
redis:
image: redis/redis-stack:latest
- restart: always
container_name: redis
+ restart: always
ports:
- "6379:6379"
- "8001:8001"
@@ -20,8 +20,8 @@ services:
postgres:
image: postgres:15
- restart: always
container_name: postgres
+ restart: always
ports:
- "5432:5432"
environment:
@@ -76,6 +76,7 @@ services:
extends:
file: docker-compose.${ECR_OR_BUILD}.yml
service: quadratic-client
+ container_name: client
environment:
VITE_DEBUG: 1
VITE_QUADRATIC_API_URL: ${QUADRATIC_API_URL_EXTERNAL}
@@ -94,12 +95,12 @@ services:
interval: 10s
timeout: 5s
restart: "always"
- depends_on:
- quadratic-api:
- condition: service_started
volumes:
- ./docker/client:/client
- ./docker/client/config/default.conf:/etc/nginx/conf.d/default.conf
+ depends_on:
+ quadratic-api:
+ condition: service_started
profiles:
- quadratic-client
- frontend
@@ -113,6 +114,7 @@ services:
extends:
file: docker-compose.${ECR_OR_BUILD}.yml
service: quadratic-api
+ container_name: api
environment:
CORS: "*"
DATABASE_URL: ${DATABASE_DSN}
@@ -155,6 +157,7 @@ services:
extends:
file: docker-compose.${ECR_OR_BUILD}.yml
service: quadratic-multiplayer
+ container_name: multiplayer
environment:
RUST_LOG: ${RUST_LOG}
HOST: ${QUADRATIC_MULTIPLAYER_HOST}
@@ -176,6 +179,8 @@ services:
depends_on:
redis:
condition: service_healthy
+ quadratic-client:
+ condition: service_healthy
quadratic-api:
condition: service_started
profiles:
@@ -191,6 +196,7 @@ services:
extends:
file: docker-compose.${ECR_OR_BUILD}.yml
service: quadratic-files
+ container_name: files
environment:
RUST_LOG: ${RUST_LOG}
HOST: ${QUADRATIC_FILES_HOST}
@@ -218,13 +224,15 @@ services:
restart: "always"
ports:
- "3002:3002"
+ volumes:
+ - ./docker/file-storage:/file-storage
depends_on:
redis:
condition: service_healthy
+ quadratic-client:
+ condition: service_healthy
quadratic-api:
condition: service_started
- volumes:
- - ./docker/file-storage:/file-storage
profiles:
- quadratic-files
- backend
@@ -238,6 +246,7 @@ services:
extends:
file: docker-compose.${ECR_OR_BUILD}.yml
service: quadratic-connection
+ container_name: connection
environment:
RUST_LOG: ${RUST_LOG}
HOST: ${QUADRATIC_CONNECTION_HOST}
@@ -251,6 +260,9 @@ services:
restart: "always"
ports:
- "3003:3003"
+ depends_on:
+ quadratic-client:
+ condition: service_healthy
profiles:
- quadratic-connection
- backend
@@ -264,7 +276,7 @@ services:
build:
context: .
dockerfile: Dockerfile
- container_name: quadratic-dev
+ container_name: dev
environment:
# common
ENVIRONMENT: ${ENVIRONMENT}
@@ -342,6 +354,7 @@ services:
ory-auth:
image: oryd/kratos:v1.2.0
+ container_name: ory-auth
ports:
- "4433:4433" # public
- "4434:4434" # admin
@@ -366,6 +379,7 @@ services:
ory-auth-migrate:
image: oryd/kratos:v1.2.0
+ container_name: ory-auth-migrate
command: migrate -c /etc/config/kratos/kratos.yml sql -e --yes
volumes:
- ./docker/ory-auth/config:/etc/config/kratos
@@ -385,6 +399,7 @@ services:
ory-auth-node:
image: oryd/kratos-selfservice-ui-node:v1.2.0
+ container_name: ory-auth-node
ports:
- "4455:4455"
environment:
@@ -406,6 +421,7 @@ services:
ory-auth-mail:
image: oryd/mailslurper:latest-smtps
+ container_name: ory-auth-mail
ports:
- "1025:1025"
- "4436:4436"
diff --git a/package.json b/package.json
index a92627d20a..e9af37827c 100644
--- a/package.json
+++ b/package.json
@@ -59,12 +59,13 @@
"copy:esbuild": "cp -f node_modules/esbuild-wasm/esbuild.wasm quadratic-client/public/",
"gen:pyright:initialization": "npm run gen:pyright:worker --workspace=quadratic-kernels/python-wasm",
"gen:pyright:worker": "npm run gen:pyright:worker --workspace=quadratic-kernels/python-wasm",
- "docker:base": "ECR_OR_BUILD=build docker compose --profile base --env-file .env.docker up",
- "docker:dev": "ECR_OR_BUILD=build docker compose --profile dev --env-file .env.docker up -d && docker attach quadratic-dev",
- "docker:build": "ECR_OR_BUILD=build docker compose --profile all --env-file .env.docker up",
- "docker:ecr": "ECR_OR_BUILD=ecr docker compose --profile all --env-file .env.docker up",
- "docker:down": "ECR_OR_BUILD=build docker compose --profile all --env-file .env.docker down",
- "docker:clean": "npm run docker:down && docker system prune -af"
+ "docker:base": "ECR_OR_BUILD=build DEV=false docker compose --profile base --env-file .env.docker up",
+ "docker:dev": "ECR_OR_BUILD=build DEV=true docker compose --profile dev --env-file .env.docker up -d && docker attach quadratic-dev",
+ "docker:build": "ECR_OR_BUILD=build DEV=false docker compose --profile all --env-file .env.docker up",
+ "docker:build:dev": "ECR_OR_BUILD=build DEV=true docker compose --profile all --env-file .env.docker up",
+ "docker:ecr": "ECR_OR_BUILD=ecr DEV=false docker compose --profile all --env-file .env.docker up",
+ "docker:down": "ECR_OR_BUILD=build DEV=false docker compose --profile all --env-file .env.docker down",
+ "docker:clean": "npm run docker:down && docker builder prune -af && docker system prune -af --volumes"
},
"dependencies": {
"@ory/kratos-client": "^1.2.1",
diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile
index d5d53f910f..11b0628541 100644
--- a/quadratic-client/Dockerfile
+++ b/quadratic-client/Dockerfile
@@ -1,7 +1,32 @@
###########################################################################################
-# Builder 1 - Build wasm (core)
+# Builder 1 - Install dependencies
###########################################################################################
-FROM node:18 AS wasm-builder
+FROM node:18 AS dependencies-builder
+
+WORKDIR /quadratic
+
+# Copy updateAlertVersion.json
+COPY updateAlertVersion.json .
+
+# Copy all package.json files
+COPY package.json .
+COPY package-lock.json .
+COPY ./quadratic-kernels/python-wasm/package*.json ./quadratic-kernels/python-wasm/
+COPY ./quadratic-core/package*.json ./quadratic-core/
+COPY ./quadratic-rust-client/package*.json ./quadratic-rust-client/
+COPY ./quadratic-shared/package*.json ./quadratic-shared/
+COPY ./quadratic-client/package*.json ./quadratic-client/
+
+# Install npm dependencies
+RUN npm install --no-audit --no-fund
+
+
+###########################################################################################
+# Builder 2 - Build core
+###########################################################################################
+FROM node:18 AS core-builder
+
+ARG DEV
# Install rustup
RUN echo 'Installing rustup...' && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
@@ -23,17 +48,21 @@ WORKDIR /quadratic
# trigger rebuild if updateAlertVersion.json changes
COPY updateAlertVersion.json .
-# Copy required files for building wasm
+# Copy required files for building core
COPY package.json .
COPY ./quadratic-core/. ./quadratic-core/
COPY ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts
-# Build wasm
-RUN echo 'Building wasm...' && npm run build --workspace=quadratic-core
+# Build core
+RUN if [ "$DEV" = "true" ]; then \
+ echo 'Building core in dev mode...' && npm run build:dev --workspace=quadratic-core; \
+ else \
+ echo 'Building core...' && npm run build --workspace=quadratic-core; \
+ fi
###########################################################################################
-# Builder 2 - Build TS/Rust types
+# Builder 3 - Build TS/Rust types
###########################################################################################
FROM node:18 AS ts-rust-types-builder
@@ -67,10 +96,12 @@ RUN echo 'Building TS/Rust types...' && npm run export_types --workspace=quadrat
###########################################################################################
-# Builder 3 - Build rust client
+# Builder 4 - Build rust client
###########################################################################################
FROM node:18 AS rust-client-builder
+ARG DEV
+
# Install rustup
RUN echo 'Installing rustup...' && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"
@@ -100,11 +131,15 @@ COPY ./quadratic-rust-client/. ./quadratic-rust-client/
# Build the rust-client
ARG GIT_COMMIT
ENV GIT_COMMIT=$GIT_COMMIT
-RUN echo 'Building rust-client...' && npm run build --workspace=quadratic-rust-client
+RUN if [ "$DEV" = "true" ]; then \
+ echo 'Building rust-client in dev mode...' && npm run build:dev --workspace=quadratic-rust-client; \
+ else \
+ echo 'Building rust-client...' && npm run build --workspace=quadratic-rust-client; \
+ fi
###########################################################################################
-# Builder 4 - Combine all builder files and build the client
+# Builder 5 - Combine all builder files and build the client
###########################################################################################
FROM node:18 AS vite-build
@@ -113,30 +148,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends python-is-pytho
WORKDIR /quadratic
-# Copy updateAlertVersion.json
+# Copy all files
COPY updateAlertVersion.json .
-
-# Copy all package.json files
COPY package.json .
COPY package-lock.json .
-COPY ./quadratic-kernels/python-wasm/package*.json ./quadratic-kernels/python-wasm/
-COPY ./quadratic-core/package*.json ./quadratic-core/
-COPY ./quadratic-rust-client/package*.json ./quadratic-rust-client/
-COPY ./quadratic-shared/package*.json ./quadratic-shared/
-COPY ./quadratic-client/package*.json ./quadratic-client/
-
-# Install npm dependencies
-RUN npm install --no-audit --no-fund
-
-# Copy rest of the files
COPY ./quadratic-core/. ./quadratic-core/
COPY ./quadratic-rust-client/. ./quadratic-rust-client/
COPY ./quadratic-kernels/python-wasm/. ./quadratic-kernels/python-wasm/
COPY ./quadratic-shared/. ./quadratic-shared/
COPY ./quadratic-client/. ./quadratic-client/
-# Copy files from wasm, ts/rust types, and rust client builders
-COPY --from=wasm-builder /quadratic/quadratic-client/src/app/quadratic-core/. ./quadratic-client/src/app/quadratic-core
+# Copy files from core, ts/rust types, and rust client builders
+COPY --from=dependencies-builder /quadratic/. ./
+COPY --from=core-builder /quadratic/quadratic-client/src/app/quadratic-core/. ./quadratic-client/src/app/quadratic-core
COPY --from=ts-rust-types-builder /quadratic/quadratic-client/src/app/quadratic-core-types/. ./quadratic-client/src/app/quadratic-core-types
COPY --from=rust-client-builder /quadratic/quadratic-client/src/app/quadratic-rust-client/. ./quadratic-client/src/app/quadratic-rust-client
diff --git a/quadratic-connection/package.json b/quadratic-connection/package.json
index 9b0d0459f6..1c08696861 100644
--- a/quadratic-connection/package.json
+++ b/quadratic-connection/package.json
@@ -15,7 +15,7 @@
"coverage:gen": "CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='coverage/cargo-test-%p-%m.profraw' cargo test",
"coverage:html": "grcov . --binary-path ./target/debug/deps/ -s . -t html --branch --ignore-not-existing --ignore '../*' --ignore '/*' -o coverage/html",
"coverage:view": "open coverage/html/index.html",
- "docker:up": "ECR_OR_BUILD=build docker compose -f ../docker-compose.yml --profile quadratic-connection-db --env-file ../.env.docker up -d --wait && sleep 10",
- "docker:down": "ECR_OR_BUILD=build docker compose -f ../docker-compose.yml --profile quadratic-connection-db --env-file ../.env.docker down"
+ "docker:up": "ECR_OR_BUILD=build DEV=false docker compose -f ../docker-compose.yml --profile quadratic-connection-db --env-file ../.env.docker up -d --wait && sleep 10",
+ "docker:down": "ECR_OR_BUILD=build DEV=false docker compose -f ../docker-compose.yml --profile quadratic-connection-db --env-file ../.env.docker down"
}
}
diff --git a/quadratic-core/Cargo.toml b/quadratic-core/Cargo.toml
index f840b1871b..cf32dc3db2 100644
--- a/quadratic-core/Cargo.toml
+++ b/quadratic-core/Cargo.toml
@@ -103,6 +103,11 @@ harness = false
[package.metadata.wasm-pack.profile.release]
wasm-opt = ['-Os', '-g'] # TODO: -g seems to fix the name mangling problem
+[package.metadata.wasm-pack.profile.release.wasm-bindgen]
+debug-js-glue = false
+demangle-name-section = true
+dwarf-debug-info = false
+
[package.metadata.wasm-pack.profile.dev]
wasm-opt = false
@@ -112,14 +117,9 @@ demangle-name-section = true
dwarf-debug-info = true
[package.metadata.wasm-pack.profile.profiling]
-wasm-opt = ['-O', '-g']
+wasm-opt = ['-O0', '-g']
[package.metadata.wasm-pack.profile.profiling.wasm-bindgen]
debug-js-glue = false
demangle-name-section = true
dwarf-debug-info = true
-
-[package.metadata.wasm-pack.profile.release.wasm-bindgen]
-debug-js-glue = false
-demangle-name-section = true
-dwarf-debug-info = false
diff --git a/quadratic-core/package.json b/quadratic-core/package.json
index 1f5028f447..95b63a1c65 100644
--- a/quadratic-core/package.json
+++ b/quadratic-core/package.json
@@ -6,6 +6,7 @@
"start": "cargo watch -s 'wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs'",
"performance": "cargo watch -s 'wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs'",
"build": "wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs",
+ "build:dev": "wasm-pack build --profiling --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs",
"export_types": "cargo run --bin export_types --features js",
"coverage": "npm run coverage:clean && npm run coverage:wasm:gen && npm run coverage:wasm:html && npm run coverage:wasm:view",
"coverage:wasm:gen": "CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='coverage/cargo-test-%p-%m.profraw' cargo test",
diff --git a/quadratic-multiplayer/package.json b/quadratic-multiplayer/package.json
index 8a98dbe325..58703f8153 100644
--- a/quadratic-multiplayer/package.json
+++ b/quadratic-multiplayer/package.json
@@ -14,8 +14,8 @@
"coverage:gen": "CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='coverage/cargo-test-%p-%m.profraw' cargo test",
"coverage:html": "grcov . --binary-path ./target/debug/deps/ -s . -t html --branch --ignore-not-existing --ignore '../*' --ignore '/*' -o coverage/html",
"coverage:view": "open coverage/html/index.html",
- "docker:up": "ECR_OR_BUILD=build docker compose -f ../docker-compose.yml --profile base --env-file ../.env.docker up -d --wait",
- "docker:down": "ECR_OR_BUILD=build docker compose -f ../docker-compose.yml --profile base --env-file ../.env.docker down -v",
- "docker:test": "ECR_OR_BUILD=build docker compose kill && npm run docker:up && npm run test && npm run docker:down"
+ "docker:up": "ECR_OR_BUILD=build DEV=false docker compose -f ../docker-compose.yml --profile base --env-file ../.env.docker up -d --wait",
+ "docker:down": "ECR_OR_BUILD=build DEV=false docker compose -f ../docker-compose.yml --profile base --env-file ../.env.docker down -v",
+ "docker:test": "ECR_OR_BUILD=build DEV=false docker compose kill && npm run docker:up && npm run test && npm run docker:down"
}
}
diff --git a/quadratic-rust-client/Cargo.toml b/quadratic-rust-client/Cargo.toml
index 04efc1204a..9c4f153432 100644
--- a/quadratic-rust-client/Cargo.toml
+++ b/quadratic-rust-client/Cargo.toml
@@ -26,6 +26,11 @@ crate-type = ["cdylib", "rlib"]
[package.metadata.wasm-pack.profile.release]
wasm-opt = ['-Os', '-g'] # TODO: -g seems to fix the name mangling problem
+[package.metadata.wasm-pack.profile.release.wasm-bindgen]
+debug-js-glue = false
+demangle-name-section = true
+dwarf-debug-info = false
+
[package.metadata.wasm-pack.profile.dev]
wasm-opt = false
@@ -35,14 +40,9 @@ demangle-name-section = true
dwarf-debug-info = true
[package.metadata.wasm-pack.profile.profiling]
-wasm-opt = ['-O', '-g']
+wasm-opt = ['-O0', '-g']
[package.metadata.wasm-pack.profile.profiling.wasm-bindgen]
debug-js-glue = false
demangle-name-section = true
dwarf-debug-info = true
-
-[package.metadata.wasm-pack.profile.release.wasm-bindgen]
-debug-js-glue = false
-demangle-name-section = true
-dwarf-debug-info = false
diff --git a/quadratic-rust-client/package.json b/quadratic-rust-client/package.json
index ae80d519d4..554a59c4e5 100644
--- a/quadratic-rust-client/package.json
+++ b/quadratic-rust-client/package.json
@@ -3,6 +3,7 @@
"private": true,
"scripts": {
"build": "wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs",
+ "build:dev": "wasm-pack build --profiling --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs",
"dev": "cargo watch -s 'wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs'",
"dev:perf": "cargo watch -s 'wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs'"
}
From 98cd263ca60d8c82ee9c6879d6a5bf2cbd83168b Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 28 Dec 2024 07:00:11 +0530
Subject: [PATCH 115/155] debug label
---
.github/workflows/staging-build-deploy-images.yml | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/.github/workflows/staging-build-deploy-images.yml b/.github/workflows/staging-build-deploy-images.yml
index 201bc6bae7..093c8b293a 100644
--- a/.github/workflows/staging-build-deploy-images.yml
+++ b/.github/workflows/staging-build-deploy-images.yml
@@ -25,6 +25,10 @@ jobs:
- name: Check for build:dev label
id: check-dev
run: |
+ # Debug: Print all labels
+ echo "Labels: ${{ toJSON(github.event.pull_request.labels) }}"
+ echo "DEV: ${{ contains(github.event.pull_request.labels.*.name, 'build:dev') }}"
+
if [[ ${{ contains(github.event.pull_request.labels.*.name, 'build:dev') }} == true ]]; then
echo "DEV=true" >> $GITHUB_OUTPUT
else
From 3c58055f109345b449074fb28ddbb984f7739161 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 28 Dec 2024 07:03:27 +0530
Subject: [PATCH 116/155] remove debug info
---
.github/workflows/staging-build-deploy-images.yml | 4 ----
1 file changed, 4 deletions(-)
diff --git a/.github/workflows/staging-build-deploy-images.yml b/.github/workflows/staging-build-deploy-images.yml
index 093c8b293a..201bc6bae7 100644
--- a/.github/workflows/staging-build-deploy-images.yml
+++ b/.github/workflows/staging-build-deploy-images.yml
@@ -25,10 +25,6 @@ jobs:
- name: Check for build:dev label
id: check-dev
run: |
- # Debug: Print all labels
- echo "Labels: ${{ toJSON(github.event.pull_request.labels) }}"
- echo "DEV: ${{ contains(github.event.pull_request.labels.*.name, 'build:dev') }}"
-
if [[ ${{ contains(github.event.pull_request.labels.*.name, 'build:dev') }} == true ]]; then
echo "DEV=true" >> $GITHUB_OUTPUT
else
From 0be3725a4dfbcf583f53427759c160d16f46ba8b Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 28 Dec 2024 07:13:47 +0530
Subject: [PATCH 117/155] try
---
.github/workflows/staging-build-deploy-images.yml | 14 ++++++++++++--
1 file changed, 12 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/staging-build-deploy-images.yml b/.github/workflows/staging-build-deploy-images.yml
index 201bc6bae7..862b958eb2 100644
--- a/.github/workflows/staging-build-deploy-images.yml
+++ b/.github/workflows/staging-build-deploy-images.yml
@@ -12,11 +12,21 @@ jobs:
build_images:
permissions:
contents: read
- runs-on: blacksmith-4vcpu-ubuntu-2204
+ runs-on: ${{ matrix.runner }}
timeout-minutes: 30
strategy:
matrix:
- service: [client, api, multiplayer, files, connection]
+ include:
+ - service: client
+ runner: blacksmith-8vcpu-ubuntu-2204
+ - service: api
+ runner: blacksmith-2vcpu-ubuntu-2204
+ - service: connection
+ runner: blacksmith-4vcpu-ubuntu-2204
+ - service: files
+ runner: blacksmith-4vcpu-ubuntu-2204
+ - service: multiplayer
+ runner: blacksmith-4vcpu-ubuntu-2204
fail-fast: true
steps:
- name: Checkout code
From 797e20693c43a7b75bed0c0e40c2521034e7e90d Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 28 Dec 2024 07:25:43 +0530
Subject: [PATCH 118/155] build cache label
---
.../workflows/staging-build-deploy-images.yml | 36 ++++++++++++++++---
1 file changed, 32 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/staging-build-deploy-images.yml b/.github/workflows/staging-build-deploy-images.yml
index 862b958eb2..965fcb255c 100644
--- a/.github/workflows/staging-build-deploy-images.yml
+++ b/.github/workflows/staging-build-deploy-images.yml
@@ -33,7 +33,7 @@ jobs:
uses: actions/checkout@v4
- name: Check for build:dev label
- id: check-dev
+ id: build-dev
run: |
if [[ ${{ contains(github.event.pull_request.labels.*.name, 'build:dev') }} == true ]]; then
echo "DEV=true" >> $GITHUB_OUTPUT
@@ -41,6 +41,15 @@ jobs:
echo "DEV=false" >> $GITHUB_OUTPUT
fi
+ - name: Check for build:cache:disabled label
+ id: build-cache
+ run: |
+ if [[ ${{ contains(github.event.pull_request.labels.*.name, 'build:cache:disabled') }} == true ]]; then
+ echo "CACHE_DISABLED=true" >> $GITHUB_OUTPUT
+ else
+ echo "CACHE_DISABLED=false" >> $GITHUB_OUTPUT
+ fi
+
- name: Generate Build Metadata
id: build-metadata
run: |
@@ -87,8 +96,27 @@ jobs:
image=moby/buildkit:latest
network=host
- - name: Build and push
- # uses: useblacksmith/build-push-action@v1
+ - name: Build and push (with cache)
+ if: steps.build-cache.outputs.CACHE_DISABLED == 'false'
+ uses: useblacksmith/build-push-action@v1
+ with:
+ context: .
+ file: quadratic-${{ matrix.service }}/Dockerfile
+ push: true
+ tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }}
+ build-args: |
+ BUILDKIT_INLINE_CACHE=1
+ BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }}
+ GIT_SHA_SHORT=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
+ BRANCH_NAME=${{ steps.build-metadata.outputs.BRANCH_NAME }}
+ PR_NUMBER=${{ github.event.pull_request.number }}
+ DEV=${{ steps.build-dev.outputs.DEV }}
+ labels: |
+ org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}
+ org.opencontainers.image.revision=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
+
+ - name: Build and push (without cache)
+ if: steps.build-cache.outputs.CACHE_DISABLED == 'true'
uses: docker/build-push-action@v6
with:
context: .
@@ -101,7 +129,7 @@ jobs:
GIT_SHA_SHORT=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
BRANCH_NAME=${{ steps.build-metadata.outputs.BRANCH_NAME }}
PR_NUMBER=${{ github.event.pull_request.number }}
- DEV=${{ steps.check-dev.outputs.DEV }}
+ DEV=${{ steps.build-dev.outputs.DEV }}
labels: |
org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}
org.opencontainers.image.revision=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
From a5715cd7a6f3bd611287793faa9cdb75a40a0faa Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 28 Dec 2024 07:33:20 +0530
Subject: [PATCH 119/155] limit image build trigger to build labels
---
.github/workflows/staging-build-deploy-images.yml | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/.github/workflows/staging-build-deploy-images.yml b/.github/workflows/staging-build-deploy-images.yml
index 965fcb255c..382e7500ef 100644
--- a/.github/workflows/staging-build-deploy-images.yml
+++ b/.github/workflows/staging-build-deploy-images.yml
@@ -10,6 +10,10 @@ concurrency:
jobs:
build_images:
+ if: |
+ github.event.action != 'labeled' && github.event.action != 'unlabeled' ||
+ ((github.event.action == 'labeled' || github.event.action == 'unlabeled') &&
+ (startsWith(github.event.label.name, 'build:')))
permissions:
contents: read
runs-on: ${{ matrix.runner }}
From 995599d51aca57c67317bb27e3a8e9d1ad50c1ea Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 28 Dec 2024 09:02:30 +0530
Subject: [PATCH 120/155] cancel-in-progress
---
.github/workflows/staging-build-deploy-images.yml | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/staging-build-deploy-images.yml b/.github/workflows/staging-build-deploy-images.yml
index 382e7500ef..b2708934d0 100644
--- a/.github/workflows/staging-build-deploy-images.yml
+++ b/.github/workflows/staging-build-deploy-images.yml
@@ -6,14 +6,11 @@ on:
concurrency:
group: pr-${{ github.event.pull_request.number }}-build-images
- cancel-in-progress: true
+ cancel-in-progress: ${{ (github.event.action != 'labeled' && github.event.action != 'unlabeled') || ((github.event.action == 'labeled' || github.event.action == 'unlabeled') && startsWith(github.event.label.name, 'build:')) }}
jobs:
build_images:
- if: |
- github.event.action != 'labeled' && github.event.action != 'unlabeled' ||
- ((github.event.action == 'labeled' || github.event.action == 'unlabeled') &&
- (startsWith(github.event.label.name, 'build:')))
+ if: (github.event.action != 'labeled' && github.event.action != 'unlabeled') || ((github.event.action == 'labeled' || github.event.action == 'unlabeled') && (startsWith(github.event.label.name, 'build:')))
permissions:
contents: read
runs-on: ${{ matrix.runner }}
From 547dbdbf711851b33a92234cd9e070e9be8ce51f Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 28 Dec 2024 09:20:16 +0530
Subject: [PATCH 121/155] comment on deployment
---
.../workflows/staging-build-deploy-images.yml | 31 +++++++++++++++++++
.github/workflows/staging-create-stack.yml | 25 ++++++++-------
2 files changed, 44 insertions(+), 12 deletions(-)
diff --git a/.github/workflows/staging-build-deploy-images.yml b/.github/workflows/staging-build-deploy-images.yml
index b2708934d0..22a14b2dcd 100644
--- a/.github/workflows/staging-build-deploy-images.yml
+++ b/.github/workflows/staging-build-deploy-images.yml
@@ -229,3 +229,34 @@ jobs:
aws ssm wait command-executed \
--command-id "$COMMAND_ID" \
--instance-id ${{ steps.get-instance.outputs.instance_id }}
+
+ - name: Find Build & Deploy Images Comment
+ uses: peter-evans/find-comment@v3
+ id: staging-build-deploy-images-comment
+ with:
+ issue-number: ${{ github.event.pull_request.number }}
+ comment-author: "github-actions[bot]"
+ body-includes: "Staging Build & Deploy Images"
+
+ - name: Create or update comment on deployment success
+ if: success()
+ uses: peter-evans/create-or-update-comment@v3
+ with:
+ comment-id: ${{ steps.staging-build-deploy-images-comment.outputs.comment-id }}
+ issue-number: ${{ github.event.pull_request.number }}
+ body: |
+ ## Staging Build & Deploy Images
+ Staging images built and deployed successfully.
+ Last deployed at: $(date -u +'%Y-%m-%dT%H:%M:%SZ')
+ edit-mode: replace
+
+ - name: Create or update comment on deployment failure
+ if: failure()
+ uses: peter-evans/create-or-update-comment@v3
+ with:
+ comment-id: ${{ steps.staging-build-deploy-images-comment.outputs.comment-id }}
+ issue-number: ${{ github.event.pull_request.number }}
+ body: |
+ ## Staging Build & Deploy Images
+ The staging build & deploy images encountered an error. Please check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
+ edit-mode: replace
diff --git a/.github/workflows/staging-create-stack.yml b/.github/workflows/staging-create-stack.yml
index a78e6dfa14..6441d4fafb 100644
--- a/.github/workflows/staging-create-stack.yml
+++ b/.github/workflows/staging-create-stack.yml
@@ -76,32 +76,33 @@ jobs:
exit 1
fi
- - name: Find Comment
+ - name: Find Create Stack Comment
uses: peter-evans/find-comment@v3
- id: staging-comment
+ id: staging-create-stack-comment
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: "github-actions[bot]"
- body-includes: "Staging Deployment"
+ body-includes: "Staging Stack"
- - name: Create or update comment on build success
- if: success() && steps.create-stack.outputs.WebsiteURL != ''
+ - name: Create or update comment on create stack success
+ if: success()
uses: peter-evans/create-or-update-comment@v3
with:
- comment-id: ${{ steps.staging-comment.outputs.comment-id }}
+ comment-id: ${{ steps.staging-create-stack-comment.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: |
- ## Staging Deployment
- Staging URL: https://${{ steps.create-stack.outputs.WebsiteURL }}
+ ## Staging Stack
+ Staging stack created successfully.
+ URL: https://${{ steps.create-stack.outputs.WebsiteURL }}
edit-mode: replace
- - name: Create or update comment on build failure
+ - name: Create or update comment on create stack failure
if: failure()
uses: peter-evans/create-or-update-comment@v3
with:
- comment-id: ${{ steps.staging-comment.outputs.comment-id }}
+ comment-id: ${{ steps.staging-create-stack-comment.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: |
- ## Staging Deployment
- The staging deployment encountered an error. Please check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
+ ## Staging Stack
+ The staging stack encountered an error. Please check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
edit-mode: replace
From 09fbdec60cc68ff9860cfa756124e9de67988348 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 28 Dec 2024 09:29:48 +0530
Subject: [PATCH 122/155] add permission for comment
---
.github/workflows/staging-build-deploy-images.yml | 6 ++++++
.github/workflows/staging-create-stack.yml | 1 +
2 files changed, 7 insertions(+)
diff --git a/.github/workflows/staging-build-deploy-images.yml b/.github/workflows/staging-build-deploy-images.yml
index 22a14b2dcd..8648080c4b 100644
--- a/.github/workflows/staging-build-deploy-images.yml
+++ b/.github/workflows/staging-build-deploy-images.yml
@@ -10,6 +10,7 @@ concurrency:
jobs:
build_images:
+ name: Build Images - Staging
if: (github.event.action != 'labeled' && github.event.action != 'unlabeled') || ((github.event.action == 'labeled' || github.event.action == 'unlabeled') && (startsWith(github.event.label.name, 'build:')))
permissions:
contents: read
@@ -136,7 +137,12 @@ jobs:
org.opencontainers.image.revision=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
deploy_images:
+ name: Deploy Images - Staging
needs: build_images
+ permissions:
+ contents: read
+ issues: write
+ pull-requests: write
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 30
env:
diff --git a/.github/workflows/staging-create-stack.yml b/.github/workflows/staging-create-stack.yml
index 6441d4fafb..e18450de34 100644
--- a/.github/workflows/staging-create-stack.yml
+++ b/.github/workflows/staging-create-stack.yml
@@ -12,6 +12,7 @@ jobs:
name: Create Stack - Staging
permissions:
contents: read
+ issues: write
pull-requests: write
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 30
From 59fdd5ce0716d8e4ffd7e243c047b3ac27c1aa7e Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 28 Dec 2024 09:40:58 +0530
Subject: [PATCH 123/155] fix deploy time
---
.github/workflows/staging-build-deploy-images.yml | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/staging-build-deploy-images.yml b/.github/workflows/staging-build-deploy-images.yml
index 8648080c4b..145d7f2e64 100644
--- a/.github/workflows/staging-build-deploy-images.yml
+++ b/.github/workflows/staging-build-deploy-images.yml
@@ -236,6 +236,11 @@ jobs:
--command-id "$COMMAND_ID" \
--instance-id ${{ steps.get-instance.outputs.instance_id }}
+ - name: Generate Deploy Metadata
+ id: deploy-metadata
+ run: |
+ echo "DEPLOY_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
+
- name: Find Build & Deploy Images Comment
uses: peter-evans/find-comment@v3
id: staging-build-deploy-images-comment
@@ -253,7 +258,7 @@ jobs:
body: |
## Staging Build & Deploy Images
Staging images built and deployed successfully.
- Last deployed at: $(date -u +'%Y-%m-%dT%H:%M:%SZ')
+ Last deployed at: ${{ steps.deploy-metadata.outputs.DEPLOY_TIME }}
edit-mode: replace
- name: Create or update comment on deployment failure
From e5911ea5575eb59beb993e64d4b1a319c6aa0220 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 28 Dec 2024 09:52:26 +0530
Subject: [PATCH 124/155] format time
---
.../workflows/staging-build-deploy-images.yml | 17 ++++++++++-------
.github/workflows/staging-create-stack.yml | 15 +++++++++------
2 files changed, 19 insertions(+), 13 deletions(-)
diff --git a/.github/workflows/staging-build-deploy-images.yml b/.github/workflows/staging-build-deploy-images.yml
index 145d7f2e64..dd2f12f703 100644
--- a/.github/workflows/staging-build-deploy-images.yml
+++ b/.github/workflows/staging-build-deploy-images.yml
@@ -239,7 +239,7 @@ jobs:
- name: Generate Deploy Metadata
id: deploy-metadata
run: |
- echo "DEPLOY_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
+ echo "DEPLOY_TIME=$(date -u +'%B %d, %Y at %H:%M UTC')" >> $GITHUB_OUTPUT
- name: Find Build & Deploy Images Comment
uses: peter-evans/find-comment@v3
@@ -247,7 +247,7 @@ jobs:
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: "github-actions[bot]"
- body-includes: "Staging Build & Deploy Images"
+ body-includes: "Staging - Build & Deploy Images"
- name: Create or update comment on deployment success
if: success()
@@ -256,9 +256,10 @@ jobs:
comment-id: ${{ steps.staging-build-deploy-images-comment.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: |
- ## Staging Build & Deploy Images
- Staging images built and deployed successfully.
- Last deployed at: ${{ steps.deploy-metadata.outputs.DEPLOY_TIME }}
+ ## Staging - Build & Deploy Images
+ ✅ Staging images built and deployed successfully.
+
+ 🕒 Last deployed: ${{ steps.deploy-metadata.outputs.DEPLOY_TIME }}
edit-mode: replace
- name: Create or update comment on deployment failure
@@ -268,6 +269,8 @@ jobs:
comment-id: ${{ steps.staging-build-deploy-images-comment.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: |
- ## Staging Build & Deploy Images
- The staging build & deploy images encountered an error. Please check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
+ ## Staging - Build & Deploy Images
+ ❌ The staging build & deploy images encountered an error.
+
+ 🔍 Please check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
edit-mode: replace
diff --git a/.github/workflows/staging-create-stack.yml b/.github/workflows/staging-create-stack.yml
index e18450de34..9c687cbb1b 100644
--- a/.github/workflows/staging-create-stack.yml
+++ b/.github/workflows/staging-create-stack.yml
@@ -83,7 +83,7 @@ jobs:
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: "github-actions[bot]"
- body-includes: "Staging Stack"
+ body-includes: "Staging - Create Stack"
- name: Create or update comment on create stack success
if: success()
@@ -92,9 +92,10 @@ jobs:
comment-id: ${{ steps.staging-create-stack-comment.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: |
- ## Staging Stack
- Staging stack created successfully.
- URL: https://${{ steps.create-stack.outputs.WebsiteURL }}
+ ## Staging - Create Stack
+ ✅ Staging stack created successfully.
+
+ 🔗 URL: https://${{ steps.create-stack.outputs.WebsiteURL }}
edit-mode: replace
- name: Create or update comment on create stack failure
@@ -104,6 +105,8 @@ jobs:
comment-id: ${{ steps.staging-create-stack-comment.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: |
- ## Staging Stack
- The staging stack encountered an error. Please check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
+ ## Staging - Create Stack
+ ❌ The staging stack creation encountered an error.
+
+ 🔍 Please check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
edit-mode: replace
From b61b56ac9f6e42b6c03d99ac0c0b0b15241b3f72 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 28 Dec 2024 10:32:36 +0530
Subject: [PATCH 125/155] fix create / delete stack comment
---
.github/workflows/staging-create-stack.yml | 8 +++----
.github/workflows/staging-delete-stack.yml | 27 ++++++++++++++++------
2 files changed, 24 insertions(+), 11 deletions(-)
diff --git a/.github/workflows/staging-create-stack.yml b/.github/workflows/staging-create-stack.yml
index 9c687cbb1b..92c9127df4 100644
--- a/.github/workflows/staging-create-stack.yml
+++ b/.github/workflows/staging-create-stack.yml
@@ -83,7 +83,7 @@ jobs:
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: "github-actions[bot]"
- body-includes: "Staging - Create Stack"
+ body-includes: "Staging - Stack"
- name: Create or update comment on create stack success
if: success()
@@ -92,10 +92,10 @@ jobs:
comment-id: ${{ steps.staging-create-stack-comment.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: |
- ## Staging - Create Stack
+ ## Staging - Stack
✅ Staging stack created successfully.
- 🔗 URL: https://${{ steps.create-stack.outputs.WebsiteURL }}
+ 🔗 URL: ${{ steps.create-stack.outputs.WebsiteURL }}
edit-mode: replace
- name: Create or update comment on create stack failure
@@ -105,7 +105,7 @@ jobs:
comment-id: ${{ steps.staging-create-stack-comment.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: |
- ## Staging - Create Stack
+ ## Staging - Stack
❌ The staging stack creation encountered an error.
🔍 Please check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
diff --git a/.github/workflows/staging-delete-stack.yml b/.github/workflows/staging-delete-stack.yml
index 1ce6993807..97744532d7 100644
--- a/.github/workflows/staging-delete-stack.yml
+++ b/.github/workflows/staging-delete-stack.yml
@@ -44,19 +44,32 @@ jobs:
- name: Find Comment
uses: peter-evans/find-comment@v3
- id: staging-comment
+ id: staging-delete-stack-comment
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: "github-actions[bot]"
- body-includes: "Staging Deployment"
+ body-includes: "Staging - Stack"
- - name: Update comment
- if: steps.staging-comment.outputs.comment-id != ''
+ - name: Update comment on delete stack success
+ if: success() && steps.staging-delete-stack-comment.outputs.comment-id != ''
uses: peter-evans/create-or-update-comment@v3
with:
- comment-id: ${{ steps.staging-comment.outputs.comment-id }}
+ comment-id: ${{ steps.staging-delete-stack-comment.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: |
- ## Staging Deployment
- Stack deleted.
+ ## Staging - Stack
+ ✅ Stack deleted.
+ edit-mode: replace
+
+ - name: Update comment on delete stack failure
+ if: failure() && steps.staging-delete-stack-comment.outputs.comment-id != ''
+ uses: peter-evans/create-or-update-comment@v3
+ with:
+ comment-id: ${{ steps.staging-delete-stack-comment.outputs.comment-id }}
+ issue-number: ${{ github.event.pull_request.number }}
+ body: |
+ ## Staging - Stack
+ ❌ Stack deletion encountered an error.\
+
+ 🔍 Please check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
edit-mode: replace
From 92fab90af12efbc9c57f5c586a582fd2686d4478 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 29 Dec 2024 05:33:23 +0530
Subject: [PATCH 126/155] upgrade ci actions, try fixing codecov
---
.github/workflows/ci.yml | 102 ++++++++++++++++++++++--------------
docker-compose.yml | 4 ++
quadratic-client/Dockerfile | 4 +-
3 files changed, 69 insertions(+), 41 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 306bd0770c..4906bedc02 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -20,15 +20,18 @@ jobs:
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- - uses: actions/setup-node@v4
+
+ - uses: useblacksmith/setup-node@v5
with:
node-version: 18
+ cache: "npm"
- name: Set up Rust
- uses: moonrepo/setup-rust@v1
+ uses: actions-rs/toolchain@v1
with:
components: clippy, llvm-tools-preview
- cache: false
+
+ - uses: useblacksmith/rust-cache@v3
- name: Install grcov
run: if ! which grcov; then cargo install grcov; fi
@@ -52,21 +55,21 @@ jobs:
cd quadratic-core
cargo test -- --test-threads=4
- - name: Generate coverage for quadratic-core
- env:
- RUSTC_BOOTSTRAP: 1
- run: |
- grcov $(find . -name "grcov-*.profraw" -print) \
- --branch \
- --ignore-not-existing \
- --binary-path ./target/debug/ \
- -s . \
- -t lcov \
- --ignore "/*" \
- --ignore "./src/wasm_bindings/*" \
- --ignore "./src/bin/*" \
- --ignore "./docker/*" \
- -o lcov.info
+ # - name: Generate coverage for quadratic-core
+ # env:
+ # RUSTC_BOOTSTRAP: 1
+ # run: |
+ # grcov $(find . -name "grcov-*.profraw" -print) \
+ # --branch \
+ # --ignore-not-existing \
+ # --binary-path ./target/debug/ \
+ # -s . \
+ # -t lcov \
+ # --ignore "/*" \
+ # --ignore "./src/wasm_bindings/*" \
+ # --ignore "./src/bin/*" \
+ # --ignore "./docker/*" \
+ # -o lcov.info
- name: Upload coverage reports to Codecov quadratic-core
uses: codecov/codecov-action@v3
@@ -78,15 +81,18 @@ jobs:
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- - uses: actions/setup-node@v4
+
+ - uses: useblacksmith/setup-node@v5
with:
node-version: 18
+ cache: "npm"
- name: Set up Rust
- uses: moonrepo/setup-rust@v1
+ uses: actions-rs/toolchain@v1
with:
components: clippy, llvm-tools-preview
- cache: false
+
+ - uses: useblacksmith/rust-cache@v3
- name: Install grcov
run: if ! which grcov; then cargo install grcov; fi
@@ -114,6 +120,7 @@ jobs:
env:
RUSTC_BOOTSTRAP: 1
run: |
+ cd quadratic-multiplayer
grcov $(find . -name "grcov-*.profraw" -print) \
--branch \
--ignore-not-existing \
@@ -124,7 +131,6 @@ jobs:
--ignore "./src/wasm_bindings/*" \
--ignore "./src/bin/*" \
--ignore "./docker/*" \
- --ignore "./../docker/*" \
-o lcov.info
- name: Upload coverage reports to Codecov quadratic-multiplayer
@@ -137,15 +143,18 @@ jobs:
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- - uses: actions/setup-node@v4
+
+ - uses: useblacksmith/setup-node@v5
with:
node-version: 18
+ cache: "npm"
- name: Set up Rust
- uses: moonrepo/setup-rust@v1
+ uses: actions-rs/toolchain@v1
with:
components: clippy, llvm-tools-preview
- cache: false
+
+ - uses: useblacksmith/rust-cache@v3
- name: Install grcov
run: if ! which grcov; then cargo install grcov; fi
@@ -174,17 +183,21 @@ jobs:
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- - uses: actions/setup-node@v4
+
+ - uses: useblacksmith/setup-node@v5
with:
node-version: 18
- - uses: actions/setup-python@v5
+ cache: "npm"
+
+ - uses: useblacksmith/setup-python@v6
with:
python-version: "3.11.3"
+ cache: "pip"
- name: Set up Rust
- uses: moonrepo/setup-rust@v1
- with:
- channel: "nightly"
+ uses: actions-rs/toolchain@v1
+
+ - uses: useblacksmith/rust-cache@v3
- uses: jetli/wasm-pack-action@v0.4.0
@@ -210,10 +223,13 @@ jobs:
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- - uses: actions/setup-node@v4
+
+ - uses: useblacksmith/setup-node@v5
with:
node-version: 18
- - uses: actions/setup-python@v5
+ cache: "npm"
+
+ - uses: useblacksmith/setup-python@v6
with:
python-version: "3.11.3"
cache: "pip"
@@ -228,9 +244,11 @@ jobs:
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- - uses: actions/setup-node@v4
+
+ - uses: useblacksmith/setup-node@v5
with:
node-version: 18
+ cache: "npm"
- name: Run npm test:ci in quadratic-api
run: |
@@ -243,11 +261,13 @@ jobs:
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
+
- name: Set up Rust
- uses: moonrepo/setup-rust@v1
+ uses: actions-rs/toolchain@v1
with:
components: clippy
- cache: false
+
+ - uses: useblacksmith/rust-cache@v3
- name: Run cargo clippy in quadratic-core
run: |
@@ -259,13 +279,17 @@ jobs:
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- - uses: actions/setup-node@v4
+
+ - uses: useblacksmith/setup-node@v5
with:
node-version: 18
+ cache: "npm"
+
- name: Set up Rust
- uses: moonrepo/setup-rust@v1
- with:
- cache: false
+ uses: actions-rs/toolchain@v1
+
+ - uses: useblacksmith/rust-cache@v3
+
- uses: jetli/wasm-pack-action@v0.4.0
with:
version: "latest"
diff --git a/docker-compose.yml b/docker-compose.yml
index bcc56c948d..a861a37f46 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -263,6 +263,8 @@ services:
depends_on:
quadratic-client:
condition: service_healthy
+ quadratic-api:
+ condition: service_started
profiles:
- quadratic-connection
- backend
@@ -410,6 +412,8 @@ services:
CSRF_COOKIE_NAME: ${KRATOS_CSRF_COOKIE_NAME}
CSRF_COOKIE_SECRET: ${KRATOS_CSRF_COOKIE_SECRET}
restart: on-failure
+ depends_on:
+ - ory-auth
profiles:
- ory
- dev
diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile
index 11b0628541..ccbd973bb9 100644
--- a/quadratic-client/Dockerfile
+++ b/quadratic-client/Dockerfile
@@ -141,7 +141,7 @@ RUN if [ "$DEV" = "true" ]; then \
###########################################################################################
# Builder 5 - Combine all builder files and build the client
###########################################################################################
-FROM node:18 AS vite-build
+FROM node:18 AS vite-builder
# Install python
RUN apt-get update && apt-get install -y --no-install-recommends python-is-python3 python3-pip
@@ -186,7 +186,7 @@ RUN echo 'Building front-end...' && npm run build --workspace=quadratic-client
###########################################################################################
FROM nginx:stable-alpine
-COPY --from=vite-build /quadratic/build /usr/share/nginx/html
+COPY --from=vite-builder /quadratic/build /usr/share/nginx/html
EXPOSE 80 443 3000
From 36f0810378cfffed7f61db958583470b0bb65330 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 29 Dec 2024 05:37:19 +0530
Subject: [PATCH 127/155] add toolchain
---
.github/workflows/ci.yml | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4906bedc02..1da0f07272 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -29,7 +29,9 @@ jobs:
- name: Set up Rust
uses: actions-rs/toolchain@v1
with:
+ toolchain: stable
components: clippy, llvm-tools-preview
+ override: true
- uses: useblacksmith/rust-cache@v3
@@ -90,7 +92,9 @@ jobs:
- name: Set up Rust
uses: actions-rs/toolchain@v1
with:
+ toolchain: stable
components: clippy, llvm-tools-preview
+ override: true
- uses: useblacksmith/rust-cache@v3
@@ -152,7 +156,9 @@ jobs:
- name: Set up Rust
uses: actions-rs/toolchain@v1
with:
+ toolchain: stable
components: clippy, llvm-tools-preview
+ override: true
- uses: useblacksmith/rust-cache@v3
@@ -196,6 +202,9 @@ jobs:
- name: Set up Rust
uses: actions-rs/toolchain@v1
+ with:
+ toolchain: stable
+ override: true
- uses: useblacksmith/rust-cache@v3
@@ -265,7 +274,9 @@ jobs:
- name: Set up Rust
uses: actions-rs/toolchain@v1
with:
+ toolchain: stable
components: clippy
+ override: true
- uses: useblacksmith/rust-cache@v3
@@ -287,6 +298,10 @@ jobs:
- name: Set up Rust
uses: actions-rs/toolchain@v1
+ with:
+ toolchain: stable
+ components: clippy
+ override: true
- uses: useblacksmith/rust-cache@v3
From 5ea49f98e3f20f5c3a9113a383382120277bf690 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 29 Dec 2024 05:48:34 +0530
Subject: [PATCH 128/155] fix target path
---
.github/workflows/ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1da0f07272..5056b6521b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -128,7 +128,7 @@ jobs:
grcov $(find . -name "grcov-*.profraw" -print) \
--branch \
--ignore-not-existing \
- --binary-path ./target/debug/ \
+ --binary-path ../target/debug/ \
-s . \
-t lcov \
--ignore "/*" \
From d9a649443fb52f177cce8f44167d04fe9d08cc2a Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 29 Dec 2024 05:55:10 +0530
Subject: [PATCH 129/155] try
---
.github/workflows/ci.yml | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5056b6521b..baf53c95b8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -128,7 +128,7 @@ jobs:
grcov $(find . -name "grcov-*.profraw" -print) \
--branch \
--ignore-not-existing \
- --binary-path ../target/debug/ \
+ --binary-path ../target/debug/ \
-s . \
-t lcov \
--ignore "/*" \
@@ -204,6 +204,7 @@ jobs:
uses: actions-rs/toolchain@v1
with:
toolchain: stable
+ target: wasm32-unknown-unknown
override: true
- uses: useblacksmith/rust-cache@v3
@@ -301,6 +302,7 @@ jobs:
with:
toolchain: stable
components: clippy
+ target: wasm32-unknown-unknown
override: true
- uses: useblacksmith/rust-cache@v3
From cbd184d8e2d2443ab58ec40c0fb858f386de47f7 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 29 Dec 2024 06:23:23 +0530
Subject: [PATCH 130/155] fix mysql
---
.github/workflows/ci.yml | 53 ++++++++++++++++++++++++++++------------
docker-compose.yml | 3 +--
2 files changed, 39 insertions(+), 17 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index baf53c95b8..7455804750 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -57,21 +57,22 @@ jobs:
cd quadratic-core
cargo test -- --test-threads=4
- # - name: Generate coverage for quadratic-core
- # env:
- # RUSTC_BOOTSTRAP: 1
- # run: |
- # grcov $(find . -name "grcov-*.profraw" -print) \
- # --branch \
- # --ignore-not-existing \
- # --binary-path ./target/debug/ \
- # -s . \
- # -t lcov \
- # --ignore "/*" \
- # --ignore "./src/wasm_bindings/*" \
- # --ignore "./src/bin/*" \
- # --ignore "./docker/*" \
- # -o lcov.info
+ - name: Generate coverage for quadratic-core
+ env:
+ RUSTC_BOOTSTRAP: 1
+ run: |
+ cd quadratic-core
+ grcov $(find . -name "grcov-*.profraw" -print) \
+ --branch \
+ --ignore-not-existing \
+ --binary-path ../target/debug/ \
+ -s . \
+ -t lcov \
+ --ignore "/*" \
+ --ignore "./src/wasm_bindings/*" \
+ --ignore "./src/bin/*" \
+ --ignore "./docker/*" \
+ -o lcov.info
- name: Upload coverage reports to Codecov quadratic-core
uses: codecov/codecov-action@v3
@@ -184,6 +185,28 @@ jobs:
cd quadratic-connection
npm run test:docker:ci
+ - name: Generate coverage for quadratic-connection
+ env:
+ RUSTC_BOOTSTRAP: 1
+ run: |
+ cd quadratic-connection
+ grcov $(find . -name "grcov-*.profraw" -print) \
+ --branch \
+ --ignore-not-existing \
+ --binary-path ../target/debug/ \
+ -s . \
+ -t lcov \
+ --ignore "/*" \
+ --ignore "./src/wasm_bindings/*" \
+ --ignore "./src/bin/*" \
+ --ignore "./docker/*" \
+ -o lcov.info
+
+ - name: Upload coverage reports to Codecov quadratic-connection
+ uses: codecov/codecov-action@v3
+ env:
+ CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
test_unit:
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 10
diff --git a/docker-compose.yml b/docker-compose.yml
index a861a37f46..7c3b77d568 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -470,7 +470,6 @@ services:
image: mysql:8
restart: always
container_name: mysql-connection
- user: "999:999"
ports:
- 3306:3306
environment:
@@ -484,7 +483,7 @@ services:
timeout: 5s
retries: 5
volumes:
- - ./docker/mysql-connection/data:/var/lib/mysql:rw
+ - ./docker/mysql-connection/data:/var/lib/mysql
- ./docker/mysql-connection/scripts:/docker-entrypoint-initdb.d/
profiles:
- quadratic-connection
From eb52c0e3b7779497e106e053616c8f603c28973a Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 29 Dec 2024 06:41:00 +0530
Subject: [PATCH 131/155] use docker volume for mysql
---
.github/workflows/ci.yml | 45 ++++++++++++++++++++--------------------
docker-compose.yml | 5 ++++-
2 files changed, 26 insertions(+), 24 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7455804750..e145760ccc 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -61,7 +61,6 @@ jobs:
env:
RUSTC_BOOTSTRAP: 1
run: |
- cd quadratic-core
grcov $(find . -name "grcov-*.profraw" -print) \
--branch \
--ignore-not-existing \
@@ -75,7 +74,7 @@ jobs:
-o lcov.info
- name: Upload coverage reports to Codecov quadratic-core
- uses: codecov/codecov-action@v3
+ uses: codecov/codecov-action@v5
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
@@ -121,27 +120,27 @@ jobs:
cd quadratic-multiplayer
npm run docker:test
- - name: Generate coverage for quadratic-multiplayer
- env:
- RUSTC_BOOTSTRAP: 1
- run: |
- cd quadratic-multiplayer
- grcov $(find . -name "grcov-*.profraw" -print) \
- --branch \
- --ignore-not-existing \
- --binary-path ../target/debug/ \
- -s . \
- -t lcov \
- --ignore "/*" \
- --ignore "./src/wasm_bindings/*" \
- --ignore "./src/bin/*" \
- --ignore "./docker/*" \
- -o lcov.info
-
- - name: Upload coverage reports to Codecov quadratic-multiplayer
- uses: codecov/codecov-action@v3
- env:
- CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+ # - name: Generate coverage for quadratic-multiplayer
+ # env:
+ # RUSTC_BOOTSTRAP: 1
+ # run: |
+ # cd quadratic-multiplayer
+ # grcov $(find . -name "grcov-*.profraw" -print) \
+ # --branch \
+ # --ignore-not-existing \
+ # --binary-path ../target/debug/ \
+ # -s . \
+ # -t lcov \
+ # --ignore "/*" \
+ # --ignore "./src/wasm_bindings/*" \
+ # --ignore "./src/bin/*" \
+ # --ignore "./docker/*" \
+ # -o lcov.info
+
+ # - name: Upload coverage reports to Codecov quadratic-multiplayer
+ # uses: codecov/codecov-action@v3
+ # env:
+ # CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
test_connection:
runs-on: blacksmith-2vcpu-ubuntu-2204
diff --git a/docker-compose.yml b/docker-compose.yml
index 7c3b77d568..3a1d42b76b 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -483,7 +483,7 @@ services:
timeout: 5s
retries: 5
volumes:
- - ./docker/mysql-connection/data:/var/lib/mysql
+ - mysql-connection:/var/lib/mysql
- ./docker/mysql-connection/scripts:/docker-entrypoint-initdb.d/
profiles:
- quadratic-connection
@@ -541,6 +541,9 @@ services:
volumes:
docker:
name: docker
+ mysql-connection:
+ name: mysql-connection
+ driver: local
networks:
host:
From b8ff7a09b3fee85b658af182384668cd29859a9e Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 29 Dec 2024 06:53:42 +0530
Subject: [PATCH 132/155] use root for mssql
---
.github/workflows/ci.yml | 47 ++++++++++++++++++++--------------------
docker-compose.yml | 2 +-
2 files changed, 25 insertions(+), 24 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e145760ccc..a40b01c4e8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -61,10 +61,11 @@ jobs:
env:
RUSTC_BOOTSTRAP: 1
run: |
+ cd quadratic-core
grcov $(find . -name "grcov-*.profraw" -print) \
--branch \
--ignore-not-existing \
- --binary-path ../target/debug/ \
+ --binary-path ./target/debug/ \
-s . \
-t lcov \
--ignore "/*" \
@@ -120,27 +121,27 @@ jobs:
cd quadratic-multiplayer
npm run docker:test
- # - name: Generate coverage for quadratic-multiplayer
- # env:
- # RUSTC_BOOTSTRAP: 1
- # run: |
- # cd quadratic-multiplayer
- # grcov $(find . -name "grcov-*.profraw" -print) \
- # --branch \
- # --ignore-not-existing \
- # --binary-path ../target/debug/ \
- # -s . \
- # -t lcov \
- # --ignore "/*" \
- # --ignore "./src/wasm_bindings/*" \
- # --ignore "./src/bin/*" \
- # --ignore "./docker/*" \
- # -o lcov.info
-
- # - name: Upload coverage reports to Codecov quadratic-multiplayer
- # uses: codecov/codecov-action@v3
- # env:
- # CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+ - name: Generate coverage for quadratic-multiplayer
+ env:
+ RUSTC_BOOTSTRAP: 1
+ run: |
+ cd quadratic-multiplayer
+ grcov $(find . -name "grcov-*.profraw" -print) \
+ --branch \
+ --ignore-not-existing \
+ --binary-path ../target/debug/ \
+ -s . \
+ -t lcov \
+ --ignore "/*" \
+ --ignore "./src/wasm_bindings/*" \
+ --ignore "./src/bin/*" \
+ --ignore "./docker/*" \
+ -o lcov.info
+
+ - name: Upload coverage reports to Codecov quadratic-multiplayer
+ uses: codecov/codecov-action@v5
+ env:
+ CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
test_connection:
runs-on: blacksmith-2vcpu-ubuntu-2204
@@ -202,7 +203,7 @@ jobs:
-o lcov.info
- name: Upload coverage reports to Codecov quadratic-connection
- uses: codecov/codecov-action@v3
+ uses: codecov/codecov-action@v5
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
diff --git a/docker-compose.yml b/docker-compose.yml
index 3a1d42b76b..e74e038790 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -494,7 +494,7 @@ services:
image: mcr.microsoft.com/mssql/server:2022-latest
restart: always
container_name: mssql-connection
- user: mssql
+ user: root
ports:
- 1433:1433
environment:
From 4a56e452d7f2f5baa757d290ff2afbbd815665f4 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 29 Dec 2024 07:00:44 +0530
Subject: [PATCH 133/155] core codecov typo
---
.github/workflows/ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a40b01c4e8..ca388703e0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -65,7 +65,7 @@ jobs:
grcov $(find . -name "grcov-*.profraw" -print) \
--branch \
--ignore-not-existing \
- --binary-path ./target/debug/ \
+ --binary-path ../target/debug/ \
-s . \
-t lcov \
--ignore "/*" \
From d6fabc98bc0ebda438b810e81c1e4733a5d47749 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 29 Dec 2024 08:43:12 +0530
Subject: [PATCH 134/155] include rust-shared and rust-client in codecov
---
.github/workflows/ci.yml | 8 ++++----
quadratic-connection/package.json | 4 ++--
quadratic-multiplayer/package.json | 2 +-
3 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ca388703e0..15484f52d0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -62,7 +62,7 @@ jobs:
RUSTC_BOOTSTRAP: 1
run: |
cd quadratic-core
- grcov $(find . -name "grcov-*.profraw" -print) \
+ grcov $(find . ../quadratic-rust-client ../quadratic-rust-shared -name "grcov-*.profraw" -print) \
--branch \
--ignore-not-existing \
--binary-path ../target/debug/ \
@@ -126,7 +126,7 @@ jobs:
RUSTC_BOOTSTRAP: 1
run: |
cd quadratic-multiplayer
- grcov $(find . -name "grcov-*.profraw" -print) \
+ grcov $(find . ../quadratic-rust-shared -name "grcov-*.profraw" -print) \
--branch \
--ignore-not-existing \
--binary-path ../target/debug/ \
@@ -183,14 +183,14 @@ jobs:
CARGO_BUILD_JOBS: 4
run: |
cd quadratic-connection
- npm run test:docker:ci
+ npm run docker:test
- name: Generate coverage for quadratic-connection
env:
RUSTC_BOOTSTRAP: 1
run: |
cd quadratic-connection
- grcov $(find . -name "grcov-*.profraw" -print) \
+ grcov $(find . ../quadratic-rust-shared -name "grcov-*.profraw" -print) \
--branch \
--ignore-not-existing \
--binary-path ../target/debug/ \
diff --git a/quadratic-connection/package.json b/quadratic-connection/package.json
index 1c08696861..14d7c157f9 100644
--- a/quadratic-connection/package.json
+++ b/quadratic-connection/package.json
@@ -9,13 +9,13 @@
"dev": "RUST_LOG=info cargo watch -x 'run'",
"test": "cargo test",
"test:watch": "RUST_LOG=info cargo watch -x 'test'",
- "test:docker:ci": "npm run docker:up && npm run test && npm run docker:down",
"lint": "cargo clippy --all-targets --all-features -- -D warnings",
"coverage": "npm run coverage:gen && npm run coverage:html && npm run coverage:view",
"coverage:gen": "CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='coverage/cargo-test-%p-%m.profraw' cargo test",
"coverage:html": "grcov . --binary-path ./target/debug/deps/ -s . -t html --branch --ignore-not-existing --ignore '../*' --ignore '/*' -o coverage/html",
"coverage:view": "open coverage/html/index.html",
"docker:up": "ECR_OR_BUILD=build DEV=false docker compose -f ../docker-compose.yml --profile quadratic-connection-db --env-file ../.env.docker up -d --wait && sleep 10",
- "docker:down": "ECR_OR_BUILD=build DEV=false docker compose -f ../docker-compose.yml --profile quadratic-connection-db --env-file ../.env.docker down"
+ "docker:down": "ECR_OR_BUILD=build DEV=false docker compose -f ../docker-compose.yml --profile quadratic-connection-db --env-file ../.env.docker down",
+ "docker:test": "ECR_OR_BUILD=build DEV=true docker compose kill && npm run docker:up && npm run test && npm run docker:down"
}
}
diff --git a/quadratic-multiplayer/package.json b/quadratic-multiplayer/package.json
index 58703f8153..a0f62645c8 100644
--- a/quadratic-multiplayer/package.json
+++ b/quadratic-multiplayer/package.json
@@ -16,6 +16,6 @@
"coverage:view": "open coverage/html/index.html",
"docker:up": "ECR_OR_BUILD=build DEV=false docker compose -f ../docker-compose.yml --profile base --env-file ../.env.docker up -d --wait",
"docker:down": "ECR_OR_BUILD=build DEV=false docker compose -f ../docker-compose.yml --profile base --env-file ../.env.docker down -v",
- "docker:test": "ECR_OR_BUILD=build DEV=false docker compose kill && npm run docker:up && npm run test && npm run docker:down"
+ "docker:test": "ECR_OR_BUILD=build DEV=true docker compose kill && npm run docker:up && npm run test && npm run docker:down"
}
}
From 138cf0a59942021e9d2aa5eade4f799d2b21c020 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 29 Dec 2024 11:39:03 +0530
Subject: [PATCH 135/155] docker dev
---
Dockerfile => Dockerfile.dev | 2 +-
dev/cli.js | 1 +
dev/cli.ts | 2 ++
dev/control.js | 8 +++++++
dev/control.ts | 8 +++++++
dev/input.js | 3 +++
dev/input.ts | 4 ++++
docker-compose.yml | 36 ++++++++++++++++++++---------
package.json | 5 ++--
quadratic-connection/src/config.rs | 19 ++++++++++++++-
quadratic-files/src/config.rs | 18 ++++++++++++++-
quadratic-multiplayer/src/config.rs | 18 ++++++++++++++-
12 files changed, 107 insertions(+), 17 deletions(-)
rename Dockerfile => Dockerfile.dev (98%)
diff --git a/Dockerfile b/Dockerfile.dev
similarity index 98%
rename from Dockerfile
rename to Dockerfile.dev
index e5aeace60f..6cd7e48e9b 100644
--- a/Dockerfile
+++ b/Dockerfile.dev
@@ -25,4 +25,4 @@ RUN echo 'Installing cargo-watch...' && cargo install cargo-watch
WORKDIR /quadratic
-CMD ["bash", "-c", "source ~/.bashrc && npm install --no-audit --no-fund && npm run compile --workspace=quadratic-shared && npm run dev"]
\ No newline at end of file
+CMD ["bash", "-c", "source ~/.bashrc && npm install --no-audit --no-fund && npm run compile --workspace=quadratic-shared && npm run dev:docker"]
\ No newline at end of file
diff --git a/dev/cli.js b/dev/cli.js
index b37db69a5a..7d722ff75a 100644
--- a/dev/cli.js
+++ b/dev/cli.js
@@ -29,6 +29,7 @@ export class CLI {
.option("-O, --rustClient", "Hide RustClient")
.option("-E, --hideRustClient", "Hide RustClient")
.option("-L, --servicesLocal", "Set Redis & Postgres as running locally")
+ .option("-D, --dockerDev", "Run dev in docker")
.option("-d, --dark", "Use dark theme")
.showHelpAfterError();
program.parse();
diff --git a/dev/cli.ts b/dev/cli.ts
index b6bdaa3d64..ccf1ffaffa 100644
--- a/dev/cli.ts
+++ b/dev/cli.ts
@@ -23,6 +23,7 @@ export class CLI {
hidePython: boolean;
hideRustClient: boolean;
servicesLocal: boolean;
+ dockerDev: boolean;
dark: boolean;
};
@@ -62,6 +63,7 @@ export class CLI {
.option("-O, --rustClient", "Hide RustClient")
.option("-E, --hideRustClient", "Hide RustClient")
.option("-L, --servicesLocal", "Set Redis & Postgres as running locally")
+ .option("-D, --dockerDev", "Run dev in docker")
.option("-d, --dark", "Use dark theme")
.showHelpAfterError();
diff --git a/dev/control.js b/dev/control.js
index 1dd5175b20..f11609ce9c 100644
--- a/dev/control.js
+++ b/dev/control.js
@@ -525,6 +525,10 @@ export class Control {
return new Promise((resolve) => {
if (this.quitting)
resolve(false);
+ const dockerDev = this.cli.options.dockerDev;
+ if (dockerDev) {
+ resolve(true);
+ }
const servicesLocal = this.cli.options.servicesLocal;
const redis = servicesLocal
? spawn("redis-cli", ["ping"])
@@ -543,6 +547,10 @@ export class Control {
return new Promise((resolve) => {
if (this.quitting)
resolve(false);
+ const dockerDev = this.cli.options.dockerDev;
+ if (dockerDev) {
+ resolve(true);
+ }
const servicesLocal = this.cli.options.servicesLocal;
const postgres = servicesLocal
? spawn("pg_isready")
diff --git a/dev/control.ts b/dev/control.ts
index 5af7e840e0..484e63c5b0 100644
--- a/dev/control.ts
+++ b/dev/control.ts
@@ -612,6 +612,10 @@ export class Control {
isRedisRunning(): Promise {
return new Promise((resolve) => {
if (this.quitting) resolve(false);
+ const dockerDev = this.cli.options.dockerDev;
+ if (dockerDev) {
+ resolve(true);
+ }
const servicesLocal = this.cli.options.servicesLocal;
const redis = servicesLocal
? spawn("redis-cli", ["ping"])
@@ -630,6 +634,10 @@ export class Control {
isPostgresRunning(): Promise {
return new Promise((resolve) => {
if (this.quitting) resolve(false);
+ const dockerDev = this.cli.options.dockerDev;
+ if (dockerDev) {
+ resolve(true);
+ }
const servicesLocal = this.cli.options.servicesLocal;
const postgres = servicesLocal
? spawn("pg_isready")
diff --git a/dev/input.js b/dev/input.js
index d8836222e1..ad1ecf0989 100644
--- a/dev/input.js
+++ b/dev/input.js
@@ -139,6 +139,9 @@ export class Input {
this.cli.options.servicesLocal = !this.cli.options.servicesLocal;
this.control.checkServices();
break;
+ case "D":
+ this.cli.options.dockerDev = !this.cli.options.dockerDev;
+ break;
}
};
}
diff --git a/dev/input.ts b/dev/input.ts
index 3cc411fd6a..584ce6f22e 100644
--- a/dev/input.ts
+++ b/dev/input.ts
@@ -148,6 +148,10 @@ export class Input {
this.cli.options.servicesLocal = !this.cli.options.servicesLocal;
this.control.checkServices();
break;
+ case "D":
+ this.cli.options.dockerDev = !this.cli.options.dockerDev;
+ this.control.checkServices();
+ break;
}
};
}
diff --git a/docker-compose.yml b/docker-compose.yml
index e74e038790..f6e5bb608a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -277,21 +277,21 @@ services:
quadratic-dev:
build:
context: .
- dockerfile: Dockerfile
- container_name: dev
+ dockerfile: Dockerfile.dev
+ container_name: quadratic-dev
environment:
# common
ENVIRONMENT: ${ENVIRONMENT}
RUST_LOG: ${RUST_LOG}
STORAGE_TYPE: ${STORAGE_TYPE}
QUADRATIC_API_URI: ${QUADRATIC_API_URL_INTERNAL}
- AUTH0_JWKS_URI: ${JWKS_URI}
M2M_AUTH_TOKEN: ${M2M_AUTH_TOKEN}
PUBSUB_HOST: ${PUBSUB_HOST}
PUBSUB_PORT: ${PUBSUB_PORT}
PUBSUB_PASSWORD: ${PUBSUB_PASSWORD}
PUBSUB_ACTIVE_CHANNELS: ${PUBSUB_ACTIVE_CHANNELS}
- HOST: 0.0.0.0
+ HOST: ""
+ PORT: ""
# client
VITE_DEBUG: 1
VITE_QUADRATIC_API_URL: ${QUADRATIC_API_URL_EXTERNAL}
@@ -314,12 +314,11 @@ services:
QUADRATIC_FILE_URI_PUBLIC: ${QUADRATIC_FILES_URL_EXTERNAL}
LICENSE_KEY: ${LICENSE_KEY}
# multiplayer
- # PORT: ${QUADRATIC_MULTIPLAYER_PORT}
HEARTBEAT_CHECK_S: ${QUADRATIC_MULTIPLAYER_HEARTBEAT_CHECK_S}
HEARTBEAT_TIMEOUT_S: ${QUADRATIC_MULTIPLAYER_HEARTBEAT_TIMEOUT_S}
AUTHENTICATE_JWT: true
+ AUTH0_JWKS_URI: ${JWKS_URI}
# files
- # PORT: ${QUADRATIC_FILES_PORT}
FILE_CHECK_S: ${QUADRATIC_FILES_FILE_CHECK_S}
FILES_PER_CHECK: ${QUADRATIC_FILES_FILES_PER_CHECK}
TRUNCATE_FILE_CHECK_S: ${QUADRATIC_FILES_TRUNCATE_FILE_CHECK_S}
@@ -328,7 +327,6 @@ services:
STORAGE_DIR: ${STORAGE_DIR}
STORAGE_ENCRYPTION_KEYS: ${ENCRYPTION_KEY}
# connection
- # PORT: ${QUADRATIC_CONNECTION_PORT}
MAX_RESPONSE_BYTES: ${QUADRATIC_CONNECTION_MAX_RESPONSE_BYTES}
STATIC_IPS: ${QUADRATIC_CONNECTION_STATIC_IPS}
ports:
@@ -338,10 +336,22 @@ services:
- "3002:3002"
- "3003:3003"
volumes:
+ - quadratic-dev-cache:/root/.cache
+ - quadratic-dev-cargo:/root/.cargo
- ./:/quadratic
depends_on:
- - postgres
- - redis
+ postgres:
+ condition: service_healthy
+ redis:
+ condition: service_healthy
+ ory-auth:
+ condition: service_started
+ ory-auth-migrate:
+ condition: service_started
+ ory-auth-node:
+ condition: service_started
+ ory-auth-mail:
+ condition: service_started
profiles:
- quadratic-dev
- dev
@@ -539,8 +549,12 @@ services:
- quadratic-connection-db-mssql
volumes:
- docker:
- name: docker
+ quadratic-dev-cache:
+ name: quadratic-dev-cache
+ driver: local
+ quadratic-dev-cargo:
+ name: quadratic-dev-cargo
+ driver: local
mysql-connection:
name: mysql-connection
driver: local
diff --git a/package.json b/package.json
index e9af37827c..ae38e49279 100644
--- a/package.json
+++ b/package.json
@@ -25,8 +25,9 @@
"start:services-local": "node dev -L",
"dev": "node dev -l",
"dev:services-local": "node dev -l -L",
+ "dev:docker": "node dev -l -D",
"api:start": "npm start --workspace=quadratic-api",
- "clean": "npm exec --workspaces -- npx rimraf node_modules && npx rimraf node_modules",
+ "clean": "npm exec --workspaces -- npx rimraf node_modules && npx rimraf target && npx rimraf node_modules && cargo clean",
"lint:client": "cd quadratic-client && npm run lint:ts && npm run lint:eslint && lint:prettier && lint:clippy",
"build:wasm:types": "cd quadratic-core && cargo run --bin export_types --features js",
"watch:wasm:javascript": "cd quadratic-core && cargo watch -s 'wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs'",
@@ -54,7 +55,7 @@
"lint:clippy": "cd quadratic-core && cargo clippy --all-targets --all-features -- -D warnings",
"lint:clippy:fix": "cd quadratic-core && cargo clippy --all-targets --all-features --fix -- -D warnings",
"heroku-postbuild": "npm run build --workspace=quadratic-api",
- "kill": "kill-port 3000 && kill-port 3001 && kill-port 8001 && kill-port 8000",
+ "kill": "kill-port 3000 && kill-port 8000 && kill-port 3001 && kill-port 3002 && kill-port 3003 && kill-port 6379 && kill-port 8001 && kill-port 5432 && kill-port 4566 && kill-port 80 && kill-port 443",
"prisma:migrate": "cd quadratic-api && npm run prisma:migrate",
"copy:esbuild": "cp -f node_modules/esbuild-wasm/esbuild.wasm quadratic-client/public/",
"gen:pyright:initialization": "npm run gen:pyright:worker --workspace=quadratic-kernels/python-wasm",
diff --git a/quadratic-connection/src/config.rs b/quadratic-connection/src/config.rs
index dcf8912f4c..126e0bfe10 100644
--- a/quadratic-connection/src/config.rs
+++ b/quadratic-connection/src/config.rs
@@ -13,7 +13,9 @@ use serde::Deserialize;
#[allow(dead_code)]
#[derive(Deserialize, Debug)]
pub(crate) struct Config {
+ #[serde(default = "default_host")]
pub(crate) host: String,
+ #[serde(default = "default_port")]
pub(crate) port: String,
pub(crate) environment: Environment,
@@ -24,6 +26,14 @@ pub(crate) struct Config {
pub(crate) static_ips: Vec,
}
+fn default_host() -> String {
+ "0.0.0.0".to_string()
+}
+
+fn default_port() -> String {
+ "3003".to_string()
+}
+
/// Load the global configuration from the environment into Config.
pub(crate) fn config() -> Result {
let filename = if cfg!(test) { ".env.test" } else { ".env" };
@@ -31,7 +41,14 @@ pub(crate) fn config() -> Result {
dotenv::from_filename(filename).ok();
dotenv().ok();
- let config = envy::from_env::().map_err(|e| ConnectionError::Config(e.to_string()))?;
+ let mut config =
+ envy::from_env::().map_err(|e| ConnectionError::Config(e.to_string()))?;
+ if config.host.is_empty() {
+ config.host = default_host();
+ }
+ if config.port.is_empty() {
+ config.port = default_port();
+ }
Ok(config)
}
diff --git a/quadratic-files/src/config.rs b/quadratic-files/src/config.rs
index ea0061e32f..5e16175c9e 100644
--- a/quadratic-files/src/config.rs
+++ b/quadratic-files/src/config.rs
@@ -21,7 +21,9 @@ pub(crate) enum StorageType {
#[allow(dead_code)]
#[derive(Deserialize, Debug)]
pub(crate) struct Config {
+ #[serde(default = "default_host")]
pub(crate) host: String,
+ #[serde(default = "default_port")]
pub(crate) port: String,
pub(crate) file_check_s: i64,
pub(crate) files_per_check: i64,
@@ -53,6 +55,14 @@ pub(crate) struct Config {
pub(crate) storage_encryption_keys: Option>,
}
+fn default_host() -> String {
+ "0.0.0.0".to_string()
+}
+
+fn default_port() -> String {
+ "3002".to_string()
+}
+
/// Load the global configuration from the environment into Config.
pub(crate) fn config() -> Result {
let filename = if cfg!(test) { ".env.test" } else { ".env" };
@@ -61,7 +71,13 @@ pub(crate) fn config() -> Result {
dotenv::from_filename(filename).ok();
dotenv().ok();
- let config = envy::from_env::().map_err(|e| FilesError::Config(e.to_string()))?;
+ let mut config = envy::from_env::().map_err(|e| FilesError::Config(e.to_string()))?;
+ if config.host.is_empty() {
+ config.host = default_host();
+ }
+ if config.port.is_empty() {
+ config.port = default_port();
+ }
Ok(config)
}
diff --git a/quadratic-multiplayer/src/config.rs b/quadratic-multiplayer/src/config.rs
index c1188753a6..f9b39202de 100644
--- a/quadratic-multiplayer/src/config.rs
+++ b/quadratic-multiplayer/src/config.rs
@@ -12,7 +12,9 @@ use serde::Deserialize;
#[derive(Deserialize, Debug)]
pub(crate) struct Config {
+ #[serde(default = "default_host")]
pub(crate) host: String,
+ #[serde(default = "default_port")]
pub(crate) port: String,
pub(crate) heartbeat_check_s: i64,
pub(crate) authenticate_jwt: bool,
@@ -29,6 +31,14 @@ pub(crate) struct Config {
pub(crate) m2m_auth_token: String,
}
+fn default_host() -> String {
+ "0.0.0.0".to_string()
+}
+
+fn default_port() -> String {
+ "3001".to_string()
+}
+
/// Load the global configuration from the environment into Config.
pub(crate) fn config() -> Result {
let filename = if cfg!(test) { ".env.test" } else { ".env" };
@@ -37,7 +47,13 @@ pub(crate) fn config() -> Result {
dotenv::from_filename(filename).ok();
dotenv().ok();
- let config = envy::from_env::().map_err(|e| MpError::Config(e.to_string()))?;
+ let mut config = envy::from_env::().map_err(|e| MpError::Config(e.to_string()))?;
+ if config.host.is_empty() {
+ config.host = default_host();
+ }
+ if config.port.is_empty() {
+ config.port = default_port();
+ }
Ok(config)
}
From 96052582c9e7b757aa3d6d62c6fc839c2adc271f Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sun, 29 Dec 2024 13:14:51 +0530
Subject: [PATCH 136/155] node dev working in docker
---
.vscode/settings.json | 2 +-
docker-compose.yml | 43 +++++++++++++++++++++++++++++--------------
2 files changed, 30 insertions(+), 15 deletions(-)
diff --git a/.vscode/settings.json b/.vscode/settings.json
index 40c306c509..491c77dc17 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -132,7 +132,7 @@
"editor.defaultFormatter": "esbenp.prettier-vscode"
},
"[dockercompose]": {
- "editor.defaultFormatter": "esbenp.prettier-vscode"
+ "editor.defaultFormatter": "ms-azuretools.vscode-docker"
},
"[javascript]": {
"editor.defaultFormatter": "esbenp.prettier-vscode"
diff --git a/docker-compose.yml b/docker-compose.yml
index f6e5bb608a..d4324c3d4a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -12,7 +12,7 @@ services:
test: ["CMD", "redis-cli", "ping"]
interval: "5s"
volumes:
- - ./docker/redis/data:/data
+ - redis-data:/data
profiles:
- base
- dev
@@ -35,7 +35,7 @@ services:
timeout: 5s
retries: 5
volumes:
- - ./docker/postgres/data:/var/lib/postgresql/data
+ - postgres-data:/var/lib/postgresql/data
- ./docker/postgres/scripts:/docker-entrypoint-initdb.d
profiles:
- base
@@ -338,12 +338,14 @@ services:
volumes:
- quadratic-dev-cache:/root/.cache
- quadratic-dev-cargo:/root/.cargo
+ - quadratic-dev-node_modules:/quadratic/node_modules
+ - quadratic-dev-target:/quadratic/target
- ./:/quadratic
depends_on:
- postgres:
- condition: service_healthy
redis:
condition: service_healthy
+ postgres:
+ condition: service_healthy
ory-auth:
condition: service_started
ory-auth-migrate:
@@ -469,7 +471,7 @@ services:
timeout: 5s
retries: 5
volumes:
- - ./docker/postgres-connection/data:/var/lib/postgresql/data
+ - postgres-connection-data:/var/lib/postgresql/data
- ./docker/postgres-connection/scripts:/docker-entrypoint-initdb.d
profiles:
- quadratic-connection
@@ -493,7 +495,7 @@ services:
timeout: 5s
retries: 5
volumes:
- - mysql-connection:/var/lib/mysql
+ - mysql-connection-data:/var/lib/mysql
- ./docker/mysql-connection/scripts:/docker-entrypoint-initdb.d/
profiles:
- quadratic-connection
@@ -512,7 +514,7 @@ services:
MSSQL_SA_PASSWORD: yourStrong(!)Password
MSSQL_PID: Express
volumes:
- - ./docker/mssql-connection/data:/var/opt/mssql
+ - mssql-connection-data:/var/opt/mssql
- ./docker/mssql-connection/scripts:/docker-entrypoint-initdb.d/
healthcheck:
test:
@@ -537,27 +539,40 @@ services:
start_period: 30s
retries: 5
command: >
- bash -c "
- /opt/mssql/bin/sqlservr &
- sleep 10 &&
- /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'yourStrong(!)Password' -i /docker-entrypoint-initdb.d/create_db.sql -C -N -t 30 &&
- /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'yourStrong(!)Password' -i /docker-entrypoint-initdb.d/seed_db.sql -C -N -t 30 &&
- tail -f /dev/null"
+ bash -c " /opt/mssql/bin/sqlservr & sleep 10 && /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'yourStrong(!)Password' -i /docker-entrypoint-initdb.d/create_db.sql -C -N -t 30 && /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'yourStrong(!)Password' -i /docker-entrypoint-initdb.d/seed_db.sql -C -N -t 30 && tail -f /dev/null"
profiles:
- quadratic-connection
- quadratic-connection-db
- quadratic-connection-db-mssql
volumes:
+ redis-data:
+ name: redis-data
+ driver: local
+ postgres-data:
+ name: postgres-data
+ driver: local
quadratic-dev-cache:
name: quadratic-dev-cache
driver: local
quadratic-dev-cargo:
name: quadratic-dev-cargo
driver: local
- mysql-connection:
+ quadratic-dev-node_modules:
+ name: quadratic-dev-node_modules
+ driver: local
+ quadratic-dev-target:
+ name: quadratic-dev-target
+ driver: local
+ postgres-connection-data:
+ name: postgres-connection-data
+ driver: local
+ mysql-connection-data:
name: mysql-connection
driver: local
+ mssql-connection-data:
+ name: mssql-connection-data
+ driver: local
networks:
host:
From dd6989cd9e40beff64c3d1e6f1ee555e30088f36 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 31 Dec 2024 05:12:33 +0530
Subject: [PATCH 137/155] try parallel builds on vercel
---
docker-compose.yml | 62 +++++++--------------------------
infra/client/build-client-ci.sh | 44 +++++++++++++++--------
2 files changed, 42 insertions(+), 64 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index d4324c3d4a..1a121b4bfc 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -9,10 +9,10 @@ services:
- "6379:6379"
- "8001:8001"
healthcheck:
- test: ["CMD", "redis-cli", "ping"]
+ test: [ "CMD", "redis-cli", "ping" ]
interval: "5s"
volumes:
- - redis-data:/data
+ - ./docker/redis/data:/data:rw
profiles:
- base
- dev
@@ -30,12 +30,12 @@ services:
POSTGRES_PASSWORD: postgres
ADDITIONAL_DATABASES: kratos
healthcheck:
- test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"]
+ test: [ "CMD-SHELL", "pg_isready -U postgres -d postgres" ]
interval: 10s
timeout: 5s
retries: 5
volumes:
- - postgres-data:/var/lib/postgresql/data
+ - ./docker/postgres/data:/var/lib/postgresql/data:rw
- ./docker/postgres/scripts:/docker-entrypoint-initdb.d
profiles:
- base
@@ -63,8 +63,8 @@ services:
- EXTRA_CORS_ALLOWED_ORIGINS=*
volumes:
- "./docker/localstack/scripts/init-aws.sh:/etc/localstack/init/ready.d/init-aws.sh" # ready hook
- - "./docker/localstack/data:/var/lib/localstack"
- - "/var/run/docker.sock:/var/run/docker.sock"
+ - "./docker/localstack/data:/var/lib/localstack:rw"
+ - "/var/run/docker.sock:/var/run/docker.sock:rw"
profiles:
- base
networks:
@@ -91,7 +91,7 @@ services:
sh -c "/client/scripts/replace_env_vars.sh &&
nginx -g \"daemon off;\""
healthcheck:
- test: ["CMD-SHELL", "curl -f http://host.docker.internal:3000/ || exit 1"]
+ test: [ "CMD-SHELL", "curl -f http://host.docker.internal:3000/ || exit 1" ]
interval: 10s
timeout: 5s
restart: "always"
@@ -225,7 +225,7 @@ services:
ports:
- "3002:3002"
volumes:
- - ./docker/file-storage:/file-storage
+ - ./docker/file-storage:/file-storage:rw
depends_on:
redis:
condition: service_healthy
@@ -336,11 +336,7 @@ services:
- "3002:3002"
- "3003:3003"
volumes:
- - quadratic-dev-cache:/root/.cache
- - quadratic-dev-cargo:/root/.cargo
- - quadratic-dev-node_modules:/quadratic/node_modules
- - quadratic-dev-target:/quadratic/target
- - ./:/quadratic
+ - ./:/quadratic:rw
depends_on:
redis:
condition: service_healthy
@@ -466,7 +462,7 @@ services:
PGUSER: user
POSTGRES_PASSWORD: password
healthcheck:
- test: ["CMD-SHELL", "pg_isready -U user -d postgres-connection"]
+ test: [ "CMD-SHELL", "pg_isready -U user -d postgres-connection" ]
interval: 10s
timeout: 5s
retries: 5
@@ -490,7 +486,7 @@ services:
MYSQL_PASSWORD: password
MYSQL_ROOT_PASSWORD: password
healthcheck:
- test: ["CMD", "mysqladmin", "ping", "-h", "localhost"]
+ test: [ "CMD", "mysqladmin", "ping", "-h", "localhost" ]
interval: 10s
timeout: 5s
retries: 5
@@ -517,23 +513,7 @@ services:
- mssql-connection-data:/var/opt/mssql
- ./docker/mssql-connection/scripts:/docker-entrypoint-initdb.d/
healthcheck:
- test:
- [
- "CMD",
- "/opt/mssql-tools18/bin/sqlcmd",
- "-S",
- "localhost",
- "-U",
- "sa",
- "-P",
- "yourStrong(!)Password",
- "-Q",
- "SELECT 1",
- "-C",
- "-N",
- "-t",
- "30",
- ]
+ test: [ "CMD", "/opt/mssql-tools18/bin/sqlcmd", "-S", "localhost", "-U", "sa", "-P", "yourStrong(!)Password", "-Q", "SELECT 1", "-C", "-N", "-t", "30" ]
interval: 10s
timeout: 5s
start_period: 30s
@@ -546,24 +526,6 @@ services:
- quadratic-connection-db-mssql
volumes:
- redis-data:
- name: redis-data
- driver: local
- postgres-data:
- name: postgres-data
- driver: local
- quadratic-dev-cache:
- name: quadratic-dev-cache
- driver: local
- quadratic-dev-cargo:
- name: quadratic-dev-cargo
- driver: local
- quadratic-dev-node_modules:
- name: quadratic-dev-node_modules
- driver: local
- quadratic-dev-target:
- name: quadratic-dev-target
- driver: local
postgres-connection-data:
name: postgres-connection-data
driver: local
diff --git a/infra/client/build-client-ci.sh b/infra/client/build-client-ci.sh
index 19af1fb2b3..665a38dc36 100755
--- a/infra/client/build-client-ci.sh
+++ b/infra/client/build-client-ci.sh
@@ -20,22 +20,38 @@ source "$HOME/.cargo/env"
echo 'Installing wasm-pack...'
curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
+# Start parallel builds
+(
+ # Task 1: Install dependencies
+ (
+ echo 'Installing dependencies...'
+ npm ci
+ ) &
+
+ # Task 2: Build core
+ (
+ echo 'Building core...'
+ npm run build --workspace=quadratic-core -- --frozen
+ ) &
+
+ # Task 3: Build TS/Rust types
+ (
+ echo 'Building TS/Rust types...'
+ npm run export_types --workspace=quadratic-core -- --frozen
+ ) &
+
+ # Task 4: Build rust client
+ (
+ echo 'Building rust-client...'
+ npm run build --workspace=quadratic-rust-client -- --frozen
+ ) &
+
+ # Wait for all background tasks to complete
+ wait
+)
+
echo 'Packaging quadratic_py'
./quadratic-kernels/python-wasm/package.sh --no-poetry
-cd quadratic-core
-
-echo 'Building wasm...'
-wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core
-
-echo 'Exporting TS/Rust types...'
-cargo run --bin export_types
-
-cd ..
-
-echo 'Building quadratic-rust-client...'
-npm run build --workspace=quadratic-rust-client
-
echo 'Building front-end...'
-npm ci
npm run build --workspace=quadratic-client
From 2a503e362e08970f8bea0635594ab739eadf1e53 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 31 Dec 2024 05:16:40 +0530
Subject: [PATCH 138/155] wasm32-unknown-unknown
---
infra/client/build-client-ci.sh | 3 +++
quadratic-client/Dockerfile | 6 +++---
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/infra/client/build-client-ci.sh b/infra/client/build-client-ci.sh
index 665a38dc36..3d4e57cb71 100755
--- a/infra/client/build-client-ci.sh
+++ b/infra/client/build-client-ci.sh
@@ -20,6 +20,9 @@ source "$HOME/.cargo/env"
echo 'Installing wasm-pack...'
curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
+echo 'Installing wasm32-unknown-unknown target...'
+rustup target add wasm32-unknown-unknown
+
# Start parallel builds
(
# Task 1: Install dependencies
diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile
index ccbd973bb9..bb208ac268 100644
--- a/quadratic-client/Dockerfile
+++ b/quadratic-client/Dockerfile
@@ -41,7 +41,7 @@ RUN echo 'Installing wasm-pack...' && curl https://rustwasm.github.io/wasm-pack/
RUN echo 'wasm-pack version:' && wasm-pack --version
# Install wasm32-unknown-unknown target
-RUN rustup target add wasm32-unknown-unknown
+RUN echo 'Installing wasm32-unknown-unknown target...' && rustup target add wasm32-unknown-unknown
WORKDIR /quadratic
@@ -79,7 +79,7 @@ RUN echo 'Installing wasm-pack...' && curl https://rustwasm.github.io/wasm-pack/
RUN echo 'wasm-pack version:' && wasm-pack --version
# Install wasm32-unknown-unknown target
-RUN rustup target add wasm32-unknown-unknown
+RUN echo 'Installing wasm32-unknown-unknown target...' && rustup target add wasm32-unknown-unknown
WORKDIR /quadratic
@@ -115,7 +115,7 @@ RUN echo 'Installing wasm-pack...' && curl https://rustwasm.github.io/wasm-pack/
RUN echo 'wasm-pack version:' && wasm-pack --version
# Install wasm32-unknown-unknown target
-RUN rustup target add wasm32-unknown-unknown
+RUN echo 'Installing wasm32-unknown-unknown target...' && rustup target add wasm32-unknown-unknown
WORKDIR /quadratic
From 810435c01f49099f268d373cc169d809bd9b93a3 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Tue, 31 Dec 2024 05:40:24 +0530
Subject: [PATCH 139/155] revert parallel, not working on vercel
---
infra/client/build-client-ci.sh | 47 ++++++++++-----------------------
vercel.json | 2 +-
2 files changed, 15 insertions(+), 34 deletions(-)
diff --git a/infra/client/build-client-ci.sh b/infra/client/build-client-ci.sh
index 3d4e57cb71..4cb8cbae08 100755
--- a/infra/client/build-client-ci.sh
+++ b/infra/client/build-client-ci.sh
@@ -20,41 +20,22 @@ source "$HOME/.cargo/env"
echo 'Installing wasm-pack...'
curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
-echo 'Installing wasm32-unknown-unknown target...'
-rustup target add wasm32-unknown-unknown
-
-# Start parallel builds
-(
- # Task 1: Install dependencies
- (
- echo 'Installing dependencies...'
- npm ci
- ) &
-
- # Task 2: Build core
- (
- echo 'Building core...'
- npm run build --workspace=quadratic-core -- --frozen
- ) &
-
- # Task 3: Build TS/Rust types
- (
- echo 'Building TS/Rust types...'
- npm run export_types --workspace=quadratic-core -- --frozen
- ) &
-
- # Task 4: Build rust client
- (
- echo 'Building rust-client...'
- npm run build --workspace=quadratic-rust-client -- --frozen
- ) &
-
- # Wait for all background tasks to complete
- wait
-)
-
echo 'Packaging quadratic_py'
./quadratic-kernels/python-wasm/package.sh --no-poetry
+cd quadratic-core
+
+echo 'Building wasm...'
+wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core
+
+echo 'Exporting TS/Rust types...'
+cargo run --bin export_types
+
+cd ..
+
+echo 'Building quadratic-rust-client...'
+npm run build --workspace=quadratic-rust-client
+
echo 'Building front-end...'
+npm ci --no-audit --no-fund
npm run build --workspace=quadratic-client
diff --git a/vercel.json b/vercel.json
index 7aa121e10d..82f21b4ba5 100644
--- a/vercel.json
+++ b/vercel.json
@@ -1,6 +1,6 @@
{
"buildCommand": "./infra/client/build-client-ci.sh",
- "installCommand": "npm install",
+ "installCommand": "npm install --no-audit --no-fund",
"outputDirectory": "build",
"headers": [
{
From a8875a44a5daec7488bdfcb50c75c97be411c0c1 Mon Sep 17 00:00:00 2001
From: David DiMaria
Date: Wed, 1 Jan 2025 10:16:20 -0700
Subject: [PATCH 140/155] Use COMMUNITY_A1_FILE_UPDATE_URL in the js console
output
---
.../worker/javascript/runner/javascriptLibrary.ts | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/quadratic-client/src/app/web-workers/javascriptWebWorker/worker/javascript/runner/javascriptLibrary.ts b/quadratic-client/src/app/web-workers/javascriptWebWorker/worker/javascript/runner/javascriptLibrary.ts
index 91afde45e2..6d7996cceb 100644
--- a/quadratic-client/src/app/web-workers/javascriptWebWorker/worker/javascript/runner/javascriptLibrary.ts
+++ b/quadratic-client/src/app/web-workers/javascriptWebWorker/worker/javascript/runner/javascriptLibrary.ts
@@ -1,3 +1,5 @@
+import { COMMUNITY_A1_FILE_UPDATE_URL } from '@/shared/constants/urls';
+
declare var self: WorkerGlobalScope & typeof globalThis;
declare global {
@@ -351,7 +353,13 @@ export class q {
* Show a conversion error message when the user tries to use an old function.
*/
static conversionError(oldFunc: string, newFunc: string): void {
- const message = oldFunc + ' functionality is no longer supported. Use ' + newFunc + ' instead.';
+ const message =
+ oldFunc +
+ ' functionality is no longer supported. Use ' +
+ newFunc +
+ ' instead. Refer to the documentation at ' +
+ COMMUNITY_A1_FILE_UPDATE_URL +
+ ' for more details.';
throw new Error(message);
}
}
From 6bbe37600527585cd4b25c007ba5742e23b3d5a6 Mon Sep 17 00:00:00 2001
From: David DiMaria
Date: Wed, 1 Jan 2025 11:22:00 -0700
Subject: [PATCH 141/155] Use COMMUNITY_A1_FILE_UPDATE_URL in the js console
output and python
---
.../worker/javascript/javascriptCompile.ts | 10 +++++++++-
.../javascript/runner/generateJavascriptForRunner.ts | 2 +-
.../javascript/runner/generatedJavascriptForEditor.ts | 7 ++++++-
.../worker/javascript/runner/javascriptLibrary.ts | 5 +----
.../web-workers/pythonWebWorker/worker/pythonClient.ts | 1 -
.../quadratic_py/quadratic_api/quadratic.py | 3 ++-
6 files changed, 19 insertions(+), 9 deletions(-)
diff --git a/quadratic-client/src/app/web-workers/javascriptWebWorker/worker/javascript/javascriptCompile.ts b/quadratic-client/src/app/web-workers/javascriptWebWorker/worker/javascript/javascriptCompile.ts
index 71efe50ea6..29e39b420d 100644
--- a/quadratic-client/src/app/web-workers/javascriptWebWorker/worker/javascript/javascriptCompile.ts
+++ b/quadratic-client/src/app/web-workers/javascriptWebWorker/worker/javascript/javascriptCompile.ts
@@ -10,6 +10,7 @@ import { getJavascriptXHROverride } from '@/app/web-workers/javascriptWebWorker/
import * as esbuild from 'esbuild-wasm';
import { LINE_NUMBER_VAR } from './javascript';
import { javascriptLibrary } from './runner/generateJavascriptForRunner';
+import { COMMUNITY_A1_FILE_UPDATE_URL } from '@/shared/constants/urls';
export interface JavascriptTransformedCode {
imports: string;
code: string;
@@ -74,12 +75,18 @@ export function prepareJavascriptCode(
const code = withLineNumbers ? javascriptAddLineNumberVars(transform) : transform.code;
const javascriptXHROverride = getJavascriptXHROverride(proxyUrl, jwt);
const javascriptFetchOverride = getJavascriptFetchOverride(proxyUrl, jwt);
+ let replacedJavascriptLibrary = javascriptLibrary;
+ replacedJavascriptLibrary = replacedJavascriptLibrary.replace('{x:0,y:0}', `{x:${x},y:${y}}`); // replace the pos() with the correct x,y coordinates
+ replacedJavascriptLibrary = replacedJavascriptLibrary.replace(
+ '{COMMUNITY_A1_FILE_UPDATE_URL}',
+ COMMUNITY_A1_FILE_UPDATE_URL
+ ); // replace the COMMUNITY_A1_FILE_UPDATE_URL with the correct url
const compiledCode =
javascriptXHROverride +
javascriptFetchOverride +
transform.imports +
(withLineNumbers ? `let ${LINE_NUMBER_VAR} = 0;` : '') +
- javascriptLibrary.replace('{x:0,y:0}', `{x:${x},y:${y}}`) + // replace the pos() with the correct x,y coordinates
+ replacedJavascriptLibrary +
'(async() => {try{' +
'let results = await (async () => {' +
code +
@@ -90,5 +97,6 @@ export function prepareJavascriptCode(
} });` +
`} catch (e) { const error = e.message; const stack = e.stack; self.postMessage({ type: "error", error, stack, console: javascriptConsole.output() }); }` +
'})();';
+
return compiledCode;
}
diff --git a/quadratic-client/src/app/web-workers/javascriptWebWorker/worker/javascript/runner/generateJavascriptForRunner.ts b/quadratic-client/src/app/web-workers/javascriptWebWorker/worker/javascript/runner/generateJavascriptForRunner.ts
index ac2b36f300..eaad9d1ff4 100644
--- a/quadratic-client/src/app/web-workers/javascriptWebWorker/worker/javascript/runner/generateJavascriptForRunner.ts
+++ b/quadratic-client/src/app/web-workers/javascriptWebWorker/worker/javascript/runner/generateJavascriptForRunner.ts
@@ -1,4 +1,4 @@
// Generated file from ./compileJavascriptRunner.mjs
-export const javascriptLibrary = `function convertNullToUndefined(arr){return arr.map(subArr=>subArr.map(element=>element===null?void 0:element))}function lineNumber(){try{throw new Error}catch(e){const match=e.stack.split("\\n")[3].match(/:(\\d+):(\\d+)/);if(match)return match[1]}}const createConversionError=(funcName,a1Params,oldFuncParams,sheetName)=>{const oldFunc=funcName+"("+oldFuncParams+")";let params=a1Params;sheetName&&(params=sheetName+":"+params);const newFunc="q.cells('"+params+"')";q.conversionError(oldFunc,newFunc)},getCellsConversionError=(funcName,x0,y0,x1,y1,sheetName)=>{const a1_0=q.toA1(x0,y0),a1_1=q.toA1(x1,y1);let oldFuncParams=x0+", "+y0+", "+x1;y1&&(oldFuncParams+=", "+y1),sheetName&&(oldFuncParams+=", "+sheetName),createConversionError(funcName,a1_0+":"+a1_1,oldFuncParams,sheetName)},getCellConversionError=(funcName,x,y,sheetName)=>{const a1=q.toA1(x,y);let oldFuncParams=x+", "+y;sheetName&&(oldFuncParams+=", "+sheetName),createConversionError(funcName,a1,oldFuncParams,sheetName)},getCells=(x0,y0,x1,y1,sheetName)=>{getCellsConversionError("getCells",x0,y0,x1,y1,sheetName)},cells=(x0,y0,x1,y1,sheetName)=>{getCellsConversionError("cells",x0,y0,x1,y1,sheetName)},getCellsWithHeadings=(x0,y0,x1,y1,sheetName)=>{getCellsConversionError("getCellsWithHeadings",x0,y0,x1,y1,sheetName)},getCell=(x,y,sheetName)=>{getCellConversionError("getCell",x,y,sheetName)},cell=(x,y,sheetName)=>{getCellConversionError("cell",x,y,sheetName)},c=(x,y,sheetName)=>{getCellConversionError("c",x,y,sheetName)},pos=()=>({x:0,y:0}),relCell=(deltaX,deltaY)=>{const a1=q.toA1(deltaX,deltaY,!1);let oldFuncParams=deltaX+", "+deltaY;createConversionError("relCell",a1,oldFuncParams)},relCells=(deltaX0,deltaY0,deltaX1,deltaY1)=>{const a1_0=q.toA1(deltaX0,deltaY0,!1),a1_1=q.toA1(deltaX1,deltaY1,!1),oldFuncParams=deltaX0+", "+deltaY0+", "+deltaX1+", "+deltaY1;createConversionError("relCells",a1_0+":"+a1_1,oldFuncParams)},rc=relCell;class q{static cells(a1){if(typeof a1!="string"){const line=lineNumber();throw new Error("q.cell requires at least 1 argument, received q.cell("+a1+")"+(line!==void 0?" at line "+(line-1):""))}try{let sharedBuffer=new SharedArrayBuffer(12),int32View=new Int32Array(sharedBuffer,0,3);Atomics.store(int32View,0,0),self.postMessage({type:"getCellsA1Length",sharedBuffer,a1});let result=Atomics.wait(int32View,0,0);const length=int32View[1];if(result!=="ok"||length===0)return[];const id=int32View[2];if(sharedBuffer=new SharedArrayBuffer(4+length),int32View=new Int32Array(sharedBuffer,0,1),Atomics.store(int32View,0,0),self.postMessage({type:"getCellsData",id,sharedBuffer}),result=Atomics.wait(int32View,0,0),result!=="ok")return[];let uint8View=new Uint8Array(sharedBuffer,4,length);const nonSharedBuffer=new ArrayBuffer(uint8View.byteLength),nonSharedView=new Uint8Array(nonSharedBuffer);nonSharedView.set(uint8View),sharedBuffer=void 0,int32View=void 0,uint8View=void 0;const resultsStringified=new TextDecoder().decode(nonSharedView),results=JSON.parse(resultsStringified),cells2=convertNullToUndefined(results.cells);if(cells2.forEach(row=>{row.forEach((cell2,i)=>{typeof cell2=="string"&&cell2.startsWith("___date___")&&(row[i]=new Date(parseInt(cell2.substring(10))))})}),!results.two_dimensional){if(cells2.length===1&&cells2[0].length===1)return cells2[0][0];if(cells2.every(row=>row.length===1))return cells2.map(row=>row[0]);if(cells2.length===1)return cells2[0]}return cells2}catch(e){console.warn("[javascriptLibrary] q error",e)}return[]}static toA1(x,y,absolute=!0){let column="";if(!absolute){const p=pos();x=x+p.x,y!==void 0&&(y=y+p.y)}for(;x>0;)x--,column=String.fromCharCode(x%26+65)+column,x=Math.floor(x/26);return column+y}static pos(){return pos()}static conversionError(oldFunc,newFunc){const message=oldFunc+" functionality is no longer supported. Use "+newFunc+" instead.";throw new Error(message)}}const TAB=" ";class JavascriptConsole{oldConsoleLog;logs=[];constructor(){this.oldConsoleLog=console.log,console.log=this.consoleMap,console.warn=this.consoleMap}log(...args){this.oldConsoleLog(args)}consoleMap=(...args)=>{args=args.map(a=>this.mapArgument(a)),this.logs.push(...args)};reset(){this.logs=[]}push(s){Array.isArray(s)?this.logs.push(...s):this.logs.push(s)}output(){return this.logs.length?this.logs.join(""):null}tab=n=>Array(n).fill(TAB).join("");mapArgument(a,level=0){if(Array.isArray(a)){if(a.length===0)return"Array: []\\n";let s="Array: [\\n";for(let i=0;isubArr.map(element=>element===null?void 0:element))}function lineNumber(){try{throw new Error}catch(e){const match=e.stack.split("\\n")[3].match(/:(\\d+):(\\d+)/);if(match)return match[1]}}const createConversionError=(funcName,a1Params,oldFuncParams,sheetName)=>{const oldFunc=funcName+"("+oldFuncParams+")";let params=a1Params;sheetName&&(params=sheetName+":"+params);const newFunc="q.cells('"+params+"')";q.conversionError(oldFunc,newFunc)},getCellsConversionError=(funcName,x0,y0,x1,y1,sheetName)=>{const a1_0=q.toA1(x0,y0),a1_1=q.toA1(x1,y1);let oldFuncParams=x0+", "+y0+", "+x1;y1&&(oldFuncParams+=", "+y1),sheetName&&(oldFuncParams+=", "+sheetName),createConversionError(funcName,a1_0+":"+a1_1,oldFuncParams,sheetName)},getCellConversionError=(funcName,x,y,sheetName)=>{const a1=q.toA1(x,y);let oldFuncParams=x+", "+y;sheetName&&(oldFuncParams+=", "+sheetName),createConversionError(funcName,a1,oldFuncParams,sheetName)},getCells=(x0,y0,x1,y1,sheetName)=>{getCellsConversionError("getCells",x0,y0,x1,y1,sheetName)},cells=(x0,y0,x1,y1,sheetName)=>{getCellsConversionError("cells",x0,y0,x1,y1,sheetName)},getCellsWithHeadings=(x0,y0,x1,y1,sheetName)=>{getCellsConversionError("getCellsWithHeadings",x0,y0,x1,y1,sheetName)},getCell=(x,y,sheetName)=>{getCellConversionError("getCell",x,y,sheetName)},cell=(x,y,sheetName)=>{getCellConversionError("cell",x,y,sheetName)},c=(x,y,sheetName)=>{getCellConversionError("c",x,y,sheetName)},pos=()=>({x:0,y:0}),relCell=(deltaX,deltaY)=>{const a1=q.toA1(deltaX,deltaY,!1);let oldFuncParams=deltaX+", "+deltaY;createConversionError("relCell",a1,oldFuncParams)},relCells=(deltaX0,deltaY0,deltaX1,deltaY1)=>{const a1_0=q.toA1(deltaX0,deltaY0,!1),a1_1=q.toA1(deltaX1,deltaY1,!1),oldFuncParams=deltaX0+", "+deltaY0+", "+deltaX1+", "+deltaY1;createConversionError("relCells",a1_0+":"+a1_1,oldFuncParams)},rc=relCell;class q{static cells(a1){if(typeof a1!="string"){const line=lineNumber();throw new Error("q.cell requires at least 1 argument, received q.cell("+a1+")"+(line!==void 0?" at line "+(line-1):""))}try{let sharedBuffer=new SharedArrayBuffer(12),int32View=new Int32Array(sharedBuffer,0,3);Atomics.store(int32View,0,0),self.postMessage({type:"getCellsA1Length",sharedBuffer,a1});let result=Atomics.wait(int32View,0,0);const length=int32View[1];if(result!=="ok"||length===0)return[];const id=int32View[2];if(sharedBuffer=new SharedArrayBuffer(4+length),int32View=new Int32Array(sharedBuffer,0,1),Atomics.store(int32View,0,0),self.postMessage({type:"getCellsData",id,sharedBuffer}),result=Atomics.wait(int32View,0,0),result!=="ok")return[];let uint8View=new Uint8Array(sharedBuffer,4,length);const nonSharedBuffer=new ArrayBuffer(uint8View.byteLength),nonSharedView=new Uint8Array(nonSharedBuffer);nonSharedView.set(uint8View),sharedBuffer=void 0,int32View=void 0,uint8View=void 0;const resultsStringified=new TextDecoder().decode(nonSharedView),results=JSON.parse(resultsStringified),cells2=convertNullToUndefined(results.cells);if(cells2.forEach(row=>{row.forEach((cell2,i)=>{typeof cell2=="string"&&cell2.startsWith("___date___")&&(row[i]=new Date(parseInt(cell2.substring(10))))})}),!results.two_dimensional){if(cells2.length===1&&cells2[0].length===1)return cells2[0][0];if(cells2.every(row=>row.length===1))return cells2.map(row=>row[0]);if(cells2.length===1)return cells2[0]}return cells2}catch(e){console.warn("[javascriptLibrary] q error",e)}return[]}static toA1(x,y,absolute=!0){let column="";if(!absolute){const p=pos();x=x+p.x,y!==void 0&&(y=y+p.y)}for(;x>0;)x--,column=String.fromCharCode(x%26+65)+column,x=Math.floor(x/26);return column+y}static pos(){return pos()}static conversionError(oldFunc,newFunc){const message=oldFunc+" functionality is no longer supported. Use "+newFunc+" instead. Refer to the documentation at {COMMUNITY_A1_FILE_UPDATE_URL} for more details.";throw new Error(message)}}const TAB=" ";class JavascriptConsole{oldConsoleLog;logs=[];constructor(){this.oldConsoleLog=console.log,console.log=this.consoleMap,console.warn=this.consoleMap}log(...args){this.oldConsoleLog(args)}consoleMap=(...args)=>{args=args.map(a=>this.mapArgument(a)),this.logs.push(...args)};reset(){this.logs=[]}push(s){Array.isArray(s)?this.logs.push(...s):this.logs.push(s)}output(){return this.logs.length?this.logs.join(""):null}tab=n=>Array(n).fill(TAB).join("");mapArgument(a,level=0){if(Array.isArray(a)){if(a.length===0)return"Array: []\\n";let s="Array: [\\n";for(let i=0;i str:
return f"{column}{y}"
def _conversion_error(old: str, new: str, raise_exception: bool = True):
- output = f"{old} functionality is no longer supported. Use {new} instead, though this may be different if your sheet had negative values. Refer to the documentation at https://docs.quadratic.app/python-api/ for more details."
+ url = "https://community.quadratichq.com/t/switch-to-a1-notation-from-x-y-coordinates/"
+ output = f"{old} functionality is no longer supported. Use {new} instead, though this may be different if your sheet had negative values. Refer to the documentation at {url} for more details."
if raise_exception:
raise Exception(output)
From f48314853041b0e34868a4d952635f8b95d9ff40 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Thu, 2 Jan 2025 16:30:26 +0530
Subject: [PATCH 142/155] switch from custom ami to selfhost repo init script
---
.env.docker | 2 +-
.../workflows/staging-build-deploy-images.yml | 2 +-
.vscode/settings.json | 1 +
infra/aws-cloudformation/ami/ami.sh | 28 --
.../ami/quadratic-selfhost/.env.docker | 89 -----
.../ami/quadratic-selfhost/docker-compose.yml | 303 ------------------
.../docker/admin/config/openapi.yaml | 48 ---
.../docker/caddy/config/Caddyfile | 27 --
.../docker/client/config/default.conf | 24 --
.../docker/client/scripts/replace_env_vars.sh | 34 --
.../ory-auth/config/identity.schema.json | 47 ---
.../docker/ory-auth/config/kratos.yml | 138 --------
.../docker/postgres/scripts/init.sh | 18 --
.../ami/quadratic-selfhost/init.sh | 133 --------
.../ami/quadratic-selfhost/start.sh | 12 -
.../ami/quadratic-selfhost/stop.sh | 8 -
.../aws-cloudformation/quadratic-selfhost.yml | 45 ++-
package.json | 2 +-
18 files changed, 44 insertions(+), 917 deletions(-)
delete mode 100644 infra/aws-cloudformation/ami/ami.sh
delete mode 100644 infra/aws-cloudformation/ami/quadratic-selfhost/.env.docker
delete mode 100644 infra/aws-cloudformation/ami/quadratic-selfhost/docker-compose.yml
delete mode 100644 infra/aws-cloudformation/ami/quadratic-selfhost/docker/admin/config/openapi.yaml
delete mode 100644 infra/aws-cloudformation/ami/quadratic-selfhost/docker/caddy/config/Caddyfile
delete mode 100644 infra/aws-cloudformation/ami/quadratic-selfhost/docker/client/config/default.conf
delete mode 100755 infra/aws-cloudformation/ami/quadratic-selfhost/docker/client/scripts/replace_env_vars.sh
delete mode 100644 infra/aws-cloudformation/ami/quadratic-selfhost/docker/ory-auth/config/identity.schema.json
delete mode 100644 infra/aws-cloudformation/ami/quadratic-selfhost/docker/ory-auth/config/kratos.yml
delete mode 100644 infra/aws-cloudformation/ami/quadratic-selfhost/docker/postgres/scripts/init.sh
delete mode 100755 infra/aws-cloudformation/ami/quadratic-selfhost/init.sh
delete mode 100755 infra/aws-cloudformation/ami/quadratic-selfhost/start.sh
delete mode 100755 infra/aws-cloudformation/ami/quadratic-selfhost/stop.sh
diff --git a/.env.docker b/.env.docker
index 7edccc20bf..cdcc5d013e 100644
--- a/.env.docker
+++ b/.env.docker
@@ -36,7 +36,7 @@ KRATOS_CSRF_COOKIE_NAME=__HOST-localhost-x-csrf-token
KRATOS_CSRF_COOKIE_SECRET=changeme
# caddy
-CADDY_IN_DOCKER_COMPOSE=true
+CADDY_IN_DOCKER_COMPOSE=false
# client
QUADRATIC_CLIENT_IN_DOCKER_COMPOSE=true
diff --git a/.github/workflows/staging-build-deploy-images.yml b/.github/workflows/staging-build-deploy-images.yml
index dd2f12f703..ac406ef502 100644
--- a/.github/workflows/staging-build-deploy-images.yml
+++ b/.github/workflows/staging-build-deploy-images.yml
@@ -73,7 +73,7 @@ jobs:
- name: Define repository name
id: repo-name
run: |
- echo "REPO_NAME=quadratic-${{ matrix.service }}-development" >> $GITHUB_OUTPUT
+ echo "REPO_NAME=quadratic-${{ matrix.service }}" >> $GITHUB_OUTPUT
- name: Create Private ECR Repository
id: create-ecr
diff --git a/.vscode/settings.json b/.vscode/settings.json
index 491c77dc17..dc61fb5b29 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -3,6 +3,7 @@
"cSpell.words": [
"actix",
"autoclean",
+ "awscli",
"awscliv",
"ayush",
"bigdecimal",
diff --git a/infra/aws-cloudformation/ami/ami.sh b/infra/aws-cloudformation/ami/ami.sh
deleted file mode 100644
index eb5b52bda9..0000000000
--- a/infra/aws-cloudformation/ami/ami.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-# script used to build quadratic-preview-ami-base, all ami images have this script pre-run
-
-#!/bin/bash
-
-# Update and install dependencies
-sudo apt-get update
-sudo apt-get install -y apt-transport-https ca-certificates curl software-properties-common unzip jq
-
-# Install AWS CLI v2
-curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-unzip awscliv2.zip
-sudo ./aws/install
-rm -rf aws awscliv2.zip
-
-# Install Docker
-curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-sudo apt-get update
-sudo apt-get install -y docker-ce docker-ce-cli containerd.io
-
-# Install Docker Compose
-sudo curl -L "https://github.com/docker/compose/releases/download/v2.21.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
-sudo chmod +x /usr/local/bin/docker-compose
-
-# Configure Docker
-sudo systemctl enable docker
-sudo systemctl start docker
-sudo usermod -aG docker ubuntu
\ No newline at end of file
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/.env.docker b/infra/aws-cloudformation/ami/quadratic-selfhost/.env.docker
deleted file mode 100644
index a5922e79e0..0000000000
--- a/infra/aws-cloudformation/ami/quadratic-selfhost/.env.docker
+++ /dev/null
@@ -1,89 +0,0 @@
-# global
-ENVIRONMENT=docker
-RUST_LOG=info
-
-# Your license key for Quadratic. Get one here https://selfhost.quadratichq.com/
-LICENSE_KEY="#LICENSE_KEY#"
-
-# postgres database
-DATABASE_IN_DOCKER_COMPOSE=true
-DATABASE_DSN=postgresql://postgres:postgres@host.docker.internal:5432/postgres
-
-# pubsub
-PUBSUB_IN_DOCKER_COMPOSE=true
-PUBSUB_HOST=host.docker.internal
-PUBSUB_PORT=6379
-PUBSUB_PASSWORD=""
-PUBSUB_ACTIVE_CHANNELS=active_channels
-PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL=processed_transactions
-
-# auth: ory or auth0
-AUTH_TYPE=ory
-JWKS_URI=http://host.docker.internal:3000/.well-known/jwks.json
-M2M_AUTH_TOKEN=M2M_AUTH_TOKEN
-ENCRYPTION_KEY=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc
-
-# auth=ory
-ORY_IN_DOCKER_COMPOSE=true
-ORY_DSN=postgresql://postgres:postgres@host.docker.internal:5432/kratos?sslmode=disable
-ORY_LOG_LEVEL=trace
-ORY_ADMIN_HOST=http://host.docker.internal:4434
-KRATOS_URL_INTERNAL=http://host.docker.internal:4433/
-KRATOS_URL_EXTERNAL=https://ory.#HOST#/
-KRATOS_NODE_PORT=4455
-KRATOS_COOKIE_SECRET=changeme
-KRATOS_CSRF_COOKIE_NAME=__HOST-#HOST#-x-csrf-token
-KRATOS_CSRF_COOKIE_SECRET=changeme
-
-# caddy
-CADDY_IN_DOCKER_COMPOSE=true
-
-# client
-QUADRATIC_CLIENT_IN_DOCKER_COMPOSE=true
-
-# api
-QUADRATIC_API_IN_DOCKER_COMPOSE=true
-QUADRATIC_API_URL_EXTERNAL=https://api.#HOST#
-QUADRATIC_API_URL_INTERNAL=http://host.docker.internal:8000
-
-# multiplayer
-QUADRATIC_MULTIPLAYER_IN_DOCKER_COMPOSE=true
-QUADRATIC_MULTIPLAYER_HOST=0.0.0.0
-QUADRATIC_MULTIPLAYER_PORT=3001
-QUADRATIC_MULTIPLAYER_HEARTBEAT_CHECK_S=3
-QUADRATIC_MULTIPLAYER_HEARTBEAT_TIMEOUT_S=600
-QUADRATIC_MULTIPLAYER_URL_EXTERNAL=wss://multiplayer.#HOST#/ws
-QUADRATIC_MULTIPLAYER_URL_INTERNAL=ws://host.docker.internal:3001
-
-# files
-QUADRATIC_FILES_IN_DOCKER_COMPOSE=true
-QUADRATIC_FILES_HOST=0.0.0.0
-QUADRATIC_FILES_PORT=3002
-QUADRATIC_FILES_FILE_CHECK_S=5
-QUADRATIC_FILES_FILES_PER_CHECK=1000
-QUADRATIC_FILES_TRUNCATE_FILE_CHECK_S=60
-QUADRATIC_FILES_TRUNCATE_TRANSACTION_AGE_DAYS=5
-QUADRATIC_FILES_URL_EXTERNAL=https://files.#HOST#
-QUADRATIC_FILES_URL_INTERNAL=http://host.docker.internal:3002
-
-# connection
-QUADRATIC_CONNECTION_IN_DOCKER_COMPOSE=true
-QUADRATIC_CONNECTION_HOST=0.0.0.0
-QUADRATIC_CONNECTION_PORT=3003
-QUADRATIC_CONNECTION_URL_EXTERNAL=https://connection.#HOST#
-QUADRATIC_CONNECTION_URL_INTERNAL=http://host.docker.internal:3003
-QUADRATIC_CONNECTION_MAX_RESPONSE_BYTES=15728640 # 15MB
-QUADRATIC_CONNECTION_STATIC_IPS=0.0.0.0,127.0.0.1
-
-# stripe
-STRIPE_SECRET_KEY=STRIPE_SECRET_KEY
-STRIPE_WEBHOOK_SECRET=STRIPE_WEBHOOK_SECRET
-
-# storage: s3 or file-system
-STORAGE_TYPE=file-system
-
-# storage=file-system
-STORAGE_DIR=/file-storage
-
-# storage=s3
-AWS_S3_BUCKET_NAME=quadratic-api-docker
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/docker-compose.yml b/infra/aws-cloudformation/ami/quadratic-selfhost/docker-compose.yml
deleted file mode 100644
index 7fc0e95117..0000000000
--- a/infra/aws-cloudformation/ami/quadratic-selfhost/docker-compose.yml
+++ /dev/null
@@ -1,303 +0,0 @@
-services:
- # base services - redis, postgres
-
- redis:
- image: redis/redis-stack:latest
- restart: always
- ports:
- - "6379:6379"
- - "8001:8001"
- healthcheck:
- test: ["CMD", "redis-cli", "ping"]
- interval: "5s"
- volumes:
- - ./docker/redis/data:/data
- profiles:
- - pubsub
-
- postgres:
- image: postgres:15
- restart: always
- ports:
- - "5432:5432"
- environment:
- POSTGRES_USER: postgres
- PGUSER: postgres
- POSTGRES_PASSWORD: postgres
- ADDITIONAL_DATABASES: kratos
- healthcheck:
- test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"]
- interval: 10s
- timeout: 5s
- retries: 5
- volumes:
- - ./docker/postgres/data:/var/lib/postgresql/data
- - ./docker/postgres/scripts:/docker-entrypoint-initdb.d
- profiles:
- - database
-
- caddy:
- image: caddy:latest
- ports:
- - "80:80"
- - "443:443"
- volumes:
- - ./docker/caddy/config/Caddyfile:/etc/caddy/Caddyfile
- - ./docker/caddy/data:/data/caddy
- profiles:
- - caddy
- networks:
- - host
- extra_hosts:
- - "host.docker.internal:host-gateway"
-
- # quadratic services - client, api, multiplayer, files, connection
-
- quadratic-client:
- image: ${ECR_URL}/quadratic-client-development:${IMAGE_TAG}
- environment:
- VITE_DEBUG: 1
- VITE_QUADRATIC_API_URL: ${QUADRATIC_API_URL_EXTERNAL}
- VITE_QUADRATIC_MULTIPLAYER_URL: ${QUADRATIC_MULTIPLAYER_URL_EXTERNAL}
- VITE_QUADRATIC_CONNECTION_URL: ${QUADRATIC_CONNECTION_URL_EXTERNAL}
- VITE_AUTH_TYPE: ${AUTH_TYPE}
- VITE_STORAGE_TYPE: ${STORAGE_TYPE}
- VITE_ORY_HOST: ${KRATOS_URL_EXTERNAL}
- ports:
- - "3000:80"
- command: >
- sh -c "/client/scripts/replace_env_vars.sh &&
- nginx -g \"daemon off;\""
- healthcheck:
- test: ["CMD-SHELL", "curl -f http://host.docker.internal:3000/ || exit 1"]
- interval: 10s
- timeout: 5s
- restart: "always"
- depends_on:
- quadratic-api:
- condition: service_started
- volumes:
- - ./docker/client:/client
- - ./docker/client/config/default.conf:/etc/nginx/conf.d/default.conf
- profiles:
- - quadratic_client
- networks:
- - host
- extra_hosts:
- - "host.docker.internal:host-gateway"
-
- quadratic-api:
- image: ${ECR_URL}/quadratic-api-development:${IMAGE_TAG}
- environment:
- CORS: "*"
- DATABASE_URL: ${DATABASE_DSN}
- ENVIRONMENT: ${ENVIRONMENT}
- STRIPE_SECRET_KEY: ${STRIPE_SECRET_KEY}
- STRIPE_WEBHOOK_SECRET: ${STRIPE_WEBHOOK_SECRET}
- OPENAI_API_KEY: ${OPENAI_API_KEY}
- ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY}
- EXA_API_KEY: ${EXA_API_KEY}
- AWS_S3_REGION: ${AWS_S3_REGION}
- AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
- AWS_S3_ACCESS_KEY_ID: ${AWS_S3_ACCESS_KEY_ID}
- AWS_S3_SECRET_ACCESS_KEY: ${AWS_S3_SECRET_ACCESS_KEY}
- M2M_AUTH_TOKEN: ${M2M_AUTH_TOKEN}
- ENCRYPTION_KEY: ${ENCRYPTION_KEY}
- AUTH_TYPE: ${AUTH_TYPE}
- ORY_JWKS_URI: ${JWKS_URI}
- ORY_ADMIN_HOST: ${ORY_ADMIN_HOST}
- STORAGE_TYPE: ${STORAGE_TYPE}
- QUADRATIC_FILE_URI: ${QUADRATIC_FILES_URL_INTERNAL}
- QUADRATIC_FILE_URI_PUBLIC: ${QUADRATIC_FILES_URL_EXTERNAL}
- LICENSE_KEY: ${LICENSE_KEY}
- restart: "always"
- ports:
- - "8000:8000"
- command: bash -c "npx prisma migrate deploy --schema quadratic-api/prisma/schema.prisma && npm run start:prod --workspace=quadratic-api"
- depends_on:
- postgres:
- condition: service_healthy
- profiles:
- - quadratic_api
- networks:
- - host
- extra_hosts:
- - "host.docker.internal:host-gateway"
-
- quadratic-multiplayer:
- image: ${ECR_URL}/quadratic-multiplayer-development:${IMAGE_TAG}
- environment:
- RUST_LOG: ${RUST_LOG}
- HOST: ${QUADRATIC_MULTIPLAYER_HOST}
- PORT: ${QUADRATIC_MULTIPLAYER_PORT}
- HEARTBEAT_CHECK_S: ${QUADRATIC_MULTIPLAYER_HEARTBEAT_CHECK_S}
- HEARTBEAT_TIMEOUT_S: ${QUADRATIC_MULTIPLAYER_HEARTBEAT_TIMEOUT_S}
- QUADRATIC_API_URI: ${QUADRATIC_API_URL_INTERNAL}
- M2M_AUTH_TOKEN: ${M2M_AUTH_TOKEN}
- ENVIRONMENT: ${ENVIRONMENT}
- PUBSUB_HOST: ${PUBSUB_HOST}
- PUBSUB_PORT: ${PUBSUB_PORT}
- PUBSUB_PASSWORD: ${PUBSUB_PASSWORD}
- PUBSUB_ACTIVE_CHANNELS: ${PUBSUB_ACTIVE_CHANNELS}
- AUTH0_JWKS_URI: ${JWKS_URI}
- AUTHENTICATE_JWT: true
- restart: "always"
- ports:
- - "3001:3001"
- depends_on:
- redis:
- condition: service_healthy
- quadratic-api:
- condition: service_started
- profiles:
- - quadratic_multiplayer
- networks:
- - host
- extra_hosts:
- - "host.docker.internal:host-gateway"
-
- quadratic-files:
- image: ${ECR_URL}/quadratic-files-development:${IMAGE_TAG}
- environment:
- RUST_LOG: ${RUST_LOG}
- HOST: ${QUADRATIC_FILES_HOST}
- PORT: ${QUADRATIC_FILES_PORT}
- FILE_CHECK_S: ${QUADRATIC_FILES_FILE_CHECK_S}
- FILES_PER_CHECK: ${QUADRATIC_FILES_FILES_PER_CHECK}
- TRUNCATE_FILE_CHECK_S: ${QUADRATIC_FILES_TRUNCATE_FILE_CHECK_S}
- TRUNCATE_TRANSACTION_AGE_DAYS: ${QUADRATIC_FILES_TRUNCATE_TRANSACTION_AGE_DAYS}
- ENVIRONMENT: ${ENVIRONMENT}
- AUTH0_JWKS_URI: ${JWKS_URI}
- QUADRATIC_API_URI: ${QUADRATIC_API_URL_INTERNAL}
- M2M_AUTH_TOKEN: ${M2M_AUTH_TOKEN}
- PUBSUB_HOST: ${PUBSUB_HOST}
- PUBSUB_PORT: ${PUBSUB_PORT}
- PUBSUB_PASSWORD: ${PUBSUB_PASSWORD}
- PUBSUB_ACTIVE_CHANNELS: ${PUBSUB_ACTIVE_CHANNELS}
- PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL: ${PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL}
- STORAGE_TYPE: ${STORAGE_TYPE}
- AWS_S3_REGION: ${AWS_S3_REGION}
- AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
- AWS_S3_ACCESS_KEY_ID: ${AWS_S3_ACCESS_KEY_ID}
- AWS_S3_SECRET_ACCESS_KEY: ${AWS_S3_SECRET_ACCESS_KEY}
- STORAGE_DIR: ${STORAGE_DIR}
- STORAGE_ENCRYPTION_KEYS: ${ENCRYPTION_KEY}
- restart: "always"
- ports:
- - "3002:3002"
- depends_on:
- redis:
- condition: service_healthy
- quadratic-api:
- condition: service_started
- volumes:
- - ./docker/file-storage:/file-storage
- profiles:
- - quadratic_files
- networks:
- - host
- extra_hosts:
- - "host.docker.internal:host-gateway"
-
- quadratic-connection:
- image: ${ECR_URL}/quadratic-connection-development:${IMAGE_TAG}
- environment:
- RUST_LOG: ${RUST_LOG}
- HOST: ${QUADRATIC_CONNECTION_HOST}
- PORT: ${QUADRATIC_CONNECTION_PORT}
- ENVIRONMENT: ${ENVIRONMENT}
- AUTH0_JWKS_URI: ${JWKS_URI}
- QUADRATIC_API_URI: ${QUADRATIC_API_URL_INTERNAL}
- M2M_AUTH_TOKEN: ${M2M_AUTH_TOKEN}
- MAX_RESPONSE_BYTES: ${QUADRATIC_CONNECTION_MAX_RESPONSE_BYTES}
- STATIC_IPS: ${QUADRATIC_CONNECTION_STATIC_IPS}
- restart: "always"
- ports:
- - "3003:3003"
- profiles:
- - quadratic_connection
- networks:
- - host
- extra_hosts:
- - "host.docker.internal:host-gateway"
-
- # auth service - ory
-
- ory-auth:
- image: oryd/kratos:v1.2.0
- ports:
- - "4433:4433" # public
- - "4434:4434" # admin
- command: serve -c /etc/config/kratos/kratos.yml --dev --watch-courier
- volumes:
- - ./docker/ory-auth/config:/etc/config/kratos
- environment:
- DSN: ${ORY_DSN}
- LOG_LEVEL: ${ORY_LOG_LEVEL}
- restart: unless-stopped
- depends_on:
- - postgres
- - ory-auth-migrate
- profiles:
- - ory
- networks:
- - host
- extra_hosts:
- - "host.docker.internal:host-gateway"
-
- ory-auth-migrate:
- image: oryd/kratos:v1.2.0
- command: migrate -c /etc/config/kratos/kratos.yml sql -e --yes
- volumes:
- - ./docker/ory-auth/config:/etc/config/kratos
- environment:
- DSN: ${ORY_DSN}
- restart: on-failure
- depends_on:
- - postgres
- profiles:
- - ory
- networks:
- - host
- extra_hosts:
- - "host.docker.internal:host-gateway"
-
- ory-auth-node:
- image: oryd/kratos-selfservice-ui-node:v1.2.0
- ports:
- - "4455:4455"
- environment:
- PORT: ${KRATOS_NODE_PORT}
- KRATOS_PUBLIC_URL: ${KRATOS_URL_INTERNAL}
- KRATOS_BROWSER_URL: ${KRATOS_URL_EXTERNAL}
- COOKIE_SECRET: ${KRATOS_COOKIE_SECRET}
- CSRF_COOKIE_NAME: ${KRATOS_CSRF_COOKIE_NAME}
- CSRF_COOKIE_SECRET: ${KRATOS_CSRF_COOKIE_SECRET}
- restart: on-failure
- profiles:
- - ory
- networks:
- - host
- extra_hosts:
- - "host.docker.internal:host-gateway"
-
- ory-auth-mail:
- image: oryd/mailslurper:latest-smtps
- ports:
- - "1025:1025"
- - "4436:4436"
- - "4437:4437"
- - "8080:8080"
- profiles:
- - ory
- networks:
- - host
- extra_hosts:
- - "host.docker.internal:host-gateway"
-
-volumes:
- docker:
- name: docker
-
-networks:
- host:
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/admin/config/openapi.yaml b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/admin/config/openapi.yaml
deleted file mode 100644
index ff6fd4fe92..0000000000
--- a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/admin/config/openapi.yaml
+++ /dev/null
@@ -1,48 +0,0 @@
-openapi: 3.0.3
-info:
- title: My API
- version: '1.0'
- x-logo:
- url: ''
-paths:
- /license/{licenseKey}:
- post:
- tags: []
- operationId: license
- parameters:
- - name: licenseKey
- in: path
- required: true
- deprecated: false
- example: sdaf
- schema:
- type: string
- x-last-modified: 1724376657454
- responses:
- '200':
- content:
- application/json:
- schema:
- type: object
- properties:
- limits:
- type: object
- properties:
- seats:
- type: number
- example: 5
- x-last-modified: 1724376729775
- description: ''
- headers: {}
- links: {}
- x-last-modified: 1724376538356
-components:
- securitySchemes: {}
- schemas: {}
- headers: {}
- responses: {}
- parameters: {}
-tags: []
-servers:
- - url: https://api.example.io
-security: []
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/caddy/config/Caddyfile b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/caddy/config/Caddyfile
deleted file mode 100644
index 3d44781eb6..0000000000
--- a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/caddy/config/Caddyfile
+++ /dev/null
@@ -1,27 +0,0 @@
-#HOST# {
- reverse_proxy http://host.docker.internal:3000
-}
-
-api.#HOST# {
- reverse_proxy http://host.docker.internal:8000
-}
-
-multiplayer.#HOST# {
- reverse_proxy http://host.docker.internal:3001
-}
-
-files.#HOST# {
- reverse_proxy http://host.docker.internal:3002
-}
-
-connection.#HOST# {
- reverse_proxy http://host.docker.internal:3003
-}
-
-ory.#HOST# {
- reverse_proxy http://host.docker.internal:4433
-}
-
-ory-node.#HOST# {
- reverse_proxy http://host.docker.internal:4455
-}
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/client/config/default.conf b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/client/config/default.conf
deleted file mode 100644
index a58529d83b..0000000000
--- a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/client/config/default.conf
+++ /dev/null
@@ -1,24 +0,0 @@
-server {
- listen 80;
- server_name localhost;
-
- root /usr/share/nginx/html;
- index index.html;
-
- location / {
- try_files $uri $uri/ /index.html =404;
- }
-
- location ~* \.(?:css|js|json|gif|png|jpg|jpeg|svg|ico)$ {
- expires 1y;
- access_log off;
- add_header Cache-Control "public, no-transform";
-
- add_header Cross-Origin-Opener-Policy "same-origin";
- add_header Cross-Origin-Embedder-Policy "require-corp";
- }
-
- # Add CORS headers to all requests
- add_header Cross-Origin-Opener-Policy "same-origin";
- add_header Cross-Origin-Embedder-Policy "require-corp";
-}
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/client/scripts/replace_env_vars.sh b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/client/scripts/replace_env_vars.sh
deleted file mode 100755
index 60160a8ccf..0000000000
--- a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/client/scripts/replace_env_vars.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/sh
-
-escape_for_sed() {
- input="$1"
- printf '%s\n' "$input" | sed -e 's/[\/&]/\\&/g'
-}
-
-replace_env_vars() {
- vite_vars=""
-
- for env_var in $(env); do
- case "$env_var" in
- VITE_*)
- vite_vars="$vite_vars $env_var"
- ;;
- esac
- done
-
- find "/usr/share/nginx/html/assets" -type f -name "*.js" | xargs grep -l "VITE_" | while read file; do
-
- for env_var in $vite_vars; do
- var="$(echo "$env_var" | cut -d'=' -f1)"
- val="$(echo "$env_var" | cut -d'=' -f2-)"
- appended_var="${var}_VAL"
- escaped_val=$(escape_for_sed "$val")
-
- # echo "Replacing $appended_var with $escaped_val in $file"
- sed -i "s/${appended_var}/${escaped_val}/g" "$file"
- done
- done
-}
-
-echo "Replacing .env values in $ENV_PATH"
-replace_env_vars
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/ory-auth/config/identity.schema.json b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/ory-auth/config/identity.schema.json
deleted file mode 100644
index a953fc68ec..0000000000
--- a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/ory-auth/config/identity.schema.json
+++ /dev/null
@@ -1,47 +0,0 @@
-{
- "$id": "https://schemas.ory.sh/presets/kratos/quickstart/email-password/identity.schema.json",
- "$schema": "http://json-schema.org/draft-07/schema#",
- "title": "Person",
- "type": "object",
- "properties": {
- "traits": {
- "type": "object",
- "properties": {
- "email": {
- "type": "string",
- "format": "email",
- "title": "E-Mail",
- "minLength": 3,
- "ory.sh/kratos": {
- "credentials": {
- "password": {
- "identifier": true
- }
- },
- "verification": {
- "via": "email"
- },
- "recovery": {
- "via": "email"
- }
- }
- },
- "name": {
- "type": "object",
- "properties": {
- "first": {
- "title": "First Name",
- "type": "string"
- },
- "last": {
- "title": "Last Name",
- "type": "string"
- }
- }
- }
- },
- "required": ["email"],
- "additionalProperties": false
- }
- }
-}
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/ory-auth/config/kratos.yml b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/ory-auth/config/kratos.yml
deleted file mode 100644
index 810ec81a24..0000000000
--- a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/ory-auth/config/kratos.yml
+++ /dev/null
@@ -1,138 +0,0 @@
-# https://raw.githubusercontent.com/ory/kratos/v1.2.0/.schemastore/config.schema.json
-version: v1.2.0
-
-dsn: memory
-
-serve:
- public:
- base_url: https://ory.#HOST#/
- cors:
- enabled: true
- allowed_origins:
- - https://#HOST#
- allowed_methods:
- - POST
- - GET
- - PUT
- - PATCH
- - DELETE
- allowed_headers:
- - Authorization
- - Access-Control-Allow-Origin
- - Cookie
- - Content-Type
- exposed_headers:
- - Content-Type
- - Set-Cookie
- admin:
- base_url: http://kratos:4434/
-
-selfservice:
- default_browser_return_url: https://#HOST#
- allowed_return_urls:
- - https://#HOST#
- - https://ory-node.#HOST#
- - https://#HOST#:3000
- - https://#HOST#:19006/Callback
- - exp://localhost:8081/--/Callback
-
- methods:
- password:
- enabled: true
- totp:
- config:
- issuer: Kratos
- enabled: true
- lookup_secret:
- enabled: true
- link:
- enabled: true
- code:
- enabled: true
-
- flows:
- error:
- ui_url: https://ory-node.#HOST#/error
-
- settings:
- ui_url: https://ory-node.#HOST#/settings
- privileged_session_max_age: 15m
- required_aal: highest_available
-
- recovery:
- enabled: true
- ui_url: https://ory-node.#HOST#/recovery
- use: code
-
- verification:
- # we disable verification for self-hosting
- enabled: false
- ui_url: https://ory-node.#HOST#/verification
- use: code
- after:
- default_browser_return_url: https://#HOST#/login-result
-
- logout:
- after:
- default_browser_return_url: https://ory-node.#HOST#/login
-
- login:
- ui_url: https://ory-node.#HOST#/login
- lifespan: 10m
-
- registration:
- lifespan: 10m
- ui_url: https://ory-node.#HOST#/registration
- after:
- password:
- default_browser_return_url: https://#HOST#/login-result
- hooks:
- - hook: session
- - hook: show_verification_ui
- default_browser_return_url: https://#HOST#/login-result
-
-session:
- whoami:
- tokenizer:
- templates:
- jwt_template:
- jwks_url: http://host.docker.internal:3000/.well-known/jwks.json
- # claims_mapper_url: base64://... # A JsonNet template for modifying the claims
- ttl: 24h # 24 hours (defaults to 10 minutes)
-cookies:
- domain: "#HOST#"
- path: /
- same_site: Lax
-
-log:
- level: warning
- format: json
- redaction_text: ""
- leak_sensitive_values: false
-
-secrets:
- cookie:
- - PLEASE-CHANGE-ME-I-AM-VERY-INSECURE
- cipher:
- - 32-LONG-SECRET-NOT-SECURE-AT-ALL
-
-ciphers:
- algorithm: xchacha20-poly1305
-
-hashers:
- algorithm: bcrypt
- bcrypt:
- cost: 8
-
-identity:
- default_schema_id: default
- schemas:
- - id: default
- url: file:///etc/config/kratos/identity.schema.json
-
-courier:
- smtp:
- connection_uri: smtps://test:test@host.docker.internal:1025/?skip_ssl_verify=true
-
-feature_flags:
- use_continue_with_transitions: true
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/postgres/scripts/init.sh b/infra/aws-cloudformation/ami/quadratic-selfhost/docker/postgres/scripts/init.sh
deleted file mode 100644
index 5e5b12df77..0000000000
--- a/infra/aws-cloudformation/ami/quadratic-selfhost/docker/postgres/scripts/init.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -e
-set -u
-
-function create_user_and_database() {
- local database=$1
- echo "Creating database '$database' with user '$POSTGRES_USER'"
- psql -c "CREATE DATABASE $database;" || { echo "Failed to create database '$database'"; exit 1; }
- echo "Database '$database' created"
-}
-
-if [ -n "$ADDITIONAL_DATABASES" ]; then
- for i in ${ADDITIONAL_DATABASES//,/ }
- do
- create_user_and_database $i
- done
-fi
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/init.sh b/infra/aws-cloudformation/ami/quadratic-selfhost/init.sh
deleted file mode 100755
index d6b70c69cf..0000000000
--- a/infra/aws-cloudformation/ami/quadratic-selfhost/init.sh
+++ /dev/null
@@ -1,133 +0,0 @@
-#!/bin/bash
-
-# Self-Hosting Initialization
-#
-# Usage:
-#
-# ./init.sh 83f0ebdf-eafb-4c8d-bd7b-04ea07d61b7f localhost
-#
-#
-# Flow:
-#
-# First, check to see if there is a VERSION file, if so, use that version.
-# If not, then check for the first command line argument, if so, use that version.
-# Else, prompt the user.
-#
-# First, check to see if there is a HOST file, if so, use that host.
-# If not, then check for the first command line argument, if so, use that host.
-# Else, prompt the user.
-
-REPO="https://github.com/quadratichq/quadratic-selfhost.git"
-SELF_HOSTING_URI="https://selfhost.quadratichq.com/"
-INVALID_LICENSE_KEY="Invalid license key."
-PROFILE=""
-LICENSE_KEY=""
-HOST=""
-
-get_license_key() {
- read -p "Enter your license key (Get one for free instantly at $SELF_HOSTING_URI): " user_input
-
- if [[ $user_input =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then
- echo $user_input
- else
- echo $INVALID_LICENSE_KEY
- return 1
- fi
-}
-
-get_host() {
- read -p "What public host name or public IP address are you using for this setup (e.g. localhost, app.quadratic.com, or other): " user_input
-
- # TODO: validate host
- echo $user_input
-}
-
-parse_profile() {
- # automatically export all variables
- set -a
- [[ -f ".env.docker" ]] && source .env.docker
- # disable auto export
- set +a
-
- values=()
- variables=(
- "DATABASE_IN_DOCKER_COMPOSE"
- "PUBSUB_IN_DOCKER_COMPOSE"
- "CADDY_IN_DOCKER_COMPOSE"
- "ORY_IN_DOCKER_COMPOSE"
- "QUADRATIC_CLIENT_IN_DOCKER_COMPOSE"
- "QUADRATIC_API_IN_DOCKER_COMPOSE"
- "QUADRATIC_MULTIPLAYER_IN_DOCKER_COMPOSE"
- "QUADRATIC_FILES_IN_DOCKER_COMPOSE"
- "QUADRATIC_FILES_URL_INTERNAL"
- "QUADRATIC_FILES_URL_EXTERNAL"
- "QUADRATIC_CONNECTION_IN_DOCKER_COMPOSE"
- )
-
- for var_name in "${variables[@]}"; do
- local var_value=$(eval echo \$$var_name)
-
- if [ "$var_value" == "true" ]; then
- # store the lowercase variable name
- var_name_stripped=$(echo "$var_name" | sed 's/_IN_DOCKER_COMPOSE//g')
- var_name_lower=$(echo "$var_name_stripped" | awk '{print tolower($0)}')
- values+=("--profile ${var_name_lower}")
- fi
- done
-
- echo "${values[@]}"
-}
-
-checkout() {
- git clone $REPO
- cd quadratic-selfhost
- git checkout
-}
-
-
-if [ -f "quadratic-selfhost/LICENSE_KEY" ]; then
- LICENSE_KEY=$( LICENSE_KEY
-
-# write docker compose profile to PROFILE file
-PROFILE=$(parse_profile)
-touch PROFILE
-echo $PROFILE > PROFILE
-
-# write host to HOST file
-touch HOST
-echo $HOST > HOST
-
-# remove the init.sh script
-# rm ../init.sh
-
-# adding .bak for compatibility with both GNU (Linux) and BSD (MacOS) sed
-sed -i.bak "s/#LICENSE_KEY#/$LICENSE_KEY/g" ".env.docker"
-sed -i.bak "s/#HOST#/$HOST/g" ".env.docker"
-sed -i.bak "s/#HOST#/$HOST/g" "docker/ory-auth/config/kratos.yml"
-sed -i.bak "s/#HOST#/$HOST/g" "docker/caddy/config/Caddyfile"
-
-rm .env.docker.bak
-rm docker/ory-auth/config/kratos.yml.bak
-rm docker/caddy/config/Caddyfile.bak
-
-sh start.sh
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/start.sh b/infra/aws-cloudformation/ami/quadratic-selfhost/start.sh
deleted file mode 100755
index 68ea70654e..0000000000
--- a/infra/aws-cloudformation/ami/quadratic-selfhost/start.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/sh
-
-# read the value of PROFILE from the file
-PROFILE=$(cat PROFILE)
-
-start() {
- docker compose $PROFILE --env-file .env.docker down -v --remove-orphans
- docker compose $PROFILE --env-file .env.docker up -d
- docker system prune -f
-}
-
-start
\ No newline at end of file
diff --git a/infra/aws-cloudformation/ami/quadratic-selfhost/stop.sh b/infra/aws-cloudformation/ami/quadratic-selfhost/stop.sh
deleted file mode 100755
index 2403c44b1b..0000000000
--- a/infra/aws-cloudformation/ami/quadratic-selfhost/stop.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh
-
-stop() {
- docker compose --profile "*" --env-file .env.docker down -v --remove-orphans
- docker system prune -af
-}
-
-stop
\ No newline at end of file
diff --git a/infra/aws-cloudformation/quadratic-selfhost.yml b/infra/aws-cloudformation/quadratic-selfhost.yml
index cd3520c050..b5416d940d 100644
--- a/infra/aws-cloudformation/quadratic-selfhost.yml
+++ b/infra/aws-cloudformation/quadratic-selfhost.yml
@@ -67,7 +67,7 @@ Resources:
- Key: Name
Value: !Sub "${ImageTag}"
InstanceType: m6a.large
- ImageId: ami-094c4f62c0fffae5e # quadratic-selfhost-base-ami
+ ImageId: ami-0b8c6b923777519db # AMI for us-west-2 (Ubuntu 22.04 LTS amd64)
IamInstanceProfile: !Ref EC2InstanceProfile
SecurityGroups:
- !Ref SecurityGroup
@@ -80,6 +80,41 @@ Resources:
Fn::Base64: !Sub |
#!/bin/bash
+ # Update and install dependencies
+ sudo apt-get update
+ sudo apt-get install -y apt-transport-https ca-certificates curl software-properties-common unzip jq awscli
+
+ # Install AWS CLI v2
+ curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+ unzip awscliv2.zip
+ sudo ./aws/install
+ rm -rf aws awscliv2.zip
+
+ # Install Docker
+ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+ echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+ sudo apt-get update
+ sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+ sudo curl -L "https://github.com/docker/compose/releases/download/v2.21.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+ sudo chmod +x /usr/local/bin/docker-compose
+ sudo chown ubuntu /var/run/docker.sock
+ sudo systemctl enable docker
+ sudo systemctl start docker
+ sudo usermod -aG docker ubuntu
+
+ # Download Quadratic initialization script
+ curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic-selfhost/main/init-aws-staging.sh -o init.sh
+
+ # Fetch License Key
+ LICENSE_KEY="$(aws ssm get-parameter --name "/quadratic-development/QUADRATIC_LICENSE_KEY" --with-decryption --query "Parameter.Value" --output text)"
+
+ # Generate Domain Name
+ DOMAIN_NAME="${ImageTag}.quadratic-selfhost.com"
+
+ # Run Quadratic initialization script
+ chmod +x init.sh
+ ./init.sh $LICENSE_KEY $DOMAIN_NAME
+
# Append environment variables to .env file
cat << EOF >> /quadratic-selfhost/.env.docker
@@ -87,7 +122,7 @@ Resources:
AWS_S3_ACCESS_KEY_ID=$(aws ssm get-parameter --name "/quadratic-development/AWS_S3_ACCESS_KEY_ID" --with-decryption --query "Parameter.Value" --output text)
AWS_S3_SECRET_ACCESS_KEY=$(aws ssm get-parameter --name "/quadratic-development/AWS_S3_SECRET_ACCESS_KEY" --with-decryption --query "Parameter.Value" --output text)
- # AI API Keys
+ # ai api keys
OPENAI_API_KEY=$(aws ssm get-parameter --name "/quadratic-development/OPENAI_API_KEY" --with-decryption --query "Parameter.Value" --output text)
ANTHROPIC_API_KEY=$(aws ssm get-parameter --name "/quadratic-development/ANTHROPIC_API_KEY" --with-decryption --query "Parameter.Value" --output text)
EXA_API_KEY=$(aws ssm get-parameter --name "/quadratic-development/EXA_API_KEY" --with-decryption --query "Parameter.Value" --output text)
@@ -104,11 +139,11 @@ Resources:
EOF
chmod +x /quadratic-selfhost/login.sh
+ chmod +x /quadratic-selfhost/start.sh
+
cd /quadratic-selfhost
./login.sh
-
- # Run Quadratic initialization script
- ./init.sh "$(aws ssm get-parameter --name "/quadratic-development/QUADRATIC_LICENSE_KEY" --with-decryption --query "Parameter.Value" --output text)" "${ImageTag}.quadratic-selfhost.com"
+ ./start.sh
# Global Accelerator
GlobalAccelerator:
diff --git a/package.json b/package.json
index ae38e49279..3aef9843cb 100644
--- a/package.json
+++ b/package.json
@@ -66,7 +66,7 @@
"docker:build:dev": "ECR_OR_BUILD=build DEV=true docker compose --profile all --env-file .env.docker up",
"docker:ecr": "ECR_OR_BUILD=ecr DEV=false docker compose --profile all --env-file .env.docker up",
"docker:down": "ECR_OR_BUILD=build DEV=false docker compose --profile all --env-file .env.docker down",
- "docker:clean": "npm run docker:down && docker builder prune -af && docker system prune -af --volumes"
+ "docker:clean": "docker system prune -af && docker builder prune -af && docker volume prune -af"
},
"dependencies": {
"@ory/kratos-client": "^1.2.1",
From a4827fcad350db7bc9c69f53bf9d4ca555e2dbf2 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Fri, 3 Jan 2025 02:58:46 +0530
Subject: [PATCH 143/155] quadratic-preview
---
.env.docker | 3 +++
docker-compose.yml | 6 +++---
infra/aws-cloudformation/quadratic-selfhost.yml | 17 ++++++++---------
3 files changed, 14 insertions(+), 12 deletions(-)
diff --git a/.env.docker b/.env.docker
index cdcc5d013e..e7f1bdf11d 100644
--- a/.env.docker
+++ b/.env.docker
@@ -48,6 +48,7 @@ QUADRATIC_API_URL_INTERNAL=http://host.docker.internal:8000
# multiplayer
QUADRATIC_MULTIPLAYER_IN_DOCKER_COMPOSE=true
+QUADRATIC_MULTIPLAYER_RUST_LOG=info
QUADRATIC_MULTIPLAYER_HOST=0.0.0.0
QUADRATIC_MULTIPLAYER_PORT=3001
QUADRATIC_MULTIPLAYER_HEARTBEAT_CHECK_S=3
@@ -57,6 +58,7 @@ QUADRATIC_MULTIPLAYER_URL_INTERNAL=ws://host.docker.internal:3001
# files
QUADRATIC_FILES_IN_DOCKER_COMPOSE=true
+QUADRATIC_FILES_RUST_LOG=info
QUADRATIC_FILES_HOST=0.0.0.0
QUADRATIC_FILES_PORT=3002
QUADRATIC_FILES_FILE_CHECK_S=5
@@ -68,6 +70,7 @@ QUADRATIC_FILES_URL_INTERNAL=http://host.docker.internal:3002
# connection
QUADRATIC_CONNECTION_IN_DOCKER_COMPOSE=true
+QUADRATIC_CONNECTION_RUST_LOG=info
QUADRATIC_CONNECTION_HOST=0.0.0.0
QUADRATIC_CONNECTION_PORT=3003
QUADRATIC_CONNECTION_URL_EXTERNAL=http://localhost:3003
diff --git a/docker-compose.yml b/docker-compose.yml
index 1a121b4bfc..17632930ad 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -159,7 +159,7 @@ services:
service: quadratic-multiplayer
container_name: multiplayer
environment:
- RUST_LOG: ${RUST_LOG}
+ RUST_LOG: ${QUADRATIC_MULTIPLAYER_RUST_LOG}
HOST: ${QUADRATIC_MULTIPLAYER_HOST}
PORT: ${QUADRATIC_MULTIPLAYER_PORT}
HEARTBEAT_CHECK_S: ${QUADRATIC_MULTIPLAYER_HEARTBEAT_CHECK_S}
@@ -198,7 +198,7 @@ services:
service: quadratic-files
container_name: files
environment:
- RUST_LOG: ${RUST_LOG}
+ RUST_LOG: ${QUADRATIC_FILES_RUST_LOG}
HOST: ${QUADRATIC_FILES_HOST}
PORT: ${QUADRATIC_FILES_PORT}
FILE_CHECK_S: ${QUADRATIC_FILES_FILE_CHECK_S}
@@ -248,7 +248,7 @@ services:
service: quadratic-connection
container_name: connection
environment:
- RUST_LOG: ${RUST_LOG}
+ RUST_LOG: ${QUADRATIC_CONNECTION_RUST_LOG}
HOST: ${QUADRATIC_CONNECTION_HOST}
PORT: ${QUADRATIC_CONNECTION_PORT}
ENVIRONMENT: ${ENVIRONMENT}
diff --git a/infra/aws-cloudformation/quadratic-selfhost.yml b/infra/aws-cloudformation/quadratic-selfhost.yml
index b5416d940d..a5410a2c08 100644
--- a/infra/aws-cloudformation/quadratic-selfhost.yml
+++ b/infra/aws-cloudformation/quadratic-selfhost.yml
@@ -1,5 +1,5 @@
AWSTemplateFormatVersion: 2010-09-09
-Description: Quadratic Preview - DockerDeployment Template
+Description: Quadratic Preview - Docker Deployment Template
Parameters:
ImageTag:
@@ -109,15 +109,14 @@ Resources:
LICENSE_KEY="$(aws ssm get-parameter --name "/quadratic-development/QUADRATIC_LICENSE_KEY" --with-decryption --query "Parameter.Value" --output text)"
# Generate Domain Name
- DOMAIN_NAME="${ImageTag}.quadratic-selfhost.com"
+ DOMAIN_NAME="${ImageTag}.quadratic-preview.com"
# Run Quadratic initialization script
chmod +x init.sh
./init.sh $LICENSE_KEY $DOMAIN_NAME
# Append environment variables to .env file
- cat << EOF >> /quadratic-selfhost/.env.docker
-
+ cat << EOF >> /quadratic-selfhost/.env
AWS_S3_REGION=$(aws ssm get-parameter --name "/quadratic-development/AWS_S3_REGION" --with-decryption --query "Parameter.Value" --output text)
AWS_S3_ACCESS_KEY_ID=$(aws ssm get-parameter --name "/quadratic-development/AWS_S3_ACCESS_KEY_ID" --with-decryption --query "Parameter.Value" --output text)
AWS_S3_SECRET_ACCESS_KEY=$(aws ssm get-parameter --name "/quadratic-development/AWS_S3_SECRET_ACCESS_KEY" --with-decryption --query "Parameter.Value" --output text)
@@ -190,8 +189,8 @@ Resources:
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
Properties:
- HostedZoneId: Z08322143IM80L9CZZQV5
- Name: !Sub "${ImageTag}.quadratic-selfhost.com."
+ HostedZoneId: Z0126430TJ1UYIMO3SYX
+ Name: !Sub "${ImageTag}.quadratic-preview.com."
Type: A
AliasTarget:
DNSName: !GetAtt GlobalAccelerator.DnsName
@@ -204,8 +203,8 @@ Resources:
DeletionPolicy: Delete
UpdateReplacePolicy: Delete
Properties:
- HostedZoneId: Z08322143IM80L9CZZQV5
- Name: !Sub "*.${ImageTag}.quadratic-selfhost.com."
+ HostedZoneId: Z0126430TJ1UYIMO3SYX
+ Name: !Sub "*.${ImageTag}.quadratic-preview.com."
Type: A
AliasTarget:
DNSName: !GetAtt GlobalAccelerator.DnsName
@@ -215,4 +214,4 @@ Resources:
Outputs:
WebsiteURL:
Description: Website URL
- Value: !Sub "https://${ImageTag}.quadratic-selfhost.com"
+ Value: !Sub "https://${ImageTag}.quadratic-preview.com"
From f38235326f0ef9306dbb199d6094e409ef8ce366 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Fri, 3 Jan 2025 03:23:46 +0530
Subject: [PATCH 144/155] undo qa-wolf concurrency
---
.github/workflows/qa-wolf-notify.yml | 3 ---
1 file changed, 3 deletions(-)
diff --git a/.github/workflows/qa-wolf-notify.yml b/.github/workflows/qa-wolf-notify.yml
index b03148cc0a..4db6ff0c3c 100644
--- a/.github/workflows/qa-wolf-notify.yml
+++ b/.github/workflows/qa-wolf-notify.yml
@@ -5,9 +5,6 @@ on:
branches:
- qa
-concurrency:
- group: qa-wolf-notify
-
jobs:
qa-wolf-notify:
runs-on: blacksmith-2vcpu-ubuntu-2204
From 936ae77f05cb8ab61ab8a4ffb3d7b4b16963dac2 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Fri, 3 Jan 2025 04:22:52 +0530
Subject: [PATCH 145/155] re-arrange builder steps
---
quadratic-client/Dockerfile | 22 +++++++++++++---------
1 file changed, 13 insertions(+), 9 deletions(-)
diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile
index bb208ac268..25fcb695a9 100644
--- a/quadratic-client/Dockerfile
+++ b/quadratic-client/Dockerfile
@@ -148,22 +148,26 @@ RUN apt-get update && apt-get install -y --no-install-recommends python-is-pytho
WORKDIR /quadratic
-# Copy all files
+# Copy updateAlertVersion.json
COPY updateAlertVersion.json .
-COPY package.json .
-COPY package-lock.json .
-COPY ./quadratic-core/. ./quadratic-core/
-COPY ./quadratic-rust-client/. ./quadratic-rust-client/
-COPY ./quadratic-kernels/python-wasm/. ./quadratic-kernels/python-wasm/
-COPY ./quadratic-shared/. ./quadratic-shared/
-COPY ./quadratic-client/. ./quadratic-client/
# Copy files from core, ts/rust types, and rust client builders
-COPY --from=dependencies-builder /quadratic/. ./
COPY --from=core-builder /quadratic/quadratic-client/src/app/quadratic-core/. ./quadratic-client/src/app/quadratic-core
COPY --from=ts-rust-types-builder /quadratic/quadratic-client/src/app/quadratic-core-types/. ./quadratic-client/src/app/quadratic-core-types
COPY --from=rust-client-builder /quadratic/quadratic-client/src/app/quadratic-rust-client/. ./quadratic-client/src/app/quadratic-rust-client
+# Copy package.json and package-lock.json
+COPY package.json .
+COPY package-lock.json .
+
+# Copy npm dependencies from dependencies-builder
+COPY --from=dependencies-builder /quadratic/. ./
+
+# Copy remaining files
+COPY ./quadratic-kernels/python-wasm/. ./quadratic-kernels/python-wasm/
+COPY ./quadratic-shared/. ./quadratic-shared/
+COPY ./quadratic-client/. ./quadratic-client/
+
# Run the packaging script for quadratic_py
RUN ./quadratic-kernels/python-wasm/package.sh --no-poetry
From e038c6a1613c8740adc9a71269008666ef1c7d26 Mon Sep 17 00:00:00 2001
From: Jim Nielsen
Date: Thu, 2 Jan 2025 16:30:52 -0700
Subject: [PATCH 146/155] don't show empty grid message on mobile
---
.../src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx b/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
index bc91b026b8..6dfdeeb7ce 100644
--- a/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
+++ b/quadratic-client/src/app/gridGL/HTMLGrid/EmptyGridMessage.tsx
@@ -10,6 +10,7 @@ import { useFileImport } from '@/app/ui/hooks/useFileImport';
import { useFileRouteLoaderData } from '@/shared/hooks/useFileRouteLoaderData';
import { Button } from '@/shared/shadcn/ui/button';
import { useEffect, useRef, useState } from 'react';
+import { isMobile } from 'react-device-detect';
import { useSetRecoilState } from 'recoil';
const fileHasData = () => sheets.sheets.filter((sheet) => sheet.bounds.type === 'nonEmpty').length > 0;
@@ -48,6 +49,10 @@ export function EmptyGridMessage() {
return null;
}
+ if (isMobile) {
+ return null;
+ }
+
return (
From 538efdda14432a770c0f2923a5c8292d55e95467 Mon Sep 17 00:00:00 2001
From: Jim Nielsen
Date: Thu, 2 Jan 2025 16:31:13 -0700
Subject: [PATCH 147/155] bump up the toast
so it doesn't display over the sheet bar and instead just the grid itself
---
.../src/shared/components/GlobalSnackbarProvider.tsx | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/quadratic-client/src/shared/components/GlobalSnackbarProvider.tsx b/quadratic-client/src/shared/components/GlobalSnackbarProvider.tsx
index eb274459d0..e64d2eb556 100644
--- a/quadratic-client/src/shared/components/GlobalSnackbarProvider.tsx
+++ b/quadratic-client/src/shared/components/GlobalSnackbarProvider.tsx
@@ -170,6 +170,10 @@ export function GlobalSnackbarProvider({ children }: { children: React.ReactElem
backgroundColor: 'hsl(var(--foreground))',
color: 'hsl(var(--background))',
},
+ // Override this so it sits over the sheetbar
+ '&.MuiSnackbar-anchorOriginBottomCenter': {
+ bottom: '66px !important',
+ },
}}
anchorOrigin={{ vertical: 'bottom', horizontal: 'center' }}
autoHideDuration={stayOpen ? null : DURATION}
From 5476d8ca99b468233bb9144b60346aa0faf3b700 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Fri, 3 Jan 2025 05:26:46 +0530
Subject: [PATCH 148/155] try 16
---
.github/workflows/staging-build-deploy-images.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/staging-build-deploy-images.yml b/.github/workflows/staging-build-deploy-images.yml
index ac406ef502..0903d7be57 100644
--- a/.github/workflows/staging-build-deploy-images.yml
+++ b/.github/workflows/staging-build-deploy-images.yml
@@ -20,7 +20,7 @@ jobs:
matrix:
include:
- service: client
- runner: blacksmith-8vcpu-ubuntu-2204
+ runner: blacksmith-16vcpu-ubuntu-2204
- service: api
runner: blacksmith-2vcpu-ubuntu-2204
- service: connection
From 26d45d8d2845958f9a6c53935ec8b1e9a96c1db2 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Fri, 3 Jan 2025 06:22:02 +0530
Subject: [PATCH 149/155] unbounded marching ants
---
.../UI/cellHighlights/cellHighlightsDraw.ts | 47 +++++++++++--------
1 file changed, 27 insertions(+), 20 deletions(-)
diff --git a/quadratic-client/src/app/gridGL/UI/cellHighlights/cellHighlightsDraw.ts b/quadratic-client/src/app/gridGL/UI/cellHighlights/cellHighlightsDraw.ts
index c30c613bab..2d8276c545 100644
--- a/quadratic-client/src/app/gridGL/UI/cellHighlights/cellHighlightsDraw.ts
+++ b/quadratic-client/src/app/gridGL/UI/cellHighlights/cellHighlightsDraw.ts
@@ -79,6 +79,9 @@ export function drawDashedRectangleMarching(options: {
const maxX = selectionRect.right - offset;
const maxY = selectionRect.bottom - offset;
+ const boundedMaxX = Math.min(maxX, bounds.right);
+ const boundedMaxY = Math.min(maxY, bounds.bottom);
+
if (!noFill) {
g.clear();
}
@@ -88,7 +91,7 @@ export function drawDashedRectangleMarching(options: {
});
if (!noFill) {
g.beginFill(color, FILL_ALPHA);
- g.drawRect(minX, minY, maxX - minX, maxY - minY);
+ g.drawRect(minX, minY, boundedMaxX - minX, boundedMaxY - minY);
g.endFill();
}
@@ -111,34 +114,38 @@ export function drawDashedRectangleMarching(options: {
let wrapAmount = 0;
// draw top line
- for (let x = minX + march; x <= maxX - DASHED / 2; x += DASHED) {
- g.moveTo(clamp(x, minX, maxX), minY);
- g.lineTo(clamp(x + DASHED / 2, minX, maxX), minY);
- wrapAmount = x - (maxX - DASHED / 2);
+ for (let x = minX + march; x <= boundedMaxX - DASHED / 2; x += DASHED) {
+ g.moveTo(clamp(x, minX, boundedMaxX), minY);
+ g.lineTo(clamp(x + DASHED / 2, minX, boundedMaxX), minY);
+ wrapAmount = x - (boundedMaxX - DASHED / 2);
}
- // draw right line
- for (let y = minY + wrapAmount; y <= maxY - DASHED / 2; y += DASHED) {
- if (y + DASHED / 2 > minY + DASHED_THICKNESS) {
- g.moveTo(maxX, clamp(y, minY, maxY));
- g.lineTo(maxX, clamp(y + DASHED / 2, minY, maxY));
- wrapAmount = y + DASHED / 2 - maxY;
+ if (maxX <= boundedMaxX) {
+ // draw right line
+ for (let y = minY + wrapAmount; y <= boundedMaxY - DASHED / 2; y += DASHED) {
+ if (y + DASHED / 2 > minY + DASHED_THICKNESS) {
+ g.moveTo(boundedMaxX, clamp(y, minY, boundedMaxY));
+ g.lineTo(boundedMaxX, clamp(y + DASHED / 2, minY, boundedMaxY));
+ wrapAmount = y + DASHED / 2 - boundedMaxY;
+ }
}
}
- // draw bottom line
- for (let x = maxX - wrapAmount; x >= minX + DASHED / 2; x -= DASHED) {
- if (x - DASHED / 2 < maxX - DASHED_THICKNESS) {
- g.moveTo(clamp(x - DASHED / 2, minX, maxX - DASHED_THICKNESS), maxY - DASHED_THICKNESS);
- g.lineTo(clamp(x, minX, maxX), maxY - DASHED_THICKNESS);
+ if (maxY <= boundedMaxY) {
+ // draw bottom line
+ for (let x = boundedMaxX - wrapAmount; x >= minX + DASHED / 2; x -= DASHED) {
+ if (x - DASHED / 2 < boundedMaxX - DASHED_THICKNESS) {
+ g.moveTo(clamp(x - DASHED / 2, minX, boundedMaxX - DASHED_THICKNESS), boundedMaxY - DASHED_THICKNESS);
+ g.lineTo(clamp(x, minX, boundedMaxX), boundedMaxY - DASHED_THICKNESS);
+ }
+ wrapAmount = minX - x - DASHED / 2;
}
- wrapAmount = minX - x - DASHED / 2;
}
// draw left line
- for (let y = maxY - wrapAmount; y >= minY + DASHED / 2; y -= DASHED) {
- g.moveTo(minX + DASHED_THICKNESS, clamp(y - DASHED / 2, minY, maxY));
- g.lineTo(minX + DASHED_THICKNESS, clamp(y, minY, maxY));
+ for (let y = boundedMaxY - wrapAmount; y >= minY + DASHED / 2; y -= DASHED) {
+ g.moveTo(minX + DASHED_THICKNESS, clamp(y - DASHED / 2, minY, boundedMaxY));
+ g.lineTo(minX + DASHED_THICKNESS, clamp(y, minY, boundedMaxY));
}
return true;
From 780ebb0a618bba78242f30fbae6658b8d242da92 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Fri, 3 Jan 2025 07:24:14 +0530
Subject: [PATCH 150/155] update bounds on code run
---
.vscode/settings.json | 2 +
.../src/controller/execution/run_code/mod.rs | 2 +-
quadratic-core/src/controller/send_render.rs | 16 ------
quadratic-core/src/grid/sheet/bounds.rs | 55 +------------------
4 files changed, 4 insertions(+), 71 deletions(-)
diff --git a/.vscode/settings.json b/.vscode/settings.json
index 1d8a0dafbb..668f82a699 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -41,12 +41,14 @@
"pixi",
"pixiapp",
"Plotly",
+ "proptest",
"pulumi",
"pyimport",
"rects",
"RELCELL",
"relcells",
"reqwest",
+ "rfind",
"scrollend",
"shadcn",
"Signin",
diff --git a/quadratic-core/src/controller/execution/run_code/mod.rs b/quadratic-core/src/controller/execution/run_code/mod.rs
index b9e112d7ed..58b1e038bb 100644
--- a/quadratic-core/src/controller/execution/run_code/mod.rs
+++ b/quadratic-core/src/controller/execution/run_code/mod.rs
@@ -81,7 +81,7 @@ impl GridController {
transaction.add_from_code_run(sheet_id, pos, &old_code_run);
transaction.add_from_code_run(sheet_id, pos, &new_code_run);
- self.send_updated_bounds_rect(&sheet_rect, false);
+ self.send_updated_bounds(sheet_rect.sheet_id);
transaction.add_dirty_hashes_from_sheet_rect(sheet_rect);
if transaction.is_user() {
if let Some(sheet) = self.try_sheet(sheet_id) {
diff --git a/quadratic-core/src/controller/send_render.rs b/quadratic-core/src/controller/send_render.rs
index c86c814372..360be26f52 100644
--- a/quadratic-core/src/controller/send_render.rs
+++ b/quadratic-core/src/controller/send_render.rs
@@ -213,22 +213,6 @@ impl GridController {
}
}
- pub fn send_updated_bounds_rect(&mut self, sheet_rect: &SheetRect, format: bool) {
- let recalculated = if let Some(sheet) = self.try_sheet_mut(sheet_rect.sheet_id) {
- sheet.recalculate_add_bounds((*sheet_rect).into(), format)
- } else {
- false
- };
-
- if cfg!(target_family = "wasm") && recalculated {
- if let Some(sheet) = self.try_sheet(sheet_rect.sheet_id) {
- if let Ok(sheet_info) = serde_json::to_string(&SheetBounds::from(sheet)) {
- crate::wasm_bindings::js::jsSheetBoundsUpdate(sheet_info);
- }
- }
- };
- }
-
/// Recalculates sheet bounds, and if changed then sends to TS.
pub fn send_updated_bounds(&mut self, sheet_id: SheetId) {
let recalculated = if let Some(sheet) = self.try_sheet_mut(sheet_id) {
diff --git a/quadratic-core/src/grid/sheet/bounds.rs b/quadratic-core/src/grid/sheet/bounds.rs
index 6898a7ccbd..d430861db0 100644
--- a/quadratic-core/src/grid/sheet/bounds.rs
+++ b/quadratic-core/src/grid/sheet/bounds.rs
@@ -51,23 +51,6 @@ impl Sheet {
|| old_format_bounds != self.format_bounds.to_bounds_rect()
}
- /// Adds a SheetRect to the bounds of the sheet.
- ///
- /// Returns whether any of the sheet's bounds has changed
- pub fn recalculate_add_bounds(&mut self, rect: Rect, format: bool) -> bool {
- if format {
- let old_format_bounds = self.format_bounds.to_bounds_rect();
- self.format_bounds.add(rect.min);
- self.format_bounds.add(rect.max);
- old_format_bounds != self.format_bounds.to_bounds_rect()
- } else {
- let old_data_bounds = self.format_bounds.to_bounds_rect();
- self.data_bounds.add(rect.min);
- self.data_bounds.add(rect.max);
- old_data_bounds != self.data_bounds.to_bounds_rect()
- }
- }
-
/// Returns whether the sheet is completely empty.
pub fn is_empty(&self) -> bool {
self.data_bounds.is_empty() && self.format_bounds.is_empty()
@@ -521,7 +504,7 @@ mod test {
},
CellAlign, CellWrap, CodeCellLanguage, GridBounds, Sheet,
},
- A1Selection, Array, CellValue, Pos, Rect, SheetPos, SheetRect,
+ A1Selection, Array, CellValue, Pos, Rect, SheetPos,
};
use proptest::proptest;
use std::collections::HashMap;
@@ -924,42 +907,6 @@ mod test {
assert_eq!(sheet.row_bounds(2, false), Some((1, 1)));
}
- #[test]
- fn send_updated_bounds_rect() {
- let mut gc = GridController::test();
- let sheet_id = gc.sheet_ids()[0];
- gc.set_cell_value(
- SheetPos {
- x: 1,
- y: 2,
- sheet_id,
- },
- "test".to_string(),
- None,
- );
- let sheet = gc.sheet(sheet_id);
- assert_eq!(
- sheet.data_bounds,
- GridBounds::NonEmpty(Rect::from_numbers(1, 2, 1, 1))
- );
- gc.set_bold(
- &A1Selection::from_rect(SheetRect::from_numbers(3, 5, 1, 1, sheet_id)),
- true,
- None,
- )
- .unwrap();
-
- let sheet = gc.sheet(sheet_id);
- assert_eq!(
- sheet.bounds(true),
- GridBounds::NonEmpty(Rect::from_numbers(1, 2, 1, 1))
- );
- assert_eq!(
- sheet.bounds(false),
- GridBounds::NonEmpty(Rect::from_numbers(1, 2, 3, 4))
- );
- }
-
#[test]
fn test_row_bounds_with_formats() {
let mut gc = GridController::test();
From 2b4ea8a3a9af928ea9635a5a87c19412cb86b6ec Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Fri, 3 Jan 2025 07:39:27 +0530
Subject: [PATCH 151/155] clippy
---
.../src/controller/execution/run_code/get_cells.rs | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/quadratic-core/src/controller/execution/run_code/get_cells.rs b/quadratic-core/src/controller/execution/run_code/get_cells.rs
index df5ca1b741..6c5e2d6c39 100644
--- a/quadratic-core/src/controller/execution/run_code/get_cells.rs
+++ b/quadratic-core/src/controller/execution/run_code/get_cells.rs
@@ -514,31 +514,31 @@ mod test {
let result = gc
.calculation_get_cells_a1(transaction_id.to_string(), "B:".to_string(), None)
.unwrap();
- assert_eq!(result.two_dimensional, true);
+ assert!(result.two_dimensional);
let result = gc
.calculation_get_cells_a1(transaction_id.to_string(), "B".to_string(), None)
.unwrap();
- assert_eq!(result.two_dimensional, false);
+ assert!(!result.two_dimensional);
let result = gc
.calculation_get_cells_a1(transaction_id.to_string(), "2:".to_string(), None)
.unwrap();
- assert_eq!(result.two_dimensional, true);
+ assert!(result.two_dimensional);
let result = gc
.calculation_get_cells_a1(transaction_id.to_string(), "2".to_string(), None)
.unwrap();
- assert_eq!(result.two_dimensional, false);
+ assert!(!result.two_dimensional);
let result = gc
.calculation_get_cells_a1(transaction_id.to_string(), "D5:E5".to_string(), None)
.unwrap();
- assert_eq!(result.two_dimensional, false);
+ assert!(!result.two_dimensional);
let result = gc
.calculation_get_cells_a1(transaction_id.to_string(), "D5:".to_string(), None)
.unwrap();
- assert_eq!(result.two_dimensional, true);
+ assert!(result.two_dimensional);
}
}
From 627249590484a8e3597d443a43ac37be26a0e862 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Fri, 3 Jan 2025 15:41:50 +0530
Subject: [PATCH 152/155] fix JSONDecodeError
---
.../UI/cellHighlights/cellHighlightsDraw.ts | 55 ++++++++++---------
.../pythonWebWorker/worker/pythonClient.ts | 7 ++-
2 files changed, 35 insertions(+), 27 deletions(-)
diff --git a/quadratic-client/src/app/gridGL/UI/cellHighlights/cellHighlightsDraw.ts b/quadratic-client/src/app/gridGL/UI/cellHighlights/cellHighlightsDraw.ts
index 2d8276c545..8bb23104fa 100644
--- a/quadratic-client/src/app/gridGL/UI/cellHighlights/cellHighlightsDraw.ts
+++ b/quadratic-client/src/app/gridGL/UI/cellHighlights/cellHighlightsDraw.ts
@@ -15,6 +15,9 @@ export function drawDashedRectangle(options: { g: Graphics; color: number; isSel
return;
}
+ const boundedRight = Math.min(selectionRect.right, bounds.right);
+ const boundedBottom = Math.min(selectionRect.bottom, bounds.bottom);
+
g.lineStyle({
width: CURSOR_THICKNESS,
color,
@@ -22,9 +25,9 @@ export function drawDashedRectangle(options: { g: Graphics; color: number; isSel
texture: generatedTextures.dashedHorizontal,
});
g.moveTo(selectionRect.left, selectionRect.top);
- g.lineTo(Math.min(selectionRect.right, bounds.right), selectionRect.top);
+ g.lineTo(boundedRight, selectionRect.top);
if (selectionRect.bottom <= bounds.bottom) {
- g.moveTo(Math.min(selectionRect.right, bounds.right), selectionRect.bottom);
+ g.moveTo(boundedRight, selectionRect.bottom);
g.lineTo(selectionRect.left, selectionRect.bottom);
}
@@ -34,10 +37,10 @@ export function drawDashedRectangle(options: { g: Graphics; color: number; isSel
alignment: 0.5,
texture: generatedTextures.dashedVertical,
});
- g.moveTo(selectionRect.left, Math.min(selectionRect.bottom, bounds.bottom));
+ g.moveTo(selectionRect.left, boundedBottom);
g.lineTo(selectionRect.left, selectionRect.top);
if (selectionRect.right <= bounds.right) {
- g.moveTo(selectionRect.right, Math.min(selectionRect.bottom, bounds.bottom));
+ g.moveTo(selectionRect.right, boundedBottom);
g.lineTo(selectionRect.right, selectionRect.top);
}
@@ -50,8 +53,8 @@ export function drawDashedRectangle(options: { g: Graphics; color: number; isSel
g.drawRect(
selectionRect.left,
selectionRect.top,
- Math.min(selectionRect.right, bounds.right) - selectionRect.left,
- Math.min(selectionRect.bottom, bounds.bottom) - selectionRect.top
+ boundedRight - selectionRect.left,
+ boundedBottom - selectionRect.top
);
g.endFill();
}
@@ -79,8 +82,8 @@ export function drawDashedRectangleMarching(options: {
const maxX = selectionRect.right - offset;
const maxY = selectionRect.bottom - offset;
- const boundedMaxX = Math.min(maxX, bounds.right);
- const boundedMaxY = Math.min(maxY, bounds.bottom);
+ const boundedRight = Math.min(maxX, bounds.right);
+ const boundedBottom = Math.min(maxY, bounds.bottom);
if (!noFill) {
g.clear();
@@ -91,7 +94,7 @@ export function drawDashedRectangleMarching(options: {
});
if (!noFill) {
g.beginFill(color, FILL_ALPHA);
- g.drawRect(minX, minY, boundedMaxX - minX, boundedMaxY - minY);
+ g.drawRect(minX, minY, boundedRight - minX, boundedBottom - minY);
g.endFill();
}
@@ -114,38 +117,38 @@ export function drawDashedRectangleMarching(options: {
let wrapAmount = 0;
// draw top line
- for (let x = minX + march; x <= boundedMaxX - DASHED / 2; x += DASHED) {
- g.moveTo(clamp(x, minX, boundedMaxX), minY);
- g.lineTo(clamp(x + DASHED / 2, minX, boundedMaxX), minY);
- wrapAmount = x - (boundedMaxX - DASHED / 2);
+ for (let x = minX + march; x <= boundedRight - DASHED / 2; x += DASHED) {
+ g.moveTo(clamp(x, minX, boundedRight), minY);
+ g.lineTo(clamp(x + DASHED / 2, minX, boundedRight), minY);
+ wrapAmount = x - (boundedRight - DASHED / 2);
}
- if (maxX <= boundedMaxX) {
+ if (maxX <= boundedRight) {
// draw right line
- for (let y = minY + wrapAmount; y <= boundedMaxY - DASHED / 2; y += DASHED) {
+ for (let y = minY + wrapAmount; y <= boundedBottom - DASHED / 2; y += DASHED) {
if (y + DASHED / 2 > minY + DASHED_THICKNESS) {
- g.moveTo(boundedMaxX, clamp(y, minY, boundedMaxY));
- g.lineTo(boundedMaxX, clamp(y + DASHED / 2, minY, boundedMaxY));
- wrapAmount = y + DASHED / 2 - boundedMaxY;
+ g.moveTo(boundedRight, clamp(y, minY, boundedBottom));
+ g.lineTo(boundedRight, clamp(y + DASHED / 2, minY, boundedBottom));
+ wrapAmount = y + DASHED / 2 - boundedBottom;
}
}
}
- if (maxY <= boundedMaxY) {
+ if (maxY <= boundedBottom) {
// draw bottom line
- for (let x = boundedMaxX - wrapAmount; x >= minX + DASHED / 2; x -= DASHED) {
- if (x - DASHED / 2 < boundedMaxX - DASHED_THICKNESS) {
- g.moveTo(clamp(x - DASHED / 2, minX, boundedMaxX - DASHED_THICKNESS), boundedMaxY - DASHED_THICKNESS);
- g.lineTo(clamp(x, minX, boundedMaxX), boundedMaxY - DASHED_THICKNESS);
+ for (let x = boundedRight - wrapAmount; x >= minX + DASHED / 2; x -= DASHED) {
+ if (x - DASHED / 2 < boundedRight - DASHED_THICKNESS) {
+ g.moveTo(clamp(x - DASHED / 2, minX, boundedRight - DASHED_THICKNESS), boundedBottom - DASHED_THICKNESS);
+ g.lineTo(clamp(x, minX, boundedRight), boundedBottom - DASHED_THICKNESS);
}
wrapAmount = minX - x - DASHED / 2;
}
}
// draw left line
- for (let y = boundedMaxY - wrapAmount; y >= minY + DASHED / 2; y -= DASHED) {
- g.moveTo(minX + DASHED_THICKNESS, clamp(y - DASHED / 2, minY, boundedMaxY));
- g.lineTo(minX + DASHED_THICKNESS, clamp(y, minY, boundedMaxY));
+ for (let y = boundedBottom - wrapAmount; y >= minY + DASHED / 2; y -= DASHED) {
+ g.moveTo(minX + DASHED_THICKNESS, clamp(y - DASHED / 2, minY, boundedBottom));
+ g.lineTo(minX + DASHED_THICKNESS, clamp(y, minY, boundedBottom));
}
return true;
diff --git a/quadratic-client/src/app/web-workers/pythonWebWorker/worker/pythonClient.ts b/quadratic-client/src/app/web-workers/pythonWebWorker/worker/pythonClient.ts
index 591b904c74..a930689447 100644
--- a/quadratic-client/src/app/web-workers/pythonWebWorker/worker/pythonClient.ts
+++ b/quadratic-client/src/app/web-workers/pythonWebWorker/worker/pythonClient.ts
@@ -14,7 +14,11 @@ declare var self: WorkerGlobalScope & typeof globalThis & {};
class PythonClient {
private id = 0;
private waitingForResponse: Record = {};
- env: Record = {};
+ private _env: Record = {};
+
+ get env() {
+ return this._env;
+ }
start() {
self.onmessage = this.handleMessage;
@@ -38,6 +42,7 @@ class PythonClient {
break;
case 'clientPythonInit':
+ this._env = e.data.env;
return;
default:
From 932fe0ef674aa0b0d7d946c1d765c7e2c7f25491 Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 4 Jan 2025 02:31:01 +0530
Subject: [PATCH 153/155] dk feedback
---
.github/workflows/ci.yml | 2 +-
.github/workflows/production-publish-images.yml | 1 -
.github/workflows/staging-create-stack.yml | 2 +-
.../{quadratic-selfhost.yml => quadratic-preview.yml} | 0
4 files changed, 2 insertions(+), 3 deletions(-)
rename infra/aws-cloudformation/{quadratic-selfhost.yml => quadratic-preview.yml} (100%)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 15484f52d0..168a7dfdc3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -55,7 +55,7 @@ jobs:
CARGO_BUILD_JOBS: 4
run: |
cd quadratic-core
- cargo test -- --test-threads=4
+ cargo test -- --test-threads=2
- name: Generate coverage for quadratic-core
env:
diff --git a/.github/workflows/production-publish-images.yml b/.github/workflows/production-publish-images.yml
index 4d9801d17a..79c20b0abd 100644
--- a/.github/workflows/production-publish-images.yml
+++ b/.github/workflows/production-publish-images.yml
@@ -7,7 +7,6 @@ on:
concurrency:
group: production-publish-images
- cancel-in-progress: true
jobs:
publish_images:
diff --git a/.github/workflows/staging-create-stack.yml b/.github/workflows/staging-create-stack.yml
index 92c9127df4..86593bb257 100644
--- a/.github/workflows/staging-create-stack.yml
+++ b/.github/workflows/staging-create-stack.yml
@@ -59,7 +59,7 @@ jobs:
uses: aws-actions/aws-cloudformation-github-deploy@v1
with:
name: ${{ env.STACK_NAME }}
- template: infra/aws-cloudformation/quadratic-selfhost.yml
+ template: infra/aws-cloudformation/quadratic-preview.yml
parameter-overrides: >-
ImageTag=pr-${{ github.event.pull_request.number }}
capabilities: CAPABILITY_IAM
diff --git a/infra/aws-cloudformation/quadratic-selfhost.yml b/infra/aws-cloudformation/quadratic-preview.yml
similarity index 100%
rename from infra/aws-cloudformation/quadratic-selfhost.yml
rename to infra/aws-cloudformation/quadratic-preview.yml
From dfc2464fb2aef21a492840c5dc85d969c3017fda Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 4 Jan 2025 02:35:15 +0530
Subject: [PATCH 154/155] ci concurrency
---
.github/workflows/ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 168a7dfdc3..c1c8b26bb7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -11,7 +11,7 @@ on:
pull_request:
concurrency:
- group: ci
+ group: ci-${{ github.event.pull_request.number || github.ref_name }}
cancel-in-progress: true
jobs:
From e63fcc1509ed681de995b5c44b95dd8ca466a20a Mon Sep 17 00:00:00 2001
From: AyushAgrawal-A2
Date: Sat, 4 Jan 2025 02:50:42 +0530
Subject: [PATCH 155/155] rename DEV to CLIENT_DEV
---
.env.docker | 2 +-
.github/workflows/staging-build-deploy-images.yml | 8 ++++----
.vscode/settings.json | 3 ++-
docker-compose.build.yml | 2 +-
package.json | 12 ++++++------
quadratic-client/Dockerfile | 8 ++++----
quadratic-connection/package.json | 6 +++---
quadratic-multiplayer/package.json | 6 +++---
8 files changed, 24 insertions(+), 23 deletions(-)
diff --git a/.env.docker b/.env.docker
index e7f1bdf11d..60921cf12a 100644
--- a/.env.docker
+++ b/.env.docker
@@ -103,7 +103,7 @@ EXA_API_KEY=EXA_API_KEY
ECR_OR_BUILD=build
# build client in without wasm-opt
-DEV=false
+CLIENT_DEV=false
ECR_URL=public.ecr.aws/z7e3d4w1
IMAGE_TAG=latest
\ No newline at end of file
diff --git a/.github/workflows/staging-build-deploy-images.yml b/.github/workflows/staging-build-deploy-images.yml
index 0903d7be57..8c3f414278 100644
--- a/.github/workflows/staging-build-deploy-images.yml
+++ b/.github/workflows/staging-build-deploy-images.yml
@@ -38,9 +38,9 @@ jobs:
id: build-dev
run: |
if [[ ${{ contains(github.event.pull_request.labels.*.name, 'build:dev') }} == true ]]; then
- echo "DEV=true" >> $GITHUB_OUTPUT
+ echo "CLIENT_DEV=true" >> $GITHUB_OUTPUT
else
- echo "DEV=false" >> $GITHUB_OUTPUT
+ echo "CLIENT_DEV=false" >> $GITHUB_OUTPUT
fi
- name: Check for build:cache:disabled label
@@ -112,7 +112,7 @@ jobs:
GIT_SHA_SHORT=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
BRANCH_NAME=${{ steps.build-metadata.outputs.BRANCH_NAME }}
PR_NUMBER=${{ github.event.pull_request.number }}
- DEV=${{ steps.build-dev.outputs.DEV }}
+ CLIENT_DEV=${{ steps.build-dev.outputs.CLIENT_DEV }}
labels: |
org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}
org.opencontainers.image.revision=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
@@ -131,7 +131,7 @@ jobs:
GIT_SHA_SHORT=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
BRANCH_NAME=${{ steps.build-metadata.outputs.BRANCH_NAME }}
PR_NUMBER=${{ github.event.pull_request.number }}
- DEV=${{ steps.build-dev.outputs.DEV }}
+ CLIENT_DEV=${{ steps.build-dev.outputs.CLIENT_DEV }}
labels: |
org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}
org.opencontainers.image.revision=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }}
diff --git a/.vscode/settings.json b/.vscode/settings.json
index 2caf40cd2f..b8cf8b846f 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -53,6 +53,7 @@
"nonblank",
"Northbridge",
"openai",
+ "opencontainers",
"opensans",
"oryd",
"peekable",
@@ -68,8 +69,8 @@
"RELCELL",
"relcells",
"reqwest",
- "rustup",
"rfind",
+ "rustup",
"scrollend",
"selfhost",
"selfhosted",
diff --git a/docker-compose.build.yml b/docker-compose.build.yml
index 55877651d1..8bec62417c 100644
--- a/docker-compose.build.yml
+++ b/docker-compose.build.yml
@@ -8,7 +8,7 @@ services:
context: .
dockerfile: quadratic-client/Dockerfile
args:
- DEV: ${DEV}
+ CLIENT_DEV: ${CLIENT_DEV}
quadratic-api:
build:
diff --git a/package.json b/package.json
index 3aef9843cb..02570cac58 100644
--- a/package.json
+++ b/package.json
@@ -60,12 +60,12 @@
"copy:esbuild": "cp -f node_modules/esbuild-wasm/esbuild.wasm quadratic-client/public/",
"gen:pyright:initialization": "npm run gen:pyright:worker --workspace=quadratic-kernels/python-wasm",
"gen:pyright:worker": "npm run gen:pyright:worker --workspace=quadratic-kernels/python-wasm",
- "docker:base": "ECR_OR_BUILD=build DEV=false docker compose --profile base --env-file .env.docker up",
- "docker:dev": "ECR_OR_BUILD=build DEV=true docker compose --profile dev --env-file .env.docker up -d && docker attach quadratic-dev",
- "docker:build": "ECR_OR_BUILD=build DEV=false docker compose --profile all --env-file .env.docker up",
- "docker:build:dev": "ECR_OR_BUILD=build DEV=true docker compose --profile all --env-file .env.docker up",
- "docker:ecr": "ECR_OR_BUILD=ecr DEV=false docker compose --profile all --env-file .env.docker up",
- "docker:down": "ECR_OR_BUILD=build DEV=false docker compose --profile all --env-file .env.docker down",
+ "docker:base": "ECR_OR_BUILD=build CLIENT_DEV=false docker compose --profile base --env-file .env.docker up",
+ "docker:dev": "ECR_OR_BUILD=build CLIENT_DEV=true docker compose --profile dev --env-file .env.docker up -d && docker attach quadratic-dev",
+ "docker:build": "ECR_OR_BUILD=build CLIENT_DEV=false docker compose --profile all --env-file .env.docker up",
+ "docker:build:dev": "ECR_OR_BUILD=build CLIENT_DEV=true docker compose --profile all --env-file .env.docker up",
+ "docker:ecr": "ECR_OR_BUILD=ecr CLIENT_DEV=false docker compose --profile all --env-file .env.docker up",
+ "docker:down": "ECR_OR_BUILD=build CLIENT_DEV=false docker compose --profile all --env-file .env.docker down",
"docker:clean": "docker system prune -af && docker builder prune -af && docker volume prune -af"
},
"dependencies": {
diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile
index 25fcb695a9..711aa49731 100644
--- a/quadratic-client/Dockerfile
+++ b/quadratic-client/Dockerfile
@@ -26,7 +26,7 @@ RUN npm install --no-audit --no-fund
###########################################################################################
FROM node:18 AS core-builder
-ARG DEV
+ARG CLIENT_DEV
# Install rustup
RUN echo 'Installing rustup...' && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
@@ -54,7 +54,7 @@ COPY ./quadratic-core/. ./quadratic-core/
COPY ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts
# Build core
-RUN if [ "$DEV" = "true" ]; then \
+RUN if [ "$CLIENT_DEV" = "true" ]; then \
echo 'Building core in dev mode...' && npm run build:dev --workspace=quadratic-core; \
else \
echo 'Building core...' && npm run build --workspace=quadratic-core; \
@@ -100,7 +100,7 @@ RUN echo 'Building TS/Rust types...' && npm run export_types --workspace=quadrat
###########################################################################################
FROM node:18 AS rust-client-builder
-ARG DEV
+ARG CLIENT_DEV
# Install rustup
RUN echo 'Installing rustup...' && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
@@ -131,7 +131,7 @@ COPY ./quadratic-rust-client/. ./quadratic-rust-client/
# Build the rust-client
ARG GIT_COMMIT
ENV GIT_COMMIT=$GIT_COMMIT
-RUN if [ "$DEV" = "true" ]; then \
+RUN if [ "$CLIENT_DEV" = "true" ]; then \
echo 'Building rust-client in dev mode...' && npm run build:dev --workspace=quadratic-rust-client; \
else \
echo 'Building rust-client...' && npm run build --workspace=quadratic-rust-client; \
diff --git a/quadratic-connection/package.json b/quadratic-connection/package.json
index 14d7c157f9..4c4b9fa5dc 100644
--- a/quadratic-connection/package.json
+++ b/quadratic-connection/package.json
@@ -14,8 +14,8 @@
"coverage:gen": "CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='coverage/cargo-test-%p-%m.profraw' cargo test",
"coverage:html": "grcov . --binary-path ./target/debug/deps/ -s . -t html --branch --ignore-not-existing --ignore '../*' --ignore '/*' -o coverage/html",
"coverage:view": "open coverage/html/index.html",
- "docker:up": "ECR_OR_BUILD=build DEV=false docker compose -f ../docker-compose.yml --profile quadratic-connection-db --env-file ../.env.docker up -d --wait && sleep 10",
- "docker:down": "ECR_OR_BUILD=build DEV=false docker compose -f ../docker-compose.yml --profile quadratic-connection-db --env-file ../.env.docker down",
- "docker:test": "ECR_OR_BUILD=build DEV=true docker compose kill && npm run docker:up && npm run test && npm run docker:down"
+ "docker:up": "ECR_OR_BUILD=build CLIENT_DEV=false docker compose -f ../docker-compose.yml --profile quadratic-connection-db --env-file ../.env.docker up -d --wait && sleep 10",
+ "docker:down": "ECR_OR_BUILD=build CLIENT_DEV=false docker compose -f ../docker-compose.yml --profile quadratic-connection-db --env-file ../.env.docker down",
+ "docker:test": "ECR_OR_BUILD=build CLIENT_DEV=true docker compose kill && npm run docker:up && npm run test && npm run docker:down"
}
}
diff --git a/quadratic-multiplayer/package.json b/quadratic-multiplayer/package.json
index a0f62645c8..33a7790ca8 100644
--- a/quadratic-multiplayer/package.json
+++ b/quadratic-multiplayer/package.json
@@ -14,8 +14,8 @@
"coverage:gen": "CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='coverage/cargo-test-%p-%m.profraw' cargo test",
"coverage:html": "grcov . --binary-path ./target/debug/deps/ -s . -t html --branch --ignore-not-existing --ignore '../*' --ignore '/*' -o coverage/html",
"coverage:view": "open coverage/html/index.html",
- "docker:up": "ECR_OR_BUILD=build DEV=false docker compose -f ../docker-compose.yml --profile base --env-file ../.env.docker up -d --wait",
- "docker:down": "ECR_OR_BUILD=build DEV=false docker compose -f ../docker-compose.yml --profile base --env-file ../.env.docker down -v",
- "docker:test": "ECR_OR_BUILD=build DEV=true docker compose kill && npm run docker:up && npm run test && npm run docker:down"
+ "docker:up": "ECR_OR_BUILD=build CLIENT_DEV=false docker compose -f ../docker-compose.yml --profile base --env-file ../.env.docker up -d --wait",
+ "docker:down": "ECR_OR_BUILD=build CLIENT_DEV=false docker compose -f ../docker-compose.yml --profile base --env-file ../.env.docker down -v",
+ "docker:test": "ECR_OR_BUILD=build CLIENT_DEV=true docker compose kill && npm run docker:up && npm run test && npm run docker:down"
}
}