From 782928ef7283fde20d668c3140caf04da844d1cf Mon Sep 17 00:00:00 2001 From: Guillaume Thibault Date: Sun, 14 Apr 2024 18:26:13 -0400 Subject: [PATCH 01/29] [stu-337-test-profile] chore: Working prototype --- captain/main.py | 4 +- captain/routes/test_profile.py | 72 +++++++++++++++++++ captain/routes/update.py | 40 ----------- src/api/index.ts | 6 +- src/main/utils.ts | 59 +++++++-------- src/renderer/hooks/useTestSequencerProject.ts | 40 +++++++++++ src/renderer/lib/api.ts | 11 +++ .../components/CloudPanel.tsx | 36 ++++++++++ 8 files changed, 190 insertions(+), 78 deletions(-) create mode 100644 captain/routes/test_profile.py delete mode 100644 captain/routes/update.py diff --git a/captain/main.py b/captain/main.py index da3600b56..0eadf446d 100644 --- a/captain/main.py +++ b/captain/main.py @@ -6,7 +6,7 @@ devices, flowchart, key, - update, + test_profile, ws, log, test_sequence, @@ -42,7 +42,7 @@ async def startup_event(app: FastAPI): app.include_router(flowchart.router) app.include_router(log.router) app.include_router(key.router) -app.include_router(update.router) +app.include_router(test_profile.router) app.include_router(blocks.router) app.include_router(devices.router) app.include_router(test_sequence.router) diff --git a/captain/routes/test_profile.py b/captain/routes/test_profile.py new file mode 100644 index 000000000..d6100b28b --- /dev/null +++ b/captain/routes/test_profile.py @@ -0,0 +1,72 @@ +import logging +import traceback +import subprocess +from typing import Annotated +from fastapi import APIRouter, Header, Response +import os +from flojoy_cloud.client import json +from captain.utils.blocks_path import get_flojoy_dir + + +router = APIRouter(tags=["test_profile"]) + + +@router.get("/test_profile/install/") +async def install(url: Annotated[str, Header()]): + """ + Download a git repo to the local machine if it's doesn't exist + verify its state + - Currently done for Github. (infer that the repo doesn't contain space) + - Private repo is not (directly) supported + TODO: + - [ ] Verify no change was done (git stash if so) + - [ ] Only clone the head commit (no history) + """ + try: + profile_name = url.split("/")[-1].strip(".git") + logging.info(f"Profile name: {profile_name}") + + # Check if Git is install + status, _ = subprocess.getstatusoutput(["git", "--version"]) + if status != 0: + raise NotImplementedError("Git is not found on you system") + print("Git is installed") + + # Find the output folder + profiles_dir = os.path.join(get_flojoy_dir(), f"test_profiles{os.sep}") + + # Create the dir if it doesn't exist + if not os.path.exists(profiles_dir): + os.makedirs(profiles_dir) + print(f"Created {profiles_dir}") + else: + print(f"{profiles_dir} already exists") + + # Find the profile + profile_root = os.path.join(profiles_dir, profile_name) + if not os.path.exists(profile_root): + # Clone the repo if it doesn't exist + status, _ = subprocess.getstatusoutput(["git", "--depth", "1", "clone", url, profile_root]) + print(f"Cloning {url} - Status: {status}") + if status != 0: + raise Exception(f"Not able to clone {url} - Error: {status}") + else: + print(f"{profile_root} already exists") + + # Get the commit ID of the local branch + branch_name = subprocess.check_output(["git", "-C", profile_root, "rev-parse", "--abbrev-ref", "HEAD"]).strip() + local_commit_id = subprocess.check_output(["git", "-C", profile_root, "rev-parse", "HEAD"]).strip().decode() + + # Return the Base Folder & the hash + print(f"Test Profile Loaded | dir:{profile_root} - branch: {branch_name} - hash: {local_commit_id}") + profile_root = profile_root.replace(os.sep, "/") + return Response(status_code=200, content=json.dumps({"profile_root": profile_root, "hash": local_commit_id})) + except Exception as e: + logging.error(f"Exception occured while installing {url}: {e}") + logging.error(traceback.format_exc()) + Response(status_code=500, content=json.dumps({"error": f"{e}"})) + + +@router.post("/test_profile/update/") +async def get_update(): + subprocess.run(["git", "pull"]) + diff --git a/captain/routes/update.py b/captain/routes/update.py deleted file mode 100644 index dbf34e882..000000000 --- a/captain/routes/update.py +++ /dev/null @@ -1,40 +0,0 @@ -import subprocess - -from fastapi import APIRouter - -router = APIRouter(tags=["update"]) - - -@router.get("/update/") -async def check_update(): - # Get the commit ID of the local branch - branch_name = ( - subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]) - .strip() - .decode() - ) - - local_commit_id = ( - subprocess.check_output(["git", "rev-parse", "HEAD"]).strip().decode() - ) - - # Get the commit ID of the remote branch - remote_commit_id = ( - subprocess.check_output( - ["git", "ls-remote", "origin", f"refs/heads/{branch_name}"] - ) - .split()[0] - .decode() - ) - - if local_commit_id != remote_commit_id: - print("Update available") - return True - else: - print("No update available") - return False - - -@router.post("/update/") -async def get_update(): - subprocess.run(["git", "pull"]) diff --git a/src/api/index.ts b/src/api/index.ts index dc9297be7..a3314bb44 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -145,14 +145,14 @@ export default { ): Promise<{ filePath: string; fileContent: string }[] | undefined> => ipcRenderer.invoke(API.openFilesPicker, allowedExtensions, title), - openAllFilesInFolderPicker: ( + openAllFilesInFolder: ( + folderPath: string, allowedExtensions: string[] = ["json"], - title: string = "Select Folder", ): Promise<{ filePath: string; fileContent: string }[] | undefined> => ipcRenderer.invoke( API.openAllFilesInFolderPicker, + folderPath, allowedExtensions, - title, ), getFileContent: (filepath: string): Promise => diff --git a/src/main/utils.ts b/src/main/utils.ts index 2272f48d0..875d011c7 100644 --- a/src/main/utils.ts +++ b/src/main/utils.ts @@ -186,46 +186,39 @@ export const openFilesPicker = ( }); }; + export const openAllFilesInFolderPicker = ( _, + folderPath: string, allowedExtensions: string[] = ["json"], - title: string = "Select Folder", -): Promise<{ filePath: string; fileContent: string }[] | undefined> => { - // Return mutiple files or all file with the allowed extensions if a folder is selected - return dialog - .showOpenDialog(global.mainWindow, { - title: title, - properties: ["openDirectory"], - }) - .then((selectedPaths) => { - if ( - selectedPaths.filePaths.length === 1 && - fs.lstatSync(selectedPaths.filePaths[0]).isDirectory() - ) { - // If a folder is selected, found all file with the allowed extensions from that folder - const folerPath = selectedPaths.filePaths[0]; - const paths: string[] = []; - fs.readdirSync(folerPath, { withFileTypes: true }).forEach((dirent) => { - if (dirent.isFile()) { - const nameAndExt = dirent.name.split("."); - const ext = nameAndExt[nameAndExt.length - 1]; - if (allowedExtensions.includes(ext)) { - paths.push(join(folerPath, dirent.name)); - } - } - }); - const files = paths.map((path) => { - return { - filePath: path.split(sep).join(posix.sep), - fileContent: fs.readFileSync(path, { encoding: "utf-8" }), - }; - }); - return files; +): { filePath: string; fileContent: string }[] | undefined => { + // Return multiple files or all files with the allowed extensions if a folder is selected + if (fs.existsSync(folderPath) && fs.lstatSync(folderPath).isDirectory()) { + // If a folder is selected, find all files with the allowed extensions from that folder + const paths: string[] = []; + fs.readdirSync(folderPath, { withFileTypes: true }).forEach((dirent) => { + if (dirent.isFile()) { + const nameAndExt = dirent.name.split("."); + const ext = nameAndExt[nameAndExt.length - 1]; + if (allowedExtensions.includes(ext)) { + paths.push(join(folderPath, dirent.name)); + } } - return undefined; }); + const files = paths.map((path) => { + return { + filePath: path.split(sep).join(posix.sep), + fileContent: fs.readFileSync(path, { encoding: "utf-8" }), + }; + }); + // Log the number of files found + return files; + } + // Log that folder doesn't exist or is not a directory + return undefined; }; + export const cleanup = async () => { const captainProcess = global.captainProcess as ChildProcess; log.info( diff --git a/src/renderer/hooks/useTestSequencerProject.ts b/src/renderer/hooks/useTestSequencerProject.ts index 3ce2a3bcc..522099fb3 100644 --- a/src/renderer/hooks/useTestSequencerProject.ts +++ b/src/renderer/hooks/useTestSequencerProject.ts @@ -102,6 +102,46 @@ export const useImportSequences = () => { return handleImport; }; +export const useLoadTestProfile = () => { + const manager = usePrepareStateManager(); + const handleImport = async (localRootFolder: string) => { + const result = await window.api.openAllFilesInFolder( + localRootFolder, + ["tjoy"], + ); + if (result === undefined) { + toast.error(`Failed to find the directory for ${localRootFolder}`); + return; + } + if (!result || result.length === 0) { + toast.error("No .tjoy file found in the selected directory"); + return; + } + const importSequences = async () => { + await Promise.all( + result.map(async (res, idx) => { + const { filePath, fileContent } = res; + const result = await importSequence( + filePath, + fileContent, + manager, + idx !== 0, + ); + if (result.isErr()) throw result.error; + }), + ); + }; + const s = result.length > 1 ? "s" : ""; + toast.promise(importSequences, { + loading: `Importing${s} sequence...`, + success: () => `Sequence${s} imported`, + error: (e) => `${e}`, + }); + }; + + return handleImport; +} + export const useCloseSequence = () => { const { isUnsaved } = useDisplayedSequenceState(); const manager = usePrepareStateManager(); diff --git a/src/renderer/lib/api.ts b/src/renderer/lib/api.ts index 0513aee20..6ae75c33c 100644 --- a/src/renderer/lib/api.ts +++ b/src/renderer/lib/api.ts @@ -282,3 +282,14 @@ export const getCloudHealth = (url: string | undefined = undefined) => { (e) => e as HTTPError, ); }; + +const TestProfile = z.object({ + profile_root: z.string(), + hash: z.string(), +}); +export type TestProfile = z.infer; + +export const installTestProfile = (url: string) => { + let options: Options = { headers: { url: url }, timeout: 60000 }; + return get("test_profile/install", TestProfile, options); +}; diff --git a/src/renderer/routes/test_sequencer_panel/components/CloudPanel.tsx b/src/renderer/routes/test_sequencer_panel/components/CloudPanel.tsx index 485ef4101..e51ffdfff 100644 --- a/src/renderer/routes/test_sequencer_panel/components/CloudPanel.tsx +++ b/src/renderer/routes/test_sequencer_panel/components/CloudPanel.tsx @@ -24,6 +24,7 @@ import { getCloudStations, getCloudUnits, getEnvironmentVariables, + installTestProfile, } from "@/renderer/lib/api"; import { toastQueryError } from "@/renderer/utils/report-error"; import { useQuery, useQueryClient } from "@tanstack/react-query"; @@ -37,6 +38,7 @@ import { getGlobalStatus } from "./DesignBar"; import { useSequencerStore } from "@/renderer/stores/sequencer"; import { useAuth } from "@/renderer/context/auth.context"; import { Autocomplete } from "@/renderer/components/ui/autocomplete"; +import { useLoadTestProfile } from "@/renderer/hooks/useTestSequencerProject"; export function CloudPanel() { const queryClient = useQueryClient(); @@ -50,6 +52,9 @@ export function CloudPanel() { const { user } = useAuth(); const { isLocked } = useDisplayedSequenceState(); const { sequences, handleUpload } = useSequencerState(); + const handleLoadProfile = useLoadTestProfile(); + const [ testProfileUrl, setTestProfileUrl ] = useState(""); + const [ currentHash, setCurrentHash ] = useState(""); const { serialNumber, isUploaded, @@ -187,6 +192,32 @@ export function CloudPanel() { enabled: projectsQuery.isSuccess, // Enable only when projectsQuery is successful }); + const installTestProfileQuery = useQuery({ + queryKey: ["profile"], + queryFn: async () => { + if (envsQuery.isSuccess && projectsQuery.isSuccess && testProfileUrl !== "") { + // dialog to ask user if they want to install test profile + const shouldContinue = window.confirm( + "Do you want to load the test profile associated with production line?", + ); + if (!shouldContinue) return; + const res = await installTestProfile(testProfileUrl); + return res.match( + (vars) => { + setCurrentHash(vars.hash); + handleLoadProfile(vars.profile_root); + }, + (e) => { + console.error(e); + toast.error(`Failed to load test profile: ${e}`); + }, + ); + } + return []; + }, + enabled: projectsQuery.isSuccess, // Enable only when projectsQuery is successful + }); + useEffect(() => { if (projectId !== "") { stationsQuery.refetch(); @@ -198,6 +229,10 @@ export function CloudPanel() { unitQuery.refetch(); }, [partVarId]); + useEffect(() => { + installTestProfileQuery.refetch(); + }, [testProfileUrl]); + useEffect(() => { const sn = serialNumber.toLowerCase(); if (sn in units) { @@ -212,6 +247,7 @@ export function CloudPanel() { setDescription(newValue.part.description); setPartNumber(newValue.part.partNumber); setPartVarId(newValue.part.partVariationId); + setTestProfileUrl("https://github.com/LatentDream/flojoy-test-fixture.git"); }; const { isEnvVarModalOpen, setIsEnvVarModalOpen } = useAppStore( From 426132f58f1b9b1f5d1fbf5bf31dad1d4c597f61 Mon Sep 17 00:00:00 2001 From: Guillaume Thibault Date: Sun, 14 Apr 2024 18:44:55 -0400 Subject: [PATCH 02/29] [stu-337-test-profile] chore: migrate logic to hooks --- src/renderer/hooks/useTestSequencerProject.ts | 50 ++++++++++++------- .../components/CloudPanel.tsx | 28 +---------- 2 files changed, 34 insertions(+), 44 deletions(-) diff --git a/src/renderer/hooks/useTestSequencerProject.ts b/src/renderer/hooks/useTestSequencerProject.ts index 522099fb3..ac53e75af 100644 --- a/src/renderer/hooks/useTestSequencerProject.ts +++ b/src/renderer/hooks/useTestSequencerProject.ts @@ -14,6 +14,8 @@ import { saveSequences, } from "@/renderer/routes/test_sequencer_panel/utils/SequenceHandler"; import { toastResultPromise } from "../utils/report-error"; +import { Result, err, ok } from "neverthrow"; +import { installTestProfile } from "../lib/api"; function usePrepareStateManager(): StateManager { const { elems, project } = useDisplayedSequenceState(); @@ -104,20 +106,33 @@ export const useImportSequences = () => { export const useLoadTestProfile = () => { const manager = usePrepareStateManager(); - const handleImport = async (localRootFolder: string) => { - const result = await window.api.openAllFilesInFolder( - localRootFolder, - ["tjoy"], - ); - if (result === undefined) { - toast.error(`Failed to find the directory for ${localRootFolder}`); - return; - } - if (!result || result.length === 0) { - toast.error("No .tjoy file found in the selected directory"); + const handleImport = async (gitRepoUrlHttp: string) => { + if (gitRepoUrlHttp === "") { return; } - const importSequences = async () => { + async function importSequences(): Promise> { + const shouldContinue = window.confirm( + "Do you want to load the test profile associated with production line?", + ); + if (!shouldContinue) { + return err(Error("User cancelled loading test profile")); + } + const res = await installTestProfile(gitRepoUrlHttp); + if (res.isErr()) { + return err(Error(`Failed to load test profile: ${res.error}`)); + } + // todo: set hash in zustand for integrity + + const result = await window.api.openAllFilesInFolder( + res.value.profile_root, + ["tjoy"], + ); + if (result === undefined) { + return err(Error(`Failed to find the directory ${res.value.profile_root}`)); + } + if (!result || result.length === 0) { + return err(Error("No .tjoy file found in the selected directory")); + } await Promise.all( result.map(async (res, idx) => { const { filePath, fileContent } = res; @@ -127,14 +142,15 @@ export const useLoadTestProfile = () => { manager, idx !== 0, ); - if (result.isErr()) throw result.error; + if (result.isErr()) + return err(result.error); }), ); + return ok(undefined); }; - const s = result.length > 1 ? "s" : ""; - toast.promise(importSequences, { - loading: `Importing${s} sequence...`, - success: () => `Sequence${s} imported`, + toastResultPromise(importSequences(), { + loading: `Importing Test Profile...`, + success: () => `Test Profile imported`, error: (e) => `${e}`, }); }; diff --git a/src/renderer/routes/test_sequencer_panel/components/CloudPanel.tsx b/src/renderer/routes/test_sequencer_panel/components/CloudPanel.tsx index e51ffdfff..b34bcb553 100644 --- a/src/renderer/routes/test_sequencer_panel/components/CloudPanel.tsx +++ b/src/renderer/routes/test_sequencer_panel/components/CloudPanel.tsx @@ -192,32 +192,6 @@ export function CloudPanel() { enabled: projectsQuery.isSuccess, // Enable only when projectsQuery is successful }); - const installTestProfileQuery = useQuery({ - queryKey: ["profile"], - queryFn: async () => { - if (envsQuery.isSuccess && projectsQuery.isSuccess && testProfileUrl !== "") { - // dialog to ask user if they want to install test profile - const shouldContinue = window.confirm( - "Do you want to load the test profile associated with production line?", - ); - if (!shouldContinue) return; - const res = await installTestProfile(testProfileUrl); - return res.match( - (vars) => { - setCurrentHash(vars.hash); - handleLoadProfile(vars.profile_root); - }, - (e) => { - console.error(e); - toast.error(`Failed to load test profile: ${e}`); - }, - ); - } - return []; - }, - enabled: projectsQuery.isSuccess, // Enable only when projectsQuery is successful - }); - useEffect(() => { if (projectId !== "") { stationsQuery.refetch(); @@ -230,7 +204,7 @@ export function CloudPanel() { }, [partVarId]); useEffect(() => { - installTestProfileQuery.refetch(); + handleLoadProfile(testProfileUrl) }, [testProfileUrl]); useEffect(() => { From aaa4d8818e199a42b5510f0b7af7ca5228b7a135 Mon Sep 17 00:00:00 2001 From: Guillaume Thibault Date: Sun, 14 Apr 2024 22:05:01 -0400 Subject: [PATCH 03/29] [stu-337-test-profile] chore: typo in cmd --- captain/routes/test_profile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/captain/routes/test_profile.py b/captain/routes/test_profile.py index d6100b28b..d7f0f0825 100644 --- a/captain/routes/test_profile.py +++ b/captain/routes/test_profile.py @@ -45,7 +45,7 @@ async def install(url: Annotated[str, Header()]): profile_root = os.path.join(profiles_dir, profile_name) if not os.path.exists(profile_root): # Clone the repo if it doesn't exist - status, _ = subprocess.getstatusoutput(["git", "--depth", "1", "clone", url, profile_root]) + status, _ = subprocess.getstatusoutput(["git", "clone", "--depth", "1", url, profile_root]) print(f"Cloning {url} - Status: {status}") if status != 0: raise Exception(f"Not able to clone {url} - Error: {status}") From eee0e392f0d498bb5c4ab570c4a90b25b866cf43 Mon Sep 17 00:00:00 2001 From: Guillaume Date: Sun, 14 Apr 2024 22:24:53 -0400 Subject: [PATCH 04/29] chore: using run with capture_output for MacOS compatibility --- captain/routes/test_profile.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/captain/routes/test_profile.py b/captain/routes/test_profile.py index d7f0f0825..9da64f274 100644 --- a/captain/routes/test_profile.py +++ b/captain/routes/test_profile.py @@ -26,10 +26,10 @@ async def install(url: Annotated[str, Header()]): logging.info(f"Profile name: {profile_name}") # Check if Git is install - status, _ = subprocess.getstatusoutput(["git", "--version"]) - if status != 0: + cmd = ["git", "--version"] + res = subprocess.run(cmd, capture_output=True) + if res.returncode != 0: raise NotImplementedError("Git is not found on you system") - print("Git is installed") # Find the output folder profiles_dir = os.path.join(get_flojoy_dir(), f"test_profiles{os.sep}") @@ -37,27 +37,30 @@ async def install(url: Annotated[str, Header()]): # Create the dir if it doesn't exist if not os.path.exists(profiles_dir): os.makedirs(profiles_dir) - print(f"Created {profiles_dir}") - else: - print(f"{profiles_dir} already exists") # Find the profile profile_root = os.path.join(profiles_dir, profile_name) if not os.path.exists(profile_root): # Clone the repo if it doesn't exist - status, _ = subprocess.getstatusoutput(["git", "clone", "--depth", "1", url, profile_root]) - print(f"Cloning {url} - Status: {status}") - if status != 0: - raise Exception(f"Not able to clone {url} - Error: {status}") + cmd = ["git", "clone", "--depth", "1", url, profile_root] + res = subprocess.run(cmd, capture_output=True) + if res.returncode != 0: + stdout = res.stdout.decode("utf-8").strip() + stderr = res.stderr.decode("utf-8").strip() + logging.error(f"Error while cloning url: {stdout} - {stderr}") + raise Exception(f"Not able to clone {url} - Error: {res.returncode}") else: - print(f"{profile_root} already exists") + # todo: check if the repo is up-to-date + pass # Get the commit ID of the local branch - branch_name = subprocess.check_output(["git", "-C", profile_root, "rev-parse", "--abbrev-ref", "HEAD"]).strip() - local_commit_id = subprocess.check_output(["git", "-C", profile_root, "rev-parse", "HEAD"]).strip().decode() + cmd = ["git", "-C", profile_root, "rev-parse", "HEAD"] + res = subprocess.run(cmd, capture_output=True) + if res.returncode != 0: + raise Exception(f"Not able to get the commit ID of the local branch - Error: {res.returncode}") + local_commit_id = res.stdout.strip().decode() # Return the Base Folder & the hash - print(f"Test Profile Loaded | dir:{profile_root} - branch: {branch_name} - hash: {local_commit_id}") profile_root = profile_root.replace(os.sep, "/") return Response(status_code=200, content=json.dumps({"profile_root": profile_root, "hash": local_commit_id})) except Exception as e: From db1ccf88b032e738cf4ab69beaf3959a1f7c91c3 Mon Sep 17 00:00:00 2001 From: Guillaume Thibault Date: Mon, 15 Apr 2024 10:16:44 -0400 Subject: [PATCH 05/29] [stu-337-test-profile] chore(test_profile): update / checkout route + DRY --- captain/routes/test_profile.py | 108 ++++++++++++++++++++++++--------- 1 file changed, 79 insertions(+), 29 deletions(-) diff --git a/captain/routes/test_profile.py b/captain/routes/test_profile.py index 9da64f274..22e2bd29e 100644 --- a/captain/routes/test_profile.py +++ b/captain/routes/test_profile.py @@ -19,30 +19,17 @@ async def install(url: Annotated[str, Header()]): - Private repo is not (directly) supported TODO: - [ ] Verify no change was done (git stash if so) - - [ ] Only clone the head commit (no history) + - [ ] Option if git is not install on the system """ try: - profile_name = url.split("/")[-1].strip(".git") - logging.info(f"Profile name: {profile_name}") - - # Check if Git is install - cmd = ["git", "--version"] - res = subprocess.run(cmd, capture_output=True) - if res.returncode != 0: - raise NotImplementedError("Git is not found on you system") - - # Find the output folder - profiles_dir = os.path.join(get_flojoy_dir(), f"test_profiles{os.sep}") - - # Create the dir if it doesn't exist - if not os.path.exists(profiles_dir): - os.makedirs(profiles_dir) + verify_git_install() + profiles_path = get_profiles_dir() + profile_path = get_profile_path_from_url(profiles_path, url) # Find the profile - profile_root = os.path.join(profiles_dir, profile_name) - if not os.path.exists(profile_root): + if not os.path.exists(profile_path): # Clone the repo if it doesn't exist - cmd = ["git", "clone", "--depth", "1", url, profile_root] + cmd = ["git", "clone", "--depth", "1", url, profile_path] res = subprocess.run(cmd, capture_output=True) if res.returncode != 0: stdout = res.stdout.decode("utf-8").strip() @@ -53,23 +40,86 @@ async def install(url: Annotated[str, Header()]): # todo: check if the repo is up-to-date pass - # Get the commit ID of the local branch - cmd = ["git", "-C", profile_root, "rev-parse", "HEAD"] + commit_hash = get_commit_hash(profile_path) + profile_path = profile_path.replace(os.sep, "/") + return Response(status_code=200, content=json.dumps({"profile_root": profile_path, "hash": commit_hash})) + except Exception as e: + logging.error(f"Exception occured while installing {url}: {e}") + logging.error(traceback.format_exc()) + Response(status_code=500, content=json.dumps({"error": f"{e}"})) + + +@router.post("/test_profile/update/") +async def get_update(url: Annotated[str, Header()]): + try: + verify_git_install() + profiles_path = get_profiles_dir() + profile_path = get_profile_path_from_url(profiles_path, url) + + cmd = ["git", "-C", profile_path, "pull"] res = subprocess.run(cmd, capture_output=True) if res.returncode != 0: - raise Exception(f"Not able to get the commit ID of the local branch - Error: {res.returncode}") - local_commit_id = res.stdout.strip().decode() + raise Exception(f"Not able to pull the repo - Error: {res.returncode}") - # Return the Base Folder & the hash - profile_root = profile_root.replace(os.sep, "/") - return Response(status_code=200, content=json.dumps({"profile_root": profile_root, "hash": local_commit_id})) + commit_hash = get_commit_hash(profile_path) + return Response(status_code=200, content=json.dumps({"profile_root": profile_path, "hash": commit_hash})) except Exception as e: logging.error(f"Exception occured while installing {url}: {e}") logging.error(traceback.format_exc()) Response(status_code=500, content=json.dumps({"error": f"{e}"})) -@router.post("/test_profile/update/") -async def get_update(): - subprocess.run(["git", "pull"]) +@router.post("/test_profile/checkout/{commit_hash}/") +async def checkout(url: Annotated[str, Header()], commit_hash: str): + try: + verify_git_install() + profiles_path = get_profiles_dir() + profile_path = get_profile_path_from_url(profiles_path, url) + await get_update(url) + curr_commit_hash = get_commit_hash(profile_path) + if curr_commit_hash != commit_hash: + cmd = ["git", "-C", profile_path, "checkout", commit_hash] + res = subprocess.run(cmd, capture_output=True) + if res.returncode != 0: + raise Exception(f"Not able to checkout the commit - Error: {res.returncode}") + + commit_hash = get_commit_hash(profile_path) + return Response(status_code=200, content=json.dumps({"profile_root": profile_path, "hash": commit_hash})) + except Exception as e: + logging.error(f"Exception occured while installing {url}: {e}") + logging.error(traceback.format_exc()) + Response(status_code=500, content=json.dumps({"error": f"{e}"})) + + +# Helper functions ------------------------------------------------------------ + + +def get_profile_path_from_url(profiles_path: str, url: str): + profile_name = url.split("/")[-1].strip(".git") + logging.info(f"Profile name: {profile_name}") + profile_root = os.path.join(profiles_path, profile_name) + return profile_root + + +def verify_git_install(): + cmd = ["git", "--version"] + res = subprocess.run(cmd, capture_output=True) + if res.returncode != 0: + raise NotImplementedError("Git is not found on you system") + + +def get_profiles_dir(): + profiles_dir = os.path.join(get_flojoy_dir(), f"test_profiles{os.sep}") + if not os.path.exists(profiles_dir): + os.makedirs(profiles_dir) + return profiles_dir + + +def get_commit_hash(profile_path: str): + # Get the commit hash of the local branch + cmd = ["git", "-C", profile_path, "rev-parse", "HEAD"] + res = subprocess.run(cmd, capture_output=True) + if res.returncode != 0: + raise Exception(f"Not able to get the commit ID of the local branch - Error: {res.returncode}") + return res.stdout.strip().decode() From b53d674f16b782d974ffd4c58a05d609b797f674 Mon Sep 17 00:00:00 2001 From: Guillaume Thibault Date: Mon, 15 Apr 2024 11:58:54 -0400 Subject: [PATCH 06/29] [stu-337-test-profile] chore: load and upload hash --- captain/routes/cloud.py | 3 ++- captain/routes/test_profile.py | 1 - src/renderer/hooks/useTestSequencerProject.ts | 10 ++++++---- src/renderer/hooks/useTestSequencerState.ts | 4 +++- src/renderer/lib/api.ts | 3 ++- .../test_sequencer_panel/components/CloudPanel.tsx | 8 ++++---- src/renderer/stores/sequencer.ts | 4 ++++ 7 files changed, 21 insertions(+), 12 deletions(-) diff --git a/captain/routes/cloud.py b/captain/routes/cloud.py index b56159545..0147fd156 100644 --- a/captain/routes/cloud.py +++ b/captain/routes/cloud.py @@ -107,7 +107,7 @@ class Project(CloudModel): updated_at: Optional[datetime.datetime] workspace_id: str part_variation_id: str - repo_Url: Optional[str] + repo_url: Optional[str] class Station(CloudModel): @@ -203,6 +203,7 @@ async def get_cloud_projects(): { "label": p.name, "value": p.id, + "repoUrl": p.repo_url, "part": await get_cloud_part_variation(p.part_variation_id), } for p in projects diff --git a/captain/routes/test_profile.py b/captain/routes/test_profile.py index 22e2bd29e..8e87e699a 100644 --- a/captain/routes/test_profile.py +++ b/captain/routes/test_profile.py @@ -93,7 +93,6 @@ async def checkout(url: Annotated[str, Header()], commit_hash: str): # Helper functions ------------------------------------------------------------ - def get_profile_path_from_url(profiles_path: str, url: str): profile_name = url.split("/")[-1].strip(".git") logging.info(f"Profile name: {profile_name}") diff --git a/src/renderer/hooks/useTestSequencerProject.ts b/src/renderer/hooks/useTestSequencerProject.ts index ac53e75af..c9b00fb31 100644 --- a/src/renderer/hooks/useTestSequencerProject.ts +++ b/src/renderer/hooks/useTestSequencerProject.ts @@ -13,9 +13,11 @@ import { closeSequence, saveSequences, } from "@/renderer/routes/test_sequencer_panel/utils/SequenceHandler"; -import { toastResultPromise } from "../utils/report-error"; +import { toastResultPromise } from "@/renderer/utils/report-error"; import { Result, err, ok } from "neverthrow"; -import { installTestProfile } from "../lib/api"; +import { installTestProfile } from "@/renderer/lib/api"; +import { useSequencerStore } from "@/renderer/stores/sequencer"; +import { useShallow } from "zustand/react/shallow"; function usePrepareStateManager(): StateManager { const { elems, project } = useDisplayedSequenceState(); @@ -106,6 +108,7 @@ export const useImportSequences = () => { export const useLoadTestProfile = () => { const manager = usePrepareStateManager(); + const setCommitHash = useSequencerStore(useShallow((state) => state.setCommitHash)); const handleImport = async (gitRepoUrlHttp: string) => { if (gitRepoUrlHttp === "") { return; @@ -121,8 +124,7 @@ export const useLoadTestProfile = () => { if (res.isErr()) { return err(Error(`Failed to load test profile: ${res.error}`)); } - // todo: set hash in zustand for integrity - + setCommitHash(res.value.hash); const result = await window.api.openAllFilesInFolder( res.value.profile_root, ["tjoy"], diff --git a/src/renderer/hooks/useTestSequencerState.ts b/src/renderer/hooks/useTestSequencerState.ts index 1dec26d78..82507b1c9 100644 --- a/src/renderer/hooks/useTestSequencerState.ts +++ b/src/renderer/hooks/useTestSequencerState.ts @@ -243,6 +243,7 @@ export function useSequencerState() { serialNumber, stationId, integrity, + commitHash, setIsUploaded, sequences, setSequences, @@ -267,6 +268,7 @@ export function useSequencerState() { serialNumber: state.serialNumber, stationId: state.stationId, integrity: state.integrity, + commitHash: state.commitHash, setIsUploaded: state.setIsUploaded, sequences: state.sequences, setSequences: state.setSequences, @@ -348,7 +350,7 @@ export function useSequencerState() { return; } const upload = async () => { - await postSession(serialNumber, stationId, integrity, aborted, "", [ + await postSession(serialNumber, stationId, integrity, aborted, commitHash, [ ...useSequencerStore.getState().cycleRuns, ]); }; diff --git a/src/renderer/lib/api.ts b/src/renderer/lib/api.ts index 6ae75c33c..990ad042d 100644 --- a/src/renderer/lib/api.ts +++ b/src/renderer/lib/api.ts @@ -179,6 +179,7 @@ export type Part = z.infer; const Project = z.object({ label: z.string(), value: z.string(), + repoUrl: z.string().nullable(), part: Part, }); export type Project = z.infer; @@ -189,7 +190,7 @@ const Station = z.object({ label: z.string(), value: z.string(), }); -export type Station = z.infer; +export type Station = z.infer; export const getCloudStations = (projectId: string) => get(`cloud/stations/${projectId}`, Station.array(), { timeout: 60000 }); diff --git a/src/renderer/routes/test_sequencer_panel/components/CloudPanel.tsx b/src/renderer/routes/test_sequencer_panel/components/CloudPanel.tsx index b34bcb553..63a74e2e6 100644 --- a/src/renderer/routes/test_sequencer_panel/components/CloudPanel.tsx +++ b/src/renderer/routes/test_sequencer_panel/components/CloudPanel.tsx @@ -53,7 +53,7 @@ export function CloudPanel() { const { isLocked } = useDisplayedSequenceState(); const { sequences, handleUpload } = useSequencerState(); const handleLoadProfile = useLoadTestProfile(); - const [ testProfileUrl, setTestProfileUrl ] = useState(""); + const [ testProfileUrl, setTestProfileUrl ] = useState(""); const [ currentHash, setCurrentHash ] = useState(""); const { serialNumber, @@ -204,7 +204,7 @@ export function CloudPanel() { }, [partVarId]); useEffect(() => { - handleLoadProfile(testProfileUrl) + handleLoadProfile(testProfileUrl === null ? "": testProfileUrl); }, [testProfileUrl]); useEffect(() => { @@ -216,12 +216,12 @@ export function CloudPanel() { } }, [serialNumber]); - const handleSetProject = (newValue: Station) => { + const handleSetProject = (newValue: Project) => { setProjectId(newValue.value); setDescription(newValue.part.description); setPartNumber(newValue.part.partNumber); setPartVarId(newValue.part.partVariationId); - setTestProfileUrl("https://github.com/LatentDream/flojoy-test-fixture.git"); + setTestProfileUrl(newValue.repoUrl); }; const { isEnvVarModalOpen, setIsEnvVarModalOpen } = useAppStore( diff --git a/src/renderer/stores/sequencer.ts b/src/renderer/stores/sequencer.ts index 1ba5bbdb1..42c9e198e 100644 --- a/src/renderer/stores/sequencer.ts +++ b/src/renderer/stores/sequencer.ts @@ -35,6 +35,7 @@ type State = { stationId: string; integrity: boolean; isUploaded: boolean; + commitHash: string; // ~~~~~~~~~~~~~~~~~~ }; @@ -52,6 +53,7 @@ type Actions = { setIntegrity: (val: boolean) => void; setStationId: (val: string) => void; setIsUploaded: (val: boolean) => void; + setCommitHash: (val: string) => void; // Cycles setCycleCount: (val: number) => void; setInfinite: (val: boolean) => void; @@ -123,6 +125,8 @@ export const useSequencerStore = create()( setIntegrity: (val) => set(() => ({ integrity: val })), isUploaded: false, setIsUploaded: (val) => set(() => ({ isUploaded: val })), + commitHash: "", + setCommitHash: (val) => set(() => ({ profileHash: val })), setWebsocketId: (val) => set((state) => { From e8685a640f93fb481f014397756243422ed30f22 Mon Sep 17 00:00:00 2001 From: Guillaume Thibault Date: Mon, 15 Apr 2024 12:13:27 -0400 Subject: [PATCH 07/29] [stu-337-test-profile] chore: remove the update route in favor of auto-update & checkout to specific hash --- captain/routes/test_profile.py | 44 +++++++++++++++------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/captain/routes/test_profile.py b/captain/routes/test_profile.py index 8e87e699a..c2f88509c 100644 --- a/captain/routes/test_profile.py +++ b/captain/routes/test_profile.py @@ -37,9 +37,8 @@ async def install(url: Annotated[str, Header()]): logging.error(f"Error while cloning url: {stdout} - {stderr}") raise Exception(f"Not able to clone {url} - Error: {res.returncode}") else: - # todo: check if the repo is up-to-date - pass - + update_to_origin_main(profile_path) + commit_hash = get_commit_hash(profile_path) profile_path = profile_path.replace(os.sep, "/") return Response(status_code=200, content=json.dumps({"profile_root": profile_path, "hash": commit_hash})) @@ -49,35 +48,18 @@ async def install(url: Annotated[str, Header()]): Response(status_code=500, content=json.dumps({"error": f"{e}"})) -@router.post("/test_profile/update/") -async def get_update(url: Annotated[str, Header()]): - try: - verify_git_install() - profiles_path = get_profiles_dir() - profile_path = get_profile_path_from_url(profiles_path, url) - - cmd = ["git", "-C", profile_path, "pull"] - res = subprocess.run(cmd, capture_output=True) - if res.returncode != 0: - raise Exception(f"Not able to pull the repo - Error: {res.returncode}") - - commit_hash = get_commit_hash(profile_path) - return Response(status_code=200, content=json.dumps({"profile_root": profile_path, "hash": commit_hash})) - except Exception as e: - logging.error(f"Exception occured while installing {url}: {e}") - logging.error(traceback.format_exc()) - Response(status_code=500, content=json.dumps({"error": f"{e}"})) - - @router.post("/test_profile/checkout/{commit_hash}/") async def checkout(url: Annotated[str, Header()], commit_hash: str): try: verify_git_install() profiles_path = get_profiles_dir() profile_path = get_profile_path_from_url(profiles_path, url) - await get_update(url) curr_commit_hash = get_commit_hash(profile_path) if curr_commit_hash != commit_hash: + cmd = ["git", "-C", profile_path, "fetch", "--all"] + res = subprocess.run(cmd, capture_output=True) + if res.returncode != 0: + raise Exception(f"Not able to fetch the repo - Error: {res.returncode}") cmd = ["git", "-C", profile_path, "checkout", commit_hash] res = subprocess.run(cmd, capture_output=True) if res.returncode != 0: @@ -93,6 +75,7 @@ async def checkout(url: Annotated[str, Header()], commit_hash: str): # Helper functions ------------------------------------------------------------ + def get_profile_path_from_url(profiles_path: str, url: str): profile_name = url.split("/")[-1].strip(".git") logging.info(f"Profile name: {profile_name}") @@ -115,10 +98,21 @@ def get_profiles_dir(): def get_commit_hash(profile_path: str): - # Get the commit hash of the local branch cmd = ["git", "-C", profile_path, "rev-parse", "HEAD"] res = subprocess.run(cmd, capture_output=True) if res.returncode != 0: raise Exception(f"Not able to get the commit ID of the local branch - Error: {res.returncode}") return res.stdout.strip().decode() + +def update_to_origin_main(profile_path: str): + cmd = ["git", "-C", profile_path, "status", "--porcelain"] + res = subprocess.run(cmd, capture_output=True) + if res.returncode != 0: + raise Exception(f"Not able to check the status of the repo - Error: {res.returncode}") + if res.stdout.strip() != b"": + raise Exception(f"Repo is not clean - {res.stdout}") + cmd = ["git", "-C", profile_path, "checkout", "origin/main"] + res = subprocess.run(cmd, capture_output=True) + if res.returncode != 0: + raise Exception(f"Not able to checkout the remote origin main - Error: {res.returncode}") From 29790f8302dcdd6ef9c9b49cc78fc723298a0d47 Mon Sep 17 00:00:00 2001 From: Guillaume Thibault Date: Mon, 15 Apr 2024 12:41:31 -0400 Subject: [PATCH 08/29] [stu-337-test-profile] chore: working live update --- captain/routes/test_profile.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/captain/routes/test_profile.py b/captain/routes/test_profile.py index c2f88509c..02b8a3630 100644 --- a/captain/routes/test_profile.py +++ b/captain/routes/test_profile.py @@ -18,10 +18,10 @@ async def install(url: Annotated[str, Header()]): - Currently done for Github. (infer that the repo doesn't contain space) - Private repo is not (directly) supported TODO: - - [ ] Verify no change was done (git stash if so) - [ ] Option if git is not install on the system """ try: + logging.info(f"Installing the profile from the url: {url}") verify_git_install() profiles_path = get_profiles_dir() profile_path = get_profile_path_from_url(profiles_path, url) @@ -38,7 +38,7 @@ async def install(url: Annotated[str, Header()]): raise Exception(f"Not able to clone {url} - Error: {res.returncode}") else: update_to_origin_main(profile_path) - + commit_hash = get_commit_hash(profile_path) profile_path = profile_path.replace(os.sep, "/") return Response(status_code=200, content=json.dumps({"profile_root": profile_path, "hash": commit_hash})) @@ -51,6 +51,7 @@ async def install(url: Annotated[str, Header()]): @router.post("/test_profile/checkout/{commit_hash}/") async def checkout(url: Annotated[str, Header()], commit_hash: str): try: + logging.info(f"Swtiching to the commit: {commit_hash} for the profile: {url}") verify_git_install() profiles_path = get_profiles_dir() profile_path = get_profile_path_from_url(profiles_path, url) @@ -106,12 +107,17 @@ def get_commit_hash(profile_path: str): def update_to_origin_main(profile_path: str): + logging.info("Updating the repo to the origin main") cmd = ["git", "-C", profile_path, "status", "--porcelain"] res = subprocess.run(cmd, capture_output=True) if res.returncode != 0: raise Exception(f"Not able to check the status of the repo - Error: {res.returncode}") if res.stdout.strip() != b"": raise Exception(f"Repo is not clean - {res.stdout}") + cmd = ["git", "-C", profile_path, "fetch", "--all"] + res = subprocess.run(cmd, capture_output=True) + if res.returncode != 0: + raise Exception(f"Not able to fetch the repo - Error: {res.returncode}") cmd = ["git", "-C", profile_path, "checkout", "origin/main"] res = subprocess.run(cmd, capture_output=True) if res.returncode != 0: From 55b07f3a99dfedf3d71889e298c0d4cd7dd8ba9a Mon Sep 17 00:00:00 2001 From: Guillaume Thibault Date: Tue, 16 Apr 2024 09:30:46 -0400 Subject: [PATCH 09/29] [stu-337-test-profile] chore: prod line -> Test Profile --- .../test_sequencer_panel/components/CloudPanel.tsx | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/renderer/routes/test_sequencer_panel/components/CloudPanel.tsx b/src/renderer/routes/test_sequencer_panel/components/CloudPanel.tsx index 63a74e2e6..2550f8676 100644 --- a/src/renderer/routes/test_sequencer_panel/components/CloudPanel.tsx +++ b/src/renderer/routes/test_sequencer_panel/components/CloudPanel.tsx @@ -18,13 +18,11 @@ import { useAppStore } from "@/renderer/stores/app"; import { useShallow } from "zustand/react/shallow"; import { Project, - Station, Unit, getCloudProjects, getCloudStations, getCloudUnits, getEnvironmentVariables, - installTestProfile, } from "@/renderer/lib/api"; import { toastQueryError } from "@/renderer/utils/report-error"; import { useQuery, useQueryClient } from "@tanstack/react-query"; @@ -54,7 +52,6 @@ export function CloudPanel() { const { sequences, handleUpload } = useSequencerState(); const handleLoadProfile = useLoadTestProfile(); const [ testProfileUrl, setTestProfileUrl ] = useState(""); - const [ currentHash, setCurrentHash ] = useState(""); const { serialNumber, isUploaded, @@ -162,7 +159,7 @@ export function CloudPanel() { }, (e) => { console.error(e); - toast.error("Failed to fetch production lines"); + toast.error("Failed to fetch test profiles"); return []; }, ); @@ -314,7 +311,7 @@ export function CloudPanel() {
-

Production Line

+

Test Profile