From ec0bf6947dd04131da969acbcb1b88faf4dc117b Mon Sep 17 00:00:00 2001 From: Ivan Date: Fri, 8 Nov 2024 13:18:27 -0800 Subject: [PATCH] Artifact Playground --- frontend/src/App.tsx | 2 + .../src/components/files/FileRenderer.tsx | 22 ++ .../src/components/files/MJCFRenderer.tsx | 220 ++++++++++++++++++ .../listing/ListingPlaygroundButton.tsx | 30 +++ .../components/listing/ListingRenderer.tsx | 16 +- .../components/pages/ArtifactPlayground.tsx | 141 +++++++++++ 6 files changed, 420 insertions(+), 11 deletions(-) create mode 100644 frontend/src/components/files/MJCFRenderer.tsx create mode 100644 frontend/src/components/listing/ListingPlaygroundButton.tsx create mode 100644 frontend/src/components/pages/ArtifactPlayground.tsx diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 5807698b..c221a7c9 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -39,6 +39,7 @@ import ResearchPage from "./components/pages/ResearchPage"; import SellerOnboarding from "./components/pages/SellerOnboarding"; import Terminal from "./components/pages/Terminal"; import TermsOfService from "./components/pages/TermsOfService"; +import ArtifactPlayground from "@/components/pages/ArtifactPlayground"; const App = () => { return ( @@ -59,6 +60,7 @@ const App = () => { } /> } /> + } /> } /> } /> diff --git a/frontend/src/components/files/FileRenderer.tsx b/frontend/src/components/files/FileRenderer.tsx index 00c861ff..09523f33 100644 --- a/frontend/src/components/files/FileRenderer.tsx +++ b/frontend/src/components/files/FileRenderer.tsx @@ -1,12 +1,23 @@ +import MJCFRenderer from "./MJCFRenderer"; import STLRenderer from "./STLRenderer"; import { UntarredFile } from "./Tarfile"; import URDFRenderer from "./URDFRenderer"; +const isMJCFFile = (content: string, filename: string): boolean => { + const extension = filename.split(".").pop()?.toLowerCase(); + if (extension !== "xml" && extension !== "mjcf") { + return false; + } + + return content.includes(""); +}; + const FileRenderer: React.FC<{ file: UntarredFile; allFiles: UntarredFile[]; }> = ({ file, allFiles }) => { const fileExtension = file.name.split(".").pop()?.toLowerCase(); + const fileContent = new TextDecoder().decode(file.content); switch (fileExtension) { case "urdf": @@ -19,6 +30,17 @@ const FileRenderer: React.FC<{ ); case "stl": return ; + case "xml": + case "mjcf": + if (isMJCFFile(fileContent, file.name)) { + return ; + } else { + return ( +
+

Invalid MJCF file format

+
+ ); + } default: return (
diff --git a/frontend/src/components/files/MJCFRenderer.tsx b/frontend/src/components/files/MJCFRenderer.tsx new file mode 100644 index 00000000..dba2c374 --- /dev/null +++ b/frontend/src/components/files/MJCFRenderer.tsx @@ -0,0 +1,220 @@ +import React, { forwardRef, useEffect, useRef, useState } from "react"; +import { UntarredFile } from "./Tarfile"; +import load_mujoco, { mujoco } from "@/lib/mujoco/mujoco_wasm.js"; +import * as THREE from "three"; +import { OrbitControls } from "three/examples/jsm/controls/OrbitControls"; + +interface MJCFRendererProps { + mjcfContent: string; + files?: UntarredFile[]; + width?: string | number; + height?: string | number; + useControls?: boolean; + showWireframe?: boolean; +} + +const MJCFRenderer = forwardRef(({ + mjcfContent, + files = [], + width = "100%", + height = "100%", + useControls = true, + showWireframe = false, +}, ref) => { + const containerRef = useRef(null); + const sceneRef = useRef(null); + const rendererRef = useRef(null); + const cameraRef = useRef(null); + const controlsRef = useRef(null); + const robotRef = useRef(null); + + const [isMujocoReady, setIsMujocoReady] = useState(false); + const [error, setError] = useState(null); + + const mujocoRef = useRef(null); + const modelRef = useRef | null>(null); + const stateRef = useRef | null>(null); + const simulationRef = useRef | null>(null); + + useEffect(() => { + const initializeScene = async () => { + if (!containerRef.current) return; + + try { + mujocoRef.current = await load_mujoco(); + console.log("MuJoCo loaded successfully"); + + // Set up file system + mujocoRef.current.FS.mkdir("/working"); + mujocoRef.current.FS.mount( + mujocoRef.current.MEMFS, + { root: "." }, + "/working" + ); + + // Create a simple test model without meshes + const testModel = ` + + + `; + + const modelPath = "/working/model.xml"; + console.log("Writing model file to:", modelPath); + mujocoRef.current.FS.writeFile(modelPath, testModel); + + // Load model + console.log("Loading model..."); + modelRef.current = new mujocoRef.current.Model(modelPath); + console.log("Model properties:", { + nq: modelRef.current.nq, + nv: modelRef.current.nv, + nbody: modelRef.current.nbody, + ngeom: modelRef.current.ngeom + }); + + stateRef.current = new mujocoRef.current.State(modelRef.current); + simulationRef.current = new mujocoRef.current.Simulation( + modelRef.current, + stateRef.current + ); + + const nq = modelRef.current.nq; + if (nq !== 7) { + throw new Error(`Unexpected number of position coordinates: ${nq}`); + } + + const initialQPos = new Float64Array(7); + initialQPos[0] = 0; // x + initialQPos[1] = 0; // y + initialQPos[2] = 1; // z + initialQPos[3] = 1; // qw + initialQPos[4] = 0; // qx + initialQPos[5] = 0; // qy + initialQPos[6] = 0; // qz + + const qpos = simulationRef.current.qpos; + for (let i = 0; i < 7; i++) { + qpos[i] = initialQPos[i]; + } + + // Initialize Three.js scene + const scene = new THREE.Scene(); + sceneRef.current = scene; + scene.background = new THREE.Color(0xf0f0f0); + + // Initialize camera + const camera = new THREE.PerspectiveCamera( + 45, + containerRef.current.clientWidth / containerRef.current.clientHeight, + 0.1, + 1000, + ); + cameraRef.current = camera; + camera.position.set(2, 2, 2); + camera.lookAt(0, 0, 0); + + // Initialize renderer + const renderer = new THREE.WebGLRenderer({ antialias: true }); + rendererRef.current = renderer; + renderer.setSize( + containerRef.current.clientWidth, + containerRef.current.clientHeight, + ); + renderer.shadowMap.enabled = true; + containerRef.current.appendChild(renderer.domElement); + + // Initialize controls + const controls = new OrbitControls(camera, renderer.domElement); + controlsRef.current = controls; + controls.enableDamping = true; + controls.dampingFactor = 0.05; + + // Add lights + const ambientLight = new THREE.AmbientLight(0x404040); + scene.add(ambientLight); + + const directionalLight = new THREE.DirectionalLight(0xffffff, 0.8); + directionalLight.position.set(1, 1, 1); + scene.add(directionalLight); + + // Add grid helper + const gridHelper = new THREE.GridHelper(10, 10); + scene.add(gridHelper); + + // Create visual representation + const robot = new THREE.Group(); + robotRef.current = robot; + scene.add(robot); + + // Animation loop + const animate = () => { + requestAnimationFrame(animate); + + // Update robot position from simulation state + if (simulationRef.current && robotRef.current) { + const qpos = simulationRef.current.qpos; + + // Update root body position and orientation + robotRef.current.position.set(qpos[0], qpos[1], qpos[2]); + + // Convert quaternion to Three.js format (w, x, y, z) + robotRef.current.quaternion.set(qpos[4], qpos[5], qpos[6], qpos[3]); + + // Step the simulation + try { + simulationRef.current.step(); + } catch (error) { + console.error("Simulation step error:", error); + } + } + + controls.update(); + renderer.render(scene, camera); + }; + animate(); + + setIsMujocoReady(true); + } catch (error) { + console.error("Error initializing scene:", error); + setError(error instanceof Error ? error.message : "Unknown error"); + } + }; + + initializeScene(); + + return () => { + if (rendererRef.current?.domElement) { + rendererRef.current.domElement.remove(); + } + }; + }, [files]); + + return ( +
+ {!isMujocoReady ? ( +
Loading MuJoCo...
+ ) : error ? ( +
Error: {error}
+ ) : null} +
+ ); +}); + +export default MJCFRenderer; diff --git a/frontend/src/components/listing/ListingPlaygroundButton.tsx b/frontend/src/components/listing/ListingPlaygroundButton.tsx new file mode 100644 index 00000000..a5c18181 --- /dev/null +++ b/frontend/src/components/listing/ListingPlaygroundButton.tsx @@ -0,0 +1,30 @@ +import { FaPlay } from "react-icons/fa"; +import { useNavigate } from "react-router-dom"; + +import { Button } from "@/components/ui/button"; +import { Artifact } from "@/components/listing/types"; + +interface Props { + artifacts: Artifact[]; +} + +const ListingPlaygroundButton = ({ artifacts }: Props) => { + const navigate = useNavigate(); + + const tgzArtifact = artifacts.find(a => a.artifact_type === 'tgz'); + + if (!tgzArtifact?.artifact_id) return null; + + return ( + + ); +}; + +export default ListingPlaygroundButton; diff --git a/frontend/src/components/listing/ListingRenderer.tsx b/frontend/src/components/listing/ListingRenderer.tsx index bcf554b8..4e4c2f52 100644 --- a/frontend/src/components/listing/ListingRenderer.tsx +++ b/frontend/src/components/listing/ListingRenderer.tsx @@ -10,6 +10,7 @@ import ListingName from "@/components/listing/ListingName"; import ListingOnshape from "@/components/listing/ListingOnshape"; import ListingRegisterRobot from "@/components/listing/ListingRegisterRobot"; import { Artifact, ListingResponse } from "@/components/listing/types"; +import ListingPlaygroundButton from "./ListingPlaygroundButton"; const ListingRenderer = ({ listing }: { listing: ListingResponse }) => { const { @@ -31,14 +32,6 @@ const ListingRenderer = ({ listing }: { listing: ListingResponse }) => { const [artifacts, setArtifacts] = useState(initialArtifacts); const [currentImageIndex, setCurrentImageIndex] = useState(0); - const addArtifacts = (newArtifacts: Artifact[]) => { - setArtifacts((prevArtifacts) => - [...newArtifacts, ...prevArtifacts].sort((a, b) => - a.is_main ? -1 : b.is_main ? 1 : 0, - ), - ); - }; - return (
{/* Main content area - flex column on mobile, row on desktop */} @@ -72,12 +65,13 @@ const ListingRenderer = ({ listing }: { listing: ListingResponse }) => {
{/* Build this robot */} -
+
+
@@ -107,7 +101,7 @@ const ListingRenderer = ({ listing }: { listing: ListingResponse }) => { dropzoneOptions={{ accept: { "image/*": [".png", ".jpg", ".jpeg"] }, }} - addArtifacts={addArtifacts} + addArtifacts={setArtifacts} /> )} @@ -117,7 +111,7 @@ const ListingRenderer = ({ listing }: { listing: ListingResponse }) => { listingId={listingId} onshapeUrl={onshapeUrl} canEdit={canEdit} - addArtifacts={addArtifacts} + addArtifacts={setArtifacts} /> )}
diff --git a/frontend/src/components/pages/ArtifactPlayground.tsx b/frontend/src/components/pages/ArtifactPlayground.tsx new file mode 100644 index 00000000..0c947a0d --- /dev/null +++ b/frontend/src/components/pages/ArtifactPlayground.tsx @@ -0,0 +1,141 @@ +import { useParams } from "react-router-dom"; +import { useEffect, useRef, useState } from "react"; +import { useAuthentication } from "@/hooks/useAuth"; +import { UntarredFile } from "@/components/files/Tarfile"; +import { parseTar } from "@/components/files/Tarfile"; +import MJCFRenderer from "@/components/files/MJCFRenderer"; +import pako from "pako"; + +const ArtifactPlayground = () => { + const { artifactId } = useParams(); + const containerRef = useRef(null); + const rendererRef = useRef(null); + const [files, setFiles] = useState([]); + const [error, setError] = useState(null); + const [isLoading, setIsLoading] = useState(true); + const auth = useAuthentication(); + + useEffect(() => { + const fetchFiles = async () => { + try { + if (!artifactId) { + throw new Error("No artifact ID provided"); + } + + const response = await auth.client.GET("/artifacts/info/{artifact_id}", { + params: { + path: { artifact_id: artifactId }, + }, + }); + + if (response.error) { + const errorMessage = response.error.detail?.[0]?.msg || "API error"; + throw new Error(errorMessage); + } + + const tgzResponse = await fetch(response.data.urls.large); + const arrayBuffer = await tgzResponse.arrayBuffer(); + const uint8Array = new Uint8Array(arrayBuffer); + const decompressed = pako.ungzip(uint8Array); + const files = parseTar(decompressed); + + setFiles(files); + setError(null); + } catch (error) { + console.error("Error fetching files:", error); + setError(error instanceof Error ? error.message : "Unknown error occurred"); + } finally { + setIsLoading(false); + } + }; + + fetchFiles(); + }, [artifactId, auth]); + + useEffect(() => { + const handleResize = () => { + const renderer = rendererRef.current; + const container = containerRef.current; + if (renderer && container) { + renderer.updateDimensions?.( + container.clientWidth, + container.clientHeight + ); + } + }; + + handleResize(); + window.addEventListener('resize', handleResize); + + return () => { + window.removeEventListener('resize', handleResize); + rendererRef.current?.cleanup?.(); + }; + }, []); + + if (isLoading) { + return
Loading...
; + } + + if (error || files.length === 0) { + return ( +
+
+

Error Loading Model

+

{error || "No files available"}

+
+
+ ); + } + + const mjcfFile = files.find(file => + (file.name.endsWith('.mjcf') || file.name.endsWith('.xml')) && + new TextDecoder().decode(file.content).includes(' +
+

Error Loading Model

+

No MJCF file found in artifact

+
+
+ ); + } + + return ( +
+
+
+ {mjcfFile && ( + + )} +
+
+ +
+

Model Controls

+ +
+ +
+
+ MJCF File: {mjcfFile.name} +
+
+ Artifact ID: {artifactId} +
+
+
+
+ ); +}; + +export default ArtifactPlayground;