From 45ae77ef448890f7193b6e8c42b7e38f6e355e49 Mon Sep 17 00:00:00 2001 From: Ivan Date: Fri, 8 Nov 2024 13:18:27 -0800 Subject: [PATCH 01/19] Artifact Playground --- frontend/src/App.tsx | 8 +- .../src/components/files/FileRenderer.tsx | 22 ++ .../src/components/files/MJCFRenderer.tsx | 220 ++++++++++++++++++ .../listing/ListingPlaygroundButton.tsx | 30 +++ .../components/listing/ListingRenderer.tsx | 16 +- .../components/pages/ArtifactPlayground.tsx | 141 +++++++++++ frontend/src/lib/types/routes.ts | 9 +- 7 files changed, 433 insertions(+), 13 deletions(-) create mode 100644 frontend/src/components/files/MJCFRenderer.tsx create mode 100644 frontend/src/components/listing/ListingPlaygroundButton.tsx create mode 100644 frontend/src/components/pages/ArtifactPlayground.tsx diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 75a32bd3..78596b96 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -13,6 +13,7 @@ import Navbar from "@/components/nav/Navbar"; import APIKeys from "@/components/pages/APIKeys"; import About from "@/components/pages/About"; import Account from "@/components/pages/Account"; +import ArtifactPlayground from "@/components/pages/ArtifactPlayground"; import Browse from "@/components/pages/Browse"; import Create from "@/components/pages/Create"; import DeleteConnect from "@/components/pages/DeleteConnect"; @@ -61,7 +62,12 @@ const App = () => { } - /> + > + } + /> + {/* General pages */} } /> diff --git a/frontend/src/components/files/FileRenderer.tsx b/frontend/src/components/files/FileRenderer.tsx index 00c861ff..09523f33 100644 --- a/frontend/src/components/files/FileRenderer.tsx +++ b/frontend/src/components/files/FileRenderer.tsx @@ -1,12 +1,23 @@ +import MJCFRenderer from "./MJCFRenderer"; import STLRenderer from "./STLRenderer"; import { UntarredFile } from "./Tarfile"; import URDFRenderer from "./URDFRenderer"; +const isMJCFFile = (content: string, filename: string): boolean => { + const extension = filename.split(".").pop()?.toLowerCase(); + if (extension !== "xml" && extension !== "mjcf") { + return false; + } + + return content.includes(""); +}; + const FileRenderer: React.FC<{ file: UntarredFile; allFiles: UntarredFile[]; }> = ({ file, allFiles }) => { const fileExtension = file.name.split(".").pop()?.toLowerCase(); + const fileContent = new TextDecoder().decode(file.content); switch (fileExtension) { case "urdf": @@ -19,6 +30,17 @@ const FileRenderer: React.FC<{ ); case "stl": return ; + case "xml": + case "mjcf": + if (isMJCFFile(fileContent, file.name)) { + return ; + } else { + return ( +
+

Invalid MJCF file format

+
+ ); + } default: return (
diff --git a/frontend/src/components/files/MJCFRenderer.tsx b/frontend/src/components/files/MJCFRenderer.tsx new file mode 100644 index 00000000..dba2c374 --- /dev/null +++ b/frontend/src/components/files/MJCFRenderer.tsx @@ -0,0 +1,220 @@ +import React, { forwardRef, useEffect, useRef, useState } from "react"; +import { UntarredFile } from "./Tarfile"; +import load_mujoco, { mujoco } from "@/lib/mujoco/mujoco_wasm.js"; +import * as THREE from "three"; +import { OrbitControls } from "three/examples/jsm/controls/OrbitControls"; + +interface MJCFRendererProps { + mjcfContent: string; + files?: UntarredFile[]; + width?: string | number; + height?: string | number; + useControls?: boolean; + showWireframe?: boolean; +} + +const MJCFRenderer = forwardRef(({ + mjcfContent, + files = [], + width = "100%", + height = "100%", + useControls = true, + showWireframe = false, +}, ref) => { + const containerRef = useRef(null); + const sceneRef = useRef(null); + const rendererRef = useRef(null); + const cameraRef = useRef(null); + const controlsRef = useRef(null); + const robotRef = useRef(null); + + const [isMujocoReady, setIsMujocoReady] = useState(false); + const [error, setError] = useState(null); + + const mujocoRef = useRef(null); + const modelRef = useRef | null>(null); + const stateRef = useRef | null>(null); + const simulationRef = useRef | null>(null); + + useEffect(() => { + const initializeScene = async () => { + if (!containerRef.current) return; + + try { + mujocoRef.current = await load_mujoco(); + console.log("MuJoCo loaded successfully"); + + // Set up file system + mujocoRef.current.FS.mkdir("/working"); + mujocoRef.current.FS.mount( + mujocoRef.current.MEMFS, + { root: "." }, + "/working" + ); + + // Create a simple test model without meshes + const testModel = ` + + + `; + + const modelPath = "/working/model.xml"; + console.log("Writing model file to:", modelPath); + mujocoRef.current.FS.writeFile(modelPath, testModel); + + // Load model + console.log("Loading model..."); + modelRef.current = new mujocoRef.current.Model(modelPath); + console.log("Model properties:", { + nq: modelRef.current.nq, + nv: modelRef.current.nv, + nbody: modelRef.current.nbody, + ngeom: modelRef.current.ngeom + }); + + stateRef.current = new mujocoRef.current.State(modelRef.current); + simulationRef.current = new mujocoRef.current.Simulation( + modelRef.current, + stateRef.current + ); + + const nq = modelRef.current.nq; + if (nq !== 7) { + throw new Error(`Unexpected number of position coordinates: ${nq}`); + } + + const initialQPos = new Float64Array(7); + initialQPos[0] = 0; // x + initialQPos[1] = 0; // y + initialQPos[2] = 1; // z + initialQPos[3] = 1; // qw + initialQPos[4] = 0; // qx + initialQPos[5] = 0; // qy + initialQPos[6] = 0; // qz + + const qpos = simulationRef.current.qpos; + for (let i = 0; i < 7; i++) { + qpos[i] = initialQPos[i]; + } + + // Initialize Three.js scene + const scene = new THREE.Scene(); + sceneRef.current = scene; + scene.background = new THREE.Color(0xf0f0f0); + + // Initialize camera + const camera = new THREE.PerspectiveCamera( + 45, + containerRef.current.clientWidth / containerRef.current.clientHeight, + 0.1, + 1000, + ); + cameraRef.current = camera; + camera.position.set(2, 2, 2); + camera.lookAt(0, 0, 0); + + // Initialize renderer + const renderer = new THREE.WebGLRenderer({ antialias: true }); + rendererRef.current = renderer; + renderer.setSize( + containerRef.current.clientWidth, + containerRef.current.clientHeight, + ); + renderer.shadowMap.enabled = true; + containerRef.current.appendChild(renderer.domElement); + + // Initialize controls + const controls = new OrbitControls(camera, renderer.domElement); + controlsRef.current = controls; + controls.enableDamping = true; + controls.dampingFactor = 0.05; + + // Add lights + const ambientLight = new THREE.AmbientLight(0x404040); + scene.add(ambientLight); + + const directionalLight = new THREE.DirectionalLight(0xffffff, 0.8); + directionalLight.position.set(1, 1, 1); + scene.add(directionalLight); + + // Add grid helper + const gridHelper = new THREE.GridHelper(10, 10); + scene.add(gridHelper); + + // Create visual representation + const robot = new THREE.Group(); + robotRef.current = robot; + scene.add(robot); + + // Animation loop + const animate = () => { + requestAnimationFrame(animate); + + // Update robot position from simulation state + if (simulationRef.current && robotRef.current) { + const qpos = simulationRef.current.qpos; + + // Update root body position and orientation + robotRef.current.position.set(qpos[0], qpos[1], qpos[2]); + + // Convert quaternion to Three.js format (w, x, y, z) + robotRef.current.quaternion.set(qpos[4], qpos[5], qpos[6], qpos[3]); + + // Step the simulation + try { + simulationRef.current.step(); + } catch (error) { + console.error("Simulation step error:", error); + } + } + + controls.update(); + renderer.render(scene, camera); + }; + animate(); + + setIsMujocoReady(true); + } catch (error) { + console.error("Error initializing scene:", error); + setError(error instanceof Error ? error.message : "Unknown error"); + } + }; + + initializeScene(); + + return () => { + if (rendererRef.current?.domElement) { + rendererRef.current.domElement.remove(); + } + }; + }, [files]); + + return ( +
+ {!isMujocoReady ? ( +
Loading MuJoCo...
+ ) : error ? ( +
Error: {error}
+ ) : null} +
+ ); +}); + +export default MJCFRenderer; diff --git a/frontend/src/components/listing/ListingPlaygroundButton.tsx b/frontend/src/components/listing/ListingPlaygroundButton.tsx new file mode 100644 index 00000000..a5c18181 --- /dev/null +++ b/frontend/src/components/listing/ListingPlaygroundButton.tsx @@ -0,0 +1,30 @@ +import { FaPlay } from "react-icons/fa"; +import { useNavigate } from "react-router-dom"; + +import { Button } from "@/components/ui/button"; +import { Artifact } from "@/components/listing/types"; + +interface Props { + artifacts: Artifact[]; +} + +const ListingPlaygroundButton = ({ artifacts }: Props) => { + const navigate = useNavigate(); + + const tgzArtifact = artifacts.find(a => a.artifact_type === 'tgz'); + + if (!tgzArtifact?.artifact_id) return null; + + return ( + + ); +}; + +export default ListingPlaygroundButton; diff --git a/frontend/src/components/listing/ListingRenderer.tsx b/frontend/src/components/listing/ListingRenderer.tsx index bcf554b8..4e4c2f52 100644 --- a/frontend/src/components/listing/ListingRenderer.tsx +++ b/frontend/src/components/listing/ListingRenderer.tsx @@ -10,6 +10,7 @@ import ListingName from "@/components/listing/ListingName"; import ListingOnshape from "@/components/listing/ListingOnshape"; import ListingRegisterRobot from "@/components/listing/ListingRegisterRobot"; import { Artifact, ListingResponse } from "@/components/listing/types"; +import ListingPlaygroundButton from "./ListingPlaygroundButton"; const ListingRenderer = ({ listing }: { listing: ListingResponse }) => { const { @@ -31,14 +32,6 @@ const ListingRenderer = ({ listing }: { listing: ListingResponse }) => { const [artifacts, setArtifacts] = useState(initialArtifacts); const [currentImageIndex, setCurrentImageIndex] = useState(0); - const addArtifacts = (newArtifacts: Artifact[]) => { - setArtifacts((prevArtifacts) => - [...newArtifacts, ...prevArtifacts].sort((a, b) => - a.is_main ? -1 : b.is_main ? 1 : 0, - ), - ); - }; - return (
{/* Main content area - flex column on mobile, row on desktop */} @@ -72,12 +65,13 @@ const ListingRenderer = ({ listing }: { listing: ListingResponse }) => {
{/* Build this robot */} -
+
+
@@ -107,7 +101,7 @@ const ListingRenderer = ({ listing }: { listing: ListingResponse }) => { dropzoneOptions={{ accept: { "image/*": [".png", ".jpg", ".jpeg"] }, }} - addArtifacts={addArtifacts} + addArtifacts={setArtifacts} /> )} @@ -117,7 +111,7 @@ const ListingRenderer = ({ listing }: { listing: ListingResponse }) => { listingId={listingId} onshapeUrl={onshapeUrl} canEdit={canEdit} - addArtifacts={addArtifacts} + addArtifacts={setArtifacts} /> )}
diff --git a/frontend/src/components/pages/ArtifactPlayground.tsx b/frontend/src/components/pages/ArtifactPlayground.tsx new file mode 100644 index 00000000..0c947a0d --- /dev/null +++ b/frontend/src/components/pages/ArtifactPlayground.tsx @@ -0,0 +1,141 @@ +import { useParams } from "react-router-dom"; +import { useEffect, useRef, useState } from "react"; +import { useAuthentication } from "@/hooks/useAuth"; +import { UntarredFile } from "@/components/files/Tarfile"; +import { parseTar } from "@/components/files/Tarfile"; +import MJCFRenderer from "@/components/files/MJCFRenderer"; +import pako from "pako"; + +const ArtifactPlayground = () => { + const { artifactId } = useParams(); + const containerRef = useRef(null); + const rendererRef = useRef(null); + const [files, setFiles] = useState([]); + const [error, setError] = useState(null); + const [isLoading, setIsLoading] = useState(true); + const auth = useAuthentication(); + + useEffect(() => { + const fetchFiles = async () => { + try { + if (!artifactId) { + throw new Error("No artifact ID provided"); + } + + const response = await auth.client.GET("/artifacts/info/{artifact_id}", { + params: { + path: { artifact_id: artifactId }, + }, + }); + + if (response.error) { + const errorMessage = response.error.detail?.[0]?.msg || "API error"; + throw new Error(errorMessage); + } + + const tgzResponse = await fetch(response.data.urls.large); + const arrayBuffer = await tgzResponse.arrayBuffer(); + const uint8Array = new Uint8Array(arrayBuffer); + const decompressed = pako.ungzip(uint8Array); + const files = parseTar(decompressed); + + setFiles(files); + setError(null); + } catch (error) { + console.error("Error fetching files:", error); + setError(error instanceof Error ? error.message : "Unknown error occurred"); + } finally { + setIsLoading(false); + } + }; + + fetchFiles(); + }, [artifactId, auth]); + + useEffect(() => { + const handleResize = () => { + const renderer = rendererRef.current; + const container = containerRef.current; + if (renderer && container) { + renderer.updateDimensions?.( + container.clientWidth, + container.clientHeight + ); + } + }; + + handleResize(); + window.addEventListener('resize', handleResize); + + return () => { + window.removeEventListener('resize', handleResize); + rendererRef.current?.cleanup?.(); + }; + }, []); + + if (isLoading) { + return
Loading...
; + } + + if (error || files.length === 0) { + return ( +
+
+

Error Loading Model

+

{error || "No files available"}

+
+
+ ); + } + + const mjcfFile = files.find(file => + (file.name.endsWith('.mjcf') || file.name.endsWith('.xml')) && + new TextDecoder().decode(file.content).includes(' +
+

Error Loading Model

+

No MJCF file found in artifact

+
+
+ ); + } + + return ( +
+
+
+ {mjcfFile && ( + + )} +
+
+ +
+

Model Controls

+ +
+ +
+
+ MJCF File: {mjcfFile.name} +
+
+ Artifact ID: {artifactId} +
+
+
+
+ ); +}; + +export default ArtifactPlayground; diff --git a/frontend/src/lib/types/routes.ts b/frontend/src/lib/types/routes.ts index 2d4d61e9..83f39f83 100644 --- a/frontend/src/lib/types/routes.ts +++ b/frontend/src/lib/types/routes.ts @@ -2,7 +2,14 @@ import { route, string } from "react-router-typesafe-routes/dom"; const ROUTES = { HOME: route(""), - PLAYGROUND: route("playground"), + PLAYGROUND: route("playground", + {}, + { + WITH_ID: route(":artifactId", { + params: { artifactId: string().defined() }, + }), + }, + ), // General pages ABOUT: route("about"), From 59484a696648527aacc7e18d31516b21beac693b Mon Sep 17 00:00:00 2001 From: Benjamin Bolte Date: Sat, 9 Nov 2024 21:15:34 -0800 Subject: [PATCH 02/19] asdf --- frontend/package-lock.json | 260 +++++++++- frontend/package.json | 1 + frontend/src/App.tsx | 22 +- .../src/components/files/FileRenderer.tsx | 17 +- .../src/components/files/MJCFRenderer.tsx | 346 ++++++------- .../src/components/files/URDFRenderer.tsx | 2 +- .../src/components/files/demo/humanoid.xml | 235 +++++++++ frontend/src/components/files/demo/simple.xml | 11 + .../src/components/files/mujoco/mujoco.ts | 213 ++++++++ .../files/{Tarfile.tsx => untar.ts} | 17 + .../listing/ListingPlaygroundButton.tsx | 30 -- .../components/listing/ListingRenderer.tsx | 4 +- frontend/src/components/nav/Navbar.tsx | 76 ++- frontend/src/components/nav/Sidebar.tsx | 2 - .../components/pages/ArtifactPlayground.tsx | 141 ------ frontend/src/components/pages/FileBrowser.tsx | 57 ++- frontend/src/components/pages/Playground.tsx | 454 ------------------ frontend/src/components/pages/Profile.tsx | 1 - .../terminal/TerminalRobotModel.tsx | 14 +- frontend/src/lib/mujoco/mujoco_wasm.d.ts | 2 +- frontend/src/lib/types/routes.ts | 13 +- frontend/src/react-app-env.d.ts | 1 + frontend/vite.config.ts | 1 + 23 files changed, 977 insertions(+), 943 deletions(-) create mode 100644 frontend/src/components/files/demo/humanoid.xml create mode 100644 frontend/src/components/files/demo/simple.xml create mode 100644 frontend/src/components/files/mujoco/mujoco.ts rename frontend/src/components/files/{Tarfile.tsx => untar.ts} (76%) delete mode 100644 frontend/src/components/listing/ListingPlaygroundButton.tsx delete mode 100644 frontend/src/components/pages/ArtifactPlayground.tsx delete mode 100644 frontend/src/components/pages/Playground.tsx diff --git a/frontend/package-lock.json b/frontend/package-lock.json index acb33816..96319cbe 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -65,6 +65,7 @@ "tailwind-merge": "^2.5.2", "tailwindcss-animate": "^1.0.7", "three": "^0.167.1", + "tsd-jsdoc": "^2.5.0", "urdf-loader": "^0.12.2", "zod": "^3.23.8", "zxcvbn": "^4.4.2" @@ -586,7 +587,6 @@ "version": "7.24.8", "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.24.8.tgz", "integrity": "sha512-pO9KhhRcuUyGnJWwyEgnRJTSIZHiT+vMD0kPeD+so0l7mxkMT19g3pjY9GTnHySck/hDzq+dtW/4VgnMkippsQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" @@ -596,7 +596,6 @@ "version": "7.24.7", "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.7.tgz", "integrity": "sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w==", - "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" @@ -661,7 +660,6 @@ "version": "7.25.3", "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.25.3.tgz", "integrity": "sha512-iLTJKDbJ4hMvFPgQwwsVoxtHyWpKKPBrxkANrSYewDPaPpT5py5yeVkgPIJ7XYXhndxJpaA3PyALSXQ7u8e/Dw==", - "dev": true, "license": "MIT", "dependencies": { "@babel/types": "^7.25.2" @@ -677,7 +675,6 @@ "version": "7.25.2", "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.25.2.tgz", "integrity": "sha512-YTnYtra7W9e6/oAZEHj0bJehPRUlLH9/fbpT5LfB0NhQXyALCRkRs3zH9v07IYhkgpqX6Z78FnuccZr/l4Fs4Q==", - "dev": true, "license": "MIT", "dependencies": { "@babel/helper-string-parser": "^7.24.8", @@ -3564,6 +3561,24 @@ "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", "dev": true }, + "node_modules/@types/linkify-it": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@types/linkify-it/-/linkify-it-5.0.0.tgz", + "integrity": "sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/markdown-it": { + "version": "12.2.3", + "resolved": "https://registry.npmjs.org/@types/markdown-it/-/markdown-it-12.2.3.tgz", + "integrity": "sha512-GKMHFfv3458yYy+v/N8gjufHO6MSZKCOXpZc5GXIWWy8uldwfmPn98vp81gZ5f9SVw8YYBctgfJ22a2d7AOMeQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/linkify-it": "*", + "@types/mdurl": "*" + } + }, "node_modules/@types/mdast": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", @@ -3573,6 +3588,13 @@ "@types/unist": "*" } }, + "node_modules/@types/mdurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@types/mdurl/-/mdurl-2.0.0.tgz", + "integrity": "sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==", + "license": "MIT", + "peer": true + }, "node_modules/@types/ms": { "version": "0.7.34", "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz", @@ -4352,7 +4374,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, "license": "Python-2.0" }, "node_modules/aria-hidden": { @@ -4619,6 +4640,13 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", + "license": "MIT", + "peer": true + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -4774,6 +4802,19 @@ ], "license": "CC-BY-4.0" }, + "node_modules/catharsis": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/catharsis/-/catharsis-0.9.0.tgz", + "integrity": "sha512-prMTQVpcns/tzFgFVkVp6ak6RykZyWb3gu8ckUpd6YkTlacOd3DXGJjIpD4Q6zJirizvaiAjSSHlOsA+6sNh2A==", + "license": "MIT", + "peer": true, + "dependencies": { + "lodash": "^4.17.15" + }, + "engines": { + "node": ">= 10" + } + }, "node_modules/ccount": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", @@ -5331,6 +5372,16 @@ "node": ">=10.13.0" } }, + "node_modules/entities": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.1.0.tgz", + "integrity": "sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w==", + "license": "BSD-2-Clause", + "peer": true, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/envinfo": { "version": "7.14.0", "resolved": "https://registry.npmjs.org/envinfo/-/envinfo-7.14.0.tgz", @@ -6532,8 +6583,7 @@ "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" }, "node_modules/graphemer": { "version": "1.4.0", @@ -7491,6 +7541,56 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/js2xmlparser": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/js2xmlparser/-/js2xmlparser-4.0.2.tgz", + "integrity": "sha512-6n4D8gLlLf1n5mNLQPRfViYzu9RATblzPEtm1SthMX1Pjao0r9YI9nw7ZIfRxQMERS87mcswrg+r/OYrPRX6jA==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "xmlcreate": "^2.0.4" + } + }, + "node_modules/jsdoc": { + "version": "3.6.11", + "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-3.6.11.tgz", + "integrity": "sha512-8UCU0TYeIYD9KeLzEcAu2q8N/mx9O3phAGl32nmHlE0LpaJL71mMkP4d+QE5zWfNt50qheHtOZ0qoxVrsX5TUg==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@babel/parser": "^7.9.4", + "@types/markdown-it": "^12.2.3", + "bluebird": "^3.7.2", + "catharsis": "^0.9.0", + "escape-string-regexp": "^2.0.0", + "js2xmlparser": "^4.0.2", + "klaw": "^3.0.0", + "markdown-it": "^12.3.2", + "markdown-it-anchor": "^8.4.1", + "marked": "^4.0.10", + "mkdirp": "^1.0.4", + "requizzle": "^0.2.3", + "strip-json-comments": "^3.1.0", + "taffydb": "2.6.2", + "underscore": "~1.13.2" + }, + "bin": { + "jsdoc": "jsdoc.js" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/jsdoc/node_modules/escape-string-regexp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=8" + } + }, "node_modules/jsesc": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", @@ -7579,6 +7679,16 @@ "node": ">=0.10.0" } }, + "node_modules/klaw": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/klaw/-/klaw-3.0.0.tgz", + "integrity": "sha512-0Fo5oir+O9jnXu5EefYbVK+mHMBeEVEy2cmctR1O1NECcCkPRreJKrS6Qt/j3KC2C148Dfo9i3pCmCMsdqGr0g==", + "license": "MIT", + "peer": true, + "dependencies": { + "graceful-fs": "^4.1.9" + } + }, "node_modules/leva": { "version": "0.9.35", "resolved": "https://registry.npmjs.org/leva/-/leva-0.9.35.tgz", @@ -7669,6 +7779,16 @@ "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", "license": "MIT" }, + "node_modules/linkify-it": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-3.0.3.tgz", + "integrity": "sha512-ynTsyrFSdE5oZ/O9GEf00kPngmOfVwazR5GKDq6EYfhlpFug3J2zybX56a2PRRpc9P+FuSoGNAwjlbDs9jJBPQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "uc.micro": "^1.0.1" + } + }, "node_modules/loader-runner": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz", @@ -7698,7 +7818,6 @@ "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "dev": true, "license": "MIT" }, "node_modules/lodash.merge": { @@ -7763,6 +7882,34 @@ "three": ">=0.134.0" } }, + "node_modules/markdown-it": { + "version": "12.3.2", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-12.3.2.tgz", + "integrity": "sha512-TchMembfxfNVpHkbtriWltGWc+m3xszaRD0CZup7GFFhzIgQqxIfn3eGj1yZpfuflzPvfkt611B2Q/Bsk1YnGg==", + "license": "MIT", + "peer": true, + "dependencies": { + "argparse": "^2.0.1", + "entities": "~2.1.0", + "linkify-it": "^3.0.1", + "mdurl": "^1.0.1", + "uc.micro": "^1.0.5" + }, + "bin": { + "markdown-it": "bin/markdown-it.js" + } + }, + "node_modules/markdown-it-anchor": { + "version": "8.6.7", + "resolved": "https://registry.npmjs.org/markdown-it-anchor/-/markdown-it-anchor-8.6.7.tgz", + "integrity": "sha512-FlCHFwNnutLgVTflOYHPW2pPcl2AACqVzExlkGQNsi4CJgqOHN7YTgDd4LuhgN1BFO3TS0vLAruV1Td6dwWPJA==", + "license": "Unlicense", + "peer": true, + "peerDependencies": { + "@types/markdown-it": "*", + "markdown-it": "*" + } + }, "node_modules/markdown-table": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.3.tgz", @@ -7773,6 +7920,19 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/marked": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz", + "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==", + "license": "MIT", + "peer": true, + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 12" + } + }, "node_modules/mdast-util-find-and-replace": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.1.tgz", @@ -8055,6 +8215,13 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/mdurl": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz", + "integrity": "sha512-/sKlQJCBYVY9Ers9hqzKou4H6V5UWc/M59TH2dvkt+84itfnq7uFOMLpOiOS4ujvHP4etln18fmIxA5R5fll0g==", + "license": "MIT", + "peer": true + }, "node_modules/merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", @@ -8752,6 +8919,19 @@ "node": ">=0.10.0" } }, + "node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "license": "MIT", + "peer": true, + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/mri": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", @@ -10270,6 +10450,16 @@ "node": ">=0.10.0" } }, + "node_modules/requizzle": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/requizzle/-/requizzle-0.2.4.tgz", + "integrity": "sha512-JRrFk1D4OQ4SqovXOgdav+K8EAhSB/LJZqCz8tbX0KObcdeM15Ss59ozWMBWmmINMagCwmqn4ZNryUGpBsl6Jw==", + "license": "MIT", + "peer": true, + "dependencies": { + "lodash": "^4.17.21" + } + }, "node_modules/resolve": { "version": "2.0.0-next.5", "resolved": "https://registry.npmjs.org/resolve/-/resolve-2.0.0-next.5.tgz", @@ -10944,7 +11134,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -11041,6 +11230,12 @@ "url": "https://opencollective.com/unts" } }, + "node_modules/taffydb": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/taffydb/-/taffydb-2.6.2.tgz", + "integrity": "sha512-y3JaeRSplks6NYQuCOj3ZFMO3j60rTwbuKCvZxsAraGYH2epusatvZ0baZYA01WsGqJBq/Dl6vOrMUJqyMj8kA==", + "peer": true + }, "node_modules/tailwind-merge": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-2.5.2.tgz", @@ -11248,7 +11443,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", - "dev": true, "license": "MIT", "engines": { "node": ">=4" @@ -11335,6 +11529,31 @@ "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==", "license": "Apache-2.0" }, + "node_modules/tsd-jsdoc": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tsd-jsdoc/-/tsd-jsdoc-2.5.0.tgz", + "integrity": "sha512-80fcJLAiUeerg4xPftp+iEEKWUjJjHk9AvcHwJqA8Zw0R4oASdu3kT/plE/Zj19QUTz8KupyOX25zStlNJjS9g==", + "license": "MIT", + "dependencies": { + "typescript": "^3.2.1" + }, + "peerDependencies": { + "jsdoc": "^3.6.3" + } + }, + "node_modules/tsd-jsdoc/node_modules/typescript": { + "version": "3.9.10", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.10.tgz", + "integrity": "sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q==", + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=4.2.0" + } + }, "node_modules/tslib": { "version": "2.6.3", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", @@ -11529,6 +11748,13 @@ "node": "*" } }, + "node_modules/uc.micro": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-1.0.6.tgz", + "integrity": "sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==", + "license": "MIT", + "peer": true + }, "node_modules/unbox-primitive": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", @@ -11545,6 +11771,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/underscore": { + "version": "1.13.7", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.7.tgz", + "integrity": "sha512-GMXzWtsc57XAtguZgaQViUOzs0KTkk8ojr3/xAxXLITqf/3EMwxC0inyETfDFjH/Krbhuep0HNbbjI9i/q3F3g==", + "license": "MIT", + "peer": true + }, "node_modules/undici-types": { "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", @@ -12346,6 +12579,13 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, + "node_modules/xmlcreate": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/xmlcreate/-/xmlcreate-2.0.4.tgz", + "integrity": "sha512-nquOebG4sngPmGPICTS5EnxqhKbCmz5Ox5hsszI2T6U5qdrJizBc+0ilYSEjTSzU0yZcmvppztXe/5Al5fUwdg==", + "license": "Apache-2.0", + "peer": true + }, "node_modules/yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index 17849b3b..a806f3f4 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -80,6 +80,7 @@ "tailwind-merge": "^2.5.2", "tailwindcss-animate": "^1.0.7", "three": "^0.167.1", + "tsd-jsdoc": "^2.5.0", "urdf-loader": "^0.12.2", "zod": "^3.23.8", "zxcvbn": "^4.4.2" diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 78596b96..4935be3f 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -13,7 +13,6 @@ import Navbar from "@/components/nav/Navbar"; import APIKeys from "@/components/pages/APIKeys"; import About from "@/components/pages/About"; import Account from "@/components/pages/Account"; -import ArtifactPlayground from "@/components/pages/ArtifactPlayground"; import Browse from "@/components/pages/Browse"; import Create from "@/components/pages/Create"; import DeleteConnect from "@/components/pages/DeleteConnect"; @@ -27,7 +26,6 @@ import Logout from "@/components/pages/Logout"; import NotFound from "@/components/pages/NotFound"; import OrderSuccess from "@/components/pages/OrderSuccess"; import OrdersPage from "@/components/pages/Orders"; -import Playground from "@/components/pages/Playground"; import PrivacyPolicy from "@/components/pages/PrivacyPolicy"; import Profile from "@/components/pages/Profile"; import ResearchPage from "@/components/pages/ResearchPage"; @@ -58,17 +56,6 @@ const App = () => { } /> - {/* Playground */} - } - > - } - /> - - {/* General pages */} } /> { /> {/* Seller */} - } - > + + } + /> } diff --git a/frontend/src/components/files/FileRenderer.tsx b/frontend/src/components/files/FileRenderer.tsx index 09523f33..fc000377 100644 --- a/frontend/src/components/files/FileRenderer.tsx +++ b/frontend/src/components/files/FileRenderer.tsx @@ -1,7 +1,7 @@ -import MJCFRenderer from "./MJCFRenderer"; -import STLRenderer from "./STLRenderer"; -import { UntarredFile } from "./Tarfile"; -import URDFRenderer from "./URDFRenderer"; +import MJCFRenderer from "@/components/files/MJCFRenderer"; +import STLRenderer from "@/components/files/STLRenderer"; +import URDFRenderer from "@/components/files/URDFRenderer"; +import { UntarredFile } from "@/components/files/untar"; const isMJCFFile = (content: string, filename: string): boolean => { const extension = filename.split(".").pop()?.toLowerCase(); @@ -23,17 +23,16 @@ const FileRenderer: React.FC<{ case "urdf": return ( ); - case "stl": - return ; case "xml": case "mjcf": if (isMJCFFile(fileContent, file.name)) { - return ; + // return ; + return ; } else { return (
@@ -41,6 +40,8 @@ const FileRenderer: React.FC<{
); } + case "stl": + return ; default: return (
diff --git a/frontend/src/components/files/MJCFRenderer.tsx b/frontend/src/components/files/MJCFRenderer.tsx index dba2c374..a71f32ec 100644 --- a/frontend/src/components/files/MJCFRenderer.tsx +++ b/frontend/src/components/files/MJCFRenderer.tsx @@ -1,220 +1,190 @@ -import React, { forwardRef, useEffect, useRef, useState } from "react"; -import { UntarredFile } from "./Tarfile"; -import load_mujoco, { mujoco } from "@/lib/mujoco/mujoco_wasm.js"; +import { useEffect, useRef, useState } from "react"; + +import humanoid from "@/components/files/demo/humanoid.xml"; +import { humanReadableError } from "@/hooks/useAlertQueue"; +import { mujoco } from "@/lib/mujoco/mujoco_wasm"; import * as THREE from "three"; import { OrbitControls } from "three/examples/jsm/controls/OrbitControls"; -interface MJCFRendererProps { - mjcfContent: string; - files?: UntarredFile[]; - width?: string | number; - height?: string | number; +import Spinner from "../ui/Spinner"; +import { + MujocoRefs, + cleanupMujoco, + initializeMujoco, + initializeThreeJS, +} from "./mujoco/mujoco"; + +interface Props { useControls?: boolean; - showWireframe?: boolean; } -const MJCFRenderer = forwardRef(({ - mjcfContent, - files = [], - width = "100%", - height = "100%", - useControls = true, - showWireframe = false, -}, ref) => { +const MJCFRenderer = ({ useControls = true }: Props) => { + const animationFrameRef = useRef(); const containerRef = useRef(null); - const sceneRef = useRef(null); - const rendererRef = useRef(null); - const cameraRef = useRef(null); - const controlsRef = useRef(null); - const robotRef = useRef(null); + // Create refs object for MuJoCo and Three.js + const refs: MujocoRefs = { + mujocoRef: useRef(null), + modelRef: useRef | null>(null), + stateRef: useRef | null>(null), + simulationRef: useRef | null>(null), + rendererRef: useRef(null), + sceneRef: useRef(null), + cameraRef: useRef(null), + controlsRef: useRef(null), + }; + + // Additional refs specific to this renderer + const leftLegRef = useRef(null); + const rightLegRef = useRef(null); + + // State management + const [leftLegAngle, setLeftLegAngle] = useState(0); + const [rightLegAngle, setRightLegAngle] = useState(0); + const isSimulatingRef = useRef(false); + const [isSimulating, setIsSimulating] = useState(false); + const mujocoTimeRef = useRef(0); const [isMujocoReady, setIsMujocoReady] = useState(false); - const [error, setError] = useState(null); - - const mujocoRef = useRef(null); - const modelRef = useRef | null>(null); - const stateRef = useRef | null>(null); - const simulationRef = useRef | null>(null); + const [showControls, setShowControls] = useState(useControls); + const [jointLimits, setJointLimits] = useState<{ + [key: string]: { min: number; max: number }; + }>({}); + const [error, setError] = useState(null); + + // Constants + const DEFAULT_TIMESTEP = 0.002; + const SWING_FREQUENCY = 2.0; + const SWING_AMPLITUDE = 0.8; + + const setupModelGeometry = () => { + if (!refs.sceneRef.current) return; + + // Setup basic geometries + const geometry = new THREE.BoxGeometry(1, 1, 1); + const material = new THREE.MeshPhongMaterial({ color: 0x808080 }); + + leftLegRef.current = new THREE.Mesh(geometry, material); + rightLegRef.current = new THREE.Mesh(geometry, material); + + refs.sceneRef.current.add(leftLegRef.current); + refs.sceneRef.current.add(rightLegRef.current); + }; + + const animate = (time: number) => { + if ( + !refs.rendererRef.current || + !refs.sceneRef.current || + !refs.cameraRef.current + ) { + return; + } + + // Update physics if simulating + if (isSimulatingRef.current && refs.simulationRef.current) { + mujocoTimeRef.current += DEFAULT_TIMESTEP; + refs.simulationRef.current.step(); + } + + // Update controls if enabled + if (useControls && refs.controlsRef.current) { + refs.controlsRef.current.update(); + } + + // Render the scene + refs.rendererRef.current.render( + refs.sceneRef.current, + refs.cameraRef.current, + ); + + // Request next frame + animationFrameRef.current = requestAnimationFrame(animate); + }; + + const stopPhysicsSimulation = () => { + isSimulatingRef.current = false; + setIsSimulating(false); + if (animationFrameRef.current) { + cancelAnimationFrame(animationFrameRef.current); + } + }; useEffect(() => { - const initializeScene = async () => { - if (!containerRef.current) return; + if (!containerRef.current) return; + (async () => { try { - mujocoRef.current = await load_mujoco(); - console.log("MuJoCo loaded successfully"); - - // Set up file system - mujocoRef.current.FS.mkdir("/working"); - mujocoRef.current.FS.mount( - mujocoRef.current.MEMFS, - { root: "." }, - "/working" - ); - - // Create a simple test model without meshes - const testModel = ` - - - `; - - const modelPath = "/working/model.xml"; - console.log("Writing model file to:", modelPath); - mujocoRef.current.FS.writeFile(modelPath, testModel); - - // Load model - console.log("Loading model..."); - modelRef.current = new mujocoRef.current.Model(modelPath); - console.log("Model properties:", { - nq: modelRef.current.nq, - nv: modelRef.current.nv, - nbody: modelRef.current.nbody, - ngeom: modelRef.current.ngeom - }); - - stateRef.current = new mujocoRef.current.State(modelRef.current); - simulationRef.current = new mujocoRef.current.Simulation( - modelRef.current, - stateRef.current - ); - - const nq = modelRef.current.nq; - if (nq !== 7) { - throw new Error(`Unexpected number of position coordinates: ${nq}`); + // Initialize MuJoCo with the humanoid model + if ( + !(await initializeMujoco({ + modelXML: humanoid, + refs, + onInitialized: () => setIsMujocoReady(true), + onError: (error) => setError(error), + })) + ) { + return; } - const initialQPos = new Float64Array(7); - initialQPos[0] = 0; // x - initialQPos[1] = 0; // y - initialQPos[2] = 1; // z - initialQPos[3] = 1; // qw - initialQPos[4] = 0; // qx - initialQPos[5] = 0; // qy - initialQPos[6] = 0; // qz - - const qpos = simulationRef.current.qpos; - for (let i = 0; i < 7; i++) { - qpos[i] = initialQPos[i]; + // Initialize Three.js scene + if ( + !(await initializeThreeJS(refs, containerRef, { + onError: (error) => setError(error), + })) + ) { + return; } - // Initialize Three.js scene - const scene = new THREE.Scene(); - sceneRef.current = scene; - scene.background = new THREE.Color(0xf0f0f0); - - // Initialize camera - const camera = new THREE.PerspectiveCamera( - 45, - containerRef.current.clientWidth / containerRef.current.clientHeight, - 0.1, - 1000, - ); - cameraRef.current = camera; - camera.position.set(2, 2, 2); - camera.lookAt(0, 0, 0); - - // Initialize renderer - const renderer = new THREE.WebGLRenderer({ antialias: true }); - rendererRef.current = renderer; - renderer.setSize( + // Add model-specific setup (bodies, geometries, etc.) + setupModelGeometry(); + + // Start animation loop + animate(performance.now()); + } catch (error) { + setError(error as Error); + } + })(); + + // Handle window resize + const handleResize = () => { + if ( + refs.rendererRef.current && + refs.cameraRef.current && + containerRef.current + ) { + refs.cameraRef.current.aspect = + containerRef.current.clientWidth / containerRef.current.clientHeight; + refs.cameraRef.current.updateProjectionMatrix(); + refs.rendererRef.current.setSize( containerRef.current.clientWidth, containerRef.current.clientHeight, ); - renderer.shadowMap.enabled = true; - containerRef.current.appendChild(renderer.domElement); - - // Initialize controls - const controls = new OrbitControls(camera, renderer.domElement); - controlsRef.current = controls; - controls.enableDamping = true; - controls.dampingFactor = 0.05; - - // Add lights - const ambientLight = new THREE.AmbientLight(0x404040); - scene.add(ambientLight); - - const directionalLight = new THREE.DirectionalLight(0xffffff, 0.8); - directionalLight.position.set(1, 1, 1); - scene.add(directionalLight); - - // Add grid helper - const gridHelper = new THREE.GridHelper(10, 10); - scene.add(gridHelper); - - // Create visual representation - const robot = new THREE.Group(); - robotRef.current = robot; - scene.add(robot); - - // Animation loop - const animate = () => { - requestAnimationFrame(animate); - - // Update robot position from simulation state - if (simulationRef.current && robotRef.current) { - const qpos = simulationRef.current.qpos; - - // Update root body position and orientation - robotRef.current.position.set(qpos[0], qpos[1], qpos[2]); - - // Convert quaternion to Three.js format (w, x, y, z) - robotRef.current.quaternion.set(qpos[4], qpos[5], qpos[6], qpos[3]); - - // Step the simulation - try { - simulationRef.current.step(); - } catch (error) { - console.error("Simulation step error:", error); - } - } - - controls.update(); - renderer.render(scene, camera); - }; - animate(); - - setIsMujocoReady(true); - } catch (error) { - console.error("Error initializing scene:", error); - setError(error instanceof Error ? error.message : "Unknown error"); } }; - initializeScene(); + window.addEventListener("resize", handleResize); return () => { - if (rendererRef.current?.domElement) { - rendererRef.current.domElement.remove(); - } + window.removeEventListener("resize", handleResize); + cleanupMujoco(refs); + stopPhysicsSimulation(); }; - }, [files]); + }, [useControls, containerRef.current]); return ( -
- {!isMujocoReady ? ( -
Loading MuJoCo...
- ) : error ? ( -
Error: {error}
- ) : null} +
+
+ {!isMujocoReady && ( +
+ +
+ )} + {error && ( +
+
{humanReadableError(error)}
+
+ )}
); -}); +}; export default MJCFRenderer; diff --git a/frontend/src/components/files/URDFRenderer.tsx b/frontend/src/components/files/URDFRenderer.tsx index f742c8e2..4eda76f2 100644 --- a/frontend/src/components/files/URDFRenderer.tsx +++ b/frontend/src/components/files/URDFRenderer.tsx @@ -10,7 +10,7 @@ import { FaUndo, } from "react-icons/fa"; -import { UntarredFile } from "@/components/files/Tarfile"; +import { UntarredFile } from "@/components/files/untar"; import * as THREE from "three"; import { OrbitControls } from "three/examples/jsm/controls/OrbitControls"; import { STLLoader } from "three/examples/jsm/loaders/STLLoader"; diff --git a/frontend/src/components/files/demo/humanoid.xml b/frontend/src/components/files/demo/humanoid.xml new file mode 100644 index 00000000..66ecc0f0 --- /dev/null +++ b/frontend/src/components/files/demo/humanoid.xml @@ -0,0 +1,235 @@ + + diff --git a/frontend/src/components/files/demo/simple.xml b/frontend/src/components/files/demo/simple.xml new file mode 100644 index 00000000..6cc33cd2 --- /dev/null +++ b/frontend/src/components/files/demo/simple.xml @@ -0,0 +1,11 @@ + + + diff --git a/frontend/src/components/files/mujoco/mujoco.ts b/frontend/src/components/files/mujoco/mujoco.ts new file mode 100644 index 00000000..7588c538 --- /dev/null +++ b/frontend/src/components/files/mujoco/mujoco.ts @@ -0,0 +1,213 @@ +import load_mujoco, { mujoco } from "@/lib/mujoco/mujoco_wasm.js"; +import * as THREE from "three"; +import { OrbitControls } from "three/examples/jsm/controls/OrbitControls"; + +export interface MujocoRefs { + mujocoRef: React.MutableRefObject; + modelRef: React.MutableRefObject | null>; + stateRef: React.MutableRefObject | null>; + simulationRef: React.MutableRefObject | null>; + rendererRef: React.MutableRefObject; + sceneRef: React.MutableRefObject; + cameraRef: React.MutableRefObject; + controlsRef: React.MutableRefObject; +} + +export interface MujocoInitOptions { + modelXML: string; + refs: MujocoRefs; + onInitialized?: () => void; + onError?: (error: Error) => void; +} + +export const initializeMujoco = async ({ + modelXML, + refs, + onInitialized, + onError, +}: MujocoInitOptions) => { + const MODEL_DIR = "/working"; + const MODEL_FILE = "model.xml"; + const MODEL_PATH = `${MODEL_DIR}/${MODEL_FILE}`; + + try { + // Load MuJoCo WASM module + refs.mujocoRef.current = await load_mujoco(); + const mj = refs.mujocoRef.current; + + // Set up file system and load XML model + // @ts-ignore + if (!mj.FS.analyzePath(MODEL_DIR).exists) { + mj.FS.mkdir(MODEL_DIR); + } + mj.FS.writeFile(MODEL_PATH, modelXML); + + const model = new mj.Model(MODEL_PATH); + const state = new mj.State(model); + const simulation = new mj.Simulation(model, state); + + // Store references + refs.modelRef.current = model; + refs.stateRef.current = state; + refs.simulationRef.current = simulation; + + onInitialized?.(); + return true; + } catch (error) { + onError?.(error as Error); + return false; + } +}; + +export interface ThreeJSInitOptions { + cameraDistance?: number; + cameraHeight?: number; + backgroundColor?: THREE.Color; + onError?: (error: Error) => void; +} + +export const initializeThreeJS = ( + refs: MujocoRefs, + containerRef: React.RefObject, + { + cameraDistance = 2.5, + cameraHeight = 1.5, + backgroundColor = new THREE.Color(0.15, 0.25, 0.35), + onError, + }: ThreeJSInitOptions = {}, +) => { + const container = containerRef.current; + + if (!container) { + onError?.(new Error("Container ref is null")); + return false; + } + + try { + // Set up renderer + const renderer = new THREE.WebGLRenderer({ + antialias: true, + alpha: true, + powerPreference: "high-performance", + }); + + if (!renderer.getContext()) { + throw new Error("Failed to get WebGL context"); + } + + renderer.setSize(container.clientWidth, container.clientHeight); + container.appendChild(renderer.domElement); + refs.rendererRef.current = renderer; + + // Set up scene + const scene = new THREE.Scene(); + scene.background = backgroundColor; + refs.sceneRef.current = scene; + + // Set up camera + const camera = new THREE.PerspectiveCamera( + 45, + container.clientWidth / container.clientHeight, + 0.001, + 100, + ); + camera.position.set( + cameraDistance * Math.cos(Math.PI / 4), + cameraHeight, + cameraDistance * Math.sin(Math.PI / 4), + ); + scene.add(camera); + refs.cameraRef.current = camera; + + // Set up controls + const controls = new OrbitControls(camera, renderer.domElement); + controls.enableDamping = true; + controls.dampingFactor = 0.1; + controls.minDistance = 1.0; + controls.maxDistance = 5.0; + controls.update(); + refs.controlsRef.current = controls; + + // Add lighting + const ambientLight = new THREE.AmbientLight(0xffffff, 0.5); + scene.add(ambientLight); + + const dirLight = new THREE.DirectionalLight(0xffffff, 0.5); + dirLight.position.set(5, 5, 5); + scene.add(dirLight); + + // Add floor + const floorGeometry = new THREE.PlaneGeometry(10, 10); + const floorMaterial = new THREE.MeshStandardMaterial({ + color: 0xe0e0e0, + roughness: 0.7, + metalness: 0.1, + }); + const floor = new THREE.Mesh(floorGeometry, floorMaterial); + floor.rotation.x = -Math.PI / 2; + floor.position.y = 0; + scene.add(floor); + + return true; + } catch (error) { + onError?.(error as Error); + + // Clean up any partially initialized resources + if (refs.rendererRef.current) { + refs.rendererRef.current.dispose(); + refs.rendererRef.current.forceContextLoss(); + refs.rendererRef.current.domElement?.remove(); + refs.rendererRef.current = null; + } + return false; + } +}; + +export const cleanupMujoco = (refs: MujocoRefs) => { + // Clean up Three.js resources + if (refs.rendererRef.current) { + refs.rendererRef.current.dispose(); + refs.rendererRef.current.forceContextLoss(); + refs.rendererRef.current.domElement.remove(); + } + + if (refs.sceneRef.current) { + refs.sceneRef.current.traverse((object) => { + if (object instanceof THREE.Mesh) { + object.geometry.dispose(); + if (object.material instanceof THREE.Material) { + object.material.dispose(); + } else if (Array.isArray(object.material)) { + object.material.forEach((material) => material.dispose()); + } + } + }); + } + + if (refs.controlsRef.current) { + refs.controlsRef.current.dispose(); + } + + // Clean up MuJoCo resources + if (refs.stateRef.current) { + refs.stateRef.current.free(); + } + if (refs.simulationRef.current) { + refs.simulationRef.current.free(); + } + if (refs.modelRef.current) { + refs.modelRef.current.free(); + } + + // Reset all refs except containerRef + refs.mujocoRef.current = null; + refs.modelRef.current = null; + refs.stateRef.current = null; + refs.simulationRef.current = null; + refs.rendererRef.current = null; + refs.sceneRef.current = null; + refs.cameraRef.current = null; + refs.controlsRef.current = null; +}; diff --git a/frontend/src/components/files/Tarfile.tsx b/frontend/src/components/files/untar.ts similarity index 76% rename from frontend/src/components/files/Tarfile.tsx rename to frontend/src/components/files/untar.ts index f35294da..026411b3 100644 --- a/frontend/src/components/files/Tarfile.tsx +++ b/frontend/src/components/files/untar.ts @@ -1,3 +1,5 @@ +import pako from "pako"; + export interface UntarredFile { name: string; content: Uint8Array; @@ -58,3 +60,18 @@ export const parseTar = (buffer: Uint8Array): UntarredFile[] => { return files; }; + +export const untarFile = async (url: string) => { + const response = await fetch(url); + const arrayBuffer = await response.arrayBuffer(); + const uint8Array = new Uint8Array(arrayBuffer); + const decompressed = pako.ungzip(uint8Array); + const files = parseTar(decompressed); + return files; +}; + +export const cleanXml = (xml: string) => { + xml = xml.replace(/<\?xml version="1.0" encoding="UTF-8"\?>/, ""); + xml = xml.replace(/\n/g, ""); + return xml; +}; diff --git a/frontend/src/components/listing/ListingPlaygroundButton.tsx b/frontend/src/components/listing/ListingPlaygroundButton.tsx deleted file mode 100644 index a5c18181..00000000 --- a/frontend/src/components/listing/ListingPlaygroundButton.tsx +++ /dev/null @@ -1,30 +0,0 @@ -import { FaPlay } from "react-icons/fa"; -import { useNavigate } from "react-router-dom"; - -import { Button } from "@/components/ui/button"; -import { Artifact } from "@/components/listing/types"; - -interface Props { - artifacts: Artifact[]; -} - -const ListingPlaygroundButton = ({ artifacts }: Props) => { - const navigate = useNavigate(); - - const tgzArtifact = artifacts.find(a => a.artifact_type === 'tgz'); - - if (!tgzArtifact?.artifact_id) return null; - - return ( - - ); -}; - -export default ListingPlaygroundButton; diff --git a/frontend/src/components/listing/ListingRenderer.tsx b/frontend/src/components/listing/ListingRenderer.tsx index 4e4c2f52..9da60f33 100644 --- a/frontend/src/components/listing/ListingRenderer.tsx +++ b/frontend/src/components/listing/ListingRenderer.tsx @@ -9,8 +9,7 @@ import ListingMetadata from "@/components/listing/ListingMetadata"; import ListingName from "@/components/listing/ListingName"; import ListingOnshape from "@/components/listing/ListingOnshape"; import ListingRegisterRobot from "@/components/listing/ListingRegisterRobot"; -import { Artifact, ListingResponse } from "@/components/listing/types"; -import ListingPlaygroundButton from "./ListingPlaygroundButton"; +import { ListingResponse } from "@/components/listing/types"; const ListingRenderer = ({ listing }: { listing: ListingResponse }) => { const { @@ -71,7 +70,6 @@ const ListingRenderer = ({ listing }: { listing: ListingResponse }) => { listingId={listingId} initialFeatured={isFeatured} /> -
diff --git a/frontend/src/components/nav/Navbar.tsx b/frontend/src/components/nav/Navbar.tsx index fec38d87..1c8bb640 100644 --- a/frontend/src/components/nav/Navbar.tsx +++ b/frontend/src/components/nav/Navbar.tsx @@ -1,11 +1,11 @@ import { useState } from "react"; import { - FaBars, - FaGithub, - FaRegFileLines, - FaRobot, - FaWpexplorer, -} from "react-icons/fa6"; + FaChevronDown, + FaChevronUp, + FaExternalLinkAlt, + FaMicroscope, +} from "react-icons/fa"; +import { FaBars, FaGithub, FaRegFileLines } from "react-icons/fa6"; import { Link, useLocation, useNavigate } from "react-router-dom"; import Logo from "@/components/Logo"; @@ -13,17 +13,12 @@ import { useFeaturedListings } from "@/components/listing/FeaturedListings"; import Sidebar from "@/components/nav/Sidebar"; import { useAuthentication } from "@/hooks/useAuth"; import ROUTES from "@/lib/types/routes"; -import { - ChevronDownIcon, - ChevronUpIcon, - ExternalLinkIcon, - MagnifyingGlassIcon, -} from "@radix-ui/react-icons"; type NavItem = { name: string; path: string; - isExternal: boolean; + icon?: JSX.Element; + isExternal?: boolean; }; const Navbar = () => { @@ -34,32 +29,28 @@ const Navbar = () => { const [showDevelopersDropdown, setShowDevelopersDropdown] = useState(false); const { featuredListings } = useFeaturedListings(); - let navItems: NavItem[] = []; + let navItems: NavItem[] = [ + { + name: "Bots", + path: ROUTES.BOTS.BROWSE.path, + }, + ]; if (isAuthenticated) { navItems = [ - { name: "Terminal", path: "/terminal", isExternal: false }, + { + name: "Terminal", + path: "/terminal", + }, ...navItems, ]; } - const technicalItems = [ - { - name: "Bots", - path: ROUTES.BOTS.BROWSE.path, - icon: , - isExternal: false, - }, - { - name: "Playground", - path: ROUTES.PLAYGROUND.path, - icon: , - isExternal: false, - }, + const technicalItems: NavItem[] = [ { name: "Research", path: ROUTES.RESEARCH.path, - icon: , + icon: , isExternal: false, }, { @@ -99,9 +90,11 @@ const Navbar = () => { rel="noopener noreferrer" >
- {icon} - {title} - +
+ {icon} +
+ {title} +
) : ( @@ -110,8 +103,10 @@ const Navbar = () => { className={`block select-none rounded-md p-2 leading-none no-underline outline-none transition-colors hover:bg-gray-1 hover:text-primary-9 focus:bg-gray-1 focus:text-primary-9 ${className}`} >
- {icon} - {title} +
+ {icon} +
+ {title}
)} @@ -180,7 +175,7 @@ const Navbar = () => { rel="noopener noreferrer" > {item.name} - + ) : ( { : "text-gray-1 hover:bg-gray-1 hover:text-primary-9" }`} > - {item.name} +
+ {item.icon} + {item.name} +
), )} @@ -204,9 +202,9 @@ const Navbar = () => { >
{ href={item.path} icon={item.icon} className="group text-gray-1" - isExternal={item.isExternal} + isExternal={item.isExternal || false} /> ))} diff --git a/frontend/src/components/nav/Sidebar.tsx b/frontend/src/components/nav/Sidebar.tsx index 65538b3e..92da073d 100644 --- a/frontend/src/components/nav/Sidebar.tsx +++ b/frontend/src/components/nav/Sidebar.tsx @@ -2,7 +2,6 @@ import { FaDiscord, FaGithub, FaTimes } from "react-icons/fa"; import { FaDownload, FaRegFileLines, - FaRobot, FaSearchengin, FaTerminal, FaWpexplorer, @@ -66,7 +65,6 @@ const Sidebar = ({ show, onClose }: SidebarProps) => { const technicalItems = [ { name: "Browse", path: "/browse", icon: }, { name: "Downloads", path: "/downloads", icon: }, - { name: "Playground", path: "/playground", icon: }, { name: "Research", path: "/research", icon: }, { name: "Docs", diff --git a/frontend/src/components/pages/ArtifactPlayground.tsx b/frontend/src/components/pages/ArtifactPlayground.tsx deleted file mode 100644 index 0c947a0d..00000000 --- a/frontend/src/components/pages/ArtifactPlayground.tsx +++ /dev/null @@ -1,141 +0,0 @@ -import { useParams } from "react-router-dom"; -import { useEffect, useRef, useState } from "react"; -import { useAuthentication } from "@/hooks/useAuth"; -import { UntarredFile } from "@/components/files/Tarfile"; -import { parseTar } from "@/components/files/Tarfile"; -import MJCFRenderer from "@/components/files/MJCFRenderer"; -import pako from "pako"; - -const ArtifactPlayground = () => { - const { artifactId } = useParams(); - const containerRef = useRef(null); - const rendererRef = useRef(null); - const [files, setFiles] = useState([]); - const [error, setError] = useState(null); - const [isLoading, setIsLoading] = useState(true); - const auth = useAuthentication(); - - useEffect(() => { - const fetchFiles = async () => { - try { - if (!artifactId) { - throw new Error("No artifact ID provided"); - } - - const response = await auth.client.GET("/artifacts/info/{artifact_id}", { - params: { - path: { artifact_id: artifactId }, - }, - }); - - if (response.error) { - const errorMessage = response.error.detail?.[0]?.msg || "API error"; - throw new Error(errorMessage); - } - - const tgzResponse = await fetch(response.data.urls.large); - const arrayBuffer = await tgzResponse.arrayBuffer(); - const uint8Array = new Uint8Array(arrayBuffer); - const decompressed = pako.ungzip(uint8Array); - const files = parseTar(decompressed); - - setFiles(files); - setError(null); - } catch (error) { - console.error("Error fetching files:", error); - setError(error instanceof Error ? error.message : "Unknown error occurred"); - } finally { - setIsLoading(false); - } - }; - - fetchFiles(); - }, [artifactId, auth]); - - useEffect(() => { - const handleResize = () => { - const renderer = rendererRef.current; - const container = containerRef.current; - if (renderer && container) { - renderer.updateDimensions?.( - container.clientWidth, - container.clientHeight - ); - } - }; - - handleResize(); - window.addEventListener('resize', handleResize); - - return () => { - window.removeEventListener('resize', handleResize); - rendererRef.current?.cleanup?.(); - }; - }, []); - - if (isLoading) { - return
Loading...
; - } - - if (error || files.length === 0) { - return ( -
-
-

Error Loading Model

-

{error || "No files available"}

-
-
- ); - } - - const mjcfFile = files.find(file => - (file.name.endsWith('.mjcf') || file.name.endsWith('.xml')) && - new TextDecoder().decode(file.content).includes(' -
-

Error Loading Model

-

No MJCF file found in artifact

-
-
- ); - } - - return ( -
-
-
- {mjcfFile && ( - - )} -
-
- -
-

Model Controls

- -
- -
-
- MJCF File: {mjcfFile.name} -
-
- Artifact ID: {artifactId} -
-
-
-
- ); -}; - -export default ArtifactPlayground; diff --git a/frontend/src/components/pages/FileBrowser.tsx b/frontend/src/components/pages/FileBrowser.tsx index 7565d5e2..4e7633b2 100644 --- a/frontend/src/components/pages/FileBrowser.tsx +++ b/frontend/src/components/pages/FileBrowser.tsx @@ -11,7 +11,7 @@ import { useTypedParams } from "react-router-typesafe-routes/dom"; import FileRenderer from "@/components/files/FileRenderer"; import FileTreeViewer from "@/components/files/FileTreeViewer"; -import { parseTar } from "@/components/files/Tarfile"; +import { UntarredFile, untarFile } from "@/components/files/untar"; import Spinner from "@/components/ui/Spinner"; import { Tooltip } from "@/components/ui/ToolTip"; import { Button } from "@/components/ui/button"; @@ -20,19 +20,13 @@ import { components } from "@/gen/api"; import { humanReadableError, useAlertQueue } from "@/hooks/useAlertQueue"; import { useAuthentication } from "@/hooks/useAuth"; import ROUTES from "@/lib/types/routes"; -import pako from "pako"; type SingleArtifactResponse = components["schemas"]["SingleArtifactResponse"]; -interface UntarredFile { - name: string; - content: Uint8Array; -} - const FileBrowser = () => { - const { artifactId } = useTypedParams(ROUTES.FILE); + const { artifactId, fileName } = useTypedParams(ROUTES.FILE); const [artifact, setArtifact] = useState(null); - const [loading, setLoading] = useState(true); + const [isLoading, setIsLoading] = useState(true); const [untarring, setUntarring] = useState(false); const [untarredFiles, setUntarredFiles] = useState([]); const [selectedFile, setSelectedFile] = useState(null); @@ -67,12 +61,15 @@ const FileBrowser = () => { if (data.urls?.large) { setUntarring(true); try { - const response = await fetch(data.urls.large); - const arrayBuffer = await response.arrayBuffer(); - const uint8Array = new Uint8Array(arrayBuffer); - const decompressed = pako.ungzip(uint8Array); - const files = parseTar(decompressed); + const files = await untarFile(data.urls.large); setUntarredFiles(files); + + if (fileName) { + const file = files.find((file) => file.name === fileName); + if (file) { + setSelectedFile(file); + } + } } catch (err) { addErrorAlert(`Error loading file: ${humanReadableError(err)}`); } finally { @@ -83,10 +80,19 @@ const FileBrowser = () => { } catch (err) { addErrorAlert(humanReadableError(err)); } finally { - setLoading(false); + setIsLoading(false); } })(); - }, [artifactId, auth.client, addErrorAlert, artifact]); + }, [artifactId, auth.client, addErrorAlert]); + + useEffect(() => { + if (fileName && untarredFiles.length > 0) { + const file = untarredFiles.find((file) => file.name === fileName); + if (file) { + setSelectedFile(file); + } + } + }, [fileName, untarredFiles]); useEffect(() => { if (artifact) { @@ -110,6 +116,9 @@ const FileBrowser = () => { const handleFileSelect = (file: UntarredFile) => { setSelectedFile(file); + navigate(ROUTES.FILE.buildPath({ artifactId, fileName: file.name }), { + replace: true, + }); }; const handleSaveEdit = async () => { @@ -139,15 +148,11 @@ const FileBrowser = () => { } }; - if (loading) { - return ( -
- -
- ); - } - - return artifact?.urls.large ? ( + return isLoading ? ( +
+ +
+ ) : artifact?.urls.large ? (
{isEditing ? ( @@ -286,7 +291,7 @@ const FileBrowser = () => {
) : (
- + Error loading artifact
); }; diff --git a/frontend/src/components/pages/Playground.tsx b/frontend/src/components/pages/Playground.tsx deleted file mode 100644 index eb6fcac6..00000000 --- a/frontend/src/components/pages/Playground.tsx +++ /dev/null @@ -1,454 +0,0 @@ -// Import necessary dependencies -import { useEffect, useRef, useState } from "react"; - -import load_mujoco, { mujoco } from "@/lib/mujoco/mujoco_wasm.js"; -import * as THREE from "three"; -import { OrbitControls } from "three/examples/jsm/controls/OrbitControls"; - -const Playground = () => { - const containerRef = useRef(null); - const mujocoRef = useRef(null); - const modelRef = useRef | null>(null); - const stateRef = useRef | null>(null); - const simulationRef = useRef | null>(null); - const rendererRef = useRef(null); - const sceneRef = useRef(null); - const cameraRef = useRef(null); - const controlsRef = useRef(null); - const leftLegRef = useRef(null); - const rightLegRef = useRef(null); - - // Add state for joint angles - const [leftLegAngle, setLeftLegAngle] = useState(0); - const [rightLegAngle, setRightLegAngle] = useState(0); - - // Add new refs for physics simulation - const isSimulatingRef = useRef(false); - - // Add simulation state - const [isSimulating, setIsSimulating] = useState(false); - - // Add mujoco time tracking - const mujocoTimeRef = useRef(0); - - // Add a state to track MuJoCo initialization - const [isMujocoReady, setIsMujocoReady] = useState(false); - - // Add a constant for the default timestep - const DEFAULT_TIMESTEP = 0.002; - - // Add these constants near the top of the component - const SWING_FREQUENCY = 2.0; // Hz - const SWING_AMPLITUDE = 0.8; // radians - - // Initialize Three.js and MuJoCo only once - useEffect(() => { - const initializeMuJoCo = async () => { - try { - // Load MuJoCo WASM module - mujocoRef.current = await load_mujoco(); - - // Set up file system and load XML model - mujocoRef.current.FS.mkdir("/working"); - mujocoRef.current.FS.mount( - mujocoRef.current.MEMFS, - { root: "." }, - "/working", - ); - - // Create a simple test model with two hinge joints - const xmlContent = ` - - `; - - mujocoRef.current.FS.writeFile("/working/model.xml", xmlContent); - - // Load model - modelRef.current = new mujocoRef.current.Model("/working/model.xml"); - - // Create initial state - stateRef.current = new mujocoRef.current.State(modelRef.current); - - // Create simulation - simulationRef.current = new mujocoRef.current.Simulation( - modelRef.current, - stateRef.current, - ); - - // Verify qpos exists - if (!simulationRef.current.qpos) { - throw new Error("MuJoCo simulation data not properly initialized"); - } - - setIsMujocoReady(true); - } catch (error) { - console.error("Error initializing MuJoCo:", error); - setIsMujocoReady(false); - } - }; - - const initializeThreeJS = () => { - const container = containerRef.current; - if (!container) return; - - // Set up Three.js scene - const scene = new THREE.Scene(); - scene.background = new THREE.Color(0.15, 0.25, 0.35); - sceneRef.current = scene; - - // Set up camera with container dimensions instead of window - const camera = new THREE.PerspectiveCamera( - 45, - container.clientWidth / container.clientHeight, - 0.001, - 100, - ); - camera.position.set(2.0, 1.7, 1.7); - scene.add(camera); - cameraRef.current = camera; - - // Set up renderer with container dimensions - const renderer = new THREE.WebGLRenderer({ antialias: true }); - renderer.setSize(container.clientWidth, container.clientHeight); - container.appendChild(renderer.domElement); - rendererRef.current = renderer; - - // Set up controls - const controls = new OrbitControls(camera, renderer.domElement); - controls.target.set(0, 0.7, 0); - controls.enableDamping = true; - controls.dampingFactor = 0.1; - controlsRef.current = controls; - - // Add ambient light - const ambientLight = new THREE.AmbientLight(0xffffff, 0.5); - scene.add(ambientLight); - - // Add floor - const floorGeometry = new THREE.PlaneGeometry(4, 4); - const floorMaterial = new THREE.MeshStandardMaterial({ - color: 0xe0e0e0, - roughness: 0.7, - metalness: 0.1, - }); - const floor = new THREE.Mesh(floorGeometry, floorMaterial); - floor.rotation.x = -Math.PI / 2; // Rotate to be horizontal - floor.position.y = 0; // Place at y=0 - scene.add(floor); - - // Add directional light for better shadows - const dirLight = new THREE.DirectionalLight(0xffffff, 0.5); - dirLight.position.set(5, 5, 5); - scene.add(dirLight); - - // Remove the existing block code and add robot parts - // Robot body - const bodyGeometry = new THREE.BoxGeometry(0.3, 0.2, 0.2); - const bodyMaterial = new THREE.MeshStandardMaterial({ color: 0x4444ff }); - const bodyMesh = new THREE.Mesh(bodyGeometry, bodyMaterial); - bodyMesh.position.y = 0.5; - scene.add(bodyMesh); - - // Robot legs - const legGeometry = new THREE.BoxGeometry(0.1, 0.3, 0.1); - // Move the geometry's origin to the top - legGeometry.translate(0, -0.15, 0); - const legMaterial = new THREE.MeshStandardMaterial({ color: 0x444444 }); - - // Left leg - const leftLeg = new THREE.Mesh(legGeometry, legMaterial); - leftLeg.position.set(-0.1, 0.4, 0); // Adjusted Y position up to match with body - scene.add(leftLeg); - leftLegRef.current = leftLeg; - - // Right leg - const rightLeg = new THREE.Mesh(legGeometry, legMaterial); - rightLeg.position.set(0.1, 0.4, 0); // Adjusted Y position up to match with body - scene.add(rightLeg); - rightLegRef.current = rightLeg; - - const animate = (timeMS: number) => { - if (leftLegRef.current && rightLegRef.current) { - if ( - isSimulatingRef.current && - simulationRef.current && - modelRef.current - ) { - try { - if (timeMS - mujocoTimeRef.current > 35.0) { - mujocoTimeRef.current = timeMS; - } - - while (mujocoTimeRef.current < timeMS) { - // Add oscillating control signals - const time = mujocoTimeRef.current / 1000.0; - const ctrl = simulationRef.current.ctrl; - - // Left leg leads by π radians (180 degrees) - ctrl[0] = - SWING_AMPLITUDE * - Math.sin(2 * Math.PI * SWING_FREQUENCY * time); - // Right leg follows - ctrl[1] = - SWING_AMPLITUDE * - Math.sin(2 * Math.PI * SWING_FREQUENCY * time + Math.PI); - - simulationRef.current.step(); - mujocoTimeRef.current += DEFAULT_TIMESTEP * 1000.0; - - if (simulationRef.current.qpos) { - const qpos = simulationRef.current.qpos; - if (leftLegRef.current) { - leftLegRef.current.userData.angle = qpos[7]; - } - if (rightLegRef.current) { - rightLegRef.current.userData.angle = qpos[8]; - } - } - } - } catch (error) { - console.error("Simulation error:", error); - isSimulatingRef.current = false; - } - } - - // Update visual representation - leftLegRef.current.rotation.x = - leftLegRef.current.userData.angle || 0; - rightLegRef.current.rotation.x = - rightLegRef.current.userData.angle || 0; - - // Update leg positions - leftLegRef.current.position.z = - Math.sin(leftLegRef.current.userData.angle || 0) * 0.15; - rightLegRef.current.position.z = - Math.sin(rightLegRef.current.userData.angle || 0) * 0.15; - } - - controlsRef.current?.update(); - rendererRef.current?.render(sceneRef.current!, cameraRef.current!); - requestAnimationFrame(animate); - }; - - animate(performance.now()); - }; - - const handleResize = () => { - const renderer = rendererRef.current; - const camera = cameraRef.current; - const container = containerRef.current; - if (renderer && camera && container) { - camera.aspect = container.clientWidth / container.clientHeight; - camera.updateProjectionMatrix(); - renderer.setSize(container.clientWidth, container.clientHeight); - } - }; - - // Initialize MuJoCo and Three.js components - initializeMuJoCo(); - initializeThreeJS(); - - // Event listener for window resizing - window.addEventListener("resize", handleResize); - - return () => { - // Clean up resources on component unmount - window.removeEventListener("resize", handleResize); - if (rendererRef.current) rendererRef.current.dispose(); - stopPhysicsSimulation(); - }; - }, []); // Empty dependency array - run only once - - // Update the effect that handles slider changes - useEffect(() => { - if (isMujocoReady && !isSimulatingRef.current && simulationRef.current) { - try { - // Safety check to ensure qpos exists - if (!simulationRef.current.qpos) { - console.warn("MuJoCo simulation data not properly initialized"); - return; - } - - // Update MuJoCo state - const qpos = simulationRef.current.qpos; - - // Clamp values to safe ranges - const maxAngle = Math.PI / 2; - const clampedLeftAngle = Math.max( - -maxAngle, - Math.min(maxAngle, leftLegAngle), - ); - const clampedRightAngle = Math.max( - -maxAngle, - Math.min(maxAngle, rightLegAngle), - ); - - qpos[7] = clampedLeftAngle; - qpos[8] = clampedRightAngle; - - // Forward the simulation to update positions - simulationRef.current.forward(); - - // Update Three.js visualization - if (leftLegRef.current) { - leftLegRef.current.userData.angle = clampedLeftAngle; - } - if (rightLegRef.current) { - rightLegRef.current.userData.angle = clampedRightAngle; - } - } catch (error) { - console.error("Error updating MuJoCo state:", error); - } - } - }, [leftLegAngle, rightLegAngle, isMujocoReady]); - - // Add physics simulation loop - const startPhysicsSimulation = () => { - if ( - !simulationRef.current || - isSimulatingRef.current || - !mujocoRef.current || - !modelRef.current - ) - return; - - try { - // Create a new state to reset the simulation - stateRef.current = new mujocoRef.current.State(modelRef.current); - simulationRef.current = new mujocoRef.current.Simulation( - modelRef.current, - stateRef.current, - ); - - // Set initial joint positions (qpos) - const qpos = simulationRef.current.qpos; - // First 7 values are for the free joint (position[3] + quaternion[4]) - qpos[0] = 0; // x position - qpos[1] = 0; // y position - qpos[2] = 0.5; // z position - qpos[3] = 1; // quaternion w - qpos[4] = 0; // quaternion x - qpos[5] = 0; // quaternion y - qpos[6] = 0; // quaternion z - // Then the leg joints - qpos[7] = leftLegAngle; // Left leg - qpos[8] = rightLegAngle; // Right leg - - // Reset velocities to zero (qvel) - const qvel = simulationRef.current.qvel; - for (let i = 0; i < qvel.length; i++) { - qvel[i] = 0; - } - - isSimulatingRef.current = true; - mujocoTimeRef.current = performance.now(); - } catch (error) { - console.error("Error starting simulation:", error); - } - }; - - const stopPhysicsSimulation = () => { - isSimulatingRef.current = false; - }; - - // Add simulation controls to the UI - return ( -
- {/* 3D Viewer */} -
-
-
- - {/* Control Panel */} -
-

Joint Controls

- - {!isMujocoReady ? ( -
Loading MuJoCo...
- ) : ( - <> - {/* Simulation button */} - - - {/* Controls */} -
-
- - setLeftLegAngle(parseFloat(e.target.value))} - className="w-full" - disabled={isSimulating} - /> -
- {((leftLegAngle * 180) / Math.PI).toFixed(1)}° -
-
- -
- - setRightLegAngle(parseFloat(e.target.value))} - className="w-full" - disabled={isSimulating} - /> -
- {((rightLegAngle * 180) / Math.PI).toFixed(1)}° -
-
-
- - )} -
-
- ); -}; - -export default Playground; diff --git a/frontend/src/components/pages/Profile.tsx b/frontend/src/components/pages/Profile.tsx index 0720e951..3bb1feeb 100644 --- a/frontend/src/components/pages/Profile.tsx +++ b/frontend/src/components/pages/Profile.tsx @@ -361,7 +361,6 @@ export const RenderProfile = (props: RenderProfileProps) => { ) : (
-

Bio

{user.bio ? (

{user.bio}

) : ( diff --git a/frontend/src/components/terminal/TerminalRobotModel.tsx b/frontend/src/components/terminal/TerminalRobotModel.tsx index f39e4f4d..34790257 100644 --- a/frontend/src/components/terminal/TerminalRobotModel.tsx +++ b/frontend/src/components/terminal/TerminalRobotModel.tsx @@ -1,21 +1,15 @@ import { useEffect, useState } from "react"; -import { parseTar } from "@/components/files/Tarfile"; import URDFRenderer from "@/components/files/URDFRenderer"; +import { UntarredFile, untarFile } from "@/components/files/untar"; import Spinner from "@/components/ui/Spinner"; import { useAlertQueue } from "@/hooks/useAlertQueue"; import { useAuthentication } from "@/hooks/useAuth"; -import pako from "pako"; interface Props { listingId: string; } -interface UntarredFile { - name: string; - content: Uint8Array; -} - const TerminalRobotModel = ({ listingId }: Props) => { const auth = useAuthentication(); const { addErrorAlert } = useAlertQueue(); @@ -58,11 +52,7 @@ const TerminalRobotModel = ({ listingId }: Props) => { setIsLoading(false); return; } - const response = await fetch(urdfUrl); - const arrayBuffer = await response.arrayBuffer(); - const uint8Array = new Uint8Array(arrayBuffer); - const decompressed = pako.ungzip(uint8Array); - const files = parseTar(decompressed); + const files = await untarFile(urdfUrl); const firstUrdfFile = files.find((file) => file.name.endsWith(".urdf")); if (!firstUrdfFile) { addErrorAlert("No URDF file found in the tarball"); diff --git a/frontend/src/lib/mujoco/mujoco_wasm.d.ts b/frontend/src/lib/mujoco/mujoco_wasm.d.ts index 96fa9e99..68917052 100644 --- a/frontend/src/lib/mujoco/mujoco_wasm.d.ts +++ b/frontend/src/lib/mujoco/mujoco_wasm.d.ts @@ -1427,7 +1427,7 @@ export interface Simulation { free() : void; /** Apply cartesian force and torque (outside xfrc_applied mechanism) */ applyForce(fx: number, fy: number, fz: number, tx: number, ty: number, tz: number, px: number, py: number, pz: number, body_id: number): void; - + /** sets perturb pos,quat in d->mocap when selected body is mocap, and in d->qpos otherwise * d->qpos written only if flg_paused and subtree root for selected body has free joint */ applyPose(bodyID: number, diff --git a/frontend/src/lib/types/routes.ts b/frontend/src/lib/types/routes.ts index 83f39f83..5d8052ba 100644 --- a/frontend/src/lib/types/routes.ts +++ b/frontend/src/lib/types/routes.ts @@ -2,14 +2,6 @@ import { route, string } from "react-router-typesafe-routes/dom"; const ROUTES = { HOME: route(""), - PLAYGROUND: route("playground", - {}, - { - WITH_ID: route(":artifactId", { - params: { artifactId: string().defined() }, - }), - }, - ), // General pages ABOUT: route("about"), @@ -48,8 +40,8 @@ const ROUTES = { PROFILE: route("profile/:id?", { params: { id: string() }, }), - FILE: route("file/:artifactId", { - params: { artifactId: string().defined() }, + FILE: route("file/:artifactId/:fileName?", { + params: { artifactId: string().defined(), fileName: string() }, }), // Sell. @@ -57,6 +49,7 @@ const ROUTES = { "sell", {}, { + DASHBOARD: route("dashboard"), ONBOARDING: route("onboarding"), DELETE: route("delete"), }, diff --git a/frontend/src/react-app-env.d.ts b/frontend/src/react-app-env.d.ts index 4ea89a5c..4645f776 100644 --- a/frontend/src/react-app-env.d.ts +++ b/frontend/src/react-app-env.d.ts @@ -3,3 +3,4 @@ declare module "*.png"; declare module "*.svg"; declare module "*.jpeg"; declare module "*.jpg"; +declare module "*.xml"; diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index f4631bfc..6dcd88e6 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -20,4 +20,5 @@ export default defineConfig({ build: { outDir: "dist", }, + assetsInclude: ["**/*.xml"], }); From 6927a6c990b5de405868c9133bbe98d88dc4f828 Mon Sep 17 00:00:00 2001 From: Benjamin Bolte Date: Sat, 9 Nov 2024 22:24:30 -0800 Subject: [PATCH 03/19] stuff --- .../src/components/files/MJCFRenderer.tsx | 145 ++++++++----- .../src/components/files/mujoco/mujoco.ts | 191 +++++++----------- 2 files changed, 171 insertions(+), 165 deletions(-) diff --git a/frontend/src/components/files/MJCFRenderer.tsx b/frontend/src/components/files/MJCFRenderer.tsx index a71f32ec..dd43bf63 100644 --- a/frontend/src/components/files/MJCFRenderer.tsx +++ b/frontend/src/components/files/MJCFRenderer.tsx @@ -1,18 +1,17 @@ import { useEffect, useRef, useState } from "react"; import humanoid from "@/components/files/demo/humanoid.xml"; -import { humanReadableError } from "@/hooks/useAlertQueue"; -import { mujoco } from "@/lib/mujoco/mujoco_wasm"; -import * as THREE from "three"; -import { OrbitControls } from "three/examples/jsm/controls/OrbitControls"; - -import Spinner from "../ui/Spinner"; import { MujocoRefs, cleanupMujoco, initializeMujoco, initializeThreeJS, -} from "./mujoco/mujoco"; +} from "@/components/files/mujoco/mujoco"; +import Spinner from "@/components/ui/Spinner"; +import { humanReadableError } from "@/hooks/useAlertQueue"; +import { mujoco } from "@/lib/mujoco/mujoco_wasm"; +import * as THREE from "three"; +import { OrbitControls } from "three/examples/jsm/controls/OrbitControls"; interface Props { useControls?: boolean; @@ -34,40 +33,89 @@ const MJCFRenderer = ({ useControls = true }: Props) => { controlsRef: useRef(null), }; - // Additional refs specific to this renderer - const leftLegRef = useRef(null); - const rightLegRef = useRef(null); - // State management - const [leftLegAngle, setLeftLegAngle] = useState(0); - const [rightLegAngle, setRightLegAngle] = useState(0); const isSimulatingRef = useRef(false); - const [isSimulating, setIsSimulating] = useState(false); const mujocoTimeRef = useRef(0); const [isMujocoReady, setIsMujocoReady] = useState(false); - const [showControls, setShowControls] = useState(useControls); - const [jointLimits, setJointLimits] = useState<{ - [key: string]: { min: number; max: number }; - }>({}); const [error, setError] = useState(null); // Constants - const DEFAULT_TIMESTEP = 0.002; - const SWING_FREQUENCY = 2.0; - const SWING_AMPLITUDE = 0.8; + const DEFAULT_TIMESTEP = 0.01; const setupModelGeometry = () => { - if (!refs.sceneRef.current) return; - - // Setup basic geometries - const geometry = new THREE.BoxGeometry(1, 1, 1); - const material = new THREE.MeshPhongMaterial({ color: 0x808080 }); - - leftLegRef.current = new THREE.Mesh(geometry, material); - rightLegRef.current = new THREE.Mesh(geometry, material); + const { sceneRef, modelRef } = refs; + if (!sceneRef.current || !modelRef.current) return; + const model = modelRef.current; + + // Loop over all geoms in the model + for (let i = 0; i < model.ngeom; i++) { + // Get geom properties from the model + const geomType = model.geom_type[i]; + const geomSize = model.geom_size.subarray(i * 3, i * 3 + 3); + const geomPos = model.geom_pos.subarray(i * 3, i * 3 + 3); + const geomMat = model.geom_mat.subarray(i * 9, i * 9 + 9); + + // Create corresponding Three.js geometry + let geometry: THREE.BufferGeometry; + switch (geomType) { + case mj.mjtGeom.mjGEOM_BOX: + geometry = new THREE.BoxGeometry( + geomSize[0] * 2, + geomSize[1] * 2, + geomSize[2] * 2, + ); + break; + case mj.mjtGeom.mjGEOM_SPHERE: + geometry = new THREE.SphereGeometry(geomSize[0], 32, 32); + break; + case mj.mjtGeom.mjGEOM_CYLINDER: + geometry = new THREE.CylinderGeometry( + geomSize[0], + geomSize[0], + geomSize[1] * 2, + 32, + ); + break; + // Add cases for other geom types as needed + default: + console.warn(`Unsupported geom type: ${geomType}`); + continue; + } - refs.sceneRef.current.add(leftLegRef.current); - refs.sceneRef.current.add(rightLegRef.current); + // Create material (you can customize this based on geom properties) + const material = new THREE.MeshPhongMaterial({ color: 0x808080 }); + + // Create mesh and set initial position and orientation + const mesh = new THREE.Mesh(geometry, material); + mesh.position.set(geomPos[0], geomPos[1], geomPos[2]); + + // Set orientation from geomMat + const rotationMatrix = new THREE.Matrix4().fromArray([ + geomMat[0], + geomMat[3], + geomMat[6], + 0, + geomMat[1], + geomMat[4], + geomMat[7], + 0, + geomMat[2], + geomMat[5], + geomMat[8], + 0, + 0, + 0, + 0, + 1, + ]); + mesh.setRotationFromMatrix(rotationMatrix); + + // Store geom index for later updates + mesh.userData.geomIndex = i; + + // Add mesh to the scene + sceneRef.current.add(mesh); + } }; const animate = (time: number) => { @@ -102,7 +150,6 @@ const MJCFRenderer = ({ useControls = true }: Props) => { const stopPhysicsSimulation = () => { isSimulatingRef.current = false; - setIsSimulating(false); if (animationFrameRef.current) { cancelAnimationFrame(animationFrameRef.current); } @@ -113,26 +160,26 @@ const MJCFRenderer = ({ useControls = true }: Props) => { (async () => { try { + if (!containerRef.current) throw new Error("Container ref is null"); + // Initialize MuJoCo with the humanoid model - if ( - !(await initializeMujoco({ - modelXML: humanoid, - refs, - onInitialized: () => setIsMujocoReady(true), - onError: (error) => setError(error), - })) - ) { - return; - } + const { mj, model, state, simulation } = await initializeMujoco({ + modelXML: humanoid, + refs, + }); + refs.mujocoRef.current = mj; + refs.modelRef.current = model; + refs.stateRef.current = state; + refs.simulationRef.current = simulation; // Initialize Three.js scene - if ( - !(await initializeThreeJS(refs, containerRef, { - onError: (error) => setError(error), - })) - ) { - return; - } + const { renderer, scene, camera, controls } = initializeThreeJS( + containerRef.current, + ); + refs.rendererRef.current = renderer; + refs.sceneRef.current = scene; + refs.cameraRef.current = camera; + refs.controlsRef.current = controls; // Add model-specific setup (bodies, geometries, etc.) setupModelGeometry(); diff --git a/frontend/src/components/files/mujoco/mujoco.ts b/frontend/src/components/files/mujoco/mujoco.ts index 7588c538..d208e2b7 100644 --- a/frontend/src/components/files/mujoco/mujoco.ts +++ b/frontend/src/components/files/mujoco/mujoco.ts @@ -24,41 +24,28 @@ export interface MujocoInitOptions { export const initializeMujoco = async ({ modelXML, - refs, onInitialized, - onError, }: MujocoInitOptions) => { const MODEL_DIR = "/working"; const MODEL_FILE = "model.xml"; const MODEL_PATH = `${MODEL_DIR}/${MODEL_FILE}`; - try { - // Load MuJoCo WASM module - refs.mujocoRef.current = await load_mujoco(); - const mj = refs.mujocoRef.current; - - // Set up file system and load XML model - // @ts-ignore - if (!mj.FS.analyzePath(MODEL_DIR).exists) { - mj.FS.mkdir(MODEL_DIR); - } - mj.FS.writeFile(MODEL_PATH, modelXML); - - const model = new mj.Model(MODEL_PATH); - const state = new mj.State(model); - const simulation = new mj.Simulation(model, state); - - // Store references - refs.modelRef.current = model; - refs.stateRef.current = state; - refs.simulationRef.current = simulation; - - onInitialized?.(); - return true; - } catch (error) { - onError?.(error as Error); - return false; + // Load MuJoCo WASM module + const mj = await load_mujoco(); + + // Set up file system and load XML model + // @ts-ignore + if (!mj.FS.analyzePath(MODEL_DIR).exists) { + mj.FS.mkdir(MODEL_DIR); } + mj.FS.writeFile(MODEL_PATH, modelXML); + + const model = new mj.Model(MODEL_PATH); + const state = new mj.State(model); + const simulation = new mj.Simulation(model, state); + + onInitialized?.(); + return { mj, model, state, simulation }; }; export interface ThreeJSInitOptions { @@ -69,100 +56,74 @@ export interface ThreeJSInitOptions { } export const initializeThreeJS = ( - refs: MujocoRefs, - containerRef: React.RefObject, + container: HTMLDivElement, { cameraDistance = 2.5, cameraHeight = 1.5, backgroundColor = new THREE.Color(0.15, 0.25, 0.35), - onError, }: ThreeJSInitOptions = {}, ) => { - const container = containerRef.current; - - if (!container) { - onError?.(new Error("Container ref is null")); - return false; + // Set up renderer + const renderer = new THREE.WebGLRenderer({ + antialias: true, + alpha: true, + powerPreference: "high-performance", + }); + + if (!renderer.getContext()) { + throw new Error("Failed to get WebGL context"); } - try { - // Set up renderer - const renderer = new THREE.WebGLRenderer({ - antialias: true, - alpha: true, - powerPreference: "high-performance", - }); - - if (!renderer.getContext()) { - throw new Error("Failed to get WebGL context"); - } - - renderer.setSize(container.clientWidth, container.clientHeight); - container.appendChild(renderer.domElement); - refs.rendererRef.current = renderer; - - // Set up scene - const scene = new THREE.Scene(); - scene.background = backgroundColor; - refs.sceneRef.current = scene; - - // Set up camera - const camera = new THREE.PerspectiveCamera( - 45, - container.clientWidth / container.clientHeight, - 0.001, - 100, - ); - camera.position.set( - cameraDistance * Math.cos(Math.PI / 4), - cameraHeight, - cameraDistance * Math.sin(Math.PI / 4), - ); - scene.add(camera); - refs.cameraRef.current = camera; - - // Set up controls - const controls = new OrbitControls(camera, renderer.domElement); - controls.enableDamping = true; - controls.dampingFactor = 0.1; - controls.minDistance = 1.0; - controls.maxDistance = 5.0; - controls.update(); - refs.controlsRef.current = controls; - - // Add lighting - const ambientLight = new THREE.AmbientLight(0xffffff, 0.5); - scene.add(ambientLight); - - const dirLight = new THREE.DirectionalLight(0xffffff, 0.5); - dirLight.position.set(5, 5, 5); - scene.add(dirLight); - - // Add floor - const floorGeometry = new THREE.PlaneGeometry(10, 10); - const floorMaterial = new THREE.MeshStandardMaterial({ - color: 0xe0e0e0, - roughness: 0.7, - metalness: 0.1, - }); - const floor = new THREE.Mesh(floorGeometry, floorMaterial); - floor.rotation.x = -Math.PI / 2; - floor.position.y = 0; - scene.add(floor); - - return true; - } catch (error) { - onError?.(error as Error); - - // Clean up any partially initialized resources - if (refs.rendererRef.current) { - refs.rendererRef.current.dispose(); - refs.rendererRef.current.forceContextLoss(); - refs.rendererRef.current.domElement?.remove(); - refs.rendererRef.current = null; - } - return false; - } + renderer.setSize(container.clientWidth, container.clientHeight); + container.appendChild(renderer.domElement); + + // Set up scene + const scene = new THREE.Scene(); + scene.background = backgroundColor; + + // Set up camera + const camera = new THREE.PerspectiveCamera( + 45, + container.clientWidth / container.clientHeight, + 0.001, + 100, + ); + camera.position.set( + cameraDistance * Math.cos(Math.PI / 4), + cameraHeight, + cameraDistance * Math.sin(Math.PI / 4), + ); + scene.add(camera); + + // Set up controls + const controls = new OrbitControls(camera, renderer.domElement); + controls.enableDamping = true; + controls.dampingFactor = 0.1; + controls.minDistance = 1.0; + controls.maxDistance = 5.0; + controls.update(); + + // Add lighting + const ambientLight = new THREE.AmbientLight(0xffffff, 0.5); + scene.add(ambientLight); + + const dirLight = new THREE.DirectionalLight(0xffffff, 0.5); + dirLight.position.set(5, 5, 5); + scene.add(dirLight); + + // Add floor + const floorGeometry = new THREE.PlaneGeometry(10, 10); + const floorMaterial = new THREE.MeshStandardMaterial({ + color: 0xe0e0e0, + roughness: 0.7, + metalness: 0.1, + }); + const floor = new THREE.Mesh(floorGeometry, floorMaterial); + floor.rotation.x = -Math.PI / 2; + floor.position.y = 0; + scene.add(floor); + + return { renderer, scene, camera, controls }; }; export const cleanupMujoco = (refs: MujocoRefs) => { @@ -189,8 +150,6 @@ export const cleanupMujoco = (refs: MujocoRefs) => { if (refs.controlsRef.current) { refs.controlsRef.current.dispose(); } - - // Clean up MuJoCo resources if (refs.stateRef.current) { refs.stateRef.current.free(); } From a4d8d9ee3fd0ceaae9850ea873088237d3a7827a Mon Sep 17 00:00:00 2001 From: Benjamin Bolte Date: Sat, 9 Nov 2024 23:42:57 -0800 Subject: [PATCH 04/19] renders... --- .../src/components/files/MJCFRenderer.tsx | 284 +++++++++++++----- .../src/components/files/demo/humanoid.xml | 1 + frontend/src/components/files/demo/simple.xml | 19 +- .../src/components/files/mujoco/mujoco.ts | 24 +- frontend/src/lib/mujoco/mujoco_wasm.d.ts | 230 ++++++++------ frontend/src/lib/mujoco/mujoco_wasm.js | 6 +- frontend/src/lib/mujoco/mujoco_wasm.wasm | Bin 1618532 -> 1609448 bytes 7 files changed, 381 insertions(+), 183 deletions(-) diff --git a/frontend/src/components/files/MJCFRenderer.tsx b/frontend/src/components/files/MJCFRenderer.tsx index dd43bf63..f25c3297 100644 --- a/frontend/src/components/files/MJCFRenderer.tsx +++ b/frontend/src/components/files/MJCFRenderer.tsx @@ -36,6 +36,8 @@ const MJCFRenderer = ({ useControls = true }: Props) => { // State management const isSimulatingRef = useRef(false); const mujocoTimeRef = useRef(0); + const tmpVecRef = useRef(new THREE.Vector3()); + const tmpQuatRef = useRef(new THREE.Quaternion()); const [isMujocoReady, setIsMujocoReady] = useState(false); const [error, setError] = useState(null); @@ -43,94 +45,222 @@ const MJCFRenderer = ({ useControls = true }: Props) => { const DEFAULT_TIMESTEP = 0.01; const setupModelGeometry = () => { - const { sceneRef, modelRef } = refs; - if (!sceneRef.current || !modelRef.current) return; + const { sceneRef, modelRef, mujocoRef, simulationRef } = refs; + if ( + !sceneRef.current || + !modelRef.current || + !mujocoRef.current || + !simulationRef.current + ) + return; const model = modelRef.current; + const mj = mujocoRef.current; + + // Create root object for MuJoCo scene + const mujocoRoot = new THREE.Group(); + mujocoRoot.name = "MuJoCo Root"; + sceneRef.current.add(mujocoRoot); - // Loop over all geoms in the model - for (let i = 0; i < model.ngeom; i++) { - // Get geom properties from the model - const geomType = model.geom_type[i]; - const geomSize = model.geom_size.subarray(i * 3, i * 3 + 3); - const geomPos = model.geom_pos.subarray(i * 3, i * 3 + 3); - const geomMat = model.geom_mat.subarray(i * 9, i * 9 + 9); - - // Create corresponding Three.js geometry - let geometry: THREE.BufferGeometry; - switch (geomType) { - case mj.mjtGeom.mjGEOM_BOX: - geometry = new THREE.BoxGeometry( - geomSize[0] * 2, - geomSize[1] * 2, - geomSize[2] * 2, - ); - break; - case mj.mjtGeom.mjGEOM_SPHERE: - geometry = new THREE.SphereGeometry(geomSize[0], 32, 32); - break; - case mj.mjtGeom.mjGEOM_CYLINDER: - geometry = new THREE.CylinderGeometry( - geomSize[0], - geomSize[0], - geomSize[1] * 2, - 32, - ); - break; - // Add cases for other geom types as needed - default: - console.warn(`Unsupported geom type: ${geomType}`); - continue; + // Create body groups first + const bodies: { [key: number]: THREE.Group } = {}; + for (let b = 0; b < model.nbody; b++) { + bodies[b] = new THREE.Group(); + bodies[b].name = `body_${b}`; + bodies[b].userData.bodyId = b; + + // Add to parent body based on MuJoCo's body hierarchy + if (b === 0) { + mujocoRoot.add(bodies[b]); + } else { + const parentId = model.body_parentid[b]; + if (bodies[parentId]) { + bodies[parentId].add(bodies[b]); + } } + } + + try { + for (let i = 0; i < model.ngeom; i++) { + // Get geom properties from the model + const geomType = model.geom_type[i]; + const geomSize = model.geom_size.subarray(i * 3, i * 3 + 3); + const geomPos = model.geom_pos.subarray(i * 3, i * 3 + 3); + + // Create corresponding Three.js geometry + let geometry: THREE.BufferGeometry; + switch (geomType) { + case mj.mjtGeom.mjGEOM_PLANE.value: + geometry = new THREE.PlaneGeometry( + geomSize[0] * 2, + geomSize[1] * 2, + ); + break; + case mj.mjtGeom.mjGEOM_SPHERE.value: + geometry = new THREE.SphereGeometry(geomSize[0], 32, 32); + break; + case mj.mjtGeom.mjGEOM_CAPSULE.value: + // Capsule is a cylinder with hemispheres at the ends + geometry = new THREE.CapsuleGeometry( + geomSize[0], + geomSize[1] * 2, + 4, + 32, + ); + break; + case mj.mjtGeom.mjGEOM_ELLIPSOID.value: + // Create a sphere and scale it to make an ellipsoid + geometry = new THREE.SphereGeometry(1, 32, 32); + geometry.scale(geomSize[0], geomSize[1], geomSize[2]); + break; + case mj.mjtGeom.mjGEOM_CYLINDER.value: + geometry = new THREE.CylinderGeometry( + geomSize[0], + geomSize[0], + geomSize[1] * 2, + 32, + ); + break; + case mj.mjtGeom.mjGEOM_BOX.value: + geometry = new THREE.BoxGeometry( + geomSize[0] * 2, + geomSize[1] * 2, + geomSize[2] * 2, + ); + break; + case mj.mjtGeom.mjGEOM_MESH.value: + // For mesh, you'll need to load the actual mesh data + console.warn("Mesh geometry requires additional mesh data loading"); + geometry = new THREE.BoxGeometry(1, 1, 1); // Placeholder + break; + case mj.mjtGeom.mjGEOM_LINE.value: + geometry = new THREE.BufferGeometry().setFromPoints([ + new THREE.Vector3(0, 0, 0), + new THREE.Vector3(geomSize[0], 0, 0), + ]); + break; + default: + console.warn(`Unsupported geom type: ${geomType}`); + continue; + } + + // Create material (you can customize this based on geom properties) + const material = new THREE.MeshPhongMaterial({ color: 0x808080 }); + + // Create mesh and add to corresponding body + const mesh = new THREE.Mesh(geometry, material); + mesh.userData.geomIndex = i; + + const bodyId = model.geom_bodyid[i]; + if (bodies[bodyId]) { + // Set local position and orientation relative to body + const pos = model.geom_pos.subarray(i * 3, i * 3 + 3); + mesh.position.set(pos[0], pos[2], -pos[1]); + + const quat = model.geom_quat.subarray(i * 4, i * 4 + 4); + mesh.quaternion.set(-quat[1], -quat[3], quat[2], -quat[0]); + + bodies[bodyId].add(mesh); + } + } + + // Update initial body positions and orientations + for (let b = 0; b < model.nbody; b++) { + if (bodies[b]) { + const pos = simulationRef.current.xpos.subarray(b * 3, b * 3 + 3); + bodies[b].position.set(pos[0], pos[2], -pos[1]); + + const quat = simulationRef.current.xquat.subarray(b * 4, b * 4 + 4); + bodies[b].quaternion.set(-quat[1], -quat[3], quat[2], -quat[0]); + } + } + + // Update world matrices from root to leaves + mujocoRoot.updateWorldMatrix(true, true); + setIsMujocoReady(true); + } catch (error) { + console.error(error); + } + }; + + const setupScene = () => { + if (!refs.sceneRef.current || !refs.cameraRef.current) return; + + // Set up scene properties + refs.sceneRef.current.background = new THREE.Color(0.15, 0.25, 0.35); + refs.sceneRef.current.fog = new THREE.Fog( + refs.sceneRef.current.background, + 30, + 50, + ); + + // Add ambient light + const ambientLight = new THREE.AmbientLight(0xffffff, 0.1); + ambientLight.name = "AmbientLight"; + refs.sceneRef.current.add(ambientLight); + + // Set up camera with wider view + refs.cameraRef.current.position.set(4.0, 3.4, 3.4); + refs.cameraRef.current.far = 100; + refs.cameraRef.current.updateProjectionMatrix(); - // Create material (you can customize this based on geom properties) - const material = new THREE.MeshPhongMaterial({ color: 0x808080 }); - - // Create mesh and set initial position and orientation - const mesh = new THREE.Mesh(geometry, material); - mesh.position.set(geomPos[0], geomPos[1], geomPos[2]); - - // Set orientation from geomMat - const rotationMatrix = new THREE.Matrix4().fromArray([ - geomMat[0], - geomMat[3], - geomMat[6], - 0, - geomMat[1], - geomMat[4], - geomMat[7], - 0, - geomMat[2], - geomMat[5], - geomMat[8], - 0, - 0, - 0, - 0, - 1, - ]); - mesh.setRotationFromMatrix(rotationMatrix); - - // Store geom index for later updates - mesh.userData.geomIndex = i; - - // Add mesh to the scene - sceneRef.current.add(mesh); + // Update OrbitControls settings + if (refs.controlsRef.current) { + refs.controlsRef.current.target.set(0, 0.7, 0); + refs.controlsRef.current.panSpeed = 2; + refs.controlsRef.current.zoomSpeed = 1; + refs.controlsRef.current.enableDamping = true; + refs.controlsRef.current.dampingFactor = 0.1; + refs.controlsRef.current.screenSpacePanning = true; + refs.controlsRef.current.update(); } }; + const updateBodyTransforms = () => { + const { modelRef, simulationRef, sceneRef } = refs; + if (!modelRef.current || !simulationRef.current || !sceneRef.current) + return; + + const mujocoRoot = sceneRef.current.getObjectByName("MuJoCo Root"); + if (!mujocoRoot) return; + + // Update body transforms + for (let b = 0; b < modelRef.current.nbody; b++) { + const body = mujocoRoot.getObjectByName(`body_${b}`); + if (body) { + const pos = simulationRef.current.xpos.subarray(b * 3, b * 3 + 3); + body.position.set(pos[0], pos[2], -pos[1]); + + const quat = simulationRef.current.xquat.subarray(b * 4, b * 4 + 4); + body.quaternion.set(-quat[1], -quat[3], quat[2], -quat[0]); + } + } + + // Update world matrices from root to leaves + mujocoRoot.updateWorldMatrix(true, true); + }; + const animate = (time: number) => { if ( !refs.rendererRef.current || !refs.sceneRef.current || !refs.cameraRef.current - ) { + ) return; - } // Update physics if simulating if (isSimulatingRef.current && refs.simulationRef.current) { - mujocoTimeRef.current += DEFAULT_TIMESTEP; - refs.simulationRef.current.step(); + const timestep = DEFAULT_TIMESTEP; + if (time - mujocoTimeRef.current > 35.0) { + mujocoTimeRef.current = time; + } + + while (mujocoTimeRef.current < time) { + refs.simulationRef.current.step(); + mujocoTimeRef.current += timestep * 1000.0; + } + + // Update body transforms after physics step + updateBodyTransforms(); } // Update controls if enabled @@ -143,8 +273,6 @@ const MJCFRenderer = ({ useControls = true }: Props) => { refs.sceneRef.current, refs.cameraRef.current, ); - - // Request next frame animationFrameRef.current = requestAnimationFrame(animate); }; @@ -163,10 +291,12 @@ const MJCFRenderer = ({ useControls = true }: Props) => { if (!containerRef.current) throw new Error("Container ref is null"); // Initialize MuJoCo with the humanoid model + const humanoidXML = await fetch(humanoid).then((res) => res.text()); const { mj, model, state, simulation } = await initializeMujoco({ - modelXML: humanoid, + modelXML: humanoidXML, refs, }); + refs.mujocoRef.current = mj; refs.modelRef.current = model; refs.stateRef.current = state; @@ -184,9 +314,13 @@ const MJCFRenderer = ({ useControls = true }: Props) => { // Add model-specific setup (bodies, geometries, etc.) setupModelGeometry(); + // Add new scene setup + setupScene(); + // Start animation loop animate(performance.now()); } catch (error) { + console.error(error); setError(error as Error); } })(); diff --git a/frontend/src/components/files/demo/humanoid.xml b/frontend/src/components/files/demo/humanoid.xml index 66ecc0f0..3f9960e1 100644 --- a/frontend/src/components/files/demo/humanoid.xml +++ b/frontend/src/components/files/demo/humanoid.xml @@ -1,3 +1,4 @@ +