diff --git a/.github/workflows/release_langtrace.yaml b/.github/workflows/release_langtrace.yaml index cf7fcce7..09b0e407 100644 --- a/.github/workflows/release_langtrace.yaml +++ b/.github/workflows/release_langtrace.yaml @@ -7,9 +7,14 @@ on: - main types: - closed + paths-ignore: + - '*.md' + - 'LICENSE' + - '.gitignore' jobs: generate-version: + if: github.event.pull_request.merged == true && !contains(github.event.pull_request.labels.*.name, 'skip-release') runs-on: ubuntu-latest outputs: new_version: ${{ steps.version.outputs.version }} diff --git a/README.md b/README.md index 9745417d..920b9395 100644 --- a/README.md +++ b/README.md @@ -133,14 +133,16 @@ Langtrace automatically captures traces from the following vendors: | Cohere | LLM | :white_check_mark: | :white_check_mark: | | Groq | LLM | :white_check_mark: | :white_check_mark: | | Perplexity | LLM | :white_check_mark: | :white_check_mark: | -| Gemini | LLM | :x: | :white_check_mark: | +| Gemini | LLM | :white_check_mark: | :white_check_mark: | +| Mistral | LLM | :x: | :white_check_mark: | | Langchain | Framework | :x: | :white_check_mark: | | LlamaIndex | Framework | :white_check_mark: | :white_check_mark: | | Langgraph | Framework | :x: | :white_check_mark: | | DSPy | Framework | :x: | :white_check_mark: | | CrewAI | Framework | :x: | :white_check_mark: | | Ollama | Framework | :x: | :white_check_mark: | -| VertexAI | Framework | :x: | :white_check_mark: | +| VertexAI | Framework | :white_check_mark: | :white_check_mark: | +| Vercel AI | Framework | :white_check_mark: | :x: | | Pinecone | Vector Database | :white_check_mark: | :white_check_mark: | | ChromaDB | Vector Database | :white_check_mark: | :white_check_mark: | | QDrant | Vector Database | :white_check_mark: | :white_check_mark: | diff --git a/app/(protected)/project/[project_id]/crewai-dash/page.tsx b/app/(protected)/project/[project_id]/crewai-dash/page.tsx new file mode 100644 index 00000000..5b71fc46 --- /dev/null +++ b/app/(protected)/project/[project_id]/crewai-dash/page.tsx @@ -0,0 +1,24 @@ +import AgentCrewsDashboard from "@/components/project/agent-crews/dashboard"; +import { authOptions } from "@/lib/auth/options"; +import { Metadata } from "next"; +import { getServerSession } from "next-auth"; +import { redirect } from "next/navigation"; + +export const metadata: Metadata = { + title: "Langtrace | Agent Crews", + description: "View all the agent crews from CrewAI.", +}; + +export default async function Page() { + const session = await getServerSession(authOptions); + if (!session || !session.user) { + redirect("/login"); + } + const user = session?.user; + + return ( + <> + + + ); +} diff --git a/app/(protected)/project/[project_id]/datasets/dataset/[dataset_id]/evaluations/page-client.tsx b/app/(protected)/project/[project_id]/datasets/dataset/[dataset_id]/evaluations/page-client.tsx index f0e4698f..c143caa2 100644 --- a/app/(protected)/project/[project_id]/datasets/dataset/[dataset_id]/evaluations/page-client.tsx +++ b/app/(protected)/project/[project_id]/datasets/dataset/[dataset_id]/evaluations/page-client.tsx @@ -40,6 +40,7 @@ import { ClipboardIcon, FlaskConical, RefreshCwIcon, + TrashIcon, } from "lucide-react"; import Link from "next/link"; import { useParams, useRouter } from "next/navigation"; @@ -121,7 +122,7 @@ export default function Evaluations() { }); const fetchExperiments = useQuery({ - queryKey: ["fetch-experiments-query"], + queryKey: ["fetch-experiments-query", projectId, datasetId, page], queryFn: async () => { const response = await fetch( `/api/run?projectId=${projectId}&datasetId=${datasetId}&page=${page}&pageSize=25` @@ -167,6 +168,36 @@ export default function Evaluations() { refetchOnWindowFocus: false, }); + const handleDeleteSelectedRuns = async () => { + if (comparisonRunIds.length === 0) return; + + try { + const response = await fetch("/api/run", { + method: "DELETE", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + runIds: comparisonRunIds, + projectId, + }), + }); + + if (!response.ok) { + const error = await response.json(); + throw new Error(error?.message || "Failed to delete evaluations"); + } + + setComparisonRunIds([]); + await fetchExperiments.refetch(); + toast.success("Selected evaluations deleted successfully."); + } catch (error) { + toast.error("Failed to delete evaluations", { + description: error instanceof Error ? error.message : String(error), + }); + } + }; + const columns: ColumnDef[] = [ { size: 50, @@ -438,6 +469,16 @@ export default function Evaluations() { {viewMetrics ? "Hide Metrics" : "View Metrics"} + + {comparisonRunIds.length > 0 && ( + + )}
diff --git a/app/(protected)/project/[project_id]/settings/general/page-client.tsx b/app/(protected)/project/[project_id]/settings/general/page-client.tsx index 56f13fcf..a36f6543 100644 --- a/app/(protected)/project/[project_id]/settings/general/page-client.tsx +++ b/app/(protected)/project/[project_id]/settings/general/page-client.tsx @@ -1,5 +1,7 @@ "use client"; +import ProjectTypesDropdown from "@/components/project/project-type-dropdown"; +import { Info } from "@/components/shared/info"; import { Spinner } from "@/components/shared/spinner"; import { Button } from "@/components/ui/button"; import { @@ -50,6 +52,7 @@ export default function ProjectView() { .min(3, { message: "Name must be 3 to 20 characters long" }) .max(20, { message: "Name must be 20 or less characters long" }), description: z.string().optional(), + type: z.string().optional(), }); const ProjectDetailsForm = useForm({ @@ -57,6 +60,7 @@ export default function ProjectView() { defaultValues: { name: "", description: "", + type: "", }, }); @@ -67,6 +71,7 @@ export default function ProjectView() { name: data.name.toLowerCase(), description: data.description.toLowerCase(), id: project.project.id, + type: data.type, }; await fetch(`/api/project?id=${projectId}`, { method: "PUT", @@ -139,6 +144,7 @@ export default function ProjectView() { description: project.project.description ? project.project.description : "", + type: project.project.type ? project.project.type : "default", }); } }, [project, ProjectDetailsForm]); @@ -157,7 +163,7 @@ export default function ProjectView() {
- Profile + Project Update your project details here @@ -209,6 +215,29 @@ export default function ProjectView() { )} /> + ( + + + Project Type + + + + + + + + )} + />
- + - - {project.name} - + {project.type === "crewai" && ( + + CrewAI Logo + {project.name} + + )} + {project.type === "default" && ( + + {project.name} + + )} {project.description} diff --git a/app/api/metrics/score/route.ts b/app/api/metrics/score/route.ts index b427df0c..9802c568 100644 --- a/app/api/metrics/score/route.ts +++ b/app/api/metrics/score/route.ts @@ -108,7 +108,7 @@ export async function POST(req: NextRequest) { if (scores[1] === 0) { entry[testId] = 0; } else { - entry[testId] = Math.round((scores[0] / scores[1]) * 100); + entry[testId] = scores[0] / scores[1]; } } ); @@ -124,16 +124,24 @@ export async function POST(req: NextRequest) { Object.entries(dateScoreMap).map(([date, scoresByTestId]) => { Object.entries(scoresByTestId as any).forEach(([testId, scores]: any) => { if (!scoresChartData[testId]) { - scoresChartData[testId] = 0; + scoresChartData[testId] = [0, 0]; } - if (scores[1] === 0) { - scoresChartData[testId] = 0; - } - scoresChartData[testId] = - Math.round((scores[0] / scores[1]) * 100) / 100; + scoresChartData[testId][0] += scores[0]; + scoresChartData[testId][1] += scores[1]; }); }); + const totalScores: any = {}; + for (const key in scoresChartData) { + const testId = key; + const scores = scoresChartData[testId]; + if (scores[1] === 0) { + totalScores[testId] = 0; + } else { + totalScores[testId] = ((scores[0] / scores[1]) * 100).toFixed(2); + } + } + metricsChartData.sort( (a, b) => new Date(a.date as string).getTime() - @@ -142,7 +150,7 @@ export async function POST(req: NextRequest) { return NextResponse.json({ metrics: metricsChartData, - scores: scoresChartData, + scores: totalScores, }); } catch (error) { return NextResponse.json( diff --git a/app/api/project/route.ts b/app/api/project/route.ts index a1f3f74f..97ee506d 100644 --- a/app/api/project/route.ts +++ b/app/api/project/route.ts @@ -56,17 +56,23 @@ export async function POST(req: NextRequest) { } } const data = await req.json(); - const { name, description, teamId } = data; + const { name, description, teamId, type } = data; let createDefaultTests = data.createDefaultTests; if (!createDefaultTests) { createDefaultTests = true; // default to true } + let projectType = type; + if (!type) { + projectType = "default"; + } + const project = await prisma.project.create({ data: { name: name, description: description, teamId: teamId, + type: projectType, }, }); @@ -97,7 +103,7 @@ export async function PUT(req: NextRequest) { } const data = await req.json(); - const { id, name, description, teamId, apiKeyHash } = data; + const { id, name, description, teamId, apiKeyHash, type } = data; const project = await prisma.project.update({ where: { @@ -108,6 +114,7 @@ export async function PUT(req: NextRequest) { description, apiKeyHash: apiKeyHash, teamId, + type, }, }); diff --git a/app/api/run/route.ts b/app/api/run/route.ts index 4820247c..0d4c6412 100644 --- a/app/api/run/route.ts +++ b/app/api/run/route.ts @@ -194,11 +194,14 @@ export async function DELETE(req: NextRequest) { } const data = await req.json(); - const { id } = data; + const { runIds, projectId } = data; - await prisma.run.delete({ + await prisma.run.deleteMany({ where: { - id, + runId: { + in: runIds, + }, + projectId: projectId, }, }); diff --git a/app/api/trace/route.ts b/app/api/trace/route.ts index 1f9f073c..eb14603a 100644 --- a/app/api/trace/route.ts +++ b/app/api/trace/route.ts @@ -26,7 +26,9 @@ export async function POST(req: NextRequest) { // Normalize and prepare data for Clickhouse let normalized = []; - if (userAgent?.toLowerCase().includes("otel-otlp")) { + if (userAgent?.toLowerCase().includes("otel-otlp") || + userAgent?.toLowerCase().includes("opentelemetry") + ) { // coming from an OTEL exporter normalized = prepareForClickhouse( normalizeOTELData(data.resourceSpans?.[0]?.scopeSpans?.[0]?.spans) diff --git a/components/annotations/chart-tabs.tsx b/components/annotations/chart-tabs.tsx index 555c8068..66f28607 100644 --- a/components/annotations/chart-tabs.tsx +++ b/components/annotations/chart-tabs.tsx @@ -38,7 +38,7 @@ export function ChartTabs({ { key: "name", operation: "EQUALS", - value: "gen_ai.content.prompt", + value: "gen_ai.content.completion", type: "event", }, ], @@ -92,7 +92,7 @@ export function ChartTabs({
{Object.keys(chartData?.scores).map( (testId: string, index: number) => { - const score = (chartData?.scores[testId] || 0) * 100; + const score = chartData?.scores[testId] || 0; const testName = tests.find((test) => testId.includes(test.id) )?.name; @@ -114,7 +114,7 @@ export function ChartTabs({ bgColor )} > - {(chartData?.scores[testId] || 0) * 100}% + {chartData?.scores[testId] || 0}% diff --git a/components/charts/inference-chart.tsx b/components/charts/inference-chart.tsx index 4f62d565..176089e4 100644 --- a/components/charts/inference-chart.tsx +++ b/components/charts/inference-chart.tsx @@ -148,7 +148,7 @@ export function AverageCostInferenceChart({

{costUsage?.count > 0 && costUsage?.cost > 0 - ? `$${(costUsage?.cost / costUsage?.count).toFixed(6)}` + ? `$${(costUsage.cost / costUsage.count).toLocaleString(undefined, { minimumFractionDigits: 6, maximumFractionDigits: 6 })}` : "$0.00"}

Average Inference Cost

@@ -159,7 +159,7 @@ export function AverageCostInferenceChart({

{costUsage?.cost > 0 - ? `$${costUsage?.cost?.toFixed(6)}` + ? `$${costUsage.cost.toLocaleString(undefined, { minimumFractionDigits: 6, maximumFractionDigits: 6 })}` : "$0.00"}

Total Cost

diff --git a/components/charts/token-chart.tsx b/components/charts/token-chart.tsx index 84c40a7a..693efff8 100644 --- a/components/charts/token-chart.tsx +++ b/components/charts/token-chart.tsx @@ -141,23 +141,44 @@ export function CostChart({

- Input Tokens Cost: ${costUsage?.input?.toFixed(6) || 0} + Input Tokens Cost: $ + {costUsage.input?.toLocaleString(undefined, { + minimumFractionDigits: 6, + maximumFractionDigits: 6, + })}

- Output Tokens Cost: ${costUsage?.output?.toFixed(6) || 0} + Output Tokens Cost: $ + {costUsage.output?.toLocaleString(undefined, { + minimumFractionDigits: 6, + maximumFractionDigits: 6, + })}

- Total Cost: ${costUsage?.total?.toFixed(6) || 0} + Total Cost: $ + {costUsage.total?.toLocaleString(undefined, { + minimumFractionDigits: 6, + maximumFractionDigits: 6, + })}

({ - date: data?.date, - "Total Cost": data?.total?.toFixed(6) || 0, - "Input Tokens Cost": data?.input?.toFixed(6) || 0, - "Output Tokens Cost": data?.output?.toFixed(6) || 0, + data={costUsage.cost?.map((data: any) => ({ + date: data.date, + "Total Cost": data.total?.toLocaleString(undefined, { + minimumFractionDigits: 6, + maximumFractionDigits: 6, + }), + "Input Tokens Cost": data.input?.toLocaleString(undefined, { + minimumFractionDigits: 6, + maximumFractionDigits: 6, + }), + "Output Tokens Cost": data.output?.toLocaleString(undefined, { + minimumFractionDigits: 6, + maximumFractionDigits: 6, + }), }))} index="date" categories={[ diff --git a/components/project/agent-crews/agents-view.tsx b/components/project/agent-crews/agents-view.tsx new file mode 100644 index 00000000..cc731024 --- /dev/null +++ b/components/project/agent-crews/agents-view.tsx @@ -0,0 +1,195 @@ +import { + Accordion, + AccordionContent, + AccordionItem, + AccordionTrigger, +} from "@/components/ui/accordion"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { CrewAIAgent, CrewAIMemory, CrewAITask, CrewAITool } from "@/lib/crewai_trace_util"; +import { cn } from "@/lib/utils"; +import { MoveDiagonal } from "lucide-react"; +import { useState } from "react"; + +export function AgentsView({ agents }: { agents: CrewAIAgent[] }) { + return ( + + {agents.map((agent, index) => ( + + {`Agent ${index + 1} - ${agent?.id}`} + +
+

ID

+

{agent?.id || "N/A"}

+

Role

+

{agent?.role || "N/A"}

+

Goal

+ +

Backstory

+ + {agent?.tools && + agent?.tools?.length > 0 && + agent?.tools?.map((tool, i) => ( + <> +

+ {`Tool ${i + 1}`} + Name +

+

+ {tool?.name || "N/A"} +

+

+ {`Tool ${i + 1}`} + Description +

+ + + ))} +

Result

+ +

Max Iter

+

{agent?.max_iter || "N/A"}

+
+ + + ))} + + ); +} + +export function TasksView({ tasks }: { tasks: CrewAITask[] }) { + return ( + + {tasks.map((task, index) => ( + + {`Task ${index + 1} - ${task?.id}`} + +
+

ID

+

{task?.id || "N/A"}

+

Description

+ +

Agent

+

{task?.agent || "N/A"}

+ {task?.tools && + task?.tools?.length > 0 && + task?.tools?.map((tool, i) => ( + <> +

+ {`Tool ${i + 1}`} + Name +

+

+ {tool?.name || "N/A"} +

+

+ {`Tool ${i + 1}`} + Description +

+ + + ))} +

Used Tools

+

+ {task?.used_tools || "N/A"} +

+

Tool Errors

+ +

Human Input

+

+ {task?.human_input || "False"} +

+

Expected Output

+ +

Result

+ +
+ + + ))} + + ); +} + +export function ToolsView({ tools }: { tools: CrewAITool[] }) { + return ( + + {tools.map((tool, index) => ( + + {`Tool ${index + 1} - ${tool?.name}`} + +
+

Name

+

{tool?.name || "N/A"}

+

Description

+ +
+ + + ))} + + ); +} + +export function MemoryView({ memory }: { memory: CrewAIMemory[] }) { + return ( + + {memory.map((mem, index) => ( + + {`Memory ${index + 1}`} + +
+

Inputs

+

{mem?.inputs || "N/A"}

+

Description

+ +
+ + + ))} + + ); +} + +function ExpandableP({ + content, + className, +}: { + content: any; + className?: string; +}) { + const [expanded, setExpanded] = useState(false); + return ( +
+

{content}

+ +
+ ); +} diff --git a/components/project/agent-crews/dashboard.tsx b/components/project/agent-crews/dashboard.tsx new file mode 100644 index 00000000..80f037d0 --- /dev/null +++ b/components/project/agent-crews/dashboard.tsx @@ -0,0 +1,430 @@ +"use client"; + +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Skeleton } from "@/components/ui/skeleton"; +import { PAGE_SIZE, RECIPE_DOCS } from "@/lib/constants"; +import { CrewAITrace, processCrewAITrace } from "@/lib/crewai_trace_util"; +import { cn } from "@/lib/utils"; +import { ArrowTopRightIcon, GearIcon } from "@radix-ui/react-icons"; +import { + BotIcon, + BrainCircuitIcon, + ChevronLeftSquareIcon, + ChevronRightSquareIcon, + FileIcon, + RefreshCwIcon, +} from "lucide-react"; +import Image from "next/image"; +import Link from "next/link"; +import { useParams } from "next/navigation"; +import { useCallback, useEffect, useState } from "react"; +import { useQuery } from "react-query"; +import { toast } from "sonner"; +import { TraceComponent } from "../traces/trace-component"; +import { AgentsView, MemoryView, TasksView, ToolsView } from "./agents-view"; +import TimelineChart from "./timeline-chart"; + +export default function AgentCrewsDashboard({ email }: { email: string }) { + const project_id = useParams()?.project_id as string; + const [page, setPage] = useState(1); + const [cachedCurrentPage, setCachedCurrentPage] = useState(1); + const [totalPages, setTotalPages] = useState(1); + const [currentData, setCurrentData] = useState([]); + const [enableFetch, setEnableFetch] = useState(false); + const [selectedTrace, setSelectedTrace] = useState(null); + const [selectedTraceIndex, setSelectedTraceIndex] = useState(0); + + useEffect(() => { + setEnableFetch(true); + }, []); + + const fetchOldTraces = () => { + if (fetchTraces.isRefetching) { + return; + } + if (page <= totalPages) { + fetchTraces.refetch(); + } + }; + + const fetchLatestTraces = () => { + if (fetchTraces.isRefetching) { + return; + } + setCachedCurrentPage(page); + setPage(1); + setEnableFetch(true); + }; + + const fetchTracesCall = useCallback( + async (pageNum: number) => { + const apiEndpoint = "/api/traces"; + + const body = { + page: pageNum, + pageSize: PAGE_SIZE, + projectId: project_id, + filters: { + filters: [ + { + key: "langtrace.service.version", + operation: "NOT_EQUALS", + value: "", + type: "attribute", + }, + ], + operation: "OR", + }, + group: true, + }; + + const response = await fetch(apiEndpoint, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }); + if (!response.ok) { + const error = await response.json(); + throw new Error(error?.message || "Failed to fetch traces"); + } + return await response.json(); + }, + [project_id] + ); + + const fetchTraces = useQuery({ + queryKey: ["fetch-traces-query", page], + queryFn: () => fetchTracesCall(page), + onSuccess: (data) => { + const newData = data?.traces?.result || []; + const metadata = data?.traces?.metadata || {}; + + setTotalPages(parseInt(metadata?.total_pages) || 1); + if (parseInt(metadata?.page) <= parseInt(metadata?.total_pages)) { + setPage(parseInt(metadata?.page) + 1); + } + + let transformedNewData: CrewAITrace[] = []; + transformedNewData = newData.map((trace: any) => { + return processCrewAITrace(trace); + }); + + if (page === 1 && currentData.length === 0) { + setCurrentData(transformedNewData); + if (transformedNewData.length > 0) + setSelectedTrace(transformedNewData[0]); + } else { + if (page === 1) { + // deduplicate transformedNewData with currentData + const currentDataIds = currentData.map((trace: any) => trace.id); + transformedNewData = transformedNewData.filter( + (trace: any) => !currentDataIds.includes(trace.id) + ); + setCurrentData((prevData: any) => [ + ...transformedNewData, + ...prevData, + ]); + setPage(cachedCurrentPage); + } else { + setCurrentData((prevData: any) => [ + ...prevData, + ...transformedNewData, + ]); + } + } + + setEnableFetch(false); + }, + onError: (error) => { + setEnableFetch(false); + toast.error("Failed to fetch traces", { + description: error instanceof Error ? error.message : String(error), + }); + }, + refetchOnWindowFocus: false, + enabled: enableFetch, + }); + + if (fetchTraces.isLoading && currentData.length === 0) { + return ; + } + + return ( +
+
+
+ CrewAI Logo +

Sessions

+
+

+ Read latest from right to left +

+
+
+
+ +

+ {fetchTraces.isFetching + ? "Fetching sessions..." + : `Fetched the last ${currentData.length} sessions`} +

+
+
+

+ Use arrow keys to navigate through traces timeline +

+ + +
+
+ {(!currentData || currentData.length === 0) && ( +
+

+ No crew sessions found. +

+ + + +
+ )} + {currentData && currentData.length > 0 && ( + + )} + {selectedTrace && ( + <> +
+ + + Session Details + + +

STATUS

+ + {selectedTrace?.status} + +

CREW ID

+ + {selectedTrace?.crew?.id || "N/A"} + +

START TIME

+ + {selectedTrace?.formatted_start_time || "N/A"} + +

TOTAL DURATION

+ + {selectedTrace?.total_duration.toLocaleString()} ms + + + + + + Usage Metrics + + +
+
+

Input

+ {selectedTrace?.input_tokens && ( + + Tokens: {selectedTrace?.input_tokens.toLocaleString()} + + )} + {selectedTrace?.input_cost && ( + + Cost: ${selectedTrace?.input_cost.toFixed(2)} + + )} +
+
+

Output

+ {selectedTrace?.output_tokens && ( + + Tokens: {selectedTrace?.output_tokens.toLocaleString()} + + )} + {selectedTrace?.output_cost && ( + + Cost: ${selectedTrace?.output_cost.toFixed(2)} + + )} +
+
+

Total

+ {selectedTrace?.total_tokens && ( + + Tokens: {selectedTrace?.total_tokens.toLocaleString()} + + )} + {selectedTrace?.total_cost && ( + + Cost: ${selectedTrace?.total_cost.toFixed(2)} + + )} +
+
+ + + + + Libraries Detected + + + {selectedTrace?.libraries.map((library, i) => { + return ( +
+

+ {library.name} +

+ + {library.version} + +
+ ); + })} + + +
+
+ + + + + Agent Details + + + + {selectedTrace?.agents.length > 0 ? ( + + ) : ( +

No agents detected

+ )} + + + + + + + Task Details + + + + {selectedTrace?.tasks.length > 0 ? ( + + ) : ( +

No tasks detected

+ )} + + +
+
+ + + + + Tool Details + + + + {selectedTrace?.tools.length > 0 ? ( + + ) : ( +

No tools detected

+ )} + + + + + + + Memory Details + + + + {selectedTrace?.memory.length > 0 ? ( + + ) : ( +

No memory detected

+ )} + + +
+ + + )} +
+ ); +} + +function PageLoading() { + return ( +
+
+
+ CrewAI Logo +

Sessions

+
+

+ Read latest from right to left +

+
+ + +
+ + + +
+
+ + +
+ + +
+ ); +} diff --git a/components/project/agent-crews/timeline-chart.tsx b/components/project/agent-crews/timeline-chart.tsx new file mode 100644 index 00000000..61ed38a6 --- /dev/null +++ b/components/project/agent-crews/timeline-chart.tsx @@ -0,0 +1,148 @@ +import { Button } from "@/components/ui/button"; +import { Card, CardContent } from "@/components/ui/card"; +import { + ChartConfig, + ChartContainer, + ChartTooltip, + ChartTooltipContent, +} from "@/components/ui/chart"; +import { CrewAITrace } from "@/lib/crewai_trace_util"; +import { cn } from "@/lib/utils"; +import { CircularProgress } from "@mui/material"; +import { ChevronLeft, ChevronRight } from "lucide-react"; +import { useEffect, useState } from "react"; +import { Bar, BarChart, CartesianGrid, Cell, XAxis } from "recharts"; + +const chartConfig = { + start_time: { + label: "start time", + color: "hsl(var(--chart-1))", + }, +} satisfies ChartConfig; + +export default function TimelineChart({ + data, + fetchOldTraces, + fetchLatestTraces, + fetching, + setSelectedTrace, + selectedTrace, + selectedTraceIndex, + setSelectedTraceIndex, +}: { + data: CrewAITrace[]; + fetchOldTraces: () => void; + fetchLatestTraces: () => void; + fetching: boolean; + setSelectedTrace: (trace: CrewAITrace) => void; + selectedTrace: CrewAITrace | null; + selectedTraceIndex: number; + setSelectedTraceIndex: (index: number) => void; +}) { + const barSize = 20; + const barGap = 4; + const groupGap = 20; + const totalGroups = data.length; + const [style, setStyle] = useState({ width: "200px", height: "200px" }); + + useEffect(() => { + // Calculate the width dynamically when the container is mounted + const width = totalGroups * (barSize + groupGap); + setStyle({ width: `${width}px`, height: "200px" }); + }, [totalGroups]); + + // use left and right keyboard shortcuts to navigate through traces using data index + useEffect(() => { + const handleKeyDown = (e: KeyboardEvent) => { + if (e.key === "ArrowRight") { + if (selectedTraceIndex > 0) { + setSelectedTrace(data[selectedTraceIndex - 1]); + setSelectedTraceIndex(selectedTraceIndex - 1); + } + } else if (e.key === "ArrowLeft") { + if (selectedTraceIndex < data.length - 1) { + setSelectedTrace(data[selectedTraceIndex + 1]); + setSelectedTraceIndex(selectedTraceIndex + 1); + } + } + }; + window.addEventListener("keydown", handleKeyDown); + return () => window.removeEventListener("keydown", handleKeyDown); + }, [selectedTraceIndex]); + + return ( + + + + + + + value.slice(-8)} + reversed + /> + } + /> + + {data.map((entry, index) => ( + { + setSelectedTrace(entry); + setSelectedTraceIndex(index); + }} + radius={4} + className={cn( + "hover:cursor-pointer hover:fill-orange-600", + selectedTrace?.id === entry.id + ? "fill-orange-600" + : "fill-blue-600" + )} + key={index} + /> + ))} + + + + + + + ); +} diff --git a/components/project/create.tsx b/components/project/create.tsx index d8a39059..277e95ff 100644 --- a/components/project/create.tsx +++ b/components/project/create.tsx @@ -25,6 +25,7 @@ import { useQueryClient } from "react-query"; import { toast } from "sonner"; import { z } from "zod"; import { Info } from "../shared/info"; +import ProjectTypesDropdown from "./project-type-dropdown"; export function Create({ teamId, @@ -43,9 +44,15 @@ export function Create({ const schema = z.object({ name: z.string().min(2, "Too short").max(30, "Too long"), description: z.string().max(100, "Too long").optional(), + type: z.string().optional(), }); const CreateProjectForm = useForm({ resolver: zodResolver(schema), + defaultValues: { + name: "", + description: "", + type: "default", + }, }); return ( @@ -77,6 +84,7 @@ export function Create({ ? data.description.toLowerCase() : "", teamId, + type: data.type || "default", }), }); await queryClient.invalidateQueries("fetch-projects-query"); @@ -142,6 +150,29 @@ export function Create({ )} /> + ( + + + Project Type + + + + + + + + )} + />
diff --git a/components/project/project-type-dropdown.tsx b/components/project/project-type-dropdown.tsx new file mode 100644 index 00000000..9df65238 --- /dev/null +++ b/components/project/project-type-dropdown.tsx @@ -0,0 +1,97 @@ +import { Button } from "@/components/ui/button"; +import { + Command, + CommandEmpty, + CommandGroup, + CommandInput, + CommandItem, + CommandList, +} from "@/components/ui/command"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@/components/ui/popover"; +import { cn } from "@/lib/utils"; +import { Check, ChevronsUpDown } from "lucide-react"; +import * as React from "react"; + +const projectTypes = [ + { + value: "default", + label: "Default", + comingSoon: false, + }, + { + value: "crewai", + label: "CrewAI", + comingSoon: false, + }, + { + value: "dspy", + label: "DSPy", + comingSoon: true, + }, + { + value: "langgraph", + label: "LangGraph", + comingSoon: true, + }, +]; + +export default function ProjectTypesDropdown({ + value, + setValue, +}: { + value: string; + setValue: (value: string) => void; +}) { + const [open, setOpen] = React.useState(false); + + return ( + + + + + + + + + No framework found. + + {projectTypes.map((framework) => ( + { + setValue(currentValue === value ? "" : currentValue); + setOpen(false); + }} + > + + {framework.label} {framework.comingSoon && "(Coming soon)"} + + ))} + + + + + + ); +} diff --git a/components/project/traces/trace-component.tsx b/components/project/traces/trace-component.tsx new file mode 100644 index 00000000..af437921 --- /dev/null +++ b/components/project/traces/trace-component.tsx @@ -0,0 +1,240 @@ +import ConversationView from "@/components/shared/conversation-view"; +import LanggraphView from "@/components/shared/langgraph-view"; +import TraceGraph, { AttributesTabs } from "@/components/traces/trace_graph"; +import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; +import { CrewAITrace } from "@/lib/crewai_trace_util"; +import { Trace } from "@/lib/trace_util"; +import { + calculateTotalTime, + convertTracesToHierarchy, +} from "@/lib/trace_utils"; +import { cn, getVendorFromSpan } from "@/lib/utils"; +import { CodeIcon, MessageCircle, NetworkIcon, XIcon } from "lucide-react"; +import { useEffect, useState } from "react"; + +export function TraceComponent({ trace }: { trace: CrewAITrace }) { + const [selectedTrace, setSelectedTrace] = useState( + trace.trace_hierarchy + ); + const [selectedVendors, setSelectedVendors] = useState( + trace.vendors + ); + const [includesLanggraph, setIncludesLanggraph] = useState(false); + const [spansView, setSpansView] = useState< + "SPANS" | "ATTRIBUTES" | "CONVERSATION" | "LANGGRAPH" + >("SPANS"); + const [span, setSpan] = useState(null); + const [attributes, setAttributes] = useState(null); + const [events, setEvents] = useState(null); + + useEffect(() => { + setSelectedTrace(trace.trace_hierarchy); + setSelectedVendors(trace.vendors); + if (trace.vendors.includes("langgraph")) setIncludesLanggraph(true); + if (!open) setSpansView("SPANS"); + }, [trace, open]); + + return ( +
+
+

Session Drilldown

+
+ +
+
+ {(spansView === "ATTRIBUTES" || + spansView === "CONVERSATION" || + spansView === "LANGGRAPH") && + span && + attributes && + events && ( +
+
+ + + {includesLanggraph && ( + + )} + +
+
+ {spansView === "ATTRIBUTES" && ( + + )} + {spansView === "CONVERSATION" && span && ( + + )} + {spansView === "LANGGRAPH" && ( + + )} +
+
+ )} +
+ ); +} + +function SpansView({ + trace, + selectedTrace, + setSelectedTrace, + selectedVendors, + setSelectedVendors, + setSpansView, + setSpan, + setAttributes, + setEvents, +}: { + trace: Trace; + selectedTrace: any[]; + setSelectedTrace: (trace: any[]) => void; + selectedVendors: string[]; + setSelectedVendors: (vendors: string[]) => void; + setSpansView: ( + spansView: "SPANS" | "ATTRIBUTES" | "CONVERSATION" | "LANGGRAPH" + ) => void; + setSpan: (span: any) => void; + setAttributes: (attributes: any) => void; + setEvents: (events: any) => void; +}) { + return ( + <> +
+
    +
  • + Tip 1: Hover over any span line to see additional attributes and + events. Attributes contain the request parameters and events contain + logs and errors. +
    • +
  • + Tip 2: Click on attributes or events to copy them to your clipboard. +
    • +
+
+ {trace.vendors.map((vendor, i) => ( +
+ { + if (checked) { + if (!selectedVendors.includes(vendor)) + setSelectedVendors([...selectedVendors, vendor]); + } else { + setSelectedVendors( + selectedVendors.filter((v) => v !== vendor) + ); + } + const traces = []; + const currVendors = [...selectedVendors]; + if (checked) currVendors.push(vendor); + else currVendors.splice(currVendors.indexOf(vendor), 1); + + // if currVendors and trace.vendors are the same, no need to filter + if (currVendors.length === trace.vendors.length) { + setSelectedTrace(trace.trace_hierarchy); + return; + } + + if (currVendors.length === 0) { + setSelectedTrace([]); + return; + } + + for (let i = 0; i < trace.sorted_trace.length; i++) { + if ( + currVendors.includes( + getVendorFromSpan(trace.sorted_trace[i]) + ) + ) + traces.push({ ...trace.sorted_trace[i] }); + } + setSelectedTrace(convertTracesToHierarchy(traces)); + }} + /> + +
+ ))} +
+
+
+ +
+ + ); +} diff --git a/components/project/traces/trace-sheet.tsx b/components/project/traces/trace-sheet.tsx index 75c5cd9c..45bce7f8 100644 --- a/components/project/traces/trace-sheet.tsx +++ b/components/project/traces/trace-sheet.tsx @@ -61,7 +61,7 @@ export function TraceSheet({ Trace Details {spansView === "SPANS" && ( -
+
-
+
{spansView === "ATTRIBUTES" && ( )} {spansView === "CONVERSATION" && span && ( - + )} {spansView === "LANGGRAPH" && ( @@ -173,7 +179,7 @@ function SpansView({ }) { return ( <> -
+
  • Tip 1: Hover over any span line to see additional attributes and diff --git a/components/project/traces/traces-table.tsx b/components/project/traces/traces-table.tsx index 731eba23..de456493 100644 --- a/components/project/traces/traces-table.tsx +++ b/components/project/traces/traces-table.tsx @@ -17,10 +17,13 @@ import { TableHeader, TableRow, } from "@/components/ui/table"; +import { Info } from "@/components/shared/info"; import { HOW_TO_GROUP_RELATED_OPERATIONS } from "@/lib/constants"; import { Trace } from "@/lib/trace_util"; import { cn } from "@/lib/utils"; import { ResetIcon } from "@radix-ui/react-icons"; +import { PropertyFilter } from "@/lib/services/query_builder_service"; +import { Switch } from "@/components/ui/switch"; import { ColumnDef, flexRender, @@ -45,6 +48,8 @@ interface TracesTableProps { refetch: () => void; paginationLoading?: boolean; scrollableDivRef?: React.RefObject; + filters: PropertyFilter[]; + setFilters: (filters: PropertyFilter[]) => void; } export function TracesTable({ @@ -56,6 +61,8 @@ export function TracesTable({ refetch, paginationLoading, scrollableDivRef, + filters, + setFilters, }: TracesTableProps) { const [tableState, setTableState] = useState({ pagination: { @@ -67,6 +74,7 @@ export function TracesTable({ const [openDropdown, setOpenDropdown] = useState(false); const [openSheet, setOpenSheet] = useState(false); const [selectedTrace, setSelectedTrace] = useState(null); + const [viewMode, setViewMode] = useState<"trace" | "prompt">("trace"); useEffect(() => { if (typeof window !== "undefined") { @@ -174,7 +182,87 @@ export function TracesTable({
+
+
+
+

Trace

+ { + const newMode = checked ? "prompt" : "trace"; + setViewMode(newMode); + if (newMode === "prompt") { + setColumnVisibility({ + start_time: true, + models: true, + inputs: true, + outputs: true, + status: false, + namespace: false, + user_ids: false, + prompt_ids: false, + vendors: false, + input_tokens: false, + output_tokens: false, + total_tokens: false, + input_cost: false, + output_cost: false, + total_cost: false, + total_duration: false, + }); + if (!filters.some((filter) => filter.value === "llm")) { + setFilters([ + ...filters, + { + key: "langtrace.service.type", + operation: "EQUALS", + value: "llm", + type: "attribute", + }, + ]); + } + } else { + setColumnVisibility({ + start_time: true, + models: true, + inputs: true, + outputs: true, + status: true, + namespace: true, + user_ids: true, + prompt_ids: true, + vendors: true, + input_tokens: true, + output_tokens: true, + total_tokens: true, + input_cost: true, + output_cost: true, + total_cost: true, + total_duration: true, + }); + setFilters( + filters.filter((filter) => filter.value !== "llm") + ); + } + }} + className="relative inline-flex items-center cursor-pointer" + > + + +

Prompt

+ +
+
Project ID: {project_id} @@ -237,7 +325,7 @@ export function TracesTable({ )} {!loading && data && data.length > 0 && ( - + {table.getHeaderGroups().map((headerGroup) => ( {headerGroup.headers.map((header) => ( diff --git a/components/project/traces/traces.tsx b/components/project/traces/traces.tsx index aebe1b95..7f5a1a9a 100644 --- a/components/project/traces/traces.tsx +++ b/components/project/traces/traces.tsx @@ -596,6 +596,8 @@ export default function Traces({ email }: { email: string }) { fetching={fetchTraces.isFetching} paginationLoading={showBottomLoader} scrollableDivRef={scrollableDivRef} + filters={filters} + setFilters={setFilters} /> No data found

; @@ -44,7 +50,12 @@ export default function ConversationView({ span }: { span: any }) { if (!prompts && !responses) return

No data found

; return ( -
+
{prompts?.length > 0 && JSON.parse(prompts).map((prompt: any, i: number) => { const role = prompt?.role ? prompt?.role?.toLowerCase() : "User"; diff --git a/components/shared/header.tsx b/components/shared/header.tsx index 9ce1eeb4..369593a8 100644 --- a/components/shared/header.tsx +++ b/components/shared/header.tsx @@ -42,7 +42,7 @@ export function Header({ email }: { email: string }) { }); return ( -
+
)} - + {!fetchUser.isLoading && fetchUser.data && (

diff --git a/components/shared/mode-toggle.tsx b/components/shared/mode-toggle.tsx index b7b1588e..c2f22387 100644 --- a/components/shared/mode-toggle.tsx +++ b/components/shared/mode-toggle.tsx @@ -23,7 +23,7 @@ export function ModeToggle() { Toggle theme - + setTheme("light")}> Light diff --git a/components/shared/nav.tsx b/components/shared/nav.tsx index 796018eb..0292da78 100644 --- a/components/shared/nav.tsx +++ b/components/shared/nav.tsx @@ -1,6 +1,9 @@ import { cn } from "@/lib/utils"; import Link from "next/link"; import { usePathname } from "next/navigation"; +import { useEffect, useState } from "react"; +import { useQuery } from "react-query"; +import { toast } from "sonner"; import { Button } from "../ui/button"; const DashboardNavLinks = [ @@ -14,51 +17,113 @@ const DashboardNavLinks = [ }, ]; -const ProjectNavLinks = (id: string) => [ - { - name: "Metrics", - href: `/project/${id}/metrics`, - }, - { - name: "Traces", - href: `/project/${id}/traces`, - }, - { - name: "Annotations", - href: `/project/${id}/annotations`, - }, - { - name: "Datasets", - href: `/project/${id}/datasets`, - }, - { - name: "Playground", - href: `/project/${id}/playground`, - }, - { - name: "Prompts", - href: `/project/${id}/prompts`, - }, - { - name: "Settings", - href: `/project/${id}/settings/general`, - }, -]; +const ProjectNavLinks = (id: string, type = "default") => { + const result = [ + { + name: "Metrics", + href: `/project/${id}/metrics`, + }, + { + name: "Traces", + href: `/project/${id}/traces`, + }, + { + name: "Annotations", + href: `/project/${id}/annotations`, + }, + { + name: "Datasets", + href: `/project/${id}/datasets`, + }, + { + name: "Playground", + href: `/project/${id}/playground`, + }, + { + name: "Prompts", + href: `/project/${id}/prompts`, + }, + { + name: "Settings", + href: `/project/${id}/settings/general`, + }, + ]; + if (type == "crewai") { + // add to the second position + result.splice(1, 0, { + name: "CrewAI Dash", + href: `/project/${id}/crewai-dash`, + }); + } + return result; +}; -export default function Nav() { +export default function Nav({}: {}) { const pathname = usePathname(); - const projectId = pathname.split("/")[2]; - let navlinks = DashboardNavLinks; - if (pathname.includes("/project/")) { - navlinks = ProjectNavLinks(projectId); + const projectId = pathname.includes("/project/") + ? pathname.split("/")[2] + : ""; + const [navlinks, setNavlinks] = useState(DashboardNavLinks); + + useEffect(() => { + if (projectId) { + setNavlinks(ProjectNavLinks(projectId)); + } else { + setNavlinks(DashboardNavLinks); + } + }, [projectId]); + + const { + data: project, + isLoading: projectLoading, + error: projectError, + } = useQuery({ + queryKey: ["fetch-project-query", projectId], + queryFn: async () => { + const response = await fetch(`/api/project?id=${projectId}`); + if (!response.ok) { + const error = await response.json(); + throw new Error(error?.message || "Failed to fetch project"); + } + const result = await response.json(); + setNavlinks( + ProjectNavLinks(projectId, result?.project?.type || "default") + ); + return result; + }, + onError: (error) => { + toast.error("Failed to fetch project", { + description: error instanceof Error ? error.message : String(error), + }); + }, + enabled: !!projectId, + }); + + if (projectLoading) { + return ( +

+ ); } + return (