diff --git a/.env b/.env index 02821c9f..a37ffcf3 100644 --- a/.env +++ b/.env @@ -31,4 +31,8 @@ NEXT_PUBLIC_ENABLE_ADMIN_LOGIN="true" # Azure AD Variables AZURE_AD_CLIENT_ID="" AZURE_AD_CLIENT_SECRET="" -AZURE_AD_TENANT_ID="" \ No newline at end of file +AZURE_AD_TENANT_ID="" + +POSTHOG_HOST="https://us.i.posthog.com" + +TELEMETRY_ENABLED="true" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 801090c8..1dcc6c41 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ WORKDIR /app ARG LANGTRACE_VERSION -RUN NEXT_PUBLIC_ENABLE_ADMIN_LOGIN=true NEXT_PUBLIC_LANGTRACE_VERSION=$LANGTRACE_VERSION npm run build +RUN POSTHOG_API_KEY=$POSTHOG_API_KEY NEXT_PUBLIC_ENABLE_ADMIN_LOGIN=true NEXT_PUBLIC_LANGTRACE_VERSION=$LANGTRACE_VERSION npm run build # Final release image FROM node:21.6-bookworm AS production diff --git a/README.md b/README.md index 920b9395..2c18bc01 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Langtrace.init({ api_key: }) OR ```typescript -import * as Langtrace from '@langtrase/typescript-sdk'; // Must precede any llm module imports +import * as Langtrace from "@langtrase/typescript-sdk"; // Must precede any llm module imports LangTrace.init(); // LANGTRACE_API_KEY as an ENVIRONMENT variable ``` @@ -119,6 +119,23 @@ docker compose down -v `-v` flag is used to delete volumes +## Telemetry + +Langtrace collects basic, non-sensitive usage data from self-hosted instances by default, which is sent to a central server (via PostHog). + +The following telemetry data is collected by us: +- Project name and type +- Team name + +This data helps us to: + +- Understand how the platform is being used to improve key features. +- Monitor overall usage for internal analysis and reporting. + +No sensitive information is gathered, and the data is not shared with third parties. + +If you prefer to disable telemetry, you can do so by setting TELEMETRY_ENABLED=false in your configuration. + --- ## Supported integrations @@ -138,6 +155,7 @@ Langtrace automatically captures traces from the following vendors: | Langchain | Framework | :x: | :white_check_mark: | | LlamaIndex | Framework | :white_check_mark: | :white_check_mark: | | Langgraph | Framework | :x: | :white_check_mark: | +| LiteLLM | Framework | :x: | :white_check_mark: | | DSPy | Framework | :x: | :white_check_mark: | | CrewAI | Framework | :x: | :white_check_mark: | | Ollama | Framework | :x: | :white_check_mark: | diff --git a/app/(protected)/layout.tsx b/app/(protected)/layout.tsx index 622e9cf9..f6c783a1 100644 --- a/app/(protected)/layout.tsx +++ b/app/(protected)/layout.tsx @@ -6,6 +6,7 @@ import { getServerSession } from "next-auth"; import { redirect } from "next/navigation"; import { Suspense } from "react"; import { PageSkeleton } from "./projects/page-client"; +import CustomPostHogProvider from "@/components/shared/posthog"; export default async function Layout({ children, @@ -19,11 +20,13 @@ export default async function Layout({ return ( }> -
-
- - {children} -
+ +
+
+ + {children} +
+ ); } diff --git a/app/(protected)/project/[project_id]/dspy-experiments/[experiment_id]/page-client.tsx b/app/(protected)/project/[project_id]/dspy-experiments/[experiment_id]/page-client.tsx new file mode 100644 index 00000000..bf60f79d --- /dev/null +++ b/app/(protected)/project/[project_id]/dspy-experiments/[experiment_id]/page-client.tsx @@ -0,0 +1,667 @@ +"use client"; +import { + DspyEvalChart, + DspyEvalChartData, +} from "@/components/charts/dspy-eval-chart"; +import { + AverageCostInferenceChart, + CountInferenceChart, +} from "@/components/charts/inference-chart"; +import { TableSkeleton } from "@/components/project/traces/table-skeleton"; +import { TraceSheet } from "@/components/project/traces/trace-sheet"; +import { GenericHoverCell } from "@/components/shared/hover-cell"; +import RowSkeleton from "@/components/shared/row-skeleton"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuCheckboxItem, + DropdownMenuContent, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { Separator } from "@/components/ui/separator"; +import { Skeleton } from "@/components/ui/skeleton"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { PAGE_SIZE } from "@/lib/constants"; +import { DspyTrace, processDspyTrace } from "@/lib/dspy_trace_util"; +import { correctTimestampFormat } from "@/lib/trace_utils"; +import { formatDateTime } from "@/lib/utils"; +import { ResetIcon } from "@radix-ui/react-icons"; +import { + ColumnDef, + flexRender, + getCoreRowModel, + useReactTable, + VisibilityState, +} from "@tanstack/react-table"; +import { ChevronDown, RefreshCwIcon } from "lucide-react"; +import { usePathname } from "next/navigation"; +import { useCallback, useEffect, useMemo, useState } from "react"; +import { useBottomScrollListener } from "react-bottom-scroll-listener"; +import { useQuery } from "react-query"; +import { toast } from "sonner"; + +export function PageClient({ email }: { email: string }) { + const pathname = usePathname(); + const project_id = pathname.split("/")[2]; + let experimentName = pathname.split("/")[4]; + // replace dashes with spaces + experimentName = experimentName.replace(/-/g, " "); + + const [data, setData] = useState([]); + const [page, setPage] = useState(1); + const [description, setDescription] = useState(""); + const [totalPages, setTotalPages] = useState(1); + const [openDropdown, setOpenDropdown] = useState(false); + const [openSheet, setOpenSheet] = useState(false); + const [selectedTrace, setSelectedTrace] = useState(null); + const [showBottomLoader, setShowBottomLoader] = useState(false); + const [enableFetch, setEnableFetch] = useState(true); + const [chartData, setChartData] = useState([]); + const [showEvalChart, setShowEvalChart] = useState(false); + + useEffect(() => { + const handleFocusChange = () => { + setPage(1); + setEnableFetch(true); + }; + + window.addEventListener("focus", handleFocusChange); + + return () => { + window.removeEventListener("focus", handleFocusChange); + }; + }, []); + + // Table state + const [tableState, setTableState] = useState({ + pagination: { + pageIndex: 0, + pageSize: 100, + }, + }); + const [columnVisibility, setColumnVisibility] = useState({}); + + const fetchTracesCall = useCallback( + async (pageNum: number) => { + const apiEndpoint = "/api/traces"; + const body = { + page: pageNum, + pageSize: PAGE_SIZE, + projectId: project_id, + filters: { + filters: [ + { + key: "experiment", + operation: "EQUALS", + value: experimentName, + type: "attribute", + }, + ], + operation: "OR", + }, + group: true, + }; + + const response = await fetch(apiEndpoint, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }); + if (!response.ok) { + const error = await response.json(); + throw new Error(error?.message || "Failed to fetch traces"); + } + return await response.json(); + }, + [project_id] + ); + + useEffect(() => { + if (typeof window !== "undefined") { + try { + const storedState = window.localStorage.getItem( + "preferences.traces.table-view" + ); + if (storedState) { + const parsedState = JSON.parse(storedState); + setTableState((prevState: any) => ({ + ...prevState, + ...parsedState, + pagination: { + ...prevState.pagination, + ...parsedState.pagination, + }, + })); + if (parsedState.columnVisibility) { + setColumnVisibility(parsedState.columnVisibility); + } + } + } catch (error) { + console.error("Error parsing stored table state:", error); + } + } + }, []); + + const fetchTraces = useQuery({ + queryKey: ["fetch-experiments-query", page, experimentName], + queryFn: () => fetchTracesCall(page), + onSuccess: (data) => { + const newData = data?.traces?.result || []; + const metadata = data?.traces?.metadata || {}; + setTotalPages(parseInt(metadata?.total_pages) || 1); + if (parseInt(metadata?.page) <= parseInt(metadata?.total_pages)) { + setPage(parseInt(metadata?.page) + 1); + } + + const transformedNewData: DspyTrace[] = newData.map((trace: any) => { + return processDspyTrace(trace); + }); + + setDescription( + transformedNewData.length > 0 + ? transformedNewData[0].experiment_description + : "" + ); + + if (page === 1) { + setData(transformedNewData); + } else { + setData((prevData) => [...prevData, ...transformedNewData]); + } + + // construct chart data + const chartData: DspyEvalChartData[] = []; + for (const trace of transformedNewData) { + if (trace.evaluated_score) { + chartData.push({ + timestamp: formatDateTime( + correctTimestampFormat(trace.start_time.toString()), + true + ), + score: trace.evaluated_score, + runId: trace.run_id, + }); + } + } + // reverse the chart data to show the latest last + chartData.reverse(); + if (page === 1) { + setChartData(chartData); + } else { + setChartData((prevData) => [...chartData, ...prevData]); + } + + setEnableFetch(false); + setShowBottomLoader(false); + }, + onError: (error) => { + setEnableFetch(false); + setShowBottomLoader(false); + toast.error("Failed to fetch traces", { + description: error instanceof Error ? error.message : String(error), + }); + }, + refetchOnWindowFocus: false, + enabled: enableFetch, + }); + + const scrollableDivRef: any = useBottomScrollListener(() => { + if (fetchTraces.isRefetching) { + return; + } + if (page <= totalPages) { + setShowBottomLoader(true); + fetchTraces.refetch(); + } + }); + + const columns: ColumnDef[] = [ + { + accessorKey: "run_id", + enableResizing: true, + header: "Run ID", + cell: ({ row }) => { + const id = row.getValue("run_id") as string; + return ( +
+ {id} +
+ ); + }, + }, + { + accessorKey: "start_time", + enableResizing: true, + header: "Start Time", + cell: ({ row }) => { + const starttime = row.getValue("start_time") as string; + return ( +
+ {formatDateTime(correctTimestampFormat(starttime), true)} +
+ ); + }, + }, + { + accessorKey: "status", + header: "Status", + cell: ({ row }) => { + const status = row.getValue("status") as string; + return ( +
+ +

+ {status} +

+
+ ); + }, + }, + { + accessorKey: "namespace", + header: "Namespace", + cell: ({ row }) => { + const namespace = row.getValue("namespace") as string; + return ( +
+

{namespace}

+
+ ); + }, + }, + { + accessorKey: "vendors", + header: "Vendors", + cell: ({ row }) => { + const vendors = row.getValue("vendors") as string[]; + return ( +
+ {vendors && + vendors.map((vendor, i) => ( + + {vendor} + + ))} +
+ ); + }, + }, + { + accessorKey: "models", + header: "Models", + cell: ({ row }) => { + const models = row.getValue("models") as string[]; + return ( +
+ {models && + models.map((model, i) => ( + + {model} + + ))} +
+ ); + }, + }, + { + accessorKey: "result", + header: "Result", + cell: ({ row }) => { + const result = row.getValue("result") as any; + return ; + }, + }, + { + accessorKey: "checkpoint", + header: "Checkpoint State", + cell: ({ row }) => { + const result = row.getValue("checkpoint") as any; + return ; + }, + }, + { + accessorKey: "input_tokens", + header: "Input Tokens", + cell: ({ row }) => { + const count = row.getValue("input_tokens") as number; + return

{count}

; + }, + }, + { + accessorKey: "output_tokens", + header: "Output Tokens", + cell: ({ row }) => { + const count = row.getValue("output_tokens") as number; + return

{count}

; + }, + }, + { + accessorKey: "total_tokens", + header: "Total Tokens", + cell: ({ row }) => { + const count = row.getValue("total_tokens") as number; + return

{count}

; + }, + }, + { + accessorKey: "input_cost", + header: "Input Cost", + cell: ({ row }) => { + const cost = row.getValue("input_cost") as number; + if (!cost) { + return null; + } + return ( +

+ {cost.toFixed(6) !== "0.000000" ? `\$${cost.toFixed(6)}` : ""} +

+ ); + }, + }, + { + accessorKey: "output_cost", + header: "Output Cost", + cell: ({ row }) => { + const cost = row.getValue("output_cost") as number; + if (!cost) { + return null; + } + return ( +

+ {cost.toFixed(6) !== "0.000000" ? `\$${cost.toFixed(6)}` : ""} +

+ ); + }, + }, + { + accessorKey: "total_cost", + header: "Total Cost", + cell: ({ row }) => { + const cost = row.getValue("total_cost") as number; + if (!cost) { + return null; + } + return ( +

+ {cost.toFixed(6) !== "0.000000" ? `\$${cost.toFixed(6)}` : ""} +

+ ); + }, + }, + { + accessorKey: "total_duration", + header: "Total Duration", + cell: ({ row }) => { + const duration = row.getValue("total_duration") as number; + if (!duration) { + return null; + } + return

{duration}ms

; + }, + }, + ]; + + const table = useReactTable({ + data, + columns, + getCoreRowModel: getCoreRowModel(), + initialState: { + ...tableState, + pagination: tableState.pagination, + columnVisibility, + }, + state: { + ...tableState, + pagination: tableState.pagination, + columnVisibility, + }, + onStateChange: (newState: any) => { + setTableState((prevState: any) => ({ + ...newState, + pagination: newState.pagination || prevState.pagination, + })); + const currState = table.getState(); + localStorage.setItem( + "preferences.traces.table-view", + JSON.stringify(currState) + ); + }, + onColumnVisibilityChange: (newVisibility) => { + setColumnVisibility(newVisibility); + const currState = table.getState(); + localStorage.setItem( + "preferences.traces.table-view", + JSON.stringify(currState) + ); + }, + enableColumnResizing: true, + columnResizeMode: "onChange", + manualPagination: true, // Add this if you're handling pagination yourself + }); + + const columnSizeVars = useMemo(() => { + const headers = table.getFlatHeaders(); + const colSizes: { [key: string]: number } = {}; + for (let i = 0; i < headers.length; i++) { + const header = headers[i]!; + colSizes[`--header-${header.id}-size`] = header.getSize(); + colSizes[`--col-${header.column.id}-size`] = header.column.getSize(); + } + return colSizes; + }, [table.getState().columnSizingInfo, table.getState().columnSizing]); + + return ( +
+
+
+ +
+

+ {experimentName} +

+ {description && ( +

+ {description} +

+ )} +
+
+
+ + + + + + {table + .getAllColumns() + .filter((column) => column.getCanHide()) + .map((column) => { + return ( + setOpenDropdown(true)} + onCheckedChange={(value) => + column.toggleVisibility(!!value) + } + > + {column.columnDef.header?.toString()} + + ); + })} + + + +
+
+ + + Eval Chart + Inference Metrics + + + {chartData.length > 0 ? ( + + ) : ( +
+

+ No evaluation data. Run dspy Evaluate() to see scores. +

+
+ )} + + +
+ + +
+ + +
+ {fetchTraces.isLoading && } + {!fetchTraces.isLoading && data && data.length > 0 && ( + + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + + {header.isPlaceholder + ? null + : flexRender( + header.column.columnDef.header, + header.getContext() + )} +
header.column.resetSize()} + onMouseDown={header.getResizeHandler()} + onTouchStart={header.getResizeHandler()} + className={`bg-muted-foreground resizer ${ + header.column.getIsResizing() ? "isResizing" : "" + }`} + /> + + ))} + + ))} + + + {fetchTraces.isFetching && ( + + {table.getFlatHeaders().map((header) => ( + + + + ))} + + )} + {table.getRowModel().rows.map((row) => ( + { + setSelectedTrace(data[row.index] as DspyTrace); + setOpenSheet(true); + }} + > + {row.getVisibleCells().map((cell) => ( + + {flexRender( + cell.column.columnDef.cell, + cell.getContext() + )} + + ))} + + ))} + +
+ )} + {selectedTrace !== null && ( + + )} + {showBottomLoader && ( +
+ + {Array.from({ length: 2 }).map((_, index) => ( + + ))} +
+ )} +
+
+ ); +} diff --git a/app/(protected)/project/[project_id]/dspy-experiments/[experiment_id]/page.tsx b/app/(protected)/project/[project_id]/dspy-experiments/[experiment_id]/page.tsx new file mode 100644 index 00000000..c0dee8b7 --- /dev/null +++ b/app/(protected)/project/[project_id]/dspy-experiments/[experiment_id]/page.tsx @@ -0,0 +1,24 @@ +import { authOptions } from "@/lib/auth/options"; +import { Metadata } from "next"; +import { getServerSession } from "next-auth"; +import { redirect } from "next/navigation"; +import { PageClient } from "./page-client"; + +export const metadata: Metadata = { + title: "Langtrace | Experiment", + description: "DSPy experiment.", +}; + +export default async function Page() { + const session = await getServerSession(authOptions); + if (!session || !session.user) { + redirect("/login"); + } + const user = session?.user; + + return ( + <> + + + ); +} diff --git a/app/(protected)/project/[project_id]/dspy-experiments/layout-client.tsx b/app/(protected)/project/[project_id]/dspy-experiments/layout-client.tsx new file mode 100644 index 00000000..eecf093e --- /dev/null +++ b/app/(protected)/project/[project_id]/dspy-experiments/layout-client.tsx @@ -0,0 +1,180 @@ +"use client"; +import { SetupInstructions } from "@/components/shared/setup-instructions"; +import Tabs from "@/components/shared/tabs"; +import { Skeleton } from "@/components/ui/skeleton"; +import { PAGE_SIZE } from "@/lib/constants"; +import { processDspyTrace } from "@/lib/dspy_trace_util"; +import { CircularProgress } from "@mui/material"; +import { usePathname } from "next/navigation"; +import { useCallback, useState } from "react"; +import { useBottomScrollListener } from "react-bottom-scroll-listener"; +import { useQuery } from "react-query"; +import { toast } from "sonner"; + +export default function LayoutClient({ + children, +}: { + children: React.ReactNode; +}) { + const pathname = usePathname(); + const project_id = pathname.split("/")[2]; + const href = `/project/${project_id}/dspy-experiments`; + + const [page, setPage] = useState(1); + const [totalPages, setTotalPages] = useState(1); + const [showBottomLoader, setShowBottomLoader] = useState(false); + const [experiments, setExperiments] = useState([]); + const [navLinks, setNavLinks] = useState([]); + + const fetchTracesCall = useCallback( + async (pageNum: number) => { + const apiEndpoint = "/api/traces"; + const body = { + page: pageNum, + pageSize: PAGE_SIZE, + projectId: project_id, + filters: { + filters: [ + { + key: "experiment", + operation: "NOT_EQUALS", + value: "", + type: "attribute", + }, + ], + operation: "OR", + }, + group: true, + }; + + const response = await fetch(apiEndpoint, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }); + if (!response.ok) { + const error = await response.json(); + throw new Error(error?.message || "Failed to fetch traces"); + } + return await response.json(); + }, + [project_id] + ); + + const fetchTraces = useQuery({ + queryKey: ["fetch-experiments-query", page], + queryFn: () => fetchTracesCall(page), + onSuccess: (data) => { + const newData = data?.traces?.result || []; + const metadata = data?.traces?.metadata || {}; + + const transformedNewData = newData.map((trace: any) => { + return processDspyTrace(trace); + }); + + const exps: string[] = []; + const navs: any[] = []; + for (const trace of transformedNewData) { + if ( + trace.experiment_name !== "" && + !exps.includes(trace.experiment_name) + ) { + exps.push(trace.experiment_name); + navs.push({ + name: trace.experiment_name, + value: + trace.experiment_name !== "" ? trace.experiment_name : "default", + href: `${href}/${trace.experiment_name}`, + }); + } + } + + // dedupe experiments from current and new data + const newExperiments = Array.from(new Set([...experiments, ...exps])); + setExperiments(newExperiments); + + // dedupe navLinks from current and new data + const newNavLinks = []; + for (const nav of navs) { + if (!navLinks.find((n) => n.value === nav.value)) { + newNavLinks.push(nav); + } + } + setNavLinks([...navLinks, ...newNavLinks]); + + // route to first experiment if no experiment is selected + if (navLinks.length > 0) { + const experimentId = navLinks[0].value; + if (!pathname.includes(experimentId)) { + window.location.href = `${href}/${experimentId}`; + } + } + + setTotalPages(parseInt(metadata?.total_pages) || 1); + if (parseInt(metadata?.page) <= parseInt(metadata?.total_pages)) { + setPage(parseInt(metadata?.page) + 1); + } + setShowBottomLoader(false); + }, + onError: (error) => { + setShowBottomLoader(false); + toast.error("Failed to fetch traces", { + description: error instanceof Error ? error.message : String(error), + }); + }, + }); + + const scrollableDivRef: any = useBottomScrollListener(() => { + if (fetchTraces.isRefetching) { + return; + } + if (page <= totalPages) { + setShowBottomLoader(true); + fetchTraces.refetch(); + } + }); + + return ( +
+
+

Experiments

+
+ {!fetchTraces.isLoading && experiments.length > 0 && ( +
+ +
+ {children} +
+
+ )} + {!fetchTraces.isLoading && experiments.length === 0 && ( +
+
+ +

+ Looking for new experiments... +

+
+ +
+ )} + {fetchTraces.isLoading && ( +
+
+ + + + +
+ +
+ )} +
+ ); +} diff --git a/app/(protected)/project/[project_id]/dspy-experiments/layout.tsx b/app/(protected)/project/[project_id]/dspy-experiments/layout.tsx new file mode 100644 index 00000000..bddf5627 --- /dev/null +++ b/app/(protected)/project/[project_id]/dspy-experiments/layout.tsx @@ -0,0 +1,11 @@ +import { Metadata } from "next"; +import LayoutClient from "./layout-client"; + +export const metadata: Metadata = { + title: "Langtrace | Experiments", + description: "Manage your DSPy experiments.", +}; + +export default function Layout({ children }: { children: React.ReactNode }) { + return ; +} diff --git a/app/(protected)/project/[project_id]/dspy-experiments/page-client.tsx b/app/(protected)/project/[project_id]/dspy-experiments/page-client.tsx new file mode 100644 index 00000000..4d0ace06 --- /dev/null +++ b/app/(protected)/project/[project_id]/dspy-experiments/page-client.tsx @@ -0,0 +1,3 @@ +export function PageClient({ email }: { email: string }) { + return
; +} diff --git a/app/(protected)/project/[project_id]/dspy-experiments/page.tsx b/app/(protected)/project/[project_id]/dspy-experiments/page.tsx new file mode 100644 index 00000000..8d08f1e1 --- /dev/null +++ b/app/(protected)/project/[project_id]/dspy-experiments/page.tsx @@ -0,0 +1,24 @@ +import { authOptions } from "@/lib/auth/options"; +import { Metadata } from "next"; +import { getServerSession } from "next-auth"; +import { redirect } from "next/navigation"; +import { PageClient } from "./page-client"; + +export const metadata: Metadata = { + title: "Langtrace | Traces", + description: "View all traces for a project.", +}; + +export default async function Page() { + const session = await getServerSession(authOptions); + if (!session || !session.user) { + redirect("/login"); + } + const user = session?.user; + + return ( + <> + + + ); +} diff --git a/app/(protected)/projects/page-client.tsx b/app/(protected)/projects/page-client.tsx index a7451b47..d7b39a7a 100644 --- a/app/(protected)/projects/page-client.tsx +++ b/app/(protected)/projects/page-client.tsx @@ -194,6 +194,18 @@ function ProjectCard({ {project.name} )} + {project.type === "dspy" && ( + + DSPy Logo + {project.name} + + )} {project.type === "default" && ( {project.name} diff --git a/app/(protected)/settings/members/page-client.tsx b/app/(protected)/settings/members/page-client.tsx index ea27aa3a..aaab18a2 100644 --- a/app/(protected)/settings/members/page-client.tsx +++ b/app/(protected)/settings/members/page-client.tsx @@ -62,6 +62,7 @@ export function InviteMember({ user }: { user: any }) { const MemberDetailsForm = useForm({ resolver: zodResolver(MemberDetailsFormSchema), }); + const { reset } = MemberDetailsForm; const sendMemberInvitation = async (data: FieldValues) => { try { @@ -92,6 +93,8 @@ export function InviteMember({ user }: { user: any }) { description: `Please have the invited person sign up with their ${data.email} email`, }); setOpen(false); + MemberDetailsForm.setValue("email", ""); + MemberDetailsForm.setValue("name", ""); } catch (error) { toast.error( "An error occurred while inviting your team member. Please try again later." diff --git a/app/api/metrics/latency/trace/route.ts b/app/api/metrics/latency/trace/route.ts index 036a2ad6..98883416 100644 --- a/app/api/metrics/latency/trace/route.ts +++ b/app/api/metrics/latency/trace/route.ts @@ -15,9 +15,22 @@ export async function GET(req: NextRequest) { const lastNHours = parseInt( req.nextUrl.searchParams.get("lastNHours") || "168" ); - const userId = req.nextUrl.searchParams.get("userId") || ""; - const model = req.nextUrl.searchParams.get("model") || ""; - const inference = req.nextUrl.searchParams.get("inference") || ""; + const userId = + req.nextUrl.searchParams.get("userId") === "undefined" + ? undefined + : (req.nextUrl.searchParams.get("userId") as string); + const model = + req.nextUrl.searchParams.get("model") === "undefined" + ? undefined + : (req.nextUrl.searchParams.get("model") as string); + const inference = + req.nextUrl.searchParams.get("inference") === "undefined" + ? "false" + : (req.nextUrl.searchParams.get("inference") as string); + const experimentId = + req.nextUrl.searchParams.get("experimentId") === "undefined" + ? undefined + : (req.nextUrl.searchParams.get("experimentId") as string); if (!projectId) { return NextResponse.json( @@ -34,6 +47,7 @@ export async function GET(req: NextRequest) { projectId, lastNHours, userId, + experimentId, model, inference === "true" ); diff --git a/app/api/metrics/usage/cost/inference/route.ts b/app/api/metrics/usage/cost/inference/route.ts index 480f1076..c855221a 100644 --- a/app/api/metrics/usage/cost/inference/route.ts +++ b/app/api/metrics/usage/cost/inference/route.ts @@ -13,9 +13,18 @@ export async function GET(req: NextRequest) { try { const projectId = req.nextUrl.searchParams.get("projectId") as string; + const experimentId = req.nextUrl.searchParams.get("experimentId") as string; + const experimentFilter: { [key: string]: string } = {}; + + if (experimentId) { + experimentFilter["experiment"] = experimentId; + } const traceService = new TraceService(); - const cost = await traceService.GetInferenceCostPerProject(projectId); + const cost = await traceService.GetInferenceCostPerProject( + projectId, + experimentFilter + ); const inferenceCount = await traceService.GetTotalTracesPerProject( projectId, true diff --git a/app/api/metrics/usage/trace/route.ts b/app/api/metrics/usage/trace/route.ts index 5282b1e8..21b79503 100644 --- a/app/api/metrics/usage/trace/route.ts +++ b/app/api/metrics/usage/trace/route.ts @@ -15,9 +15,22 @@ export async function GET(req: NextRequest) { const lastNHours = parseInt( req.nextUrl.searchParams.get("lastNHours") || "168" ); - const userId = req.nextUrl.searchParams.get("userId") || ""; - const model = req.nextUrl.searchParams.get("model") || ""; - const inference = req.nextUrl.searchParams.get("inference") || ""; + const userId = + req.nextUrl.searchParams.get("userId") === "undefined" + ? undefined + : (req.nextUrl.searchParams.get("userId") as string); + const model = + req.nextUrl.searchParams.get("model") === "undefined" + ? undefined + : (req.nextUrl.searchParams.get("model") as string); + const inference = + req.nextUrl.searchParams.get("inference") === "undefined" + ? undefined + : (req.nextUrl.searchParams.get("inference") as string); + const experimentId = + req.nextUrl.searchParams.get("experimentId") === "undefined" + ? undefined + : (req.nextUrl.searchParams.get("experimentId") as string); if (!projectId) { return NextResponse.json( @@ -33,6 +46,7 @@ export async function GET(req: NextRequest) { projectId, lastNHours, userId, + experimentId, model, inference === "true" ); diff --git a/app/api/project/route.ts b/app/api/project/route.ts index 97ee506d..27b8deb2 100644 --- a/app/api/project/route.ts +++ b/app/api/project/route.ts @@ -1,6 +1,7 @@ import { authOptions } from "@/lib/auth/options"; import { DEFAULT_TESTS } from "@/lib/constants"; import prisma from "@/lib/prisma"; +import { captureEvent } from "@/lib/services/posthog"; import { authApiKey } from "@/lib/utils"; import { getServerSession } from "next-auth"; import { redirect } from "next/navigation"; @@ -87,6 +88,13 @@ export async function POST(req: NextRequest) { }, }); } + + const session = await getServerSession(authOptions); + const userEmail = session?.user?.email ?? "anonymous"; + await captureEvent(project.id, "project_created", { + project_name: project.name, + project_type: projectType, + }); } const { apiKeyHash, ...projectWithoutApiKeyHash } = project; diff --git a/components/charts/dspy-eval-chart.tsx b/components/charts/dspy-eval-chart.tsx new file mode 100644 index 00000000..d2e2e90a --- /dev/null +++ b/components/charts/dspy-eval-chart.tsx @@ -0,0 +1,88 @@ +"use client"; + +import { CartesianGrid, Line, LineChart, XAxis, YAxis } from "recharts"; + +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { + ChartConfig, + ChartContainer, + ChartTooltip, + ChartTooltipContent, +} from "@/components/ui/chart"; +import LargeChartSkeleton from "./large-chart-skeleton"; + +export interface DspyEvalChartData { + timestamp: string; + score: number; + runId: string; +} + +const chartConfig = { + score: { + label: "Score", + color: "hsl(var(--chart-1))", + }, + runId: { + label: "Run ID", + color: "hsl(var(--chart-2))", + }, +} satisfies ChartConfig; + +export function DspyEvalChart({ + data, + isLoading, +}: { + data: DspyEvalChartData[]; + isLoading: boolean; +}) { + if (isLoading) { + return ; + } + return ( + + + Evaluated scores across runs + + + + + + + + } /> + + + + + + ); +} diff --git a/components/charts/inference-chart.tsx b/components/charts/inference-chart.tsx index 176089e4..91382637 100644 --- a/components/charts/inference-chart.tsx +++ b/components/charts/inference-chart.tsx @@ -12,10 +12,12 @@ export function CountInferenceChart({ projectId, lastNHours = 168, userId, + experimentId, model, }: { projectId: string; lastNHours?: number; + experimentId?: string; userId?: string; model?: string; }) { @@ -32,9 +34,11 @@ export function CountInferenceChart({ model, ], queryFn: async () => { - const response = await fetch( - `/api/metrics/usage/trace?projectId=${projectId}&lastNHours=${lastNHours}&userId=${userId}&model=${model}&inference=true` - ); + let url = `/api/metrics/usage/trace?projectId=${projectId}&lastNHours=${lastNHours}&userId=${userId}&model=${model}&inference=true`; + if (experimentId) { + url += `&experimentId=${experimentId}`; + } + const response = await fetch(url); if (!response.ok) { const error = await response.json(); throw new Error(error?.message || "Failed to fetch inference count"); @@ -98,31 +102,24 @@ export function CountInferenceChart({ export function AverageCostInferenceChart({ projectId, - lastNHours = 168, - userId, - model, + experimentId, }: { projectId: string; - lastNHours?: number; - userId?: string; - model?: string; + experimentId?: string; }) { const { data: costUsage, isLoading: costUsageLoading, error: costUsageError, } = useQuery({ - queryKey: [ - "fetch-metrics-inference-cost", - projectId, - lastNHours, - userId, - model, - ], + queryKey: ["fetch-metrics-inference-cost", projectId, experimentId], queryFn: async () => { - const response = await fetch( - `/api/metrics/usage/cost/inference?projectId=${projectId}` - ); + let url = `/api/metrics/usage/cost/inference?projectId=${projectId}`; + if (experimentId) { + url += `&experimentId=${experimentId}`; + } + const response = await fetch(url); + if (!response.ok) { const error = await response.json(); throw new Error( diff --git a/components/project/metrics.tsx b/components/project/metrics.tsx index e91720a5..df4ab842 100644 --- a/components/project/metrics.tsx +++ b/components/project/metrics.tsx @@ -98,12 +98,7 @@ export default function Metrics({ email }: { email: string }) { projectId={project_id} lastNHours={lastNHours} /> - +
diff --git a/components/project/project-type-dropdown.tsx b/components/project/project-type-dropdown.tsx index 9df65238..cc264b28 100644 --- a/components/project/project-type-dropdown.tsx +++ b/components/project/project-type-dropdown.tsx @@ -30,7 +30,7 @@ const projectTypes = [ { value: "dspy", label: "DSPy", - comingSoon: true, + comingSoon: false, }, { value: "langgraph", diff --git a/components/shared/hover-cell.tsx b/components/shared/hover-cell.tsx index b475f435..4eb5d882 100644 --- a/components/shared/hover-cell.tsx +++ b/components/shared/hover-cell.tsx @@ -129,3 +129,88 @@ export function HoverCell({ return null; } } + +export function GenericHoverCell({ + value, + className, + expand = false, +}: { + value: any; + className?: string; + expand?: boolean; +}) { + try { + const [expandedView, setExpandedView] = useState(expand); + const content = safeStringify(value); + + if (!content || content.length === 0) { + return null; + } + + const copyToClipboard = (e: any) => { + e.stopPropagation(); + navigator.clipboard.writeText(content); + toast.success("Copied to clipboard"); + }; + + return ( + + +
+ + {!expandedView && ( + { + e.stopPropagation(); + setExpandedView(!expandedView); + }} + /> + )} + {expandedView && ( + { + e.stopPropagation(); + setExpandedView(!expandedView); + }} + /> + )} +
+
+ + e.stopPropagation()} + > +
+ +
+
+ + + ); + } catch (e) { + return null; + } +} diff --git a/components/shared/nav.tsx b/components/shared/nav.tsx index 0292da78..603b5a83 100644 --- a/components/shared/nav.tsx +++ b/components/shared/nav.tsx @@ -55,6 +55,13 @@ const ProjectNavLinks = (id: string, type = "default") => { href: `/project/${id}/crewai-dash`, }); } + if (type == "dspy") { + // add to the second position + result.splice(1, 0, { + name: "Experiments", + href: `/project/${id}/dspy-experiments`, + }); + } return result; }; diff --git a/components/shared/posthog.tsx b/components/shared/posthog.tsx new file mode 100644 index 00000000..a6599297 --- /dev/null +++ b/components/shared/posthog.tsx @@ -0,0 +1,42 @@ +"use client"; + +import React, { useEffect, useState } from "react"; +import posthog from "posthog-js"; +import { PostHogProvider } from "posthog-js/react"; +import { Session } from "next-auth"; + +interface CustomPostHogProviderProps { + children: React.ReactNode; + session: Session | null; +} + +export default function CustomPostHogProvider({ + children, + session, +}: CustomPostHogProviderProps) { + const [telemetryEnabled, setTelemetryEnabled] = useState(false); + + useEffect(() => { + async function initializePostHog() { + const enabled = process.env.TELEMETRY_ENABLED === "true"; + setTelemetryEnabled(enabled); + + if (enabled && typeof window !== "undefined") { + posthog.init(process.env.POSTHOG_API_KEY!, { + api_host: "https://app.posthog.com", + loaded: (posthog) => { + if (process.env.NODE_ENV === "development") posthog.debug(); + }, + }); + } + } + + initializePostHog(); + }, [session]); + + if (!telemetryEnabled) { + return <>{children}; + } + + return {children}; +} diff --git a/components/shared/tabs.tsx b/components/shared/tabs.tsx index b9827562..cad73b6d 100644 --- a/components/shared/tabs.tsx +++ b/components/shared/tabs.tsx @@ -1,6 +1,7 @@ "use client"; import Link from "next/link"; import { usePathname } from "next/navigation"; +import { Skeleton } from "../ui/skeleton"; export interface Tab { name: string; @@ -8,24 +9,36 @@ export interface Tab { href: string; } -export default function Tabs({ tabs }: { tabs: Tab[] }) { +export default function Tabs({ + tabs, + paginationLoading = false, + scrollableDivRef, +}: { + tabs: Tab[]; + paginationLoading?: boolean; + scrollableDivRef?: React.RefObject; +}) { const pathname = usePathname(); return ( -
+
{tabs.map((tab, idx) => ( {tab.name} ))} + {paginationLoading && }
); } diff --git a/components/shared/vendor-metadata.tsx b/components/shared/vendor-metadata.tsx index a56a2855..769ba083 100644 --- a/components/shared/vendor-metadata.tsx +++ b/components/shared/vendor-metadata.tsx @@ -62,6 +62,10 @@ export function vendorBadgeColor(vendor: string) { return "bg-red-500"; } + if (vendor.includes("litellm")) { + return "bg-blue-500"; + } + return "bg-gray-500"; } @@ -451,6 +455,22 @@ export function VendorLogo({ ); } + if (vendor.includes("litellm")) { + const color = vendorColor("litellm"); + return ( + LiteLLM Logo + ); + } + return (
= ({ serviceName.includes("llamaindex") ) color = "bg-indigo-500"; - else if ( - span.name.includes("vercel") || - serviceName.includes("vercel") - ) + else if (span.name.includes("vercel") || serviceName.includes("vercel")) color = "bg-gray-500"; else if ( span.name.includes("embedchain") || serviceName.includes("embedchain") ) color = "bg-slate-500"; + else if (span.name.includes("litellm") || serviceName.includes("litellm")) + color = "bg-blue-500"; const fillColor = color.replace("bg-", "fill-"); const vendor = getVendorFromSpan(span as any); diff --git a/lib/auth/options.ts b/lib/auth/options.ts index 0c578ce2..383be218 100644 --- a/lib/auth/options.ts +++ b/lib/auth/options.ts @@ -1,9 +1,9 @@ import prisma from "@/lib/prisma"; import { PrismaAdapter } from "@next-auth/prisma-adapter"; import { type NextAuthOptions } from "next-auth"; +import AzureADProvider from "next-auth/providers/azure-ad"; import CredentialsProvider from "next-auth/providers/credentials"; import GoogleProvider from "next-auth/providers/google"; -import AzureADProvider from 'next-auth/providers/azure-ad'; export const authOptions: NextAuthOptions = { providers: [ @@ -64,7 +64,7 @@ export const authOptions: NextAuthOptions = { clientSecret: process.env.AZURE_AD_CLIENT_SECRET as string, tenantId: process.env.AZURE_AD_TENANT_ID as string, allowDangerousEmailAccountLinking: true, - }) + }), ], adapter: PrismaAdapter(prisma), session: { strategy: "jwt" }, diff --git a/lib/dspy_trace_util.ts b/lib/dspy_trace_util.ts new file mode 100644 index 00000000..4bfb0ddb --- /dev/null +++ b/lib/dspy_trace_util.ts @@ -0,0 +1,331 @@ +import { calculateTotalTime, convertTracesToHierarchy } from "./trace_utils"; +import { calculatePriceFromUsage } from "./utils"; + +export interface DspyTrace { + id: string; + run_id: string; + experiment_name: string; + experiment_description: string; + status: string; + namespace: string; + user_ids: string[]; + prompt_ids: string[]; + prompt_versions: string[]; + models: string[]; + vendors: string[]; + inputs: Record[]; + outputs: Record[]; + input_tokens: number; + output_tokens: number; + total_tokens: number; + input_cost: number; + output_cost: number; + total_cost: number; + start_time: number; + total_duration: number; + all_events: any[]; + sorted_trace: any[]; + trace_hierarchy: any[]; + raw_attributes: any; + result: any; + checkpoint: any; + evaluated_score?: number; +} + +export function processDspyTrace(trace: any): DspyTrace { + const traceHierarchy = convertTracesToHierarchy(trace); + const totalTime = calculateTotalTime(trace); + const startTime = trace[0].start_time; + let tokenCounts: any = {}; + let models: string[] = []; + let vendors: string[] = []; + let userIds: string[] = []; + let promptIds: string[] = []; + let promptVersions: string[] = []; + let messages: Record[] = []; + let allEvents: any[] = []; + let attributes: any = {}; + let cost = { total: 0, input: 0, output: 0 }; + let experiment_name = "default"; + let experiment_description = ""; + let run_id = "unspecified"; + let spanResult: any = {}; + let checkpoint: any = {}; + let evaluated_score: number | undefined; + // set status to ERROR if any span has an error + let status = "success"; + for (const span of trace) { + if (span.status_code === "ERROR") { + status = "error"; + break; + } + } + + for (const span of trace) { + if (span.attributes) { + // parse the attributes of the span + attributes = JSON.parse(span.attributes); + let vendor = ""; + + let resultContent = ""; + if (attributes["dspy.signature.result"]) { + resultContent = attributes["dspy.signature.result"]; + } else if (attributes["dspy.evaluate.result"]) { + resultContent = attributes["dspy.evaluate.result"]; + evaluated_score = attributes["dspy.evaluate.result"]; + } + + if (resultContent) { + try { + spanResult = JSON.parse(resultContent); + } catch (e) { + spanResult = resultContent; + } + } + + if (attributes["dspy.checkpoint"]) { + checkpoint = JSON.parse(attributes["dspy.checkpoint"]); + } + + // get the service name from the attributes + if (attributes["langtrace.service.name"]) { + vendor = attributes["langtrace.service.name"].toLowerCase(); + if (!vendors.includes(vendor)) vendors.push(vendor); + } + + // get the experiment name from the attributes + if (attributes["experiment"]) { + experiment_name = attributes["experiment"] + .toLowerCase() + .replace(/\s/g, "-"); + } + + // get the run_id from the attributes + if (attributes["run_id"]) { + run_id = attributes["run_id"].toLowerCase().replace(/\s/g, "-"); + } + + // get the experiment description from the attributes + if (attributes["description"]) { + experiment_description = attributes["description"]; + } + + // get the user_id, prompt_id, prompt_version, and model from the attributes + if (attributes["user_id"]) { + userIds.push(attributes["user_id"]); + } + if (attributes["prompt_id"]) { + promptIds.push(attributes["prompt_id"]); + } + if (attributes["prompt_version"]) { + promptVersions.push(attributes["prompt_version"]); + } + + let model = ""; + if ( + attributes["gen_ai.response.model"] || + attributes["llm.model"] || + attributes["gen_ai.request.model"] + ) { + model = + attributes["gen_ai.response.model"] || + attributes["llm.model"] || + attributes["gen_ai.request.model"]; + models.push(model); + } + // TODO(Karthik): This logic is for handling old traces that were not compatible with the gen_ai conventions. + if (attributes["llm.prompts"] && attributes["llm.responses"]) { + const message = { + prompt: attributes["llm.prompts"], + response: attributes["llm.responses"], + }; + messages.push(message); + } + + if ( + attributes["gen_ai.usage.prompt_tokens"] && + attributes["gen_ai.usage.completion_tokens"] + ) { + tokenCounts = { + input_tokens: tokenCounts.prompt_tokens + ? tokenCounts.prompt_tokens + + attributes["gen_ai.usage.prompt_tokens"] + : attributes["gen_ai.usage.prompt_tokens"], + output_tokens: tokenCounts.completion_tokens + ? tokenCounts.completion_tokens + + attributes["gen_ai.usage.completion_tokens"] + : attributes["gen_ai.usage.completion_tokens"], + total_tokens: tokenCounts.total_tokens + ? tokenCounts.total_tokens + + attributes["gen_ai.usage.prompt_tokens"] + + attributes["gen_ai.usage.completion_tokens"] + : attributes["gen_ai.usage.prompt_tokens"] + + attributes["gen_ai.usage.completion_tokens"], + }; + + // calculate the cost of the current span + const currentcost = calculatePriceFromUsage(vendor, model, tokenCounts); + + // add the cost of the current span to the total cost + cost.total += currentcost.total; + cost.input += currentcost.input; + cost.output += currentcost.output; + } else if ( + attributes["gen_ai.usage.input_tokens"] && + attributes["gen_ai.usage.output_tokens"] + ) { + tokenCounts = { + input_tokens: tokenCounts.prompt_tokens + ? tokenCounts.prompt_tokens + + attributes["gen_ai.usage.input_tokens"] + : attributes["gen_ai.usage.input_tokens"], + output_tokens: tokenCounts.completion_tokens + ? tokenCounts.completion_tokens + + attributes["gen_ai.usage.output_tokens"] + : attributes["gen_ai.usage.output_tokens"], + total_tokens: tokenCounts.total_tokens + ? tokenCounts.total_tokens + + attributes["gen_ai.usage.input_tokens"] + + attributes["gen_ai.usage.output_tokens"] + : attributes["gen_ai.usage.input_tokens"] + + attributes["gen_ai.usage.output_tokens"], + }; + const currentcost = calculatePriceFromUsage(vendor, model, tokenCounts); + // add the cost of the current span to the total cost + cost.total += currentcost.total; + cost.input += currentcost.input; + cost.output += currentcost.output; + } else if (attributes["llm.token.counts"]) { + // TODO(Karthik): This logic is for handling old traces that were not compatible with the gen_ai conventions. + const currentcounts = JSON.parse(attributes["llm.token.counts"]); + tokenCounts = { + input_tokens: tokenCounts.input_tokens + ? tokenCounts.input_tokens + currentcounts.input_tokens + : currentcounts.input_tokens, + output_tokens: tokenCounts.output_tokens + ? tokenCounts.output_tokens + currentcounts.output_tokens + : currentcounts.output_tokens, + total_tokens: tokenCounts.total_tokens + ? tokenCounts.total_tokens + currentcounts.total_tokens + : currentcounts.total_tokens, + }; + + // calculate the cost of the current span + const currentcost = calculatePriceFromUsage( + vendor, + model, + currentcounts + ); + // add the cost of the current span to the total cost + cost.total += currentcost.total; + cost.input += currentcost.input; + cost.output += currentcost.output; + } + } + + // TODO(Karthik): This logic is for handling old traces that were not compatible with the gen_ai conventions. + const message: Record = { + prompts: [], + responses: [], + }; + + if (attributes["llm.prompts"]) { + message.prompts.push(attributes["llm.prompts"]); + } + + if (attributes["llm.responses"]) { + message.responses.push(attributes["llm.responses"]); + } + + if (message.prompts.length > 0 || message.responses.length > 0) { + messages.push(message); + } + + if (span.events && span.events !== "[]") { + const events = JSON.parse(span.events); + const inputs = []; + const outputs = []; + allEvents.push(events); + + // find event with name 'gen_ai.content.prompt' + const promptEvent = events.find( + (event: any) => event.name === "gen_ai.content.prompt" + ); + if ( + promptEvent && + promptEvent["attributes"] && + promptEvent["attributes"]["gen_ai.prompt"] + ) { + inputs.push(promptEvent["attributes"]["gen_ai.prompt"]); + } + + // find event with name 'gen_ai.content.completion' + const responseEvent = events.find( + (event: any) => event.name === "gen_ai.content.completion" + ); + if ( + responseEvent && + responseEvent["attributes"] && + responseEvent["attributes"]["gen_ai.completion"] + ) { + outputs.push(responseEvent["attributes"]["gen_ai.completion"]); + } + + const message: Record = { + prompts: [], + responses: [], + }; + if (inputs.length > 0) { + message.prompts.push(...inputs); + } + if (outputs.length > 0) { + message.responses.push(...outputs); + } + if (message.prompts.length > 0 || message.responses.length > 0) { + messages.push(message); + } + } + } + + // Sort the trace based on start_time, then end_time + trace.sort((a: any, b: any) => { + if (a.start_time === b.start_time) { + return a.end_time < b.end_time ? 1 : -1; + } + return a.start_time < b.start_time ? -1 : 1; + }); + + // construct the response object + const result: DspyTrace = { + id: trace[0]?.trace_id, + run_id: run_id, + experiment_name: experiment_name, + experiment_description: experiment_description, + status: status, + namespace: traceHierarchy[0].name, + user_ids: userIds, + prompt_ids: promptIds, + prompt_versions: promptVersions, + models: models, + vendors: vendors, + inputs: messages, + outputs: messages, + all_events: allEvents, + input_tokens: tokenCounts.input_tokens, + output_tokens: tokenCounts.output_tokens, + total_tokens: tokenCounts.total_tokens, + input_cost: cost.input, + output_cost: cost.output, + total_cost: cost.total, + total_duration: totalTime, + start_time: startTime, + sorted_trace: trace, + trace_hierarchy: traceHierarchy, + raw_attributes: attributes, + result: spanResult, + checkpoint: checkpoint, + evaluated_score: evaluated_score, + }; + + return result; +} diff --git a/lib/middleware/app.ts b/lib/middleware/app.ts index 5f5eb498..14fac36e 100644 --- a/lib/middleware/app.ts +++ b/lib/middleware/app.ts @@ -1,6 +1,7 @@ import { User } from "@prisma/client"; import { getToken } from "next-auth/jwt"; import { NextRequest, NextResponse } from "next/server"; +import { captureEvent } from "../services/posthog"; export default async function AppMiddleware(req: NextRequest) { const path = req.nextUrl.pathname; @@ -36,9 +37,10 @@ export default async function AppMiddleware(req: NextRequest) { const response = await userReq.json(); const user = response.data; if (user) { + let teamName: string | undefined; if (!user.teamId) { // create a team - await fetch(`${process.env.NEXT_PUBLIC_HOST}/api/team`, { + const team = await fetch(`${process.env.NEXT_PUBLIC_HOST}/api/team`, { method: "POST", headers: { "Content-Type": "application/json", @@ -49,6 +51,12 @@ export default async function AppMiddleware(req: NextRequest) { role: "owner", status: "active", }), + }).then((res) => res.json()); + teamName = team?.data?.name; + } + if (teamName) { + await captureEvent(user.id, "team_created", { + team_name: teamName, }); } } diff --git a/lib/services/posthog.ts b/lib/services/posthog.ts new file mode 100644 index 00000000..9f4f6c39 --- /dev/null +++ b/lib/services/posthog.ts @@ -0,0 +1,37 @@ +import { PostHog } from "posthog-node"; + +let posthogClient: PostHog | null = null; + +export function getPostHogClient(): PostHog { + if (!posthogClient) { + posthogClient = new PostHog(process.env.POSTHOG_API_KEY!, { + host: process.env.POSTHOG_HOST, + flushAt: 1, + flushInterval: 0, + }); + } + return posthogClient; +} + +export async function captureEvent( + distinctId: string, + eventName: string, + properties: Record = {} +): Promise { + try { + if (process.env.TELEMETRY_ENABLED !== "false") { + const client = getPostHogClient(); + await client.capture({ + distinctId, + event: eventName, + properties: { + ...properties, + is_self_hosted: true, + }, + }); + await client.shutdown(); + } + } catch (error) { + console.error("Error capturing telemetry events:", error); + } +} diff --git a/lib/services/trace_service.ts b/lib/services/trace_service.ts index 1caf851f..4e1014ea 100644 --- a/lib/services/trace_service.ts +++ b/lib/services/trace_service.ts @@ -18,6 +18,7 @@ export interface ITraceService { project_id: string, lastNHours?: number, userId?: string, + experimentId?: string, model?: string, inference?: boolean ) => Promise; @@ -96,7 +97,10 @@ export interface ITraceService { GetUsersInProject: (project_id: string) => Promise; GetPromptsInProject: (project_id: string) => Promise; GetModelsInProject: (project_id: string) => Promise; - GetInferenceCostPerProject: (project_id: string) => Promise; + GetInferenceCostPerProject: ( + project_id: string, + attribute_filters?: { [key: string]: string } + ) => Promise; } export class TraceService implements ITraceService { @@ -107,7 +111,10 @@ export class TraceService implements ITraceService { this.queryBuilderService = new QueryBuilderService(); } - async GetInferenceCostPerProject(project_id: string): Promise { + async GetInferenceCostPerProject( + project_id: string, + attribute_filters: { [key: string]: string } = {} + ): Promise { try { const tableExists = await this.client.checkTableExists(project_id); if (!tableExists) { @@ -120,6 +127,17 @@ export class TraceService implements ITraceService { ), ]; + if (Object.keys(attribute_filters).length > 0) { + Object.keys(attribute_filters).forEach((key) => { + conditions.push( + sql.eq( + `JSONExtractString(attributes, '${key}')`, + attribute_filters[key] + ) + ); + }); + } + const query = sql .select([ `IF( @@ -407,6 +425,7 @@ export class TraceService implements ITraceService { project_id: string, lastNHours = 168, userId?: string, + experimentId?: string, model?: string, inference = false ): Promise { @@ -425,6 +444,12 @@ export class TraceService implements ITraceService { ); } + if (experimentId) { + conditions.push( + sql.eq("JSONExtractString(attributes, 'experiment')", experimentId) + ); + } + if (model) { conditions.push( sql.eq("JSONExtractString(attributes, 'llm.model')", model) diff --git a/lib/utils.ts b/lib/utils.ts index a1ee9de5..ab299edb 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -518,10 +518,46 @@ export function calculatePriceFromUsage( if (!model) return { total: 0, input: 0, output: 0 }; let costTable: CostTableEntry | undefined = undefined; - if (vendor === "openai") { + if (vendor === "litellm") { + let correctModel = model; + if (model.includes("gpt") || model.includes("o1")) { + if (model.includes("gpt-4o-mini")) { + correctModel = "gpt-4o-mini"; + } else if (model.includes("gpt-4o")) { + correctModel = "gpt-4o"; + } else if (model.includes("gpt-4")) { + correctModel = "gpt-4"; + } else if (model.includes("o1-preview")) { + correctModel = "o1-preview"; + } else if (model.includes("o1-mini")) { + correctModel = "o1-mini"; + } + costTable = OPENAI_PRICING[correctModel]; + } else if (model.includes("claude")) { + let cmodel = ""; + if (model.includes("opus")) { + cmodel = "claude-3-opus"; + } else if (model.includes("sonnet")) { + cmodel = "claude-3-sonnet"; + } else if (model.includes("haiku")) { + cmodel = "claude-3-haiku"; + } else if (model.includes("claude-2.1")) { + cmodel = "claude-2.1"; + } else if (model.includes("claude-2.0")) { + cmodel = "claude-2.0"; + } else if (model.includes("instant")) { + cmodel = "claude-instant"; + } else { + return 0; + } + costTable = ANTHROPIC_PRICING[cmodel]; + } else if (model.includes("command")) { + costTable = COHERE_PRICING[model]; + } + } else if (vendor === "openai") { // check if model is present as key in OPENAI_PRICING let correctModel = model; - if (!OPENAI_PRICING.hasOwnProperty(model)) { + if (model.includes("gpt") || model.includes("o1")) { if (model.includes("gpt-4o-mini")) { correctModel = "gpt-4o-mini"; } else if (model.includes("gpt-4o")) { @@ -757,6 +793,8 @@ export function getVendorFromSpan(span: Span): string { serviceName.includes("embedchain") ) { vendor = "embedchain"; + } else if (span.name.includes("litellm") || serviceName.includes("litellm")) { + vendor = "litellm"; } return vendor; } diff --git a/package-lock.json b/package-lock.json index 20b8e6a6..fb73a2a7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -76,6 +76,8 @@ "npm": "^10.8.2", "openai": "^4.40.0", "pdf-parse": "^1.1.1", + "posthog-js": "^1.161.3", + "posthog-node": "^4.2.0", "pretty-print-json": "^3.0.0", "prism": "^1.0.0", "prismjs": "^1.29.0", @@ -5778,8 +5780,9 @@ } }, "node_modules/axios": { - "version": "1.7.2", - "license": "MIT", + "version": "1.7.7", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", + "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.0", @@ -8130,6 +8133,11 @@ "node": "^12.20 || >= 14.13" } }, + "node_modules/fflate": { + "version": "0.4.8", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.4.8.tgz", + "integrity": "sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA==" + }, "node_modules/file-entry-cache": { "version": "6.0.1", "dev": true, @@ -14922,6 +14930,28 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/posthog-js": { + "version": "1.161.3", + "resolved": "https://registry.npmjs.org/posthog-js/-/posthog-js-1.161.3.tgz", + "integrity": "sha512-TQ77jtLemkUJUyJAPrwGay6tLqcAmXEM1IJgXOx5Tr4UohiTx8JTznzrCuh/SdwPIrbcSM1r2YPwb72XwTC3wg==", + "dependencies": { + "fflate": "^0.4.8", + "preact": "^10.19.3", + "web-vitals": "^4.0.1" + } + }, + "node_modules/posthog-node": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-4.2.0.tgz", + "integrity": "sha512-hgyCYMyzMvuF3qWMw6JvS8gT55v7Mtp5wKWcnDrw+nu39D0Tk9BXD7I0LOBp0lGlHEPaXCEVYUtviNKrhMALGA==", + "dependencies": { + "axios": "^1.7.4", + "rusha": "^0.8.14" + }, + "engines": { + "node": ">=15.0.0" + } + }, "node_modules/preact": { "version": "10.23.1", "license": "MIT", @@ -15911,6 +15941,11 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/rusha": { + "version": "0.8.14", + "resolved": "https://registry.npmjs.org/rusha/-/rusha-0.8.14.tgz", + "integrity": "sha512-cLgakCUf6PedEu15t8kbsjnwIFFR2D4RfL+W3iWFJ4iac7z4B0ZI8fxy4R3J956kAI68HclCFGL8MPoUVC3qVA==" + }, "node_modules/rw": { "version": "1.3.3", "license": "BSD-3-Clause" @@ -17459,6 +17494,11 @@ "node": ">= 8" } }, + "node_modules/web-vitals": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/web-vitals/-/web-vitals-4.2.3.tgz", + "integrity": "sha512-/CFAm1mNxSmOj6i0Co+iGFJ58OS4NRGVP+AWS/l509uIK5a1bSoIVaHz/ZumpHTfHSZBpgrJ+wjfpAOrTHok5Q==" + }, "node_modules/webidl-conversions": { "version": "3.0.1", "license": "BSD-2-Clause" diff --git a/package.json b/package.json index 2ea3aac9..e433b162 100644 --- a/package.json +++ b/package.json @@ -84,6 +84,8 @@ "npm": "^10.8.2", "openai": "^4.40.0", "pdf-parse": "^1.1.1", + "posthog-js": "^1.161.3", + "posthog-node": "^4.2.0", "pretty-print-json": "^3.0.0", "prism": "^1.0.0", "prismjs": "^1.29.0", diff --git a/public/litellm.png b/public/litellm.png new file mode 100644 index 00000000..79468969 Binary files /dev/null and b/public/litellm.png differ