WIP on tracing to google cloud

Fix project ID Replace tracing with OpenTelemetry Try linking graph compute parent span Refactor routing to normal async/await
cloudydeno · Mar 12, 2023 · 129de4f · 129de4f
1 parent ea9bfff
commit 129de4f
Show file tree

Hide file tree

Showing 8 changed files with 144 additions and 89 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM denoland/deno:alpine-1.30.1
+FROM denoland/deno:alpine-1.31.1
 RUN apk add --no-cache graphviz
 ADD fonts/ /usr/share/fonts/truetype/
 
@@ -7,4 +7,4 @@ ADD deps.ts .
 RUN deno check deps.ts
 ADD . .
 RUN deno check server.ts
-ENTRYPOINT ["deno","run","--allow-env","--allow-net","--allow-run=deno,dot","--allow-read=.","server.ts"]
+ENTRYPOINT ["deno","run","--allow-sys=hostname","--allow-env","--allow-net","--allow-run=deno,dot","--allow-read=.","server.ts"]
diff --git a/deps.ts b/deps.ts
@@ -18,3 +18,5 @@ export type {
   ModuleGraphJson,
   ModuleJson,
 } from "https://deno.land/x/[email protected]/lib/types.d.ts";
+
+export { trace, context, type Context } from "npm:@opentelemetry/api";
diff --git a/feat/dependencies-of/api.ts b/feat/dependencies-of/api.ts
@@ -4,13 +4,18 @@ import {
   readableStreamFromIterable,
   SubProcess,
   type SubprocessErrorData,
+  trace,
+  context,
+  Context,
 } from "../../deps.ts";
 
 import { templateHtml, makeErrorResponse, HtmlHeaders } from '../../lib/request-handling.ts';
 import { findModuleSlug, resolveModuleUrl } from "../../lib/resolve.ts";
 import { computeGraph, renderGraph } from "./compute.ts";
 
-export async function *handleRequest(req: Request, modSlug: string, args: URLSearchParams) {
+const tracer = trace.getTracer('dependencies-of-api');
+
+export async function handleRequest(req: Request, modSlug: string, args: URLSearchParams) {
   if (modSlug == '') {
     const url = args.get('url');
     if (!url) return;
@@ -26,7 +31,7 @@ export async function *handleRequest(req: Request, modSlug: string, args: URLSea
 
     const slug = await findModuleSlug(url);
     const location = slug + (args.toString() ? `?${args}` : '');
-    yield new Response(`302: ${location}`, {
+    return new Response(`302: ${location}`, {
       status: 302,
       headers: { location },
     });
@@ -37,17 +42,13 @@ export async function *handleRequest(req: Request, modSlug: string, args: URLSea
 
   switch (args.get('format')) {
     case 'json':
-      yield serveBufferedOutput(req, computeGraph(modUrl, args), 'application/json');
-      return;
+      return await serveBufferedOutput(req, computeGraph(modUrl, args), 'application/json');
     case 'dot':
-      yield serveBufferedOutput(req, computeGraph(modUrl, args), 'text/plain; charset=utf-8');
-      return;
+      return await serveBufferedOutput(req, computeGraph(modUrl, args), 'text/plain; charset=utf-8');
     case 'svg':
-      yield serveStreamingOutput(req, renderGraph(modUrl, ["-Tsvg"], args), 'image/svg+xml');
-      return;
+      return await serveStreamingOutput(req, renderGraph(modUrl, ["-Tsvg"], args), 'image/svg+xml');
     case null:
-      yield serveHtmlGraphPage(req, modUrl, modSlug, args);
-      return;
+      return await serveHtmlGraphPage(req, modUrl, modSlug, args);
   }
 }
 
@@ -75,56 +76,40 @@ async function serveStreamingOutput(req: Request, computation: Promise<SubProces
 
 const hideLoadMsg = `<style type="text/css">#graph-waiting { display: none; }</style>`;
 
-async function serveHtmlGraphPage(req: Request, modUrl: string, modSlug: string, args: URLSearchParams) {
-  args.set('font', 'Archivo Narrow');
-
-  // Render the basic page first, so we can error more cleanly if that fails
-  let pageHtml = '';
+async function renderModuleToHtml(modUrl: string, args: URLSearchParams) {
   try {
-    pageHtml = await templateHtml('feat/dependencies-of/public.html', {
-      module_slug: entities.encode(modSlug),
-      module_url: entities.encode(modUrl),
-      export_prefix: entities.encode(`${req.url}${req.url.includes('?') ? '&' : '?'}format=`),
-    });
-  } catch (err) {
-    return makeErrorResponse(err);
-  }
 
-  const graphPromise = ((args.get('renderer') === 'interactive')
-
-  ? computeGraph(modUrl, args, 'dot')
-      .then(data => {
-        return `
-          <div id="graph"></div>
-          <script type="text/javascript" src="https://unpkg.com/[email protected]/standalone/umd/vis-network.min.js"></script>
-          <script type="text/javascript" src="/interactive-graph.js"></script>
-          <template type="text/plain" id="graphviz_data">\n${data
-            .replace(/&/g, '&amp;')
-            .replace(/>/g, '&gt;')
-            .replace(/</g, '&lt;')
-          }</template>
-          <script type="text/javascript">
-          window.CreateModuleGraph(document.getElementById('graphviz_data').innerHTML
-            .replace(/&gt;/g, '>')
-            .replace(/&lt;/g, '<')
-            .replace(/&amp;/g, '&'));
-          </script>
-          `.replace(/^ {10}/gm, '');
-      })
-
-  : renderGraph(modUrl, ["-Tsvg"], args)
-      .then(dotProc => dotProc.captureAllOutput())
-      .then(raw => {
-        const fullSvg = new TextDecoder().decode(raw);
-        const attrs = [`id="graph"`];
-        const svgWidth = fullSvg.match(/viewBox="(?:([0-9.-]+) ){3}/)?.[1];
-        if (svgWidth) attrs.push(`style="max-width: ${parseInt(svgWidth)*2}px;"`);
-        return fullSvg
-          .slice(fullSvg.indexOf('<!--'))
-          .replace(/<svg width="[^"]+" height="[^"]+"/, '<svg '+attrs.join(' '));
-      })
-
-  ).catch(err => {
+    if (args.get('renderer') === 'interactive') {
+      const data = await computeGraph(modUrl, args, 'dot');
+      return `
+        <div id="graph"></div>
+        <script type="text/javascript" src="https://unpkg.com/[email protected]/standalone/umd/vis-network.min.js"></script>
+        <script type="text/javascript" src="/interactive-graph.js"></script>
+        <template type="text/plain" id="graphviz_data">\n${data
+          .replace(/&/g, '&amp;')
+          .replace(/>/g, '&gt;')
+          .replace(/</g, '&lt;')
+        }</template>
+        <script type="text/javascript">
+        window.CreateModuleGraph(document.getElementById('graphviz_data').innerHTML
+          .replace(/&gt;/g, '>')
+          .replace(/&lt;/g, '<')
+          .replace(/&amp;/g, '&'));
+        </script>
+        `.replace(/^ {8}/gm, '');
+    }
+
+    const dotProc = await renderGraph(modUrl, ["-Tsvg"], args);
+    const raw = await dotProc.captureAllOutput();
+    const fullSvg = new TextDecoder().decode(raw);
+    const attrs = [`id="graph"`];
+    const svgWidth = fullSvg.match(/viewBox="(?:([0-9.-]+) ){3}/)?.[1];
+    if (svgWidth) attrs.push(`style="max-width: ${parseInt(svgWidth)*2}px;"`);
+    return fullSvg
+      .slice(fullSvg.indexOf('<!--'))
+      .replace(/<svg width="[^"]+" height="[^"]+"/, '<svg '+attrs.join(' '));
+
+  } catch (err) {
     if (err.subproc) {
       const info = err.subproc as SubprocessErrorData;
       return `
@@ -140,7 +125,30 @@ async function serveHtmlGraphPage(req: Request, modUrl: string, modSlug: string,
     }
     console.error('Graph computation error:', err.stack);
     return `<div id="graph-error">${entities.encode(err.stack)}</div>`;
-  });
+  }
+}
+
+async function serveHtmlGraphPage(req: Request, modUrl: string, modSlug: string, args: URLSearchParams) {
+  args.set('font', 'Archivo Narrow');
+
+  // Render the basic page first, so we can error more cleanly if that fails
+  let pageHtml = '';
+  try {
+    pageHtml = await templateHtml('feat/dependencies-of/public.html', {
+      module_slug: entities.encode(modSlug),
+      module_url: entities.encode(modUrl),
+      export_prefix: entities.encode(`${req.url}${req.url.includes('?') ? '&' : '?'}format=`),
+    });
+  } catch (err) {
+    return makeErrorResponse(err);
+  }
+
+  const graphPromise = tracer.startActiveSpan('Compute + Render Graph', {
+    attributes: {
+      'render.mod_url': modUrl,
+      'render.params': args.toString(),
+    },
+  }, context.active(), span => renderModuleToHtml(modUrl, args).finally(() => span.end()));
 
   // Return the body in two parts, with a comment in between
   return new Response(readableStreamFromIterable((async function*() {

diff --git a/feat/shields/api.ts b/feat/shields/api.ts
@@ -6,43 +6,39 @@ import { resolveModuleUrl } from "../../lib/resolve.ts";
 import { processDenoInfo, ModuleMap } from "../../lib/module-map.ts";
 import { determineModuleAttrs } from "../../lib/module-registries.ts";
 
-export async function *handleRequest(req: Request, shieldId: string, modSlug: string) {
+export async function handleRequest(req: Request, shieldId: string, modSlug: string) {
   const modUrl = await resolveModuleUrl(modSlug);
   if (!modUrl) return;
   switch (shieldId) {
 
     case 'dep-count':
-      yield computeGraph(modUrl)
+      return await computeGraph(modUrl)
         .then(makeDepCountShield)
         .catch(makeErrorShield);
-      return;
 
     case 'updates':
-      yield computeGraph(modUrl)
+      return await computeGraph(modUrl)
         .then(makeUpdatesShield)
         .catch(makeErrorShield);
-      return;
 
     case 'cache-size':
-      yield computeGraph(modUrl)
+      return await computeGraph(modUrl)
         .then(makeCacheSizeShield)
         .catch(makeErrorShield);
-      return;
 
     case 'latest-version':
       if (modSlug.startsWith('x/')) {
-        yield makeXLatestVersionShield(modSlug.split('/')[1].split('@')[0])
+        return await makeXLatestVersionShield(modSlug.split('/')[1].split('@')[0])
           .catch(makeErrorShield);
       }
       return;
 
     case 'setup':
-      yield serveTemplatedHtml(req, 'feat/shields/public.html', {
+      return await serveTemplatedHtml(req, 'feat/shields/public.html', {
         module_slug: entities.encode(modSlug),
         module_url: entities.encode(modUrl),
         module_slug_component: encodeURIComponent(modSlug),
       });
-      return;
   }
 }
 

diff --git a/lib/module-map.ts b/lib/module-map.ts
@@ -2,6 +2,12 @@ import { filesize, ModuleGraphJson, ModuleJson } from "../deps.ts";
 import { CodeModule } from "./types.ts";
 import * as registries from "./module-registries.ts";
 
+// TODO: enable multiple ModuleMap modes
+// 1. 'per-file' graph, direct from `deno info` without collapsing
+// 2. 'pkg-overview' graph, built from per-file graph & collapses everything into their modules
+// 3. 'module-focus' graph, shows all files within one module + immediate upstream/downstreams (also emits as subgraphs)
+// per-file graphs are used as input when constructing the other graphs
+
 export class ModuleMap {
   modules = new Map<string,CodeModule>();
   mainModule: CodeModule;

diff --git a/lib/request-handling.ts b/lib/request-handling.ts
@@ -1,4 +1,4 @@
-import { http, file_server } from "../deps.ts";
+import { http, file_server, trace, context } from "../deps.ts";
 
 export const HtmlHeaders = new Headers({
   'content-type': 'text/html; charset=utf-8',
@@ -7,7 +7,20 @@ export const TextHeaders = new Headers({
   'content-type': 'text/text; charset=utf-8',
 });
 
-export async function templateHtml(templatePath: string, replacements: Record<string,string> = {}) {
+
+const tracer = trace.getTracer('html-templating');
+
+export function templateHtml(templatePath: string, replacements: Record<string,string> = {}) {
+  return tracer.startActiveSpan(`Render ${templatePath}`, {
+    attributes: {
+      'template_path': templatePath,
+    },
+  }, span =>
+    templateHtmlInner(templatePath, replacements)
+      .finally(() => span.end()));
+}
+
+async function templateHtmlInner(templatePath: string, replacements: Record<string,string> = {}) {
   const [
     template,
     globals,

diff --git a/server.ts b/server.ts
@@ -1,7 +1,8 @@
-#!/usr/bin/env -S deno run --check=all --allow-read=. --allow-net=0.0.0.0 --allow-run=deno,dot --allow-env=PORT
+#!/usr/bin/env -S deno run --watch --check --allow-sys=hostname --allow-read --allow-net --allow-run=deno,dot --allow-env
 
-import { http } from "./deps.ts";
+import { Context, context, http } from "./deps.ts";
 import { serveFont, servePublic, serveTemplatedHtml } from './lib/request-handling.ts';
+import { asyncGeneratorWithContext, httpTracer, provider } from "./tracer.ts";
 
 // The different HTTP surfaces we expose
 import * as DependenciesOf from './feat/dependencies-of/api.ts';
@@ -16,57 +17,56 @@ try {
 }
 
 console.log('Setting up on', { port });
-http.serve(async request => {
-  for await (const response of handleReq(request)) {
-    return response;
-  }
-  console.log('reached 404');
-  return new Response('404 Not Found', {
+http.serve(httpTracer(provider, async request => {
+
+  const resp = await handleReq(request);
+  return resp ?? new Response('404 Not Found', {
     status: 404,
   });
-}, {
+
+}), {
   port,
 });
 
-async function *handleReq(req: Request) {
+async function handleReq(req: Request): Promise<Response | undefined> {
   console.log(req.method, req.url);
   const url = new URL(req.url, 'http://localhost');
   const args = new URLSearchParams(url.search);
 
   { // feature: dependencies-of
     const match = url.pathname.match(/^\/dependencies-of\/(.*)$/);
     if (match && req.method === 'GET') {
-      yield* DependenciesOf.handleRequest(req, match[1], args);
+      return await DependenciesOf.handleRequest(req, match[1], args);
     }
   }
 
   { // feature: shields
     const match = url.pathname.match(/^\/shields\/([^\/]+)\/(.+)$/);
     if (match && req.method === 'GET') {
-      yield* Shields.handleRequest(req, match[1], match[2]);
+      return await Shields.handleRequest(req, match[1], match[2]);
     }
   }
 
   { // feature: registry-key
     if (url.pathname === '/registry-key' && req.method === 'GET') {
-      yield RegistryKey.handleRequest(req);
+      return await RegistryKey.handleRequest(req);
     }
   }
 
   if (url.pathname === '/') {
-    yield serveTemplatedHtml(req, 'public/index.html');
+    return serveTemplatedHtml(req, 'public/index.html');
   }
 
   if ([
     '/global.css',
     '/icon-deps.png',
     '/interactive-graph.js',
   ].includes(url.pathname)) {
-    yield servePublic(req, url.pathname);
+    return servePublic(req, url.pathname);
   }
 
   if (url.pathname.startsWith('/fonts/') &&
       url.pathname.endsWith('.woff2')) {
-    yield serveFont(req, url.pathname.slice(6));
+    return serveFont(req, url.pathname.slice(6));
   }
 }
diff --git a/tracer.ts b/tracer.ts
@@ -0,0 +1,30 @@
+import {
+  DenoTracerProvider,
+  OTLPTraceFetchExporter,
+  httpTracer,
+  DenoFetchInstrumentation,
+  SubProcessInstrumentation,
+  Resource,
+  asyncGeneratorWithContext,
+} from "https://raw.githubusercontent.com/cloudydeno/deno-observability/7a96cf859631e81df821ef8c3352b92d7f909739/tracing/mod.ts";
+import { GcpBatchSpanExporter } from "https://raw.githubusercontent.com/cloudydeno/deno-observability/7a96cf859631e81df821ef8c3352b92d7f909739/tracing/exporters/google-cloud.ts";
+import { GoogleCloudPropagator } from "https://raw.githubusercontent.com/cloudydeno/deno-observability/7a96cf859631e81df821ef8c3352b92d7f909739/tracing/propagators/google-cloud.ts";
+
+export { httpTracer, asyncGeneratorWithContext };
+
+export const provider = new DenoTracerProvider({
+  resource: new Resource({
+    'service.name': 'module-visualizer',
+    'service.version': 'adhoc',
+    'deployment.environment': 'local',
+  }),
+  propagator: new GoogleCloudPropagator(),
+  instrumentations: [
+    new DenoFetchInstrumentation(),
+    new SubProcessInstrumentation(),
+  ],
+  batchSpanProcessors: [
+    new GcpBatchSpanExporter(),
+    // new OTLPTraceFetchExporter(),
+  ],
+});