From a9c65412f838ca94f504b314a67abacf0a6bf6a4 Mon Sep 17 00:00:00 2001 From: Olaf Rosendahl Date: Fri, 17 Nov 2023 12:47:33 +0100 Subject: [PATCH] fix: pageview next on edge --- ...l.$teamSlug.$projectSlug.pageview-next.tsx | 108 ++++++++++++++++++ ...ternal.$teamSlug.$projectSlug.pageview.tsx | 86 +------------- .../api.$teamSlug.$projectSlug.pageview.tsx | 52 ++++++++- 3 files changed, 161 insertions(+), 85 deletions(-) create mode 100644 web/app/routes/api-internal.$teamSlug.$projectSlug.pageview-next.tsx diff --git a/web/app/routes/api-internal.$teamSlug.$projectSlug.pageview-next.tsx b/web/app/routes/api-internal.$teamSlug.$projectSlug.pageview-next.tsx new file mode 100644 index 0000000..857b3a7 --- /dev/null +++ b/web/app/routes/api-internal.$teamSlug.$projectSlug.pageview-next.tsx @@ -0,0 +1,108 @@ +import type { Location, Prisma, Project } from '@prisma/client'; +import type { Geo } from '@vercel/edge'; +import type { ActionFunctionArgs } from '@vercel/remix'; +import { json } from '@vercel/remix'; +import { prismaClient } from '~/prismaClient'; +import type { UserAgentData } from '~/user-agent'; +import { getProjectAndCheckPermissions } from '~/utils_api.server'; +import crypto from 'crypto'; +import { format } from 'date-fns'; +import invariant from 'tiny-invariant'; + +import type { PageviewInput } from '~/types'; + +export type PageviewRequestData = { + data: PageviewInput; + userAgentData: ReturnType; + user_hash: Awaited>; + date: string; + geo: Required>; +}; + +const parsePageviewRequestData = async (request: Request): Promise => { + const data = (await request.json()) as PageviewRequestData; + if (!data.data?.pathname) { + throw JSON.stringify({ pathname: `Pathname isn't defined` }); + } + if (!data.date) { + throw JSON.stringify({ pathname: `Date isn't defined` }); + } + if (!data.geo) { + throw JSON.stringify({ pathname: `geo isn't defined` }); + } + if (!data.userAgentData) { + throw JSON.stringify({ pathname: `userAgentData isn't defined` }); + } + if (!data.user_hash) { + throw JSON.stringify({ pathname: `user_hash isn't defined` }); + } + + return data; +}; + +export const getPageViewUserIdHash = async (ip: string, userAgent: string, date: Date) => { + invariant(process.env.SECRET_KEY, 'Expected environment variable "SECRET_KEY" to be set when tracking page visitors'); + const day = format(date, 'yyyy-MM-dd'); + const msgUint8 = new TextEncoder().encode(`${ip}_${userAgent}_${day}_${process.env.SECRET_KEY}`); // encode as (utf-8) Uint8Array + const hashBuffer = await crypto.subtle.digest('SHA-256', msgUint8); // hash the message + const hashArray = Array.from(new Uint8Array(hashBuffer)); // convert buffer to byte array + const hashHex = hashArray.map((b) => b.toString(16).padStart(2, '0')).join(''); // convert bytes to hex string + return hashHex; +}; + +export const getPageViewUserAgentData = (ua: UserAgentData): Pick => { + const browser = ua.browser.name || null; + const device = [ua.device.vendor, ua.device.model].filter(Boolean).join(' ') || null; + const os = ua.os.name || null; + return { browser, device, os }; +}; + +const getLocation = async (geo: PageviewRequestData['geo']): Promise => { + const flag_country_city: Prisma.LocationFlagCountryCityCompoundUniqueInput = { flag: geo.flag, country: geo.country, city: geo.city }; + const location: Prisma.LocationCreateWithoutPageViewsInput = { + ...flag_country_city, + latitude: Number(geo.latitude), + longitude: Number(geo.longitude), + }; + return await prismaClient.location.upsert({ + create: location, + update: {}, + where: { flag_country_city }, + }); +}; + +const trackPageviewNext = async (request: Request, project: Project) => { + const { data, date, geo, userAgentData, user_hash } = await parsePageviewRequestData(request); + + const location = await getLocation(geo); + + await prismaClient.pageViewNext.create({ + data: { + date: date, + pathname: data.pathname, + referrer: data.referrer, + user_hash, + browser: userAgentData.browser, + device: userAgentData.device, + os: userAgentData.os, + projectId: project.id, + locationId: location.id, + }, + }); +}; + +export const action = async ({ request, params }: ActionFunctionArgs) => { + try { + invariant(params.teamSlug, `Expected params.teamSlug`); + invariant(params.projectSlug, `Expected params.projectSlug`); + + const { project } = await getProjectAndCheckPermissions(request, params.teamSlug, params.projectSlug); + + await trackPageviewNext(request, project); + + return json({ ok: true }, { status: 202 }); + } catch (e) { + console.error('[API-Internal - Pageview]', e); + return json({ ok: false }, { status: 400 }); + } +}; diff --git a/web/app/routes/api-internal.$teamSlug.$projectSlug.pageview.tsx b/web/app/routes/api-internal.$teamSlug.$projectSlug.pageview.tsx index 97e3a66..ddcede1 100644 --- a/web/app/routes/api-internal.$teamSlug.$projectSlug.pageview.tsx +++ b/web/app/routes/api-internal.$teamSlug.$projectSlug.pageview.tsx @@ -1,16 +1,12 @@ -import type { Location, Prisma, Project } from '@prisma/client'; -import type { Geo } from '@vercel/edge'; -import { geolocation, ipAddress } from '@vercel/edge'; +import type { Prisma, Project } from '@prisma/client'; import type { ActionFunctionArgs } from '@vercel/remix'; import { json } from '@vercel/remix'; import { prismaClient } from '~/prismaClient'; -import type { UserAgentData } from '~/user-agent'; -import { userAgent } from '~/user-agent'; import { screenWidthToDeviceType } from '~/utils'; import { getDate, getProjectAndCheckPermissions } from '~/utils_api.server'; import { getPageViewsUsage, getPageVisitorsUsage } from '~/utils_usage.server'; import crypto from 'crypto'; -import { format, getHours } from 'date-fns'; +import { getHours } from 'date-fns'; import { getClientIPAddress } from 'remix-utils/get-client-ip-address'; import invariant from 'tiny-invariant'; @@ -123,78 +119,6 @@ const trackPageVisitor = async (request: Request, project: Project) => { }); }; -const getPageViewUserIdHash = async (ip: string, userAgent: string, date: Date) => { - invariant(process.env.SECRET_KEY, 'Expected environment variable "SECRET_KEY" to be set when tracking page visitors'); - const day = format(date, 'yyyy-MM-dd'); - const msgUint8 = new TextEncoder().encode(`${ip}_${userAgent}_${day}_${process.env.SECRET_KEY}`); // encode as (utf-8) Uint8Array - const hashBuffer = await crypto.subtle.digest('SHA-256', msgUint8); // hash the message - const hashArray = Array.from(new Uint8Array(hashBuffer)); // convert buffer to byte array - const hashHex = hashArray.map((b) => b.toString(16).padStart(2, '0')).join(''); // convert bytes to hex string - return hashHex; -}; - -const getPageViewUserAgentData = (ua: UserAgentData): Pick => { - const browser = ua.browser.name || null; - const device = [ua.device.vendor, ua.device.model].filter(Boolean).join(' ') || null; - const os = ua.os.name || null; - return { browser, device, os }; -}; - -const getLocation = async (geo: Geo): Promise => { - if (!geo.city || !geo.country || !geo.flag || !geo.latitude || !geo.longitude) { - return undefined; - } - const flag_country_city: Prisma.LocationFlagCountryCityCompoundUniqueInput = { flag: geo.flag, country: geo.country, city: geo.city }; - const location: Prisma.LocationCreateWithoutPageViewsInput = { - ...flag_country_city, - latitude: Number(geo.latitude), - longitude: Number(geo.longitude), - }; - return await prismaClient.location.upsert({ - create: location, - update: {}, - where: { flag_country_city }, - }); -}; - -const trackPageviewNext = async (request: Request, project: Project) => { - const geo = geolocation(request); - const ip = ipAddress(request); - const ua = userAgent(request); - - console.info('[API-Internal - PageViewNext]', { geo, ua, ip }); - - const location = await getLocation(geo); - - if (!location || !ip || ua.isBot) { - throw new Error( - JSON.stringify({ - location: location ? undefined : `Location could not be found`, - ip: ip ? undefined : `IP-address could not be found`, - isBot: ua.isBot ? `Bot detected` : undefined, - }), - ); - } - - const date = getDate(request); - const { browser, device, os } = getPageViewUserAgentData(ua); - const [data, user_hash] = await Promise.all([parsePageviewInput(request), getPageViewUserIdHash(ip, ua.ua, date)]); - - await prismaClient.pageViewNext.create({ - data: { - date, - pathname: data.pathname, - referrer: data.referrer, - user_hash, - browser, - device, - os, - projectId: project.id, - locationId: location.id, - }, - }); -}; - export const action = async ({ request, params }: ActionFunctionArgs) => { try { invariant(params.teamSlug, `Expected params.teamSlug`); @@ -202,11 +126,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const { project } = await getProjectAndCheckPermissions(request, params.teamSlug, params.projectSlug); - const req1 = request.clone(); - const req2 = request.clone(); - const req3 = request.clone(); - - await Promise.all([trackPageview(req1, project), trackPageVisitor(req2, project), trackPageviewNext(req3, project)]); + await Promise.all([trackPageview(request, project), trackPageVisitor(request, project)]); return json({ ok: true }, { status: 202 }); } catch (e) { diff --git a/web/app/routes/api.$teamSlug.$projectSlug.pageview.tsx b/web/app/routes/api.$teamSlug.$projectSlug.pageview.tsx index 5654189..3486225 100644 --- a/web/app/routes/api.$teamSlug.$projectSlug.pageview.tsx +++ b/web/app/routes/api.$teamSlug.$projectSlug.pageview.tsx @@ -1,21 +1,69 @@ +import { geolocation, ipAddress } from '@vercel/edge'; import type { RequestContext } from '@vercel/edge'; import type { ActionFunctionArgs } from '@vercel/remix'; import { json } from '@vercel/remix'; +import { userAgent } from '~/user-agent'; +import { getDate } from '~/utils_api.server'; import { forwardRequestToInternalApi } from '~/utils_edge.server'; import invariant from 'tiny-invariant'; +import type { PageviewInput } from '~/types'; + +import { getPageViewUserAgentData, getPageViewUserIdHash, type PageviewRequestData } from './api-internal.$teamSlug.$projectSlug.pageview-next'; + export const config = { runtime: 'edge' }; +const getPageViewNextRequest = async (request: Request): Promise => { + const geo = geolocation(request); + const ip = ipAddress(request); + const ua = userAgent(request); + const date = getDate(request); + + console.info('[API - PageViewNext]', { geo, ua, ip }); + + const geoData = geo.city && geo.country && geo.flag && geo.latitude && geo.longitude ? (geo as PageviewRequestData['geo']) : undefined; + if (!geoData || !ip || ua.isBot) { + throw new Error( + JSON.stringify({ + location: geoData ? undefined : `Location could not be found`, + ip: ip ? undefined : `IP-address could not be found`, + isBot: ua.isBot ? `Bot detected` : undefined, + }), + ); + } + + const userAgentData = getPageViewUserAgentData(ua); + const user_hash = await getPageViewUserIdHash(ip, ua.ua, date); + + return new Request('', { + body: JSON.stringify({ + data: (await request.json()) as PageviewInput, + date: date.toJSON(), + geo: geoData, + user_hash, + userAgentData, + } satisfies PageviewRequestData), + }); +}; + export const action = async ({ request, params, context }: ActionFunctionArgs) => { try { invariant(params.teamSlug, `Expected params.teamSlug`); invariant(params.projectSlug, `Expected params.projectSlug`); const ctx = context as unknown as RequestContext; + const pageViewRequest = request.clone(); + const pageViewNextRequest = await getPageViewNextRequest(request.clone()); + + const promise = Promise.all([ + forwardRequestToInternalApi(pageViewRequest, `${params.teamSlug}/${params.projectSlug}/pageview/`), + ...(pageViewNextRequest ? [forwardRequestToInternalApi(pageViewNextRequest, `${params.teamSlug}/${params.projectSlug}/pageview-next/`)] : []), + ]); + if ('waitUntil' in ctx) { - ctx.waitUntil(forwardRequestToInternalApi(request, `${params.teamSlug}/${params.projectSlug}/pageview/`)); + ctx.waitUntil(promise); } else { - await forwardRequestToInternalApi(request, `${params.teamSlug}/${params.projectSlug}/pageview/`); + await promise; } return json({ ok: true }, { status: 200 }); } catch (e) {