From 8c5316ca8fc1a6163891f7f2220f83a4e95006cf Mon Sep 17 00:00:00 2001 From: "Grayson, Matthew" Date: Fri, 15 Dec 2023 09:58:20 -0600 Subject: [PATCH 1/4] Update censys.ts endpoint and field names to match v2 search api. --- backend/src/tasks/censys.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/src/tasks/censys.ts b/backend/src/tasks/censys.ts index bdd1895f6..7338484fa 100644 --- a/backend/src/tasks/censys.ts +++ b/backend/src/tasks/censys.ts @@ -9,7 +9,7 @@ import getRootDomains from './helpers/getRootDomains'; interface CensysAPIResponse { status: string; results: { - 'parsed.names'?: string[]; + names?: string[]; }[]; metadata: { count: number; @@ -27,7 +27,7 @@ const fetchCensysData = async (rootDomain: string, page: number) => { `[censys] fetching certificates for query "${rootDomain}", page ${page}` ); const { data, status } = await axios({ - url: 'https://censys.io/api/v1/search/certificates', + url: 'https://search.censys.io/api/v2/certificates/search', method: 'POST', auth: { username: String(process.env.CENSYS_API_ID), @@ -39,7 +39,7 @@ const fetchCensysData = async (rootDomain: string, page: number) => { data: { query: rootDomain, page: page, - fields: ['parsed.names'] + fields: ['names'] } }); return data as CensysAPIResponse; @@ -64,7 +64,7 @@ export const handler = async (commandOptions: CommandOptions) => { const data = await fetchCensysData(rootDomain, page); pages = data.metadata.pages; for (const result of data.results) { - const names = result['parsed.names']; + const names = result['names']; if (!names) continue; for (const name of names) { if (name.endsWith(rootDomain)) { @@ -109,6 +109,6 @@ export const handler = async (commandOptions: CommandOptions) => { }) ); } - saveDomainsToDb(domains); + await saveDomainsToDb(domains); console.log(`[censys] done, saved or updated ${domains.length} domains`); }; From 33f96695373fdec00484af6ac697a0a61cd83e18 Mon Sep 17 00:00:00 2001 From: "Grayson, Matthew" Date: Fri, 15 Dec 2023 10:08:45 -0600 Subject: [PATCH 2/4] Remove unused imports and variables. --- backend/src/tasks/censys.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/src/tasks/censys.ts b/backend/src/tasks/censys.ts index 7338484fa..8d6842f5e 100644 --- a/backend/src/tasks/censys.ts +++ b/backend/src/tasks/censys.ts @@ -1,5 +1,5 @@ import axios from 'axios'; -import { Domain, Organization } from '../models'; +import { Domain } from '../models'; import { plainToClass } from 'class-transformer'; import * as dns from 'dns'; import saveDomainsToDb from './helpers/saveDomainsToDb'; @@ -18,7 +18,7 @@ interface CensysAPIResponse { }; } -const sleep = (milliseconds) => { +const sleep = (milliseconds: number) => { return new Promise((resolve) => setTimeout(resolve, milliseconds)); }; @@ -26,7 +26,7 @@ const fetchCensysData = async (rootDomain: string, page: number) => { console.log( `[censys] fetching certificates for query "${rootDomain}", page ${page}` ); - const { data, status } = await axios({ + const { data } = await axios({ url: 'https://search.censys.io/api/v2/certificates/search', method: 'POST', auth: { From 493d1c357b5cee9d8945a65ae50c4b23269a16a5 Mon Sep 17 00:00:00 2001 From: "Grayson, Matthew" Date: Fri, 15 Dec 2023 13:59:50 -0600 Subject: [PATCH 3/4] Remove deprecated page field from fetchCensysData(). --- backend/src/tasks/censys.ts | 42 +++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/backend/src/tasks/censys.ts b/backend/src/tasks/censys.ts index 8d6842f5e..9861a8ce9 100644 --- a/backend/src/tasks/censys.ts +++ b/backend/src/tasks/censys.ts @@ -22,9 +22,9 @@ const sleep = (milliseconds: number) => { return new Promise((resolve) => setTimeout(resolve, milliseconds)); }; -const fetchCensysData = async (rootDomain: string, page: number) => { +const fetchCensysData = async (rootDomain: string) => { console.log( - `[censys] fetching certificates for query "${rootDomain}", page ${page}` + `[censys] fetching certificates for query "${rootDomain}" from Censys...` ); const { data } = await axios({ url: 'https://search.censys.io/api/v2/certificates/search', @@ -37,8 +37,8 @@ const fetchCensysData = async (rootDomain: string, page: number) => { 'Content-Type': 'application/json' }, data: { - query: rootDomain, - page: page, + q: rootDomain, + per_page: 100, fields: ['names'] } }); @@ -59,33 +59,29 @@ export const handler = async (commandOptions: CommandOptions) => { }>(); for (const rootDomain of rootDomains) { - let pages = 1; - for (let page = 1; page <= pages; page++) { - const data = await fetchCensysData(rootDomain, page); - pages = data.metadata.pages; - for (const result of data.results) { - const names = result['names']; - if (!names) continue; - for (const name of names) { - if (name.endsWith(rootDomain)) { - foundDomains.add({ - name: name.replace('*.', ''), - organization: { id: organizationId! }, - fromRootDomain: rootDomain, - discoveredBy: { id: scanId } - }); - } + const data = await fetchCensysData(rootDomain); + for (const result of data.results) { + const names = result['names']; + if (!names) continue; + for (const name of names) { + if (name.endsWith(rootDomain)) { + foundDomains.add({ + name: name.replace('*.', ''), + organization: { id: organizationId! }, + fromRootDomain: rootDomain, + discoveredBy: { id: scanId } + }); } } - - await sleep(1000); // Wait for rate limit } + + await sleep(1000); // Wait for rate limit } // LATER: Can we just grab the cert the site is presenting, and store that? // Censys (probably doesn't know who's presenting it) // SSLyze (fetches the cert), Project Sonar (has SSL certs, but not sure how pulls domains -- from IPs) - // Project Sonar has forward & reverse DNS for finding subdomains + // Project Sonar has both forward & reverse DNS for finding subdomains // Save domains to database console.log('[censys] saving domains to database...'); From 5d71e1c962401879a0c18a0d786ce5d0ac2d89f6 Mon Sep 17 00:00:00 2001 From: "Grayson, Matthew" Date: Mon, 18 Dec 2023 13:22:28 -0600 Subject: [PATCH 4/4] Refactor CensysResponse interface to match v2 API; add deduping of domain names which reduces dns lookups by 95% in testing. --- backend/src/tasks/censys.ts | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/backend/src/tasks/censys.ts b/backend/src/tasks/censys.ts index 9861a8ce9..3817187c8 100644 --- a/backend/src/tasks/censys.ts +++ b/backend/src/tasks/censys.ts @@ -6,16 +6,20 @@ import saveDomainsToDb from './helpers/saveDomainsToDb'; import { CommandOptions } from './ecs-client'; import getRootDomains from './helpers/getRootDomains'; +// TODO: code only returns first 100 results; add cursor to API call to return all results interface CensysAPIResponse { status: string; - results: { - names?: string[]; - }[]; - metadata: { - count: number; - page: number; - pages: number; + result: { + total: number; + hits: [ + { + names?: string[]; + } + ]; }; + // links: { + // next: string; + // }; } const sleep = (milliseconds: number) => { @@ -51,6 +55,7 @@ export const handler = async (commandOptions: CommandOptions) => { console.log('Running censys on organization', organizationName); const rootDomains = await getRootDomains(organizationId!); + const uniqueNames = new Set(); //used to dedupe domain names const foundDomains = new Set<{ name: string; organization: { id: string }; @@ -60,11 +65,12 @@ export const handler = async (commandOptions: CommandOptions) => { for (const rootDomain of rootDomains) { const data = await fetchCensysData(rootDomain); - for (const result of data.results) { - const names = result['names']; + for (const hit of data.result.hits) { + const names = hit['names']; if (!names) continue; for (const name of names) { - if (name.endsWith(rootDomain)) { + if (!uniqueNames.has(name) && name.endsWith(rootDomain)) { + uniqueNames.add(name); foundDomains.add({ name: name.replace('*.', ''), organization: { id: organizationId! },