From 1a21862e02a030e9ca83c238a464af3913f78dab Mon Sep 17 00:00:00 2001 From: Shaun Cooley Date: Tue, 19 Sep 2023 13:49:57 -0700 Subject: [PATCH] New INCLUDED_DEVICETYPE_SLUGS option to limit device types/contracts Resolves: #1433 Change-type: minor --- config/confd/conf.d/env.toml | 1 + config/confd/templates/env.tmpl | 1 + src/features/contracts/contracts-directory.ts | 24 ++++++++++++++++++- .../device-types/device-types-list.ts | 14 ++++++++++- src/lib/config.ts | 18 ++++++++++++++ 5 files changed, 56 insertions(+), 2 deletions(-) diff --git a/config/confd/conf.d/env.toml b/config/confd/conf.d/env.toml index e7d28b0fc6..28e0b1b686 100644 --- a/config/confd/conf.d/env.toml +++ b/config/confd/conf.d/env.toml @@ -29,6 +29,7 @@ keys = [ "IMAGE_STORAGE_FORCE_PATH_STYLE", "IMAGE_STORAGE_PREFIX", "IMAGE_STORAGE_SECRET_KEY", + "INCLUDED_DEVICETYPE_SLUGS", "JSON_WEB_TOKEN_EXPIRY_MINUTES", "JSON_WEB_TOKEN_SECRET", "MIXPANEL_TOKEN", diff --git a/config/confd/templates/env.tmpl b/config/confd/templates/env.tmpl index f6284dba7f..77fe2f5315 100644 --- a/config/confd/templates/env.tmpl +++ b/config/confd/templates/env.tmpl @@ -8,6 +8,7 @@ COOKIE_SESSION_SECRET={{getenv "COOKIE_SESSION_SECRET"}} {{if getenv "CONTRACTS_PRIVATE_REPO_NAME"}}CONTRACTS_PRIVATE_REPO_NAME={{getenv "CONTRACTS_PRIVATE_REPO_NAME"}}{{end}} {{if getenv "CONTRACTS_PRIVATE_REPO_BRANCH"}}CONTRACTS_PRIVATE_REPO_BRANCH"CONTRACTS_PRIVATE_REPO_BRANCH"}}{{end}} {{if getenv "CONTRACTS_PRIVATE_REPO_TOKEN"}}CONTRACTS_PRIVATE_REPO_TOKEN={{getenv "CONTRACTS_PRIVATE_REPO_TOKEN"}}{{end}} +{{if getenv "INCLUDED_DEVICETYPE_SLUGS"}}INCLUDED_DEVICETYPE_SLUGS={{getenv "INCLUDED_DEVICETYPE_SLUGS"}}{{end}} DATABASE_URL=postgres://{{getenv "DB_USER"}}:{{getenv "DB_PASSWORD"}}@{{getenv "DB_HOST"}}:{{getenv "DB_PORT"}}/{{getenv "DB_NAME" "resin"}} DB_HOST={{getenv "DB_HOST"}} DB_PASSWORD={{getenv "DB_PASSWORD"}} diff --git a/src/features/contracts/contracts-directory.ts b/src/features/contracts/contracts-directory.ts index 02c8fa8e12..8628b81fef 100644 --- a/src/features/contracts/contracts-directory.ts +++ b/src/features/contracts/contracts-directory.ts @@ -11,6 +11,7 @@ import validator from 'validator'; import type { RepositoryInfo, Contract } from './index'; import { getBase64DataUri } from '../../lib/utils'; import { captureException } from '../../infra/error-handling'; +import { INCLUDED_DEVICETYPE_SLUGS } from '../../lib/config'; const pipeline = util.promisify(stream.pipeline); const exists = util.promisify(fs.exists); @@ -177,13 +178,34 @@ export const getContracts = async (type: string): Promise => { return []; } - const contractFiles = await glob( + let contractFiles = await glob( `${CONTRACTS_BASE_DIR}/**/contracts/${type}/**/*.json`, ); if (!contractFiles.length) { return []; } + // If there are explicit includes, then everything else is excluded so we need to + // filter the contractFiles list to include only contracts that are in the INCLUDED_DEVICETYPE_SLUGS map + if (INCLUDED_DEVICETYPE_SLUGS.size > 0) { + const slugRegex = new RegExp(`/contracts/${type}/([^/]*)/`); + const before = contractFiles.length; + contractFiles = contractFiles.filter((file) => { + // Get the contract slug from the file path + const match = file.match(slugRegex); + if (!match || match.length < 2) { + return false; + } + + // Check if this slug is included in the map + return INCLUDED_DEVICETYPE_SLUGS.has(match[1]); + }); + + console.log( + `INCLUDED_DEVICETYPE_SLUGS reduced ${type} contract slugs from ${before} to ${contractFiles.length}`, + ); + } + const contracts = await Promise.all( contractFiles.map(async (file) => { let contract; diff --git a/src/features/device-types/device-types-list.ts b/src/features/device-types/device-types-list.ts index 9197c53372..b8c962a415 100644 --- a/src/features/device-types/device-types-list.ts +++ b/src/features/device-types/device-types-list.ts @@ -14,6 +14,7 @@ import { multiCacheMemoizee } from '../../infra/cache'; import { DEVICE_TYPES_CACHE_LOCAL_TIMEOUT, DEVICE_TYPES_CACHE_TIMEOUT, + INCLUDED_DEVICETYPE_SLUGS, } from '../../lib/config'; export interface DeviceTypeInfo { @@ -58,7 +59,18 @@ const getFirstValidBuild = async ( export const getDeviceTypes = multiCacheMemoizee( async (): Promise> => { const result: Dictionary = {}; - const slugs = await listFolders(IMAGE_STORAGE_PREFIX); + let slugs = await listFolders(IMAGE_STORAGE_PREFIX); + + // If there are explicit includes, then everything else is excluded so we need to + // filter the slugs list to include only contracts that are in the INCLUDED_DEVICETYPE_SLUGS map + if (INCLUDED_DEVICETYPE_SLUGS.size > 0) { + const before = slugs.length; + slugs = slugs.filter((slug) => INCLUDED_DEVICETYPE_SLUGS.has(slug)); + console.log( + `INCLUDED_DEVICETYPE_SLUGS reduced device type slugs from ${before} to ${slugs.length}`, + ); + } + await Promise.all( slugs.map(async (slug) => { try { diff --git a/src/lib/config.ts b/src/lib/config.ts index 972fd48f39..8931283f2b 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -82,6 +82,24 @@ export const AUTH_RESINOS_REGISTRY_CODE = optionalVar( 'AUTH_RESINOS_REGISTRY_CODE', ); export const COOKIE_SESSION_SECRET = requiredVar('COOKIE_SESSION_SECRET'); + +/** + * null: include all device type and device contract slugs + * "x;y;z": include only the specified device type and contract slugs - note that you MUST list + * all dependent slugs as well so for hw.device-type/asus-tinker-board-s you would need + * `armv7hf;asus;tinkerboard;asus-tinker-board-s` to pick up arch.sw/armv7hf, + * hw.device-manufacturer/asus, hw.device-family/tinkerboard, and hw.device-type/asus-tinker-board-s + * For something like hw.device-type/iot-gate-imx8 you would need `aarch64;iot-gate-imx8` + * to pick up arch.sw/aarch64 and hw.device-type/iot-gate-imx8 + * (the order of the slugs in this variable does not matter) + */ +export const INCLUDED_DEVICETYPE_SLUGS = new Map( + (optionalVar('INCLUDED_DEVICETYPE_SLUGS') ?? '') + .split(';') + .filter((slug) => slug.trim().length > 0) + .map((slug) => [slug.trim(), true]), +); + export const CONTRACTS_PUBLIC_REPO_OWNER = optionalVar( 'CONTRACTS_PUBLIC_REPO_OWNER', 'balena-io',