New INCLUDED_DEVICETYPE_SLUGS option to limit device types/contracts

Resolves: balena-io#1433
Change-type: minor

config/confd/conf.d/env.toml

@@ -29,6 +29,7 @@ keys = [
 	"IMAGE_STORAGE_FORCE_PATH_STYLE",
 	"IMAGE_STORAGE_PREFIX",
 	"IMAGE_STORAGE_SECRET_KEY",
+	"INCLUDED_DEVICETYPE_SLUGS",
 	"JSON_WEB_TOKEN_EXPIRY_MINUTES",
 	"JSON_WEB_TOKEN_SECRET",
 	"MIXPANEL_TOKEN",

config/confd/templates/env.tmpl

@@ -8,6 +8,7 @@ COOKIE_SESSION_SECRET={{getenv "COOKIE_SESSION_SECRET"}}
 {{if getenv "CONTRACTS_PRIVATE_REPO_NAME"}}CONTRACTS_PRIVATE_REPO_NAME={{getenv "CONTRACTS_PRIVATE_REPO_NAME"}}{{end}}
 {{if getenv "CONTRACTS_PRIVATE_REPO_BRANCH"}}CONTRACTS_PRIVATE_REPO_BRANCH"CONTRACTS_PRIVATE_REPO_BRANCH"}}{{end}}
 {{if getenv "CONTRACTS_PRIVATE_REPO_TOKEN"}}CONTRACTS_PRIVATE_REPO_TOKEN={{getenv "CONTRACTS_PRIVATE_REPO_TOKEN"}}{{end}}
+{{if getenv "INCLUDED_DEVICETYPE_SLUGS"}}INCLUDED_DEVICETYPE_SLUGS={{getenv "INCLUDED_DEVICETYPE_SLUGS"}}{{end}}
 DATABASE_URL=postgres://{{getenv "DB_USER"}}:{{getenv "DB_PASSWORD"}}@{{getenv "DB_HOST"}}:{{getenv "DB_PORT"}}/{{getenv "DB_NAME" "resin"}}
 DB_HOST={{getenv "DB_HOST"}}
 DB_PASSWORD={{getenv "DB_PASSWORD"}}

src/features/contracts/contracts-directory.ts

@@ -11,6 +11,7 @@ import validator from 'validator';
 import type { RepositoryInfo, Contract } from './index';
 import { getBase64DataUri } from '../../lib/utils';
 import { captureException } from '../../infra/error-handling';
+import { INCLUDED_DEVICETYPE_SLUGS } from '../../lib/config';
 
 const pipeline = util.promisify(stream.pipeline);
 const exists = util.promisify(fs.exists);
@@ -177,13 +178,34 @@ export const getContracts = async (type: string): Promise<Contract[]> => {
 		return [];
 	}
 
-	const contractFiles = await glob(
+	let contractFiles = await glob(
 		`${CONTRACTS_BASE_DIR}/**/contracts/${type}/**/*.json`,
 	);
 	if (!contractFiles.length) {
 		return [];
 	}
 
+	// If there are explicit includes, then everything else is excluded so we need to
+	// filter the contractFiles list to include only contracts that are in the INCLUDED_DEVICETYPE_SLUGS map
+	if (INCLUDED_DEVICETYPE_SLUGS.size > 0) {
+		const slugRegex = new RegExp(`/contracts/${_.escapeRegExp(type)}/([^/]+)/`);
+		const before = contractFiles.length;
+		contractFiles = contractFiles.filter((file) => {
+			// Get the contract slug from the file path
+			const deviceTypeSlug = file.match(slugRegex)?.[1];
+			if (!deviceTypeSlug) {
+				return false;
+			}
+
+			// Check if this slug is included in the map
+			return INCLUDED_DEVICETYPE_SLUGS.has(deviceTypeSlug);
+		});
+
+		console.log(
+			`INCLUDED_DEVICETYPE_SLUGS reduced ${type} contract slugs from ${before} to ${contractFiles.length}`,
+		);
+	}
+
 	const contracts = await Promise.all(
 		contractFiles.map(async (file) => {
 			let contract;

src/features/device-types/device-types-list.ts

@@ -14,6 +14,7 @@ import { multiCacheMemoizee } from '../../infra/cache';
 import {
 	DEVICE_TYPES_CACHE_LOCAL_TIMEOUT,
 	DEVICE_TYPES_CACHE_TIMEOUT,
+	INCLUDED_DEVICETYPE_SLUGS,
 } from '../../lib/config';
 
 export interface DeviceTypeInfo {
@@ -58,7 +59,18 @@ const getFirstValidBuild = async (
 export const getDeviceTypes = multiCacheMemoizee(
 	async (): Promise<Dictionary<DeviceTypeInfo>> => {
 		const result: Dictionary<DeviceTypeInfo> = {};
-		const slugs = await listFolders(IMAGE_STORAGE_PREFIX);
+		let slugs = await listFolders(IMAGE_STORAGE_PREFIX);
+
+		// If there are explicit includes, then everything else is excluded so we need to
+		// filter the slugs list to include only contracts that are in the INCLUDED_DEVICETYPE_SLUGS map
+		if (INCLUDED_DEVICETYPE_SLUGS.size > 0) {
+			const before = slugs.length;
+			slugs = slugs.filter((slug) => INCLUDED_DEVICETYPE_SLUGS.has(slug));
+			console.log(
+				`INCLUDED_DEVICETYPE_SLUGS reduced device type slugs from ${before} to ${slugs.length}`,
+			);
+		}
+
 		await Promise.all(
 			slugs.map(async (slug) => {
 				try {

src/lib/config.ts

@@ -82,6 +82,24 @@ export const AUTH_RESINOS_REGISTRY_CODE = optionalVar(
 	'AUTH_RESINOS_REGISTRY_CODE',
 );
 export const COOKIE_SESSION_SECRET = requiredVar('COOKIE_SESSION_SECRET');
+
+/**
+ * null: include all device type and device contract slugs
+ * "x;y;z": include only the specified device type and contract slugs - note that you MUST list
+ *          all dependent slugs as well so for hw.device-type/asus-tinker-board-s you would need
+ *          `armv7hf;asus;tinkerboard;asus-tinker-board-s` to pick up arch.sw/armv7hf,
+ *          hw.device-manufacturer/asus, hw.device-family/tinkerboard, and hw.device-type/asus-tinker-board-s
+ * 		    For something like hw.device-type/iot-gate-imx8 you would need `aarch64;iot-gate-imx8`
+ * 			to pick up arch.sw/aarch64 and hw.device-type/iot-gate-imx8
+ *          (the order of the slugs in this variable does not matter)
+ */
+export const INCLUDED_DEVICETYPE_SLUGS = new Set(
+	optionalVar('INCLUDED_DEVICETYPE_SLUGS', '')
+		.split(';')
+		.map((slug) => slug.trim())
+		.filter((slug) => slug.length > 0),
+);
+
 export const CONTRACTS_PUBLIC_REPO_OWNER = optionalVar(
 	'CONTRACTS_PUBLIC_REPO_OWNER',
 	'balena-io',