Merge pull request #633 from Arthi-chaud/server/opti-metadata-fetching

Server: Optimize metadata fetching

front/src/models/external-id.ts

@@ -32,10 +32,6 @@ const ExternalId = yup.object({
 			 * URL to the Homepage of the provider
 			 */
 			homepage: yup.string().required(),
-			/**
-			 * API-relative route to the provider's banner
-			 */
-			banner: yup.string().required(),
 			/**
 			 * API-relative route to the provider's icon
 			 */

server/prisma/migrations/20240207162850_artist_image_url/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "artist_external_ids" ADD COLUMN     "illustration" TEXT;

server/prisma/schema.prisma

@@ -453,6 +453,8 @@ model ArtistExternalId {
 	artistId		Int
 	/// @description Short text about the related resource
 	description		String?
+	/// @description URL to an image of the artist
+	illustration	String?
 	/// @description The Provider
 	provider		Provider @relation(fields: [providerId], references: [id], onDelete: Cascade)
 	/// @description Unique numeric identifier of the provider

server/src/album/album.controller.spec.ts

@@ -354,7 +354,6 @@ describe("Album Controller", () => {
 									homepage: providerService
 										.getProviderById(provider.id)
 										.getProviderHomepage(),
-									banner: `/illustrations/providers/${provider.name}/banner`,
 									icon: `/illustrations/providers/${provider.name}/icon`,
 								},
 								description: "Album blah blah blah",

server/src/artist/artist.controller.spec.ts

@@ -191,7 +191,6 @@ describe("Artist Controller", () => {
 									homepage: providerService
 										.getProviderById(provider.id)
 										.getProviderHomepage(),
-									banner: `/illustrations/providers/${provider.name}/banner`,
 									icon: `/illustrations/providers/${provider.name}/icon`,
 								},
 								description: "Artist Desc.",

server/src/illustration/illustration.controller.ts

@@ -22,18 +22,12 @@ import {
 	Delete,
 	Get,
 	Header,
-	HttpStatus,
 	Param,
 	Post,
 	Query,
 	Response,
 } from "@nestjs/common";
-import {
-	ApiOperation,
-	ApiParam,
-	ApiPropertyOptional,
-	ApiTags,
-} from "@nestjs/swagger";
+import { ApiOperation, ApiPropertyOptional, ApiTags } from "@nestjs/swagger";
 import ArtistService from "src/artist/artist.service";
 import ReleaseService from "src/release/release.service";
 import TrackService from "src/track/track.service";
@@ -53,7 +47,6 @@ import ProviderIllustrationService from "src/providers/provider-illustration.ser
 import ProviderService from "src/providers/provider.service";
 import ProvidersSettings from "src/providers/models/providers.settings";
 import { UnknownProviderError } from "src/providers/provider.exception";
-import { MeeloException } from "src/exceptions/meelo-exception";
 import PlaylistService from "src/playlist/playlist.service";
 import PlaylistQueryParameters from "src/playlist/models/playlist.query-parameters";
 import IllustrationRepository from "./illustration.repository";
@@ -354,17 +347,12 @@ export class IllustrationController {
 	}
 
 	@ApiOperation({
-		summary: "Get a Provider's icon or banner",
-	})
-	@ApiParam({
-		name: "type",
-		enum: ["icon", "banner"],
+		summary: "Get a Provider's icon",
 	})
 	@Cached()
-	@Get("providers/:name/:type")
+	@Get("providers/:name/icon")
 	async getProviderIillustration(
 		@Param("name") providerName: string,
-		@Param("type") type: string,
 		@Query() dimensions: IllustrationDimensionsDto,
 		@Response({ passthrough: true }) res: Response,
 	) {
@@ -377,25 +365,8 @@ export class IllustrationController {
 		if (!pNameIsValid(providerName)) {
 			throw new UnknownProviderError(providerName);
 		}
-		switch (type) {
-			case "icon":
-				illustrationPath =
-					this.providerIllustrationService.buildIconPath(
-						providerName,
-					);
-				break;
-			case "banner":
-				illustrationPath =
-					this.providerIllustrationService.buildBannerPath(
-						providerName,
-					);
-				break;
-			default:
-				throw new MeeloException(
-					HttpStatus.BAD_REQUEST,
-					"Invalid Provider Illustration type",
-				);
-		}
+		illustrationPath =
+			this.providerIllustrationService.buildIconPath(providerName);
 		return this.illustrationService.streamIllustration(
 			illustrationPath,
 			`${providerName}-${parse(illustrationPath).name}`,

server/src/illustration/illustration.module.ts

@@ -32,10 +32,12 @@ import PlaylistModule from "src/playlist/playlist.module";
 import IllustrationRepository from "./illustration.repository";
 import PrismaModule from "src/prisma/prisma.module";
 import ScannerModule from "src/scanner/scanner.module";
+import { HttpModule } from "@nestjs/axios";
 
 @Module({
 	imports: [
 		PrismaModule,
+		HttpModule,
 		FileManagerModule,
 		forwardRef(() => ArtistModule),
 		forwardRef(() => AlbumModule),

server/src/illustration/illustration.repository.ts

@@ -790,41 +790,36 @@ export default class IllustrationRepository {
 	async downloadMissingArtistIllustrations() {
 		const artistsWithoutIllustrations =
 			await this.prismaService.artist.findMany({
-				where: { illustration: null },
-				include: { externalIds: true },
+				where: {
+					illustration: null,
+					externalIds: { some: { illustration: { not: null } } },
+				},
+				include: {
+					externalIds: { where: { illustration: { not: null } } },
+				},
 			});
 
 		await Promise.allSettled(
-			artistsWithoutIllustrations.map((artist) => {
-				return this.providerService.runAction(async (provider) => {
-					// We select the external id of the artist from the current provider
-					const externalIdProvider = artist.externalIds.find(
-						(id) =>
-							this.providerService.getProviderById(id.providerId)
-								.name == provider.name,
-					);
-
-					if (!externalIdProvider) {
+			artistsWithoutIllustrations.map(async (artist) => {
+				for (const url of artist.externalIds
+					.map(({ illustration }) => illustration)
+					.filter((illustration) => illustration !== null)) {
+					try {
+						const buffer =
+							await this.illustrationService.downloadIllustration(
+								url!,
+							);
+						await this.createArtistIllustration(buffer, {
+							id: artist.id,
+						});
+						this.logger.verbose(
+							`Illustration found for artist '${artist.name}'`,
+						);
 						return;
+					} catch {
+						continue;
 					}
-					const illustrationUrl =
-						await provider.getArtistIllustrationUrl(
-							externalIdProvider.value,
-						);
-
-					return this.illustrationService
-						.downloadIllustration(illustrationUrl)
-						.then((buffer) =>
-							this.createArtistIllustration(buffer, {
-								id: artist.id,
-							}),
-						)
-						.then(() =>
-							this.logger.verbose(
-								`Illustration found for artist '${artist.name}'`,
-							),
-						);
-				});
+				}
 			}),
 		);
 	}

server/src/illustration/illustration.service.ts

@@ -39,6 +39,8 @@ import mime from "mime";
 import { InvalidRequestException } from "src/exceptions/meelo-exception";
 import { ImageQuality } from "./models/illustration-quality";
 import md5 from "md5";
+import { HttpService } from "@nestjs/axios";
+import { version } from "package.json";
 
 type IllustrationExtractStatus =
 	| "extracted"
@@ -54,7 +56,10 @@ type IllustrationExtractStatus =
 export default class IllustrationService {
 	private readonly logger = new Logger(IllustrationService.name);
 
-	constructor(private fileManagerService: FileManagerService) {}
+	constructor(
+		private fileManagerService: FileManagerService,
+		private readonly httpService: HttpService,
+	) {}
 
 	/**
 	 * Downloads an illustration from a URL, and stores it in the illustration file system
@@ -63,16 +68,21 @@ export default class IllustrationService {
 	 * @param outPath path to the file to save the illustration as
 	 */
 	async downloadIllustration(illustrationURL: string) {
-		for (let iteration = 0; iteration < 100; iteration++) {
-			try {
-				const image = await Jimp.read(illustrationURL);
-
-				return image.getBufferAsync(image.getMIME());
-			} catch {
-				throw new CantDownloadIllustrationException(illustrationURL);
-			}
+		try {
+			return Buffer.from(
+				(
+					await this.httpService.axiosRef.get(illustrationURL, {
+						responseType: "arraybuffer",
+						headers: {
+							"User-Agent": "Meelo v" + version,
+						},
+					})
+				).data,
+				"binary",
+			);
+		} catch {
+			throw new CantDownloadIllustrationException(illustrationURL);
 		}
-		throw new CantDownloadIllustrationException(illustrationURL);
 	}
 
 	/**

server/src/providers/allmusic/allmusic.provider.ts

@@ -37,10 +37,6 @@ export default class AllMusicProvider
 		super("allMusic");
 	}
 
-	getProviderBannerUrl(): string {
-		return "https://d39w11zmd7f11d.cloudfront.net/wp-content/uploads/2020/09/AllMusic-inline.jpg";
-	}
-
 	getProviderIconUrl(): string {
 		return "https://upload.wikimedia.org/wikipedia/commons/thumb/a/a0/AllMusic_Logo.svg/187px-AllMusic_Logo.svg.png";
 	}
@@ -57,6 +53,17 @@ export default class AllMusicProvider
 		return "P1729";
 	}
 
+	getMusicBrainzRelationKey(): string | null {
+		return "allmusic";
+	}
+
+	parseAlbumIdentifierFromUrl(url: string): string | null {
+		return (
+			url.match(/(https:\/\/www\.)?allmusic\.com\/album\/(?<ID>mw\d+)/i)
+				?.groups?.["ID"] ?? null
+		);
+	}
+
 	getAlbumURL(albumIdentifier: string): string {
 		return `${this.getProviderHomepage()}/album/${albumIdentifier}`;
 	}

server/src/providers/discogs/discogs.provider.ts

@@ -42,17 +42,13 @@ export default class DiscogsProvider
 	}
 
 	onModuleInit() {
-		this._settings = this.settingsService.settingsValues.providers.discogs;
+		this.settings = this.settingsService.settingsValues.providers.discogs;
 	}
 
 	getProviderHomepage(): string {
 		return "https://www.discogs.com";
 	}
 
-	getProviderBannerUrl(): string {
-		return "https://st.discogs.com/7790e868083f99e9f3293cb4a33581347374b4c6/images/discogs-primary-logo.png";
-	}
-
 	getProviderIconUrl(): string {
 		return "https://upload.wikimedia.org/wikipedia/commons/thumb/6/69/Discogs_record_icon.svg/240px-Discogs_record_icon.svg.png";
 	}
@@ -78,6 +74,7 @@ export default class DiscogsProvider
 			if (isNumber(artist.id) && isString(artist.profile_plaintext)) {
 				return {
 					value: artist.id.toString(),
+					illustration: artist.images.at(0)?.uri ?? null,
 					description: artist.profile_plaintext,
 				};
 			}
@@ -86,7 +83,7 @@ export default class DiscogsProvider
 				"getArtistMetadataByIdentifier",
 				"Invalid Data Type",
 			);
-		} catch {
+		} catch (e) {
 			throw new ProviderActionFailedError(
 				this.name,
 				"getArtistMetadataByIdentifier",
@@ -149,6 +146,32 @@ export default class DiscogsProvider
 		}
 	}
 
+	getArtistWikidataIdentifierProperty() {
+		return "P1953";
+	}
+
+	getAlbumWikidataIdentifierProperty() {
+		return "P1954";
+	}
+
+	getMusicBrainzRelationKey(): string | null {
+		return "discogs";
+	}
+
+	parseArtistIdentifierFromUrl(url: string): string | null {
+		return (
+			url.match(/(https:\/\/www\.)?discogs\.com\/artist\/(?<ID>\d+)/)
+				?.groups?.["ID"] ?? null
+		);
+	}
+
+	parseAlbumIdentifierFromUrl(url: string): string | null {
+		return (
+			url.match(/(https:\/\/www\.)?discogs\.com\/master\/(?<ID>\d+)/)
+				?.groups?.["ID"] ?? null
+		);
+	}
+
 	async fetch(route: string): Promise<any> {
 		const accessToken =
 			process.env.NODE_ENV == "test"