From b00f2aabfc297b308dd26072dd23109d31ef94f7 Mon Sep 17 00:00:00 2001 From: Johan Frostmark Date: Fri, 30 Jul 2021 21:09:00 +0200 Subject: [PATCH] opt-in for thumbnail storage on s3 instead of disk i have missed that cloudflare dont cache the images as long as i originally thought --- .gitignore | 3 + pom.xml | 12 + .../delegate/CowVOResourceDelegateImpl.java | 31 +- .../service/AbstractPictureSourceService.java | 324 ++++++++++++++---- .../bff/service/CowPictureSourceService.java | 88 +++-- .../gateway/config/ApplicationProperties.java | 141 +++++++- .../gateway/config/SecurityConfiguration.java | 9 +- src/main/resources/config/application-dev.yml | 9 +- .../resources/config/application-prod.yml | 9 +- src/main/resources/config/application.yml | 5 + src/test/resources/config/application.yml | 8 +- 11 files changed, 496 insertions(+), 143 deletions(-) diff --git a/.gitignore b/.gitignore index 520491f..e008dec 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,9 @@ /target/classes/static/** /src/test/javascript/coverage/ +# Bon +/src/main/resources/config/application-dev-*.yml + ###################### # Node ###################### diff --git a/pom.xml b/pom.xml index 4b511bd..a6aa593 100644 --- a/pom.xml +++ b/pom.xml @@ -98,6 +98,7 @@ 2.2.2.RELEASE 0.4.11 1.24.1 + 1.12.32 @@ -112,6 +113,13 @@ + + com.amazonaws + aws-java-sdk-bom + ${aws-java-sdk-bom.version} + pom + import + com.graphql-java-kickstart graphql-spring-boot-starter @@ -432,6 +440,10 @@ + + com.amazonaws + aws-java-sdk-s3 + com.graphql-java-kickstart graphql-spring-boot-starter diff --git a/src/main/java/com/bonlimousin/gateway/bff/delegate/CowVOResourceDelegateImpl.java b/src/main/java/com/bonlimousin/gateway/bff/delegate/CowVOResourceDelegateImpl.java index ab8287a..45f3f40 100644 --- a/src/main/java/com/bonlimousin/gateway/bff/delegate/CowVOResourceDelegateImpl.java +++ b/src/main/java/com/bonlimousin/gateway/bff/delegate/CowVOResourceDelegateImpl.java @@ -46,9 +46,8 @@ import org.zalando.problem.Status; import javax.validation.ValidationException; -import java.io.FileInputStream; import java.io.IOException; -import java.nio.file.Path; +import java.io.InputStream; import java.text.MessageFormat; import java.time.Duration; import java.time.OffsetDateTime; @@ -223,21 +222,25 @@ public ResponseEntity> getAllPictureVOsByCow(Long earTagId, Inte ResponseEntity> response = fetchPhotosByEarTagId(earTagId, page, size, sort); List list = new ArrayList<>(); for (PhotoEntity entity : response.getBody()) { - PictureVO vo = PictureVOMapper.INSTANCE.photoEntityToPictureVO(entity); - String baseUrl = getCowImageBaseUrl(earTagId, entity.getId()); - try { - Map map = cowPictureSourceService.createPictureSourceVOs(entity, baseUrl); - vo.setSources(new ArrayList<>(map.values())); - cowPictureSourceService.asyncCreateThumbnailsOnDiskIfNotExists(entity.getId(), map); - } catch (MimeTypeException e) { - log.warn("Failed to extract image extension", e); - } + PictureVO vo = getPictureVO(earTagId, entity); list.add(vo); } long totalCount = BFFUtil.extractTotalCount(response); return BFFUtil.createResponse(list, page, size, sort, totalCount); } + private PictureVO getPictureVO(Long earTagId, PhotoEntity entity) { + PictureVO vo = PictureVOMapper.INSTANCE.photoEntityToPictureVO(entity); + String baseUrl = getCowImageBaseUrl(earTagId, entity.getId()); + try { + Map map = cowPictureSourceService.createPictureSourceVOs(entity, baseUrl); + vo.setSources(new ArrayList<>(map.values())); + } catch (MimeTypeException e) { + log.warn("Failed to extract image extension", e); + } + return vo; + } + private String getCowImageBaseUrl(long earTagId, long pictureId) { return MessageFormat.format("/api/public/cows/{0}/pictures/{1}", String.valueOf(earTagId), String.valueOf(pictureId)); @@ -265,14 +268,14 @@ public ResponseEntity getImageForCow(Long earTagId, Long pictureId, St if (!availableImageNames.contains(name)) { throw new ResponseStatusException(HttpStatus.NOT_FOUND); } - String baseUrl = getCowImageBaseUrl(earTagId, pictureId); - Path imagePath = cowPictureSourceService.getOrCreateImagePath(pictureId, name, baseUrl); + + String baseUrl = getCowImageBaseUrl(earTagId, photoEntity.getId()); CacheControl cacheControl = getImageCacheControl(cattleEntity, photoEntity); return ResponseEntity .ok() .contentType(MediaType.parseMediaType(photoEntity.getImageContentType())) .cacheControl(cacheControl) - .body(new InputStreamResource(new FileInputStream(imagePath.toFile()))); + .body(new InputStreamResource(cowPictureSourceService.createImageInputStream(photoEntity, baseUrl, name))); } catch (IOException | MimeTypeException e) { log.warn("Image with name {} for cow {} and id {} not found", name, earTagId, pictureId, e); throw new ResponseStatusException(HttpStatus.NOT_FOUND); diff --git a/src/main/java/com/bonlimousin/gateway/bff/service/AbstractPictureSourceService.java b/src/main/java/com/bonlimousin/gateway/bff/service/AbstractPictureSourceService.java index 907d73c..a3cbb96 100644 --- a/src/main/java/com/bonlimousin/gateway/bff/service/AbstractPictureSourceService.java +++ b/src/main/java/com/bonlimousin/gateway/bff/service/AbstractPictureSourceService.java @@ -1,103 +1,301 @@ package com.bonlimousin.gateway.bff.service; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.HttpMethod; +import com.amazonaws.SdkClientException; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.TransferManagerBuilder; +import com.amazonaws.services.s3.transfer.Upload; +import com.bonlimousin.gateway.config.ApplicationProperties; import com.bonlimousin.gateway.web.api.model.PictureSourceVO; import net.coobird.thumbnailator.Thumbnails; import org.apache.commons.lang3.StringUtils; import org.apache.tika.mime.MimeTypeException; import org.apache.tika.mime.MimeTypes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.io.ByteArrayInputStream; -import java.io.IOException; +import java.io.*; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.text.MessageFormat; import java.util.*; +import java.util.function.Supplier; public abstract class AbstractPictureSourceService { + private final Logger log = LoggerFactory.getLogger(AbstractPictureSourceService.class); + + protected final ApplicationProperties applicationProperties; private final String imagePrefix; private final String imageDir; + private final String imageBucketName; + private final AmazonS3 s3Client; + private final TransferManager s3tm; - protected AbstractPictureSourceService(String imageDir, String imagePrefix) { - super(); - this.imageDir = imageDir; - this.imagePrefix = imagePrefix; - } - - // Bootstrap max container widths - public enum PictureSize { - ORIGINAL(null), SMALL(540), MEDIUM(720), LARGE(960), XL(1140); + protected AbstractPictureSourceService(String imagePrefix, ApplicationProperties applicationProperties) { + this.imagePrefix = imagePrefix; + this.imageDir = applicationProperties.getThumbnails().getImageBaseDir() + "/" + imagePrefix; + this.applicationProperties = applicationProperties; + if(this.applicationProperties.getThumbnails().getImageStorage() == ApplicationProperties.ImageStorage.AWS) { + this.imageBucketName = applicationProperties.getAws().getImageBucketName(); + this.s3Client = createS3Client(applicationProperties); + this.s3tm = createS3TransferManager(s3Client); + } else { + this.imageBucketName = null; + this.s3Client = null; + this.s3tm = null; + } + } - private final Integer pixelWidth; + // Bootstrap max container widths + public enum PictureSize { + ORIGINAL(null), SMALL(540), MEDIUM(720), LARGE(960), XL(1140); - PictureSize(Integer pixelWidth) { - this.pixelWidth = pixelWidth; - } + private final Integer pixelWidth; - public Integer pixelWidth() { - return pixelWidth; - } - } + PictureSize(Integer pixelWidth) { + this.pixelWidth = pixelWidth; + } - public Map createPictureSourceVOs(T entity, String baseUrl) throws MimeTypeException { - Map map = new HashMap<>(); - for(PictureSize picSize : PictureSize.values()) { - createPictureSourceVO(entity, baseUrl, picSize).ifPresent(ps -> map.put(picSize, ps)); + public Integer pixelWidth() { + return pixelWidth; } - return map; } - public abstract Optional createPictureSourceVO(T entity, String baseUrl, PictureSize pictureSize) throws MimeTypeException; + private AmazonS3 createS3Client(ApplicationProperties applicationProperties) { + String accessKey = applicationProperties.getAws().getAccessKey(); + String secretKey = applicationProperties.getAws().getSecretKey(); + String regionName = applicationProperties.getAws().getRegionName(); + AWSCredentials awsCredentials = new BasicAWSCredentials(accessKey, secretKey); + return AmazonS3ClientBuilder + .standard() + .withRegion(regionName) + .withCredentials(new AWSStaticCredentialsProvider(awsCredentials)) + .build(); + } - public String getImageDir() { - return imageDir; + private TransferManager createS3TransferManager(AmazonS3 s3Client) { + return TransferManagerBuilder.standard() + .withS3Client(s3Client) + .build(); } + public abstract Map createPictureSourceVOs(T entity, String baseUrl) throws MimeTypeException; + + protected abstract Optional createPictureSourceVO(T entity, String baseUrl, PictureSize pictureSize) throws MimeTypeException; + + public abstract InputStream createImageInputStream(T entity, String baseUrl, String name) throws IOException, MimeTypeException; + public String getImageExtension(String contentType) throws MimeTypeException { - String imageExt = MimeTypes.getDefaultMimeTypes().forName(contentType).getExtension(); - if (StringUtils.trimToEmpty(imageExt).isEmpty()) { - throw new MimeTypeException("Unknow file-extension for content-type " + contentType); - } - return imageExt; - } - - public String getImageName(long domainId, long pictureId, PictureSize pictureSize, String imageExt) { - return pictureSize.pixelWidth() != null - ? MessageFormat.format("{0}{1}_{2}_{3}w{4}", imagePrefix, String.valueOf(domainId), - String.valueOf(pictureId), String.valueOf(pictureSize.pixelWidth()), imageExt) - : MessageFormat.format("{0}{1}_{2}{3}", imagePrefix, String.valueOf(domainId), - String.valueOf(pictureId), imageExt); - - } - - public List getImageNames(long domainId, long pictureId, String contentType) throws MimeTypeException { - String imageExt = getImageExtension(contentType); - List list = new ArrayList<>(); - for (PictureSize pictureSize : PictureSize.values()) { - list.add(getImageName(domainId, pictureId, pictureSize, imageExt)); - } - return list; - } - - public Path getImagePath(String imageName) { - return Paths.get(imageDir, imageName); - } - - public Path createThumbnailOndisk(String imageName, byte[] image, PictureSize pictureSize) throws IOException { + String imageExt = MimeTypes.getDefaultMimeTypes().forName(contentType).getExtension(); + if (StringUtils.trimToEmpty(imageExt).isEmpty()) { + throw new MimeTypeException("Unknow file-extension for content-type " + contentType); + } + return imageExt; + } + + public String getImageName(long domainId, long pictureId, PictureSize pictureSize, String imageExt) { + return pictureSize.pixelWidth() != null + ? MessageFormat.format("{0}{1}_{2}_{3}w{4}", this.imagePrefix, String.valueOf(domainId), + String.valueOf(pictureId), String.valueOf(pictureSize.pixelWidth()), imageExt) + : MessageFormat.format("{0}{1}_{2}{3}", this.imagePrefix, String.valueOf(domainId), + String.valueOf(pictureId), imageExt); + } + + public List getImageNames(long domainId, long pictureId, String contentType) throws MimeTypeException { + String imageExt = getImageExtension(contentType); + List list = new ArrayList<>(); + for (PictureSize pictureSize : PictureSize.values()) { + list.add(getImageName(domainId, pictureId, pictureSize, imageExt)); + } + return list; + } + + public byte[] createThumbnail(byte[] image, PictureSize pictureSize) throws IOException { + try (ByteArrayInputStream bais = new ByteArrayInputStream(image); ByteArrayOutputStream os = new ByteArrayOutputStream()) { + if (pictureSize.pixelWidth() != null) { + Thumbnails.of(bais).width(pictureSize.pixelWidth()).toOutputStream(os); + } else { + Thumbnails.of(bais).scale(1).toOutputStream(os); + } + return os.toByteArray(); + } + } + + public Optional generatePresignedUrl(PictureSourceVO ps, Date expiration) { + if(this.applicationProperties.getThumbnails().getImageStorage() == ApplicationProperties.ImageStorage.AWS) { + return this.generatePresignedUrlForS3(ps.getUrl(), expiration); + } else { + return Optional.of(ps.getUrl()); + } + } + + public boolean doesThumbnailExist(String imageDir, String baseUrl, String imageName) { + if(this.applicationProperties.getThumbnails().getImageStorage() == ApplicationProperties.ImageStorage.AWS) { + return doesThumbnailExistOnS3(baseUrl, imageName); + } else { + return doesThumbnailExistOnDisk(imageDir, imageName); + } + } + + public void createThumbnailsIfNotExists(Supplier originalImageBytesSupplier, Map map) { + if(this.applicationProperties.getThumbnails().getImageStorage() == ApplicationProperties.ImageStorage.AWS) { + createThumbnailsOnS3IfNotExists(originalImageBytesSupplier, map); + } else { + createThumbnailsOnDiskIfNotExists(originalImageBytesSupplier, map, this.imageDir); + } + } + + public InputStream createImageInputStream(T entity, Supplier originalImageBytesSupplier, String baseUrl, String name) throws IOException, MimeTypeException { + if(this.applicationProperties.getThumbnails().getImageStorage() == ApplicationProperties.ImageStorage.AWS) { + return this.createImageInputStreamFromS3(entity, originalImageBytesSupplier, baseUrl, name); + } else { + return this.createImageInputStreamFromDiskPath(entity, originalImageBytesSupplier, baseUrl, this.imageDir, name); + } + } + + // Disk + protected Path getImagePath(String imageDir, String imageName) { + return Paths.get(imageDir, imageName); + } + + protected boolean doesThumbnailExistOnDisk(String imageDir, String imageName) { + return Files.exists(getImagePath(imageDir, imageName)); + } + + protected InputStream createImageInputStreamFromDiskPath(T entity, Supplier originalImageBytesSupplier, String baseUrl, String imageDir, String name) throws IOException, MimeTypeException { + Path imagePath = getImagePath(imageDir, name); + if (Files.exists(imagePath)) { + return new FileInputStream(imagePath.toFile()); + } else { + for (PictureSize picSize : PictureSize.values()) { + Optional opt = createPictureSourceVO(entity, baseUrl, picSize); + if (opt.isPresent() && opt.get().getName().equals(name)) { + imagePath = storeThumbnailOnDisk(this.imageDir, name, originalImageBytesSupplier.get(), picSize); + return new FileInputStream(imagePath.toFile()); + } + } + } + throw new IOException("Image not found"); + } + + protected List createThumbnailsOnDiskIfNotExists(Supplier originalImageBytesSupplier, Map map, String imageDir) { + List names = new ArrayList<>(); + byte[] imageBytes = null; + for (Map.Entry picEntry : map.entrySet()) { + PictureSize picSize = picEntry.getKey(); + PictureSourceVO ps = picEntry.getValue(); + if (!doesThumbnailExistOnDisk(imageDir, ps.getName())) { + if (imageBytes == null) { + imageBytes = originalImageBytesSupplier.get(); + if(imageBytes == null) { + log.warn("Original image is missing for thumbnail {}", ps.getName()); + return names; + } + } + try { + storeThumbnailOnDisk(imageDir, ps.getName(), imageBytes, picSize); + names.add(ps.getName()); + } catch (IOException e) { + log.warn("Failed to store image {} on disk", ps.getName(), e); + } + } else { + names.add(ps.getName()); + } + } + return names; + } + + protected Path storeThumbnailOnDisk(String imageDir, String imageName, byte[] image, PictureSize pictureSize) throws IOException { Path path = Paths.get(imageDir, imageName); Path imageBasePath = Paths.get(imageDir); if (!Files.exists(imageBasePath)) { Files.createDirectories(imageBasePath); } - try (ByteArrayInputStream bais = new ByteArrayInputStream(image)) { - if (pictureSize.pixelWidth() != null) { - Thumbnails.of(bais).width(pictureSize.pixelWidth()).toFile(path.toFile()); - } else { - Thumbnails.of(bais).scale(1).toFile(path.toFile()); + byte[] thumbnail = createThumbnail(image, pictureSize); + Files.write(path, thumbnail); + return path; + } + + // S3 + protected InputStream createImageInputStreamFromS3(T entity, Supplier originalImageBytesSupplier, String baseUrl, String name) throws IOException, MimeTypeException { + throw new UnsupportedOperationException("Get image directly from S3..."); + } + + protected List createThumbnailsOnS3IfNotExists(Supplier originalImageBytesSupplier, Map map) { + List names = new ArrayList<>(); + byte[] imageBytes = null; + for (Map.Entry picEntry : map.entrySet()) { + PictureSize pictureSize = picEntry.getKey(); + PictureSourceVO ps = picEntry.getValue(); + String keyName = ps.getUrl(); + boolean doesPhotoExist = s3Client.doesObjectExist(this.imageBucketName, keyName); + if (!doesPhotoExist) { + if (imageBytes == null) { + imageBytes = originalImageBytesSupplier.get(); + if(imageBytes == null) { + log.warn("Original image is missing for thumbnail {}", ps.getName()); + return names; + } + } + try { + byte[] thumbnail = createThumbnail(imageBytes, pictureSize); + String contentType = ps.getContentType(); + uploadThumbnailToS3(thumbnail, contentType, this.imageBucketName, keyName); + names.add(ps.getName()); + } catch (IOException e) { + log.warn("Failed to store image {} on s3", ps.getName(), e); + } } } - return path; + return names; + } + + protected boolean doesThumbnailExistOnS3(String baseUrl, String imageName) { + String keyName = baseUrl + "/" + imageName; + return s3Client.doesObjectExist(this.imageBucketName, keyName); + } + + private void uploadThumbnailToS3(byte[] thumbnail, String contentType, String bucketName, String keyName) throws IOException { + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentType(contentType); + try (InputStream is = new ByteArrayInputStream(thumbnail)) { + Upload upload = s3tm.upload(bucketName, keyName, is, metadata); + upload.waitForCompletion(); + } catch (AmazonServiceException e) { + log.error("The call was transmitted successfully, but Amazon S3 couldn't process it, so it returned an error response.", e); + } catch (SdkClientException e) { + log.error("Amazon S3 couldn't be contacted for a response, or the client couldn't parse the response from Amazon S3.", e); + } catch (InterruptedException e) { + log.error("Something interrupted the upload", e); + } + } + + private Optional generatePresignedUrlForS3(String objectKey, Date expiration) { + String bucketName = this.applicationProperties.getAws().getImageBucketName(); + try { + GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucketName, objectKey) + .withMethod(HttpMethod.GET) + .withExpiration(expiration); + URL url = s3Client.generatePresignedUrl(generatePresignedUrlRequest); + + return Optional.of(url.toString()); + } catch (AmazonServiceException e) { + log.error("The call was transmitted successfully, but Amazon S3 couldn't process it, so it returned an error response.", e); + } catch (SdkClientException e) { + log.error("Amazon S3 couldn't be contacted for a response, or the client couldn't parse the response from Amazon S3.", e); + } + return Optional.empty(); } } diff --git a/src/main/java/com/bonlimousin/gateway/bff/service/CowPictureSourceService.java b/src/main/java/com/bonlimousin/gateway/bff/service/CowPictureSourceService.java index b207b14..96de7ce 100644 --- a/src/main/java/com/bonlimousin/gateway/bff/service/CowPictureSourceService.java +++ b/src/main/java/com/bonlimousin/gateway/bff/service/CowPictureSourceService.java @@ -5,40 +5,65 @@ import com.bonlimousin.gateway.config.ApplicationProperties; import com.bonlimousin.gateway.web.api.model.PictureSourceVO; import org.apache.tika.mime.MimeTypeException; +import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; +import java.io.InputStream; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Date; +import java.util.EnumMap; import java.util.Map; import java.util.Optional; -import java.util.concurrent.Executor; +import java.util.function.Supplier; @Service public class CowPictureSourceService extends AbstractPictureSourceService { private final Logger log = LoggerFactory.getLogger(CowPictureSourceService.class); - private final Executor taskExecutor; private final PhotoResourceApiClient photoResourceApiClient; - public CowPictureSourceService(ApplicationProperties applicationProperties, Executor taskExecutor, PhotoResourceApiClient photoResourceApiClient) { - super(applicationProperties.getImageBaseDir() + "/cow", "cow"); - this.taskExecutor = taskExecutor; + public CowPictureSourceService(ApplicationProperties applicationProperties, PhotoResourceApiClient photoResourceApiClient) { + super("cow", applicationProperties); this.photoResourceApiClient = photoResourceApiClient; } @Override - public Optional createPictureSourceVO(PhotoEntity entity, String baseUrl, PictureSize pictureSize) throws MimeTypeException { + public Map createPictureSourceVOs(PhotoEntity entity, String baseUrl) throws MimeTypeException { + Map map = new EnumMap<>(PictureSize.class); + for(PictureSize picSize : PictureSize.values()) { + createPictureSourceVO(entity, baseUrl, picSize).ifPresent(ps -> map.put(picSize, ps)); + } + + if(this.applicationProperties.getThumbnails().getImageCreationAot()) { + Supplier orgImageBytesSupplier = createOriginalBytesSupplier(entity); + this.createThumbnailsIfNotExists(orgImageBytesSupplier, map); + } + + Date expiration = new Date(); + if(entity.getVisibility() == PhotoEntity.VisibilityEnum.ANONYMOUS) { + expiration.setTime(Instant.now().plus(7, ChronoUnit.DAYS).toEpochMilli()); + } else { + expiration.setTime(Instant.now().plus(2, ChronoUnit.HOURS).toEpochMilli()); + } + map.values().forEach(ps -> this.generatePresignedUrl(ps, expiration).ifPresent(ps::setUrl)); + return map; + } + + @Override + protected Optional createPictureSourceVO(PhotoEntity entity, String baseUrl, PictureSize pictureSize) throws MimeTypeException { if (pictureSize != PictureSize.ORIGINAL && pictureSize.pixelWidth() > entity.getWidth()) { return Optional.empty(); } Integer earTagId = entity.getCattle().getEarTagId(); - String imageExt = getImageExtension(entity.getImageContentType()); - String imageName = getImageName(earTagId, entity.getId(), pictureSize, imageExt); + String imageExt = this.getImageExtension(entity.getImageContentType()); + String imageName = this.getImageName(earTagId, entity.getId(), pictureSize, imageExt); String imageUrl = baseUrl + "/" + imageName; double scale = pictureSize != PictureSize.ORIGINAL ? pictureSize.pixelWidth().doubleValue() / entity.getWidth().doubleValue() @@ -56,42 +81,15 @@ public Optional createPictureSourceVO(PhotoEntity entity, Strin return Optional.of(ps); } - public Path getOrCreateImagePath(Long pictureId, String name, String baseUrl) throws MimeTypeException, IOException { - Path imagePath = getImagePath(name); - if (!Files.exists(imagePath)) { - PhotoEntity fullPhoto = this.photoResourceApiClient.getPhotoUsingGET(pictureId).getBody(); - for (PictureSize picSize : PictureSize.values()) { - Optional opt = createPictureSourceVO(fullPhoto, baseUrl, picSize); - if (opt.isPresent() && opt.get().getName().equals(name)) { - imagePath = createThumbnailOndisk(name, fullPhoto.getImage(), picSize); - break; - } - } - } - return imagePath; - } - - public void asyncCreateThumbnailsOnDiskIfNotExists(Long pictureId, Map map) { - taskExecutor.execute(() -> { - try { - createThumbnailsOnDiskIfNotExists(pictureId, map); - } catch (MimeTypeException | IOException e) { - log.warn("Async creation of thumbnail on disk failed", e); - } - }); + @Override + public InputStream createImageInputStream(PhotoEntity entity, String baseUrl, String name) throws IOException, MimeTypeException { + return this.createImageInputStream(entity, createOriginalBytesSupplier(entity), baseUrl, name); } - public void createThumbnailsOnDiskIfNotExists(Long pictureId, Map map) throws MimeTypeException, IOException { - PhotoEntity fullPhoto = null; - for (Map.Entry picEntry : map.entrySet()) { - PictureSize picSize = picEntry.getKey(); - PictureSourceVO ps = picEntry.getValue(); - if (!Files.exists(getImagePath(ps.getName()))) { - if (fullPhoto == null) { - fullPhoto = this.photoResourceApiClient.getPhotoUsingGET(pictureId).getBody(); - } - createThumbnailOndisk(ps.getName(), fullPhoto.getImage(), picSize); - } - } + private Supplier createOriginalBytesSupplier(PhotoEntity entity) { + return () -> { + ResponseEntity photoResp = this.photoResourceApiClient.getPhotoUsingGET(entity.getId()); + return photoResp.getBody() != null ? photoResp.getBody().getImage() : null; + }; } } diff --git a/src/main/java/com/bonlimousin/gateway/config/ApplicationProperties.java b/src/main/java/com/bonlimousin/gateway/config/ApplicationProperties.java index 534bd3b..314b6db 100644 --- a/src/main/java/com/bonlimousin/gateway/config/ApplicationProperties.java +++ b/src/main/java/com/bonlimousin/gateway/config/ApplicationProperties.java @@ -12,9 +12,16 @@ */ @ConfigurationProperties(prefix = "application", ignoreUnknownFields = false) public class ApplicationProperties { - + + public enum ImageStorage { + DISK, + AWS + } + private Boolean publicAccountRegistration; - private String imageBaseDir; + private Thumbnails thumbnails; + private Web web; + private Aws aws; private Bff bff; public Boolean getPublicAccountRegistration() { @@ -24,16 +31,32 @@ public Boolean getPublicAccountRegistration() { public void setPublicAccountRegistration(Boolean publicAccountRegistration) { this.publicAccountRegistration = publicAccountRegistration; } - - public String getImageBaseDir() { - return imageBaseDir; - } - public void setImageBaseDir(String imageBaseDir) { - this.imageBaseDir = imageBaseDir; - } - - public Bff getBff() { + public Thumbnails getThumbnails() { + return thumbnails; + } + + public void setThumbnails(Thumbnails thumbnails) { + this.thumbnails = thumbnails; + } + + public Web getWeb() { + return web; + } + + public void setWeb(Web web) { + this.web = web; + } + + public Aws getAws() { + return aws; + } + + public void setAws(Aws aws) { + this.aws = aws; + } + + public Bff getBff() { return bff; } @@ -41,10 +64,100 @@ public void setBff(Bff bff) { this.bff = bff; } + public static class Thumbnails { + private String imageBaseDir; + private String imageStorage; + private Boolean imageCreationAot; + + public String getImageBaseDir() { + return imageBaseDir; + } + + public void setImageBaseDir(String imageBaseDir) { + this.imageBaseDir = imageBaseDir; + } + + public ImageStorage getImageStorage() { + return ImageStorage.valueOf(imageStorage); + } + + public void setImageStorage(String imageStorage) { + this.imageStorage = imageStorage; + } + + public Boolean getImageCreationAot() { + return imageCreationAot; + } + + public void setImageCreationAot(Boolean imageCreationAot) { + this.imageCreationAot = imageCreationAot; + } + } + + public static class Web { + private String contentSecurityPolicy; + private String featurePolicy; + + public String getContentSecurityPolicy() { + return contentSecurityPolicy; + } + + public void setContentSecurityPolicy(String contentSecurityPolicy) { + this.contentSecurityPolicy = contentSecurityPolicy; + } + + public String getFeaturePolicy() { + return featurePolicy; + } + + public void setFeaturePolicy(String featurePolicy) { + this.featurePolicy = featurePolicy; + } + } + + public static class Aws { + private String regionName; + private String accessKey; + private String secretKey; + private String imageBucketName; + + public String getRegionName() { + return regionName; + } + + public void setRegionName(String regionName) { + this.regionName = regionName; + } + + public String getAccessKey() { + return accessKey; + } + + public void setAccessKey(String accessKey) { + this.accessKey = accessKey; + } + + public String getSecretKey() { + return secretKey; + } + + public void setSecretKey(String secretKey) { + this.secretKey = secretKey; + } + + public String getImageBucketName() { + return imageBucketName; + } + + public void setImageBucketName(String imageBucketName) { + this.imageBucketName = imageBucketName; + } + } + public static class Bff { private Client client; private Resource resource; - + public Client getClient() { return client; } @@ -60,7 +173,7 @@ public Resource getResource() { public void setResource(Resource resource) { this.resource = resource; } - + public static class Client { @NotBlank private String username; @@ -236,7 +349,7 @@ public void setSourcefile(ResourceProps sourcefile) { public static class ResourceProps { @NotBlank - private String name; + private String name; private String url; public String getName() { diff --git a/src/main/java/com/bonlimousin/gateway/config/SecurityConfiguration.java b/src/main/java/com/bonlimousin/gateway/config/SecurityConfiguration.java index 0dc16a4..b777f27 100644 --- a/src/main/java/com/bonlimousin/gateway/config/SecurityConfiguration.java +++ b/src/main/java/com/bonlimousin/gateway/config/SecurityConfiguration.java @@ -25,14 +25,15 @@ public class SecurityConfiguration extends WebSecurityConfigurerAdapter { private final TokenProvider tokenProvider; - private final CorsFilter corsFilter; private final SecurityProblemSupport problemSupport; + private final ApplicationProperties applicationProperties; - public SecurityConfiguration(TokenProvider tokenProvider, CorsFilter corsFilter, SecurityProblemSupport problemSupport) { + public SecurityConfiguration(TokenProvider tokenProvider, CorsFilter corsFilter, SecurityProblemSupport problemSupport, ApplicationProperties applicationProperties) { this.tokenProvider = tokenProvider; this.corsFilter = corsFilter; this.problemSupport = problemSupport; + this.applicationProperties = applicationProperties; } @Bean @@ -65,11 +66,11 @@ public void configure(HttpSecurity http) throws Exception { .accessDeniedHandler(problemSupport) .and() .headers() - .contentSecurityPolicy("default-src 'self'; frame-src 'self' https://player.vimeo.com https://*.vimeo.com data:; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://storage.googleapis.com https://buttons.github.io https://player.vimeo.com https://*.vimeocdn.com; style-src 'self' https://fonts.googleapis.com 'unsafe-inline'; img-src 'self' https://picsum.photos https://*.picsum.photos https://*.vimeocdn.com data:; font-src 'self' https://fonts.gstatic.com data:") + .contentSecurityPolicy(this.applicationProperties.getWeb().getContentSecurityPolicy()) .and() .referrerPolicy(ReferrerPolicyHeaderWriter.ReferrerPolicy.STRICT_ORIGIN_WHEN_CROSS_ORIGIN) .and() - .featurePolicy("geolocation 'none'; midi 'none'; sync-xhr 'none'; microphone 'none'; camera 'none'; magnetometer 'none'; gyroscope 'none'; speaker 'none'; fullscreen 'self'; payment 'none'") + .featurePolicy(this.applicationProperties.getWeb().getFeaturePolicy()) .and() .frameOptions() .deny() diff --git a/src/main/resources/config/application-dev.yml b/src/main/resources/config/application-dev.yml index f8a5ed0..5b0ac51 100644 --- a/src/main/resources/config/application-dev.yml +++ b/src/main/resources/config/application-dev.yml @@ -145,7 +145,14 @@ jhipster: # =================================================================== application: - image-base-dir: target/images + thumbnails: + image-storage: DISK + image-base-dir: target/images + aws: + region-name: eu-north-1 + # access-key: defined somewhere else... + # secret-key: defined somewhere else... + image-bucket-name: bonlimousin-dev bff: client: username: gatewayclient diff --git a/src/main/resources/config/application-prod.yml b/src/main/resources/config/application-prod.yml index 34f7da5..15e48e0 100644 --- a/src/main/resources/config/application-prod.yml +++ b/src/main/resources/config/application-prod.yml @@ -157,7 +157,14 @@ jhipster: # =================================================================== application: - image-base-dir: ${java.io.tmpdir} + thumbnails: + image-storage: AWS + image-base-dir: ${java.io.tmpdir} + aws: + region-name: eu-north-1 + # access-key: defined somewhere else... + # secret-key: defined somewhere else... + image-bucket-name: bonlimousin bff: client: username: gatewayclient diff --git a/src/main/resources/config/application.yml b/src/main/resources/config/application.yml index e53d83e..595975e 100644 --- a/src/main/resources/config/application.yml +++ b/src/main/resources/config/application.yml @@ -233,6 +233,11 @@ kafka: application: public-account-registration: false + thumbnails: + image-creation-aot: true + web: + content-security-policy: "default-src 'self'; frame-src 'self' https://player.vimeo.com https://*.vimeo.com data:; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://storage.googleapis.com https://buttons.github.io https://player.vimeo.com https://*.vimeocdn.com; style-src 'self' https://fonts.googleapis.com 'unsafe-inline'; img-src 'self' https://picsum.photos https://*.picsum.photos https://*.vimeocdn.com https://bonlimousin-dev.s3.eu-north-1.amazonaws.com https://bonlimousin.s3.eu-north-1.amazonaws.com data:; font-src 'self' https://fonts.gstatic.com data:" + feature-policy: "geolocation 'none'; midi 'none'; sync-xhr 'none'; microphone 'none'; camera 'none'; magnetometer 'none'; gyroscope 'none'; speaker 'none'; fullscreen 'self'; payment 'none'" bff: resource: # bonContentService diff --git a/src/test/resources/config/application.yml b/src/test/resources/config/application.yml index 5335eb5..ae32fdc 100644 --- a/src/test/resources/config/application.yml +++ b/src/test/resources/config/application.yml @@ -139,7 +139,13 @@ kafka: application: public-account-registration: false - image-base-dir: target/images + thumbnails: + image-storage: DISK + image-base-dir: target/images + image-creation-aot: false + web: + content-security-policy: "default-src 'self'; frame-src 'self' https://player.vimeo.com https://*.vimeo.com data:; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://storage.googleapis.com https://buttons.github.io https://player.vimeo.com https://*.vimeocdn.com; style-src 'self' https://fonts.googleapis.com 'unsafe-inline'; img-src 'self' https://picsum.photos https://*.picsum.photos https://*.vimeocdn.com https://bonlimousin-dev.s3.eu-north-1.amazonaws.com https://bonlimousin.s3.eu-north-1.amazonaws.com data:; font-src 'self' https://fonts.gstatic.com data:" + feature-policy: "geolocation 'none'; midi 'none'; sync-xhr 'none'; microphone 'none'; camera 'none'; magnetometer 'none'; gyroscope 'none'; speaker 'none'; fullscreen 'self'; payment 'none'" bff: client: username: gatewayclient