From ec01411bf62c92888afdae8ffd88c0d485b8b3b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=83=AD=E5=98=89=E4=BC=9F?= <18613599970@163.com> Date: Fri, 17 May 2024 14:12:28 +0800 Subject: [PATCH 1/4] [feature] add alibaba cloud oss support --- manager/pom.xml | 6 ++ .../manager/pojo/dto/ObjectStoreDTO.java | 23 ++++- .../impl/ObjectStoreConfigServiceImpl.java | 19 ++++ .../impl/OssObjectStoreServiceImpl.java | 90 +++++++++++++++++++ web-app/src/app/pojo/ObjectStore.ts | 15 +++- .../object-store/object-store.component.html | 21 ++--- .../object-store/object-store.component.ts | 5 +- web-app/src/assets/i18n/en-US.json | 11 +++ web-app/src/assets/i18n/zh-CN.json | 11 +++ web-app/src/assets/i18n/zh-TW.json | 11 +++ 10 files changed, 199 insertions(+), 13 deletions(-) create mode 100644 manager/src/main/java/org/apache/hertzbeat/manager/service/impl/OssObjectStoreServiceImpl.java diff --git a/manager/pom.xml b/manager/pom.xml index 50b31d82fb3..dd53419f1e1 100644 --- a/manager/pom.xml +++ b/manager/pom.xml @@ -37,6 +37,7 @@ 4.3.0 3.1.37 3.23.5 + 3.17.4 @@ -198,6 +199,11 @@ + + com.aliyun.oss + aliyun-sdk-oss + ${alibaba.oss.version} + diff --git a/manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/ObjectStoreDTO.java b/manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/ObjectStoreDTO.java index 0a8a6fdb00b..07aca4ed2c6 100644 --- a/manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/ObjectStoreDTO.java +++ b/manager/src/main/java/org/apache/hertzbeat/manager/pojo/dto/ObjectStoreDTO.java @@ -54,7 +54,12 @@ public enum Type { /** * Huawei Cloud OBS */ - OBS + OBS, + + /** + * Alibaba Cloud OSS + */ + OSS, } /** @@ -73,4 +78,20 @@ public static class ObsConfig { private String savePath = "hertzbeat"; } + /** + * file oss storage configuration + */ + @Data + public static class OssConfig { + private String accessKey; + private String secretKey; + private String bucketName; + private String endpoint; + + /** + * Save path + */ + private String savePath = "hertzbeat"; + } + } diff --git a/manager/src/main/java/org/apache/hertzbeat/manager/service/impl/ObjectStoreConfigServiceImpl.java b/manager/src/main/java/org/apache/hertzbeat/manager/service/impl/ObjectStoreConfigServiceImpl.java index ed7dbef1d59..7425f863599 100644 --- a/manager/src/main/java/org/apache/hertzbeat/manager/service/impl/ObjectStoreConfigServiceImpl.java +++ b/manager/src/main/java/org/apache/hertzbeat/manager/service/impl/ObjectStoreConfigServiceImpl.java @@ -17,6 +17,7 @@ package org.apache.hertzbeat.manager.service.impl; +import com.aliyun.oss.OSSClientBuilder; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.obs.services.ObsClient; @@ -83,10 +84,28 @@ public void handler(ObjectStoreDTO config) { initObs(config); // case other object store service } + if (config.getType() == ObjectStoreDTO.Type.OSS) { + initOss(config); + } ctx.publishEvent(new ObjectStoreConfigChangeEvent(config)); } } + private void initOss(ObjectStoreDTO config) { + var ossConfig = objectMapper.convertValue(config.getConfig(), ObjectStoreDTO.OssConfig.class); + Assert.hasText(ossConfig.getAccessKey(), "cannot find oss accessKey"); + Assert.hasText(ossConfig.getSecretKey(), "cannot find oss secretKey"); + Assert.hasText(ossConfig.getEndpoint(), "cannot find oss endpoint"); + Assert.hasText(ossConfig.getBucketName(), "cannot find oss bucket name"); + + var ossClient = new OSSClientBuilder().build(ossConfig.getEndpoint(), ossConfig.getAccessKey(), ossConfig.getSecretKey()); + + beanFactory.destroySingleton(BEAN_NAME); + beanFactory.registerSingleton(BEAN_NAME, new OssObjectStoreServiceImpl(ossClient, ossConfig.getBucketName(), ossConfig.getSavePath())); + + log.info("oss store service init success."); + } + /** * init Huawei Cloud OBS */ diff --git a/manager/src/main/java/org/apache/hertzbeat/manager/service/impl/OssObjectStoreServiceImpl.java b/manager/src/main/java/org/apache/hertzbeat/manager/service/impl/OssObjectStoreServiceImpl.java new file mode 100644 index 00000000000..3f6ec362178 --- /dev/null +++ b/manager/src/main/java/org/apache/hertzbeat/manager/service/impl/OssObjectStoreServiceImpl.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hertzbeat.manager.service.impl; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.model.ListObjectsRequest; +import com.aliyun.oss.model.OSSObjectSummary; +import java.io.InputStream; +import java.util.List; +import java.util.Objects; +import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.common.constants.SignConstants; +import org.apache.hertzbeat.manager.pojo.dto.FileDTO; +import org.apache.hertzbeat.manager.pojo.dto.ObjectStoreDTO; +import org.apache.hertzbeat.manager.service.ObjectStoreService; + +/** + * Alibaba cloud storage service + */ +@Slf4j +public class OssObjectStoreServiceImpl implements ObjectStoreService { + + private final OSS ossClient; + + private final String bucketName; + private final String rootPath; + + public OssObjectStoreServiceImpl(OSS ossClient, String bucketName, String rootPath) { + this.ossClient = ossClient; + this.bucketName = bucketName; + if (rootPath.startsWith(SignConstants.RIGHT_DASH)) { + this.rootPath = rootPath.substring(1); + } else { + this.rootPath = rootPath; + } + } + + @Override + public boolean upload(String filePath, InputStream is) { + var objectKey = getObjectKey(filePath); + var response = ossClient.putObject(bucketName, objectKey, is); + return Objects.equals(response.getResponse().getStatusCode(), 200); + } + + @Override + public FileDTO download(String filePath) { + var objectKey = getObjectKey(filePath); + try { + var ossObject = ossClient.getObject(bucketName, objectKey); + return new FileDTO(filePath, ossObject.getObjectContent()); + } catch (Exception ex) { + log.warn("download file from oss error {}", objectKey); + return null; + } + } + + @Override + public List list(String dir) { + var request = new ListObjectsRequest(bucketName); + request.setPrefix(getObjectKey(dir)); + return ossClient.listObjects(request).getObjectSummaries().stream() + .map(OSSObjectSummary::getKey).toList().stream() + .map(k -> new FileDTO(k, ossClient.getObject(bucketName, k).getObjectContent())).toList(); + } + + @Override + public ObjectStoreDTO.Type type() { + return ObjectStoreDTO.Type.OSS; + } + + + private String getObjectKey(String filePath) { + return rootPath + SignConstants.RIGHT_DASH + filePath; + } +} diff --git a/web-app/src/app/pojo/ObjectStore.ts b/web-app/src/app/pojo/ObjectStore.ts index d8faed5ed85..0cdda867d43 100644 --- a/web-app/src/app/pojo/ObjectStore.ts +++ b/web-app/src/app/pojo/ObjectStore.ts @@ -32,7 +32,11 @@ export enum ObjectStoreType { /** * 华为云OBS */ - OBS = 'OBS' + OBS = 'OBS', + /** + * Alibaba Cloud OSS + */ + OSS = 'OSS', } export class ObsConfig { @@ -42,3 +46,12 @@ export class ObsConfig { endpoint!: string; savePath: string = 'hertzbeat'; } + +export class OssConfig { + accessKey!: string; + secretKey!: string; + bucketName!: string; + endpoint!: string; + savePath: string = 'hertzbeat'; +} + diff --git a/web-app/src/app/routes/setting/settings/object-store/object-store.component.html b/web-app/src/app/routes/setting/settings/object-store/object-store.component.html index 741fe310126..597a97a5630 100644 --- a/web-app/src/app/routes/setting/settings/object-store/object-store.component.html +++ b/web-app/src/app/routes/setting/settings/object-store/object-store.component.html @@ -32,16 +32,17 @@ > + - + {{ 'settings.object-store.obs.accessKey' | i18n }} - + {{ 'settings.object-store.obs.secretKey' | i18n }} - + {{ 'settings.object-store.obs.bucketName' | i18n }} - + {{ 'settings.object-store.obs.endpoint' | i18n }} - + {{ 'settings.object-store.obs.savePath' | i18n }} Date: Fri, 17 May 2024 18:10:17 +0800 Subject: [PATCH 2/4] [feature] update html support ali oss --- web-app/src/app/pojo/ObjectStore.ts | 3 +- .../object-store/object-store.component.html | 30 +++++++++++++++---- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/web-app/src/app/pojo/ObjectStore.ts b/web-app/src/app/pojo/ObjectStore.ts index 0cdda867d43..915fa9dc149 100644 --- a/web-app/src/app/pojo/ObjectStore.ts +++ b/web-app/src/app/pojo/ObjectStore.ts @@ -36,7 +36,7 @@ export enum ObjectStoreType { /** * Alibaba Cloud OSS */ - OSS = 'OSS', + OSS = 'OSS' } export class ObsConfig { @@ -54,4 +54,3 @@ export class OssConfig { endpoint!: string; savePath: string = 'hertzbeat'; } - diff --git a/web-app/src/app/routes/setting/settings/object-store/object-store.component.html b/web-app/src/app/routes/setting/settings/object-store/object-store.component.html index 597a97a5630..8f55b1776c7 100644 --- a/web-app/src/app/routes/setting/settings/object-store/object-store.component.html +++ b/web-app/src/app/routes/setting/settings/object-store/object-store.component.html @@ -42,7 +42,11 @@ Date: Fri, 17 May 2024 19:57:19 +0800 Subject: [PATCH 3/4] [feature] update support ali oss config import --- .../setting/settings/object-store/object-store.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-app/src/app/routes/setting/settings/object-store/object-store.component.ts b/web-app/src/app/routes/setting/settings/object-store/object-store.component.ts index cbb19842395..6cb02e72e92 100644 --- a/web-app/src/app/routes/setting/settings/object-store/object-store.component.ts +++ b/web-app/src/app/routes/setting/settings/object-store/object-store.component.ts @@ -22,7 +22,7 @@ import { ChangeDetectorRef, Component, Inject, OnInit } from '@angular/core'; import { NzNotificationService } from 'ng-zorro-antd/notification'; import { finalize } from 'rxjs/operators'; -import {ObjectStore, ObjectStoreType, ObsConfig, OssConfig} from '../../../../pojo/ObjectStore'; +import { ObjectStore, ObjectStoreType, ObsConfig, OssConfig } from '../../../../pojo/ObjectStore'; import { GeneralConfigService } from '../../../../service/general-config.service'; const key = 'oss'; From d494c750df5a94a79062ba08fe088452b047bef7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=83=AD=E5=98=89=E4=BC=9F?= <18613599970@163.com> Date: Sat, 18 May 2024 00:13:42 +0800 Subject: [PATCH 4/4] [feature] change show store info --- .../object-store/object-store.component.html | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/web-app/src/app/routes/setting/settings/object-store/object-store.component.html b/web-app/src/app/routes/setting/settings/object-store/object-store.component.html index 8f55b1776c7..99a2b6f9d54 100644 --- a/web-app/src/app/routes/setting/settings/object-store/object-store.component.html +++ b/web-app/src/app/routes/setting/settings/object-store/object-store.component.html @@ -44,8 +44,8 @@ [(ngModel)]="config.config.accessKey" placeholder="{{ config.type == ObjectStoreType.OBS - ? ('settings.object-store.obs.bucketName.placeholder' | i18n) - : ('settings.object-store.oss.bucketName.placeholder' | i18n) + ? ('settings.object-store.obs.accessKey.placeholder' | i18n) + : ('settings.object-store.oss.accessKey.placeholder' | i18n) }}" nz-input required @@ -64,8 +64,8 @@ [(ngModel)]="config.config.secretKey" placeholder="{{ config.type == ObjectStoreType.OBS - ? ('settings.object-store.obs.bucketName.placeholder' | i18n) - : ('settings.object-store.oss.bucketName.placeholder' | i18n) + ? ('settings.object-store.obs.secretKey.placeholder' | i18n) + : ('settings.object-store.oss.secretKey.placeholder' | i18n) }}" nz-input required @@ -104,8 +104,8 @@ [(ngModel)]="config.config.endpoint" placeholder="{{ config.type == ObjectStoreType.OBS - ? ('settings.object-store.obs.bucketName.placeholder' | i18n) - : ('settings.object-store.oss.bucketName.placeholder' | i18n) + ? ('settings.object-store.obs.endpoint.placeholder' | i18n) + : ('settings.object-store.oss.endpoint.placeholder' | i18n) }}" nz-input required @@ -124,8 +124,8 @@ [(ngModel)]="config.config.savePath" placeholder="{{ config.type == ObjectStoreType.OBS - ? ('settings.object-store.obs.bucketName.placeholder' | i18n) - : ('settings.object-store.oss.bucketName.placeholder' | i18n) + ? ('settings.object-store.obs.savePath.placeholder' | i18n) + : ('settings.object-store.oss.savePath.placeholder' | i18n) }}" nz-input required