From d826a863851aad26530c578abfe02e4a63ea7ed8 Mon Sep 17 00:00:00 2001 From: Will Kang Date: Sun, 14 Jul 2024 17:35:43 +0800 Subject: [PATCH] Init message push service by websocket. --- message-push-common/.gitignore | 13 + message-push-common/README.md | 3 + message-push-common/pom.xml | 134 +++ .../core/constant/ErrorMessageConstants.java | 18 + .../core/constant/MessagePushConstants.java | 84 ++ .../core/constant/OrderConstant.java | 21 + .../core/constant/RedisConstants.java | 37 + .../messagepush/core/dto/page/Order.java | 30 + .../core/dto/page/PageRequestDto.java | 51 + .../core/dto/page/PageResponseDto.java | 37 + .../core/dto/response/ApiResponse.java | 69 ++ .../elasticsearch/ElasticSearchConst.java | 489 ++++++++++ .../elasticsearch/ElasticsearchManager.java | 404 ++++++++ .../RestHighLevelClientContextHolder.java | 82 ++ .../elasticsearch/annotation/EntityField.java | 24 + .../elasticsearch/annotation/QueryField.java | 49 + .../core/elasticsearch/annotation/Score.java | 16 + .../elasticsearch/config/GaussConfig.java | 41 + .../elasticsearch/config/ScriptConfig.java | 25 + .../elasticsearch/domain/CombinatorWord.java | 28 + .../elasticsearch/domain/FullTextItem.java | 53 ++ .../elasticsearch/domain/FullTextQuery.java | 72 ++ .../domain/ImportantBizPosEnum.java | 88 ++ .../domain/ImportantPosEnum.java | 81 ++ .../elasticsearch/domain/Segmentation.java | 52 ++ .../elasticsearch/domain/SynonymWord.java | 24 + .../core/elasticsearch/domain/Word.java | 39 + .../elasticsearch/enumeration/OccurEnum.java | 56 ++ .../elasticsearch/enumeration/ParseEnum.java | 136 +++ .../impl/ElasticsearchManagerImpl.java | 872 ++++++++++++++++++ .../query/CustomQueryBuilder.java | 20 + .../query/FulltextQueryBuilder.java | 303 ++++++ .../elasticsearch/request/AliasAction.java | 37 + .../elasticsearch/request/BaseRequest.java | 31 + .../request/JsonAliasActionsRequest.java | 67 ++ .../request/JsonCreateIndexRequest.java | 112 +++ .../request/JsonSearchRequest.java | 218 +++++ .../request/PlainRolloverRequest.java | 44 + .../SimpleGroupByJsonSearchRequest.java | 142 +++ .../response/AggregationsResult.java | 247 +++++ .../core/elasticsearch/response/Result.java | 47 + .../core/elasticsearch/util/QueryUtil.java | 565 ++++++++++++ .../messagepush/core/enums/ResponseEnum.java | 160 ++++ .../core/event/NacosServiceUpInfo.java | 23 + .../core/event/NacosServiceUpdateEvent.java | 15 + .../core/event/NacosServiceUpdateInfo.java | 52 ++ .../core/hash/ConsistencyHashing.java | 239 +++++ .../kangspace/messagepush/core/hash/Node.java | 38 + .../messagepush/core/hash/PhysicalNode.java | 46 + .../messagepush/core/hash/VirtualNode.java | 62 ++ .../core/hash/algoithm/HashAlgorithm.java | 46 + .../hash/algoithm/KetamaHashAlgorithm.java | 45 + .../hash/repository/HashRouterRepository.java | 61 ++ .../core/http/RestMapProperties.java | 30 + .../messagepush/core/http/RestProperties.java | 58 ++ .../core/http/RestTemplateFactory.java | 163 ++++ .../messagepush/core/redis/RedisService.java | 138 +++ .../messagepush/core/util/AppGenerator.java | 83 ++ .../messagepush/core/util/ArrayUtil.java | 60 ++ .../messagepush/core/util/BeanUtil.java | 82 ++ .../core/util/ExchangeRequestUtils.java | 46 + .../messagepush/core/util/HttpUtils.java | 188 ++++ .../messagepush/core/util/IpUtils.java | 72 ++ .../messagepush/core/util/JsonUtil.java | 360 ++++++++ .../messagepush/core/util/ListUtil.java | 148 +++ .../messagepush/core/util/MD5Util.java | 36 + .../messagepush/core/util/ObjectUtil.java | 42 + .../core/util/PathVariableResolver.java | 111 +++ .../messagepush/core/util/StrUtil.java | 289 ++++++ .../websocket/WebSocketSessionManager.java | 56 ++ .../main/resources/META-INF/spring.factories | 0 .../messagepush/common/MD5UtilTest.java | 58 ++ .../common/PathVariableResolverTest.java | 59 ++ message-push-consumer/.gitignore | 13 + message-push-consumer/README.md | 19 + .../message-push-consumer-core/pom.xml | 77 ++ .../core/config/ControllerAccessConfig.java | 133 +++ .../config/ElasticSearchIndexInitial.java | 162 ++++ .../config/PushConsumerConfiguration.java | 28 + .../core/config/TaskExecutorConfig.java | 32 + .../consumer/core/config/WebMvcConfig.java | 114 +++ .../dto/request/MessagePushRequestDto.java | 33 + .../core/feign/DefaultFeignFallBack.java | 14 + .../consumer/core/feign/MessagePushWsApi.java | 32 + .../feign/MessagePushWsApiFeignFallBack.java | 24 + .../consumer/core/feign/TempFeignAPi.java | 12 + .../config/MessagePushWsApiFeignConfig.java | 42 + .../consumer/core/hash/HashRouterLoader.java | 78 ++ .../core/mq/kafka/KafkaBindingConfig.java | 15 + .../core/mq/kafka/MessagePushChannel.java | 28 + .../core/mq/kafka/MessagePushConsumer.java | 39 + .../consumer/core/redis/RedisService.java | 69 ++ .../consumer/core/service/BaseService.java | 10 + .../core/service/ElasticSearchService.java | 186 ++++ .../service/MessagePushConsumerService.java | 17 + .../service/impl/MessagePushServiceImpl.java | 99 ++ .../messagepush/consumer/core/Test.java | 16 + .../pom.xml | 113 +++ .../MessagePushConsumerApplication.java | 29 + .../src/main/resources/bootstrap.yml | 17 + .../consumer/MessagePushServiceTest.java | 117 +++ .../kangspace/messagepush/consumer/Test.java | 13 + message-push-consumer/pom.xml | 50 + message-push-rest/.gitignore | 13 + message-push-rest/README.md | 14 + .../message-push-rest-api/pom.xml | 43 + .../messagepush/rest/api/dto/ApiBaseDTO.java | 15 + .../dto/request/MessagePushRequestDTO.java | 94 ++ .../request/MessagePushRequestTimeDTO.java | 67 ++ .../message-push-rest-core/pom.xml | 84 ++ .../rest/core/auth/ApiAuthentication.java | 15 + .../rest/core/auth/AuthRequestAop.java | 96 ++ .../core/config/ControllerAccessConfig.java | 133 +++ .../rest/core/config/KafkaBindingConfig.java | 14 + .../rest/core/config/WebMvcConfig.java | 114 +++ .../rest/core/constant/AppThreadLocal.java | 39 + .../core/constant/MessagePushConstants.java | 24 + .../core/domain/entity/TemplateEntity.java | 5 + .../rest/core/feign/DefaultFeignFallBack.java | 14 + .../rest/core/feign/TempFeignAPi.java | 12 + .../rest/core/manager/TemplateManager.java | 10 + .../core/manager/TemplateManagerImpl.java | 10 + .../rest/core/mapper/TemplateMapper.java | 11 + .../rest/core/mq/kafka/KafkaMqSender.java | 46 + .../mq/kafka/KafkaMqTopicChannelMapping.java | 43 + .../core/mq/kafka/MessagePushChannel.java | 36 + .../core/properties/TemplateProperties.java | 18 + .../rest/core/service/BaseService.java | 10 + .../rest/core/service/MessagePushService.java | 24 + .../service/impl/MessagePushServiceImpl.java | 40 + .../rest/core/utils/AppGenerator.java | 83 ++ .../messagepush/rest/core/utils/IpUtil.java | 49 + .../messagepush/rest/core/utils/UidCoder.java | 72 ++ .../rest/core/utils/VariableUtils.java | 33 + .../kangspace/messagepush/rest/core/Test.java | 81 ++ .../message-push-rest-microservice/pom.xml | 114 +++ .../rest/MessagePushRestApplication.java | 29 + .../controller/MessagePushRestController.java | 54 ++ .../src/main/resources/bootstrap.yml | 19 + .../org/kangspace/messagepush/rest/Test.java | 13 + message-push-rest/pom.xml | 51 + message-push-ws-gateway/.gitignore | 13 + message-push-ws-gateway/README.md | 25 + message-push-ws-gateway/pom.xml | 167 ++++ .../MessagePushWsGatewayApplication.java | 37 + .../config/EventListenerConfiguration.java | 28 + ...MessagePushWsGatewayAutoConfiguration.java | 21 + .../MessagePushWsGatewayConfiguration.java | 220 +++++ .../MessagePushWsGatewayProperties.java | 26 + .../constant/MessagePushWsConstants.java | 17 + .../DebugWebsocketSessionCountScheduler.java | 55 ++ .../SessionCenterUserInfoParamDTO.java | 29 + .../response/SessionCenterUserInfoDTO.java | 37 + .../exception/GatewayExceptionHandler.java | 124 +++ .../exception/TokenValidateException.java | 43 + .../gateway/feign/DefaultFeignFallBack.java | 14 + .../feign/PassportSessionFeignApi.java | 29 + .../feign/PassportSessionFeignFallBack.java | 25 + .../ws/gateway/feign/TempFeignAPi.java | 12 + .../ws/gateway/filter/BaseFilter.java | 73 ++ .../ws/gateway/filter/FilterOrders.java | 33 + .../filter/GatewayWebsocketRoutingFilter.java | 225 +++++ .../gateway/filter/RequestValidateFilter.java | 146 +++ ...ocketReactiveLoadBalancerClientFilter.java | 135 +++ .../balancer/LbServiceInstanceChooser.java | 27 + .../RoundRibbonServiceInstanceChooser.java | 42 + .../balancer/UIDServiceInstanceChooser.java | 60 ++ .../balancer/WebSocketLoadBalancer.java | 72 ++ .../filter/session/SessionProxyHolder.java | 30 + .../session/WebSocketUserSessionManager.java | 167 ++++ .../hash/DebugPrintHashingScheduler.java | 50 + .../hash/RedisHashRouterRepository.java | 262 ++++++ .../ws/gateway/model/MessageRequestParam.java | 58 ++ .../nacos/NacosDynamicServerListListener.java | 79 ++ .../ws/gateway/nacos/NacosNamingService.java | 41 + .../ws/gateway/util/ExchangeRequestUtils.java | 111 +++ .../validation/PassportSessionValidator.java | 93 ++ .../ws/gateway/validation/TokenValidator.java | 21 + .../main/resources/META-INF/spring.factories | 1 + .../src/main/resources/bootstrap.yml | 24 + .../RedisHashRouterRepositoryTest.java | 56 ++ .../WebSocketUserSessionManagerTest.java | 41 + message-push-ws/.gitignore | 13 + message-push-ws/README.md | 20 + message-push-ws/message-push-ws-api/pom.xml | 25 + message-push-ws/message-push-ws-core/pom.xml | 72 ++ .../config/ControllerExceptionConfig.java | 95 ++ .../core/config/WebsocketConfiguration.java | 131 +++ .../ws/core/constant/MessagePushCmdEnum.java | 28 + .../constant/MessagePushResponseTypeEnum.java | 31 + .../core/constant/MessagePushWsConstants.java | 23 + .../DebugWebsocketSessionCountScheduler.java | 58 ++ .../ws/core/domain/dto/MessageCmdDTO.java | 22 + .../domain/dto/MessageCmdResponseDTO.java | 34 + .../domain/dto/request/LoginReqDataDTO.java | 21 + .../dto/response/HeartBeatRespDataDTO.java | 24 + .../dto/response/MessageRespDataDTO.java | 35 + .../dto/response/NormalRespDataDTO.java | 56 ++ .../core/domain/model/HttpPushMessageDTO.java | 29 + .../domain/model/MessageRequestParam.java | 36 + .../ws/core/feign/DefaultFeignFallBack.java | 14 + .../ws/core/feign/TempFeignAPi.java | 12 + .../ws/core/service/BaseService.java | 10 + .../ws/core/service/MessagePushWsService.java | 22 + .../impl/MessagePushWsServiceImpl.java | 39 + .../core/utils/MessagePushHandlerUtils.java | 28 + .../core/websocket/BaseWebcosketHandler.java | 45 + .../websocket/HttpPushMessageConsumer.java | 102 ++ .../websocket/HttpPushMessagePublisher.java | 81 ++ .../MessagePushHandshakeWebSocketService.java | 30 + .../MessagePushWebSocketHandler.java | 268 ++++++ .../ws/core/websocket/WebsocketUtils.java | 60 ++ .../core/websocket/session/SessionHolder.java | 31 + .../session/WebSocketSessionManager.java | 58 ++ .../session/WebSocketUserSessionManager.java | 161 ++++ .../messagepush/ws/core/FluxTest.java | 59 ++ .../ws/core/HttpPushMessagePublisherTest.java | 51 + .../message-push-ws-microservice/pom.xml | 101 ++ .../ws/MessagePushWsApplication.java | 27 + .../controller/MessagePushRestController.java | 40 + .../src/main/resources/bootstrap.yml | 17 + .../org/kangspace/messagepush/ws/Test.java | 13 + message-push-ws/pom.xml | 64 ++ 223 files changed, 15955 insertions(+) create mode 100644 message-push-common/.gitignore create mode 100644 message-push-common/README.md create mode 100644 message-push-common/pom.xml create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/constant/ErrorMessageConstants.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/constant/MessagePushConstants.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/constant/OrderConstant.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/constant/RedisConstants.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/dto/page/Order.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/dto/page/PageRequestDto.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/dto/page/PageResponseDto.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/dto/response/ApiResponse.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/ElasticSearchConst.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/ElasticsearchManager.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/RestHighLevelClientContextHolder.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/annotation/EntityField.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/annotation/QueryField.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/annotation/Score.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/config/GaussConfig.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/config/ScriptConfig.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/CombinatorWord.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/FullTextItem.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/FullTextQuery.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/ImportantBizPosEnum.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/ImportantPosEnum.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/Segmentation.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/SynonymWord.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/Word.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/enumeration/OccurEnum.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/enumeration/ParseEnum.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/impl/ElasticsearchManagerImpl.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/query/CustomQueryBuilder.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/query/FulltextQueryBuilder.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/AliasAction.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/BaseRequest.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/JsonAliasActionsRequest.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/JsonCreateIndexRequest.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/JsonSearchRequest.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/PlainRolloverRequest.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/SimpleGroupByJsonSearchRequest.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/response/AggregationsResult.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/response/Result.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/util/QueryUtil.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/enums/ResponseEnum.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/event/NacosServiceUpInfo.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/event/NacosServiceUpdateEvent.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/event/NacosServiceUpdateInfo.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/hash/ConsistencyHashing.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/hash/Node.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/hash/PhysicalNode.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/hash/VirtualNode.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/hash/algoithm/HashAlgorithm.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/hash/algoithm/KetamaHashAlgorithm.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/hash/repository/HashRouterRepository.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/http/RestMapProperties.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/http/RestProperties.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/http/RestTemplateFactory.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/redis/RedisService.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/util/AppGenerator.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/util/ArrayUtil.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/util/BeanUtil.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/util/ExchangeRequestUtils.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/util/HttpUtils.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/util/IpUtils.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/util/JsonUtil.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/util/ListUtil.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/util/MD5Util.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/util/ObjectUtil.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/util/PathVariableResolver.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/util/StrUtil.java create mode 100644 message-push-common/src/main/java/org/kangspace/messagepush/core/websocket/WebSocketSessionManager.java create mode 100644 message-push-common/src/main/resources/META-INF/spring.factories create mode 100644 message-push-common/src/test/java/org/kangspace/messagepush/common/MD5UtilTest.java create mode 100644 message-push-common/src/test/java/org/kangspace/messagepush/common/PathVariableResolverTest.java create mode 100644 message-push-consumer/.gitignore create mode 100644 message-push-consumer/README.md create mode 100644 message-push-consumer/message-push-consumer-core/pom.xml create mode 100644 message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/config/ControllerAccessConfig.java create mode 100644 message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/config/ElasticSearchIndexInitial.java create mode 100644 message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/config/PushConsumerConfiguration.java create mode 100644 message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/config/TaskExecutorConfig.java create mode 100644 message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/config/WebMvcConfig.java create mode 100644 message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/domain/dto/request/MessagePushRequestDto.java create mode 100644 message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/feign/DefaultFeignFallBack.java create mode 100644 message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/feign/MessagePushWsApi.java create mode 100644 message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/feign/MessagePushWsApiFeignFallBack.java create mode 100644 message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/feign/TempFeignAPi.java create mode 100644 message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/feign/config/MessagePushWsApiFeignConfig.java create mode 100644 message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/hash/HashRouterLoader.java create mode 100644 message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/mq/kafka/KafkaBindingConfig.java create mode 100644 message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/mq/kafka/MessagePushChannel.java create mode 100644 message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/mq/kafka/MessagePushConsumer.java create mode 100644 message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/redis/RedisService.java create mode 100644 message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/service/BaseService.java create mode 100644 message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/service/ElasticSearchService.java create mode 100644 message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/service/MessagePushConsumerService.java create mode 100644 message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/service/impl/MessagePushServiceImpl.java create mode 100644 message-push-consumer/message-push-consumer-core/src/test/java/org/kangspace/messagepush/consumer/core/Test.java create mode 100644 message-push-consumer/message-push-consumer-microservice/pom.xml create mode 100644 message-push-consumer/message-push-consumer-microservice/src/main/java/org/kangspace/messagepush/consumer/MessagePushConsumerApplication.java create mode 100644 message-push-consumer/message-push-consumer-microservice/src/main/resources/bootstrap.yml create mode 100644 message-push-consumer/message-push-consumer-microservice/src/test/java/org/kangspace/messagepush/consumer/MessagePushServiceTest.java create mode 100644 message-push-consumer/message-push-consumer-microservice/src/test/java/org/kangspace/messagepush/consumer/Test.java create mode 100644 message-push-consumer/pom.xml create mode 100644 message-push-rest/.gitignore create mode 100644 message-push-rest/README.md create mode 100644 message-push-rest/message-push-rest-api/pom.xml create mode 100644 message-push-rest/message-push-rest-api/src/main/java/org/kangspace/messagepush/rest/api/dto/ApiBaseDTO.java create mode 100644 message-push-rest/message-push-rest-api/src/main/java/org/kangspace/messagepush/rest/api/dto/request/MessagePushRequestDTO.java create mode 100644 message-push-rest/message-push-rest-api/src/main/java/org/kangspace/messagepush/rest/api/dto/request/MessagePushRequestTimeDTO.java create mode 100644 message-push-rest/message-push-rest-core/pom.xml create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/auth/ApiAuthentication.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/auth/AuthRequestAop.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/config/ControllerAccessConfig.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/config/KafkaBindingConfig.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/config/WebMvcConfig.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/constant/AppThreadLocal.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/constant/MessagePushConstants.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/domain/entity/TemplateEntity.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/feign/DefaultFeignFallBack.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/feign/TempFeignAPi.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/manager/TemplateManager.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/manager/TemplateManagerImpl.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/mapper/TemplateMapper.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/mq/kafka/KafkaMqSender.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/mq/kafka/KafkaMqTopicChannelMapping.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/mq/kafka/MessagePushChannel.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/properties/TemplateProperties.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/service/BaseService.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/service/MessagePushService.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/service/impl/MessagePushServiceImpl.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/utils/AppGenerator.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/utils/IpUtil.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/utils/UidCoder.java create mode 100644 message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/utils/VariableUtils.java create mode 100644 message-push-rest/message-push-rest-core/src/test/java/org/kangspace/messagepush/rest/core/Test.java create mode 100644 message-push-rest/message-push-rest-microservice/pom.xml create mode 100644 message-push-rest/message-push-rest-microservice/src/main/java/org/kangspace/messagepush/rest/MessagePushRestApplication.java create mode 100644 message-push-rest/message-push-rest-microservice/src/main/java/org/kangspace/messagepush/rest/controller/MessagePushRestController.java create mode 100644 message-push-rest/message-push-rest-microservice/src/main/resources/bootstrap.yml create mode 100644 message-push-rest/message-push-rest-microservice/src/test/java/org/kangspace/messagepush/rest/Test.java create mode 100644 message-push-rest/pom.xml create mode 100644 message-push-ws-gateway/.gitignore create mode 100644 message-push-ws-gateway/README.md create mode 100644 message-push-ws-gateway/pom.xml create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/MessagePushWsGatewayApplication.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/config/EventListenerConfiguration.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/config/MessagePushWsGatewayAutoConfiguration.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/config/MessagePushWsGatewayConfiguration.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/config/MessagePushWsGatewayProperties.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/constant/MessagePushWsConstants.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/debug/DebugWebsocketSessionCountScheduler.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/domain/dto/request/SessionCenterUserInfoParamDTO.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/domain/dto/response/SessionCenterUserInfoDTO.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/exception/GatewayExceptionHandler.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/exception/TokenValidateException.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/feign/DefaultFeignFallBack.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/feign/PassportSessionFeignApi.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/feign/PassportSessionFeignFallBack.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/feign/TempFeignAPi.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/BaseFilter.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/FilterOrders.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/GatewayWebsocketRoutingFilter.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/RequestValidateFilter.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/WebsocketReactiveLoadBalancerClientFilter.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/balancer/LbServiceInstanceChooser.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/balancer/RoundRibbonServiceInstanceChooser.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/balancer/UIDServiceInstanceChooser.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/balancer/WebSocketLoadBalancer.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/session/SessionProxyHolder.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/session/WebSocketUserSessionManager.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/hash/DebugPrintHashingScheduler.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/hash/RedisHashRouterRepository.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/model/MessageRequestParam.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/nacos/NacosDynamicServerListListener.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/nacos/NacosNamingService.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/util/ExchangeRequestUtils.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/validation/PassportSessionValidator.java create mode 100644 message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/validation/TokenValidator.java create mode 100644 message-push-ws-gateway/src/main/resources/META-INF/spring.factories create mode 100644 message-push-ws-gateway/src/main/resources/bootstrap.yml create mode 100644 message-push-ws-gateway/src/test/java/org/kangspace/messagepush/ws/gateway/RedisHashRouterRepositoryTest.java create mode 100644 message-push-ws-gateway/src/test/java/org/kangspace/messagepush/ws/gateway/WebSocketUserSessionManagerTest.java create mode 100644 message-push-ws/.gitignore create mode 100644 message-push-ws/README.md create mode 100644 message-push-ws/message-push-ws-api/pom.xml create mode 100644 message-push-ws/message-push-ws-core/pom.xml create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/config/ControllerExceptionConfig.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/config/WebsocketConfiguration.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/constant/MessagePushCmdEnum.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/constant/MessagePushResponseTypeEnum.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/constant/MessagePushWsConstants.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/debug/DebugWebsocketSessionCountScheduler.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/MessageCmdDTO.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/MessageCmdResponseDTO.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/request/LoginReqDataDTO.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/response/HeartBeatRespDataDTO.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/response/MessageRespDataDTO.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/response/NormalRespDataDTO.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/model/HttpPushMessageDTO.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/model/MessageRequestParam.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/feign/DefaultFeignFallBack.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/feign/TempFeignAPi.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/service/BaseService.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/service/MessagePushWsService.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/service/impl/MessagePushWsServiceImpl.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/utils/MessagePushHandlerUtils.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/BaseWebcosketHandler.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/HttpPushMessageConsumer.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/HttpPushMessagePublisher.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/MessagePushHandshakeWebSocketService.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/MessagePushWebSocketHandler.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/WebsocketUtils.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/session/SessionHolder.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/session/WebSocketSessionManager.java create mode 100644 message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/session/WebSocketUserSessionManager.java create mode 100644 message-push-ws/message-push-ws-core/src/test/java/org/kangspace/messagepush/ws/core/FluxTest.java create mode 100644 message-push-ws/message-push-ws-core/src/test/java/org/kangspace/messagepush/ws/core/HttpPushMessagePublisherTest.java create mode 100644 message-push-ws/message-push-ws-microservice/pom.xml create mode 100644 message-push-ws/message-push-ws-microservice/src/main/java/org/kangspace/messagepush/ws/MessagePushWsApplication.java create mode 100644 message-push-ws/message-push-ws-microservice/src/main/java/org/kangspace/messagepush/ws/controller/MessagePushRestController.java create mode 100644 message-push-ws/message-push-ws-microservice/src/main/resources/bootstrap.yml create mode 100644 message-push-ws/message-push-ws-microservice/src/test/java/org/kangspace/messagepush/ws/Test.java create mode 100644 message-push-ws/pom.xml diff --git a/message-push-common/.gitignore b/message-push-common/.gitignore new file mode 100644 index 0000000..84f2628 --- /dev/null +++ b/message-push-common/.gitignore @@ -0,0 +1,13 @@ +.idea +/message-push-common*/target +/message-push-common-*/target +/message-push-common*/target/* +/message-push-common-*/target/* + +.DS_Store +*.iml + +/.idea/* +/target/* + +!.mvn/wrapper/maven-wrapper.jar \ No newline at end of file diff --git a/message-push-common/README.md b/message-push-common/README.md new file mode 100644 index 0000000..a105c1d --- /dev/null +++ b/message-push-common/README.md @@ -0,0 +1,3 @@ +# message-push-common + +消息推送项目公共包 \ No newline at end of file diff --git a/message-push-common/pom.xml b/message-push-common/pom.xml new file mode 100644 index 0000000..8f4c6d2 --- /dev/null +++ b/message-push-common/pom.xml @@ -0,0 +1,134 @@ + + + 4.0.0 + + + org.kangspace.messagepush + message-push + ${revision} + + + message-push-common + ${revision} + + + 1.8 + 8 + 8 + + + + + + + com.alibaba.cloud + spring-cloud-starter-alibaba-nacos-discovery + true + provided + + + + + com.google.guava + guava + + + + org.apache.commons + commons-lang3 + + + + commons-collections + commons-collections + + + + com.alibaba + fastjson + + + + javax.servlet + javax.servlet-api + provided + + + + org.springframework.data + spring-data-redis + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + + + + jakarta.annotation + jakarta.annotation-api + + + + jakarta.validation + jakarta.validation-api + + + + org.springframework.boot + spring-boot-starter-web + provided + + + + org.springframework.boot + spring-boot-starter-webflux + provided + + + + cn.hutool + hutool-all + + + + + + + ${project.artifactId} + + + src/main/resources + true + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.12.4 + + true + + + + + + + + spring-milestones + Spring Milestones + https://repo.spring.io/milestone + + + maven2 + maven2 + https://repo1.maven.org/maven2 + + + + + \ No newline at end of file diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/constant/ErrorMessageConstants.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/constant/ErrorMessageConstants.java new file mode 100644 index 0000000..310d87f --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/constant/ErrorMessageConstants.java @@ -0,0 +1,18 @@ +package org.kangspace.messagepush.core.constant; + +/** + * 错误消息常量类 + * + * @author kango2gler@gmail.com + * @since 2021/10/27 + */ +public interface ErrorMessageConstants { + /** + * 协议错误 + */ + String INVALID_PROTOCOL_TYPE_MSG = "错误的请求协议,要求协议为:%s,实际为:%s"; + /** + * 请求参数错误 + */ + String INVALID_REQUEST_PARAM_MSG = "错误的请求参数,要求为:%s,实际为:%s"; +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/constant/MessagePushConstants.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/constant/MessagePushConstants.java new file mode 100644 index 0000000..05cde68 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/constant/MessagePushConstants.java @@ -0,0 +1,84 @@ +package org.kangspace.messagepush.core.constant; + +/** + * 常量类 + * + * @author kango2gler@gmail.com + * @date 2021-08-25 + */ +public interface MessagePushConstants { + + /** + * 服务路由前缀 + */ + String LB_PATH = "lb://"; + + /** + * 成功码 + */ + Integer SUCCESS_CODE = 1; + /** + * 失败码 + */ + Integer FAIL_CODE = 0; + + /** + * Response body 属性 + */ + String CACHED_RESPONSE_BODY_ATTR = "customCachedResponseBody"; + + /** + * WebSocket 支持的协议 + */ + String[] WEBSOCKET_PROTOCOLS = {"ws", "wss"}; + + /** + * UID Http请求头Key + */ + String HTTP_HEADER_UID_KEY = "uid"; + /** + * auth-app-id Http请求头,值为生成Passport Token使用的AppId + */ + String HTTP_HEADER_AUTH_APP_ID_KEY = "auth-app-id"; + + /** + * 请求参数 Exchange Attr Key + */ + String EXCHANGE_ATTR_REQUEST_PARAM = "_requestParam"; + + /** + * 一致性HASH每个物理节点的虚拟节点数 + * (每个物理节点的总虚拟节点保持在100-200,默认取:160,此处默认值:40(每个虚拟节点会生成4个节点)) + */ + int NUMBER_OF_VIRTUAL_NODE = 40; + + /** + * 消息推送Websocket服务名 + */ + String MESSAGE_WS_SERVICE_ID = "message-push-ws-microservice"; + + /** + * 用户Session绑定 ServerWebExchange属性key + */ + String USER_SESSION_HOLDER_EXCHANGE_ATTR_KEY = "user-session-holder"; + + /** + * 用户Session绑定 ServerWebExchange属性key + */ + String WEBSOCKET_TARGET_SERVICE_NODE_ATTR_KEY = "websocket-target-service-node"; + + /** + * 推送目标平台 + */ + enum PUSH_PLATFORM { + ALL, + H5, + Android, + iOS; + + @Override + public String toString() { + return super.toString().toLowerCase(); + } + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/constant/OrderConstant.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/constant/OrderConstant.java new file mode 100644 index 0000000..e0c03a4 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/constant/OrderConstant.java @@ -0,0 +1,21 @@ +package org.kangspace.messagepush.core.constant; + +/** + * 排序常量类 + * + * @author kango2gler@gmail.com + * @since 2021-04-23 + */ +public class OrderConstant { + + /** + * 顺序 + */ + public static final Integer ASC = 1; + + /** + * 逆序 + */ + public static final Integer DESC = 2; + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/constant/RedisConstants.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/constant/RedisConstants.java new file mode 100644 index 0000000..9a7e440 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/constant/RedisConstants.java @@ -0,0 +1,37 @@ +package org.kangspace.messagepush.core.constant; + +/** + * Redis相关常量类 + * + * @author kango2gler@gmail.com + * @since 2021/10/27 + */ +public interface RedisConstants { + + String DELIMTRER = ":"; + /** + * 消息推送相关key前缀 + */ + String MESSAGE_PUSH_BASE_KEY = "message_push"; + /** + * 消息推送相关Hash环key + */ + String MESSAGE_PUSH_HASH_RING_KEY = MESSAGE_PUSH_BASE_KEY + DELIMTRER + "hash_ring"; + /** + * 消息推送相关Hash环更新同步锁key + */ + String MESSAGE_PUSH_HASH_RING_STORE_LOCK_KEY = MESSAGE_PUSH_BASE_KEY + DELIMTRER + "hash_ring_store_lock"; + /** + * 消息推送相关Hash环更新同步锁超时时间,单位s + */ + long MESSAGE_PUSH_HASH_RING_STORE_LOCK_EXPIRE_SEC = 3; + + /** + * 实例用户映射 map key + * key: 服务实例 ip:port + * value: array 用户ID列表 + */ + String MESSAGE_PUSH_INSTANCE_USER_MAP_KEY = MESSAGE_PUSH_BASE_KEY + DELIMTRER + "instance_user"; + + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/dto/page/Order.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/dto/page/Order.java new file mode 100644 index 0000000..8a00291 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/dto/page/Order.java @@ -0,0 +1,30 @@ +package org.kangspace.messagepush.core.dto.page; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +/** + * 排序字段 + * + * @author kango2gler@gmail.com + * @since 2021-04-23 + */ +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +public class Order { + + /** + * 字段名 + */ + private String field; + + /** + * 排序方式，ApiConst.ASC或ApiConst.DESC + */ + private Integer sequence; + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/dto/page/PageRequestDto.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/dto/page/PageRequestDto.java new file mode 100644 index 0000000..1a4239a --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/dto/page/PageRequestDto.java @@ -0,0 +1,51 @@ +package org.kangspace.messagepush.core.dto.page; + + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import java.util.List; + +/** + * 通用列表页请求DTO + * + * @author kango2gler@gmail.com + * @since 2021-04-23 + */ +@Data +@SuperBuilder +@AllArgsConstructor +@NoArgsConstructor +public class PageRequestDto { + + /** + * 页索引，需大于等于1 + */ + @NotNull(message = "pageIndex不能为空") + @Min(value = 1, message = "pageIndex必须大于等于1") + private Integer pageNum; + + /** + * 页大小 + */ + @NotNull(message = "pageSize不能为空") + @Min(value = 1, message = "pageSize必须大于等于1") + private Integer pageSize; + + /** + * 查询字段，可使用ApiUtil.getFields辅助构建 + */ + @NotEmpty(message = "fields 字段不能为NULL和空集合") + private List fields; + + /** + * 排序字符串,可使用ApiUtil.getOrders、ApiUtil.getAscOrder、ApiUtil.getDescOrder辅助构建 + */ + private List orders; + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/dto/page/PageResponseDto.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/dto/page/PageResponseDto.java new file mode 100644 index 0000000..163fe77 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/dto/page/PageResponseDto.java @@ -0,0 +1,37 @@ +package org.kangspace.messagepush.core.dto.page; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.List; + +/** + * 通用列表页响应DTO + * + * @author kango2gler@gmail.com + * @since 2021-04-23 + */ +@Data +@SuperBuilder +@NoArgsConstructor +@AllArgsConstructor +public class PageResponseDto { + + /** + * 总记录数 + */ + private Long totalCount; + + /** + * 总页数 + */ + private Long totalPages; + + /** + * 响应记录列表 + */ + private List list; + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/dto/response/ApiResponse.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/dto/response/ApiResponse.java new file mode 100644 index 0000000..730054c --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/dto/response/ApiResponse.java @@ -0,0 +1,69 @@ +package org.kangspace.messagepush.core.dto.response; + +import org.kangspace.messagepush.core.enums.ResponseEnum; + +/** + * @author kango2gler@gmail.com + * @date 2024/7/13 + * @since + */ +public class ApiResponse { + private Integer code; + private String msg; + private T data; + + public ApiResponse() { + this.code = ResponseEnum.OK.getValue(); + } + + public ApiResponse(T data) { + this.code = ResponseEnum.OK.getValue(); + this.data = data; + } + + public ApiResponse(ResponseEnum responseEnum) { + this.code = responseEnum.getValue(); + this.msg = responseEnum.getReasonPhrase(); + this.data = null; + } + + public ApiResponse(Integer code, String msg) { + this.code = code; + this.msg = msg; + this.data = null; + } + + public ApiResponse(final Integer code, final String msg, final T data) { + this.code = code; + this.msg = msg; + this.data = data; + } + + public Integer getCode() { + return this.code; + } + + public void setCode(final Integer code) { + this.code = code; + } + + public String getMsg() { + return this.msg; + } + + public void setMsg(final String msg) { + this.msg = msg; + } + + public T getData() { + return this.data; + } + + public void setData(final T data) { + this.data = data; + } + + public String toString() { + return "ApiResponse(code=" + this.getCode() + ", msg=" + this.getMsg() + ", data=" + this.getData() + ")"; + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/ElasticSearchConst.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/ElasticSearchConst.java new file mode 100644 index 0000000..0499b77 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/ElasticSearchConst.java @@ -0,0 +1,489 @@ +package org.kangspace.messagepush.core.elasticsearch; + +/** + * @author kango2gler@gmail.com + * @date 2024/7/13 + * @since + */ +public interface ElasticSearchConst { + + /** + * 分页最大查询记录数 TODO 后续考虑使用 scroll解决查询不可超过10000的问题 + */ + int MAX_PAGE_SEARCH_COUNT = 10000; + /** + * 默认分页大小 + */ + int DEFAULT_PAGE_SIZE = 10; + /** + * es默认时间格式 + */ + String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss"; + /** + * 分词rest接口连接池名称 + */ + String REST_TEMPLATE_POOL = "elasticsearchFullText"; + + /** + * 分词rest接口分词参数名 + */ + String KW = "kw"; + + Long ZERO = 0L; + // ***************************************创建索引开始*************************************** + /** + * 索引开启状态 + */ + String STATE = "state"; + /** + * 索引开启状态-open + */ + String OPEN = "open"; + /** + * 设置项 + */ + String SETTINGS = "settings"; + /** + * 分片数 + */ + String NUMBER_OF_SHARDS = "number_of_shards"; + /** + * 副本数 + */ + String NUMBER_OF_REPLICAS = "number_of_replicas"; + /** + * 别名ALIASES + */ + String ALIASES = "aliases"; + /** + * 别名ALIAS + */ + String ALIAS = "alias"; + /** + * 别名操作 请求路径 + */ + String ALIASES_ENDPOINT = "_aliases"; + /** + * 别名actions + */ + String ALIASES_ACTIONS = "actions"; + /** + * 别名actions add + */ + String ALIASES_ACTIONS_ADD = "add"; + /** + * 别名actions remove + */ + String ALIASES_ACTIONS_REMOVE = "remove"; + /** + * 索引key + */ + String INDEX = "index"; + /** + * is_write_index + */ + String IS_WRITE_INDEX = "is_write_index"; + /** + * 映射 + */ + String MAPPINGS = "mappings"; + /** + * 属性 + */ + String PROPERTIES = "properties"; + /** + * 数据类型 + */ + String TYPE = "type"; + /** + * 数据类型 - 文本 + */ + String TEXT = "text"; + /** + * 数据类型 - 关键字 + */ + String KEYWORD = "keyword"; + /** + * 数据结构 - 有符号的8位整数, 范围: [-128 ~ 127] + */ + String BYTE = "byte"; + /** + * 数据类型 - 有符号的16位整数, 范围: [-32768 ~ 32767] + */ + String SHORT = "short"; + /** + * 数据类型 - 有符号的32位整数, 范围: [$-2^{31}$ ~ $2^{31}$-1] + */ + String INTEGER = "integer"; + /** + * 数据类型 - 有符号的32位整数, 范围: [$-2^{63}$ ~ $2^{63}$-1] + */ + String LONG = "long"; + /** + * 数据类型 - 32位单精度浮点数 + */ + String FLOAT = "float"; + /** + * 数据类型 - 64位双精度浮点数 + */ + String DOUBLE = "double"; + /** + * 数据类型 - 日期 + */ + String DATE = "date"; + /** + * 数据类型 - 布尔 + */ + String BOOLEAN = "boolean"; + /** + * 索引的分词器,一般索引和搜索用同样的分词器，如需不一样可更改 + */ + String SEARCH_ANALYZER = "search_analyzer"; + /** + * 字符串分析器,默认值:"standard" + */ + String ANALYZER = "analyzer"; + /** + * 字符串专用，查询时将term-document关系存储在内存中 + */ + String FIELDDATA = "fielddata"; + /** + * BM25/classic/boolean，主要用于文本字段的相似度算法，默认值："BM25" + */ + String SIMILARITY = "similarity"; + /** + * 默认情况字段被索引可以搜索，但没有存储原始值且不能用原始值查询，_resource包含了所有的值，当大段文本需要搜索时可以修改为true，默认值：false + */ + String STORE = "store"; + /** + * docs(只索引文档编号)/freqs(索引文档编号和词频)/positions(索引文档编号/词频/词位置)/offsets(索引文档编号/词频/词偏移量/词位置) ,被索引的字段默认用positions，其他的docs，默认值：positions/docs + */ + String INDEX_OPTIONS = "index_options"; + + // ***************************************创建索引结束*************************************** + + // ***************************************查询root开始*************************************** + + /** + * 起始行 + */ + String FROM = "from"; + /** + * 查询数量 + */ + String SIZE = "size"; + /** + * 单分片扫描终结数量（准确度换速度） + */ + String TERMINATE_AFTER = "terminate_after"; + /** + * 等待超时时间 + */ + String TIMEOUT = "timeout"; + /** + * 显示列表 + */ + String UNDERSCORE_SOURCE = "_source"; + /** + * 查询 + */ + String QUERY = "query"; + /** + * 高亮 + */ + String HIGHLIGHT = "highlight"; + /** + * 排序 + */ + String SORT = "sort"; + + /** + * 分数 + */ + String UNDERSCORE_SCORE = "_score"; + + /** + * 返回每个搜索命中的版本 + */ + String VERSION = "version"; + /** + * 关联度得分计算 + */ + String EXPLAIN = "explain"; + /** + * 允许在搜索多个索引时为每个索引配置不同的提升权重 + */ + String INDICES_BOOST = "indices_boost"; + /** + * 排除_score小于min_score中指定的最小值的文档 + */ + String MIN_SCORE = "min_score"; + /** + * 对query和post_filter阶段返回的Top-K结果执行第二个查询 + */ + String RESCORE = "rescore"; + + + // ***************************************查询root结束*************************************** + + + // ***************************************查询query开始*************************************** + /** + * 布尔。包含使用must、should、must_not、filter等。 + * "bool": { + * "must": { "match": { "email": "business opportunity" }}, + * "should": [ + * { "match": { "starred": true }}, + * { "bool": { + * "must": { "match": { "folder": "inbox" }}, + * "must_not": { "match": { "spam": true }} + * }} + * ], + * "minimum_should_match": 1 + * } + */ + String BOOL = "bool"; + /** + * 过滤 + */ + String FILTER = "filter"; + /** + * 与 + */ + String MUST = "must"; + /** + * 或 + */ + String SHOULD = "should"; + /** + * 最小匹配度，跟在should后 + */ + String MINIMUM_SHOULD_MATCH = "minimum_should_match"; + /** + * 非 + */ + String MUST_NOT = "must_not"; + /** + * 关键词项， "term": { "age":26} + */ + String TERM = "term"; + /** + * 关键词项数组， + * "terms": { + * "tag": [ "search", "full_text", "nosql" ] + * } + */ + String TERMS = "terms"; + /** + * 匹配查询， "match": { "content": "中国杭州" } + */ + String MATCH = "match"; + /** + * 
+     * 分词匹配-短语匹配查询
+     * 如: "match_phrase": { "content": "中国杭州" }
+     * 结果: 会匹配到"%中国杭州%"的数据
+     *
+     *
+ */ + String MATCH_PHRASE = "match_phrase"; + + /** + * 
+     * 分词匹配-短语匹配查询-对最后一个分词进行通配符匹配
+     * 如: "match_phrase_prefix": { "content": "abc" }
+     * 结果: 会匹配到"aaa abc def","abcdef"的数据
+     *
+     *
+ */ + String MATCH_PHRASE_PREFIX = "match_phrase_prefix"; + + /** + * 
+     * 前缀查询-对前缀后内容做匹配，类似sql中的like
+     * 如: "prefix": { "content": "abc" }
+     * 结果: 会匹配到前缀为"abc"的数据
+     *
+     *
+ */ + String PREFIX = "prefix"; + + /** + * 查询所有文档，"match_all": {} + */ + String MATCH_ALL = "match_all"; + + /** + * 分组标记 + */ + String AGGS = "aggs"; + + /** + * 指定多个字段 + * "multi_match": { + * "query": "full text search", + * "fields": [ "title", "body" ] + * } + */ + String MULTI_MATCH = "multi_match"; + /** + * 属性 + */ + String FIELDS = "fields"; + /** + * 分词项的共同前缀长度，"range": {"age": {"gte": 20,"lt": 30}} + */ + String RANGE = "range"; + /** + * 大于 + */ + String GT = "gt"; + /** + * 大于等于 + */ + String GTE = "gte"; + /** + * 小于 + */ + String LT = "lt"; + /** + * 小于等于 + */ + String LTE = "lte"; + /** + * 存在 "exists": {"field": "title"} + */ + String EXISTS = "exists"; + /** + * 属性 + */ + String FIELD = "field"; + /** + * 模糊查询 + */ + String FUZZY = "fuzzy"; + /** + * 相关性提高权重 + */ + String BOOST = "boost"; + /** + * 匹配的最小相似度 + */ + String MIN_SIMILARITY = "min_similarity"; + /** + * 分词项的共同前缀长度 + */ + String PREFIX_LENGTH = "prefix_length"; + /** + * + */ + String SOURCE = "source"; + /** + * 值 + */ + String VALUE = "value"; + /** + * + */ + String QUERY_STRING = "query_string"; + /** + * 缓存 + */ + String CACHE = "_cache"; + /** + * 方法 + */ + String FUNCTION_SCORE = "function_score"; + /** + * 方法集 + */ + String FUNCTIONS = "functions"; + /** + * 高斯衰减函数名 + */ + String GAUSS = "gauss"; + /** + * 起始值 + */ + String ORIGIN = "origin"; + /** + * 级别因子 + */ + String SCALE = "scale"; + /** + * 补偿系数 + */ + String OFFSET = "offset"; + /** + * 衰减系数 + */ + String DECAY = "decay"; + /** + * 脚本打分方法 + */ + String SCRIPT_SCORE = "script_score"; + /** + * 脚本 + */ + String SCRIPT = "script"; + /** + * 脚本行 + */ + String INLINE = "inline"; + /** + * 权重 + */ + String WEIGHT = "weight"; + /** + * 函数内部各方法怎样汇总 + */ + String SCORE_MODE = "score_mode"; + /** + * 与query查询分的计算方式 + */ + String BOOST_MODE = "boost_mode"; + /** + * score_mode或boost_mode或其他的运算方式 + */ + String MULTIPLY = "multiply"; + /** + * score_mode或boost_mode或其他的运算方式 + */ + String REPLACE = "replace"; + /** + * score_mode或boost_mode或其他的运算方式 + */ + String SUM = "sum"; + /** + * score_mode或boost_mode或其他的运算方式 + */ + String AVG = "avg"; + /** + * score_mode或boost_mode或其他的运算方式 + */ + String MAX = "max"; + /** + * score_mode或boost_mode或其他的运算方式 + */ + String MIN = "min"; + + // ***************************************查询query结束*************************************** + + // ***************************************查询其他项开始*************************************** + /** + * 顺序 + */ + String ASC = "asc"; + /** + * 逆序 + */ + String DESC = "desc"; + + /** + * 获取真实的总数开关 + */ + String TRACK_TOTAL_HITS = "track_total_hits"; + +// ***************************************查询其他项结束*************************************** + String DEFAULT = "default"; +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/ElasticsearchManager.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/ElasticsearchManager.java new file mode 100644 index 0000000..9c2c11a --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/ElasticsearchManager.java @@ -0,0 +1,404 @@ +package org.kangspace.messagepush.core.elasticsearch; + + +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.index.query.QueryBuilder; +import org.kangspace.messagepush.core.dto.page.PageResponseDto; +import org.kangspace.messagepush.core.elasticsearch.request.*; +import org.kangspace.messagepush.core.elasticsearch.response.AggregationsResult; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + +/** + * ES方法Dao层接口 + * + * @author kango2gler@gmail.com + */ +public interface ElasticsearchManager { + + /** + * 判断指定的索引名是否存在 + * + * @param index 索引名 + * @return 存在：true; 不存在：false; + * @throws IOException Io异常 + */ + Boolean existsIndex(String index) throws IOException; + + /** + * 创建索引 + * + * @param jsonIndexRequest json创建索引对象 + * @return 创建索引是否成功 + * @throws Exception 异常 + */ + Boolean createIndex(JsonCreateIndexRequest jsonIndexRequest) throws Exception; + + /** + * 删除索引(慎用) + * + * @param index 索引名 + * @return 删除索引是否成功 + * @throws IOException 异常 + */ + Boolean deleteIndex(String index) throws IOException; + + /** + * 判断指定别名是否存在 + * + * @param alias 别名 + * @return true: 存在, false: 不存在 + * @throws IOException IO异常 + */ + Boolean existsAlias(String alias) throws IOException; + + /** + * 别名操作 + * + * @param jsonAliasActionsRequest 别名操作对象 + * @return true:成功, false:失败 + * @throws IOException + */ + Boolean aliasAction(JsonAliasActionsRequest jsonAliasActionsRequest) throws IOException; + + /** + * 别名滚动索引 + * + * @param rolloverRequest 滚动请求 + * @return true: 需要滚动/滚动成功,false: 无需滚动 + * @throws IOException IO异常 + */ + Boolean rollover(PlainRolloverRequest rolloverRequest) throws IOException; + + /** + * 插入数据 + * + * @param index 索引 + * @param entity 插入实体 + * @param 实体泛型 + * @return 插入是否成功 + * @throws Exception 异常 + */ + Boolean insert(String index, T entity) throws Exception; + + /** + * 批量插入数据 + * + * @param index 索引 + * @param list 插入列表 + * @param 实体泛型 + * @return 插入是否成功 + * @throws Exception 异常 + */ + Boolean insert(String index, List list) throws Exception; + + /** + * 批量插入数据，有返回值 + * + * @param index + * @param list + * @return + * @throws Exception + */ + List batchInsert(String index, List list) throws Exception; + + /** + * 批量插入数据 + * + * @param index 索引 + * @param idKey es id主键 + * @param list 插入列表 + * @return 插入是否成功 + * @throws Exception 异常 + */ + Boolean insert(String index, String idKey, List> list) throws Exception; + + /** + * 拼装插入请求对象 + * + * @param index 索引 + * @param entity 实体 + * @param 实体泛型 + * @return 插入是否成功 + * @throws IOException IO异常 + * @throws IllegalAccessException 访问权限异常 + */ + IndexRequest getIndexRequest(String index, T entity) throws IOException, IllegalAccessException; + + /** + * 更新数据 + * + * @param index 索引 + * @param id id + * @param map 更新的数据 + * @throws Exception 异常 + */ + Boolean update(String index, String id, Map map) throws Exception; + + /** + * 更新数据 + * + * @param index 索引 + * @param id id + * @param entity 更新的实体类数据，不为null的属性都参与更新 + * @return 更新是否成功 + * @throws Exception 异常 + */ + Boolean update(String index, String id, T entity) throws Exception; + + /** + * 更新数据 + * + * @param index 索引 + * @param ids ids + * @param list 更新的数据列表 + * @return 更新是否成功 + * @throws Exception 异常 + */ + Boolean update(String index, List ids, List> list) throws Exception; + + /** + * 更新数据 + * + * @param index 索引 + * @param ids ids + * @param list 更新的数据列表 + * @return 更新是否成功 + * @throws Exception 异常 + */ + List batchUpdate(String index, List ids, List> list) throws Exception; + + /** + * 删除数据 + * + * @param index 索引 + * @param id es id + * @return 删除是否成功 + * @throws Exception 异常 + */ + Boolean delete(String index, String id) throws Exception; + + /** + * 批量删除数据 + * + * @param index 索引 + * @param ids es id列表 + * @return 删除是否成功 + * @throws Exception 异常 + */ + Boolean delete(String index, List ids) throws Exception; + + /** + * 获取一个ES数据对象 + * + * @param index 索引 + * @param id id + * @param classType 返回对象class对象 + * @param 返回对象泛型 + * @return 结果对象 + * @throws Exception 异常 + */ + T get(String index, String id, Class classType) throws Exception; + + /** + * 获取一个ES数据对象 + * + * @param index 索引 + * @param id id + * @param fields 需要返回的字段 + * @param classType 返回对象class对象 + * @param 返回对象泛型 + * @return 结果对象 + * @throws Exception 异常 + */ + T get(String index, String id, List fields, Class classType) throws Exception; + + /** + * 获取一个ES数据对象 + * + * @param jsonSearchRequest json查询对象 + * @param 返回对象泛型 + * @return 结果对象 + * @throws Exception 异常 + */ + T get(JsonSearchRequest jsonSearchRequest) throws Exception; + + /** + * 基于json脚本查询 + * + * @param jsonSearchRequest json查询对象 + * @param 返回对象泛型 + * @return 列表对象 + * @throws Exception 异常 + */ + List list(JsonSearchRequest jsonSearchRequest) throws Exception; + + /** + * 基于json脚本查询 + * + * @param jsonSearchRequest json查询对象 + * @param 返回对象泛型 + * @return 分页对象 + * @throws Exception 异常 + */ + PageResponseDto page(JsonSearchRequest jsonSearchRequest) throws Exception; + + /** + * 基于json脚本查询，并返回ES返回结果map + * + * @param jsonSearchRequest json查询对象 + * @return ES返回结果map + * @throws Exception 异常 + */ + Map map(JsonSearchRequest jsonSearchRequest) throws Exception; + + /** + * 查询数量 + * + * @param jsonSearchRequest json查询对象 + * @return 数量 + * @throws Exception 异常 + */ + Long count(JsonSearchRequest jsonSearchRequest) throws Exception; + + /** + * 批量更新/插入数据 + * + * @param index 索引 + * @param list 更新的数据列表 + * @param T 中使用{@link org.yaml.snakeyaml.events.Event.ID}指定为_doc的_id + * @return 更新是否成功 + * @throws Exception 异常 + */ + List batchUpsert(String index, List list) throws Exception; + + /** + * 批量更新/插入数据 + * + * @param index 索引 + * @param list 更新的数据列表 + * @param dataHandleOnInsert 插入时的数据处理 + * @return 更新是否成功 + * @throws Exception 异常 + */ + List batchUpsert(String index, List list, Consumer dataHandleOnInsert) throws Exception; + + /** + * 更新/插入数据 + * + * @param index 索引 + * @param id id + * @param entity 更新的实体类数据，不为null的属性都参与更新 + * @return 更新是否成功 + * @throws Exception 异常 + */ + Boolean upsert(String index, String id, T entity) throws Exception; + + /** + * 更新/插入数据 + * + * @param index 索引 + * @param id id + * @param entity 更新的实体类数据，不为null的属性都参与更新 + * @param dataHandleOnInsert 插入时的数据处理 + * @return 更新是否成功 + * @throws Exception 异常 + */ + Boolean upsert(String index, String id, T entity, Consumer dataHandleOnInsert) throws Exception; + + /** + * 简单分组查询 + * 如: + * 请求: + * + * { + * "size": 0, + * "query": { + * "match_all": {} + * }, + * "aggs": { + * "groupByIndex": { + * "terms": { + * "field": "index", + * "size": 1000 + * }, + * "aggs": { + * "groupByTime": { + * "terms": { + * "field": "time", + * "size": 1 + * } + * } + * } + * } + * } + * } + * 响应: + * { + * "took" : 2, + * "timed_out" : false, + * "_shards" : {...}, + * "hits" : {...}, + * "aggregations" : { + * "groupByIndex" : { + * "doc_count_error_upper_bound" : 0, + * "sum_other_doc_count" : 0, + * "buckets" : [ + * { + * "key" : "bcl-test-batch-upsert-index", + * "doc_count" : 10, + * "groupByTime" : { + * "doc_count_error_upper_bound" : 0, + * "sum_other_doc_count" : 9, + * "buckets" : [ + * { + * "key" : 1643018137436, + * "doc_count" : 1 + * } + * ] + * } + * } + * ] + * } + * } + * } + * + * 返回: + * [{ + * "index":bcl-test-batch-upsert-index, + * "time": 1643018137436 + * }] + * + * + * + * + * @param jsonSearchRequest + * @param + * @return T 对象列表 + * @throws Exception 异常 + */ + List simpleGroupBy(SimpleGroupByJsonSearchRequest jsonSearchRequest) throws Exception; + + /** + * 分组查询 + * + * @param jsonScript 完整的查询json语句 + * @return {@link AggregationsResult} + * @throws Exception ex + */ + AggregationsResult aggs(String[] indexes, String jsonScript) throws Exception; + + /** + * 通过查询更新数据 + * + * @param indexes 索引列表 + * @param query 查询条件 {@link QueryBuilder} + * @param updateMap 更新的参数, key: 更新的字段, value: 更新的值 + * @return 更新是否成功 + * @throws Exception ex + */ + Boolean updateByQuery(String[] indexes, QueryBuilder query, Map updateMap) throws Exception; +} \ No newline at end of file diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/RestHighLevelClientContextHolder.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/RestHighLevelClientContextHolder.java new file mode 100644 index 0000000..8002188 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/RestHighLevelClientContextHolder.java @@ -0,0 +1,82 @@ +package org.kangspace.messagepush.core.elasticsearch; + +import lombok.extern.slf4j.Slf4j; +import org.elasticsearch.client.RestHighLevelClient; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Es请求客户端上下文 + * + * @author kango2gler@gmail.com + */ +@Slf4j +public class RestHighLevelClientContextHolder { + + /** + * 存放所有的Es请求客户端 + */ + private static final Map REST_HIGH_LEVEL_CLIENT_MAP = new ConcurrentHashMap<>(); + + /** + * 线程级别的私有变量 + */ + private static final ThreadLocal HOLDER = ThreadLocal.withInitial(() -> ElasticSearchConst.DEFAULT); + + /** + * 根据名字获取当前线程的请求客户端 + * + * @return 数据源 + */ + public static RestHighLevelClient getRestHighLevelClientByKey() { + return REST_HIGH_LEVEL_CLIENT_MAP.get(HOLDER.get()); + } + + /** + * 获取当前线程的请求客户端的Key + * + * @return 数据源 + */ + public static String getRestHighLevelClientKey() { + return HOLDER.get(); + } + + /** + * 设置当前线程的请求客户端 + * + * @param restHighLevelClientKey 客户端key + */ + public static void setRestHighLevelClientKey(String restHighLevelClientKey) { + log.info("切换至{}RestHighLevelClient", restHighLevelClientKey); + HOLDER.set(restHighLevelClientKey); + } + + /** + * 设置之前RestHighLevelClient一定要先移除 + */ + public static void removeRestHighLevelClientKey() { + HOLDER.remove(); + } + + /** + * 判断指定的RestHighLevelClient当前是否存在 + * + * @param restHighLevelClientKey 连接名 + * @return true存在；false不存在 + */ + public static boolean containsRestHighLevelClientKey(String restHighLevelClientKey) { + return REST_HIGH_LEVEL_CLIENT_MAP.containsKey(restHighLevelClientKey); + } + + /** + * 添加请求客户端 + * + * @param restHighLevelClientKey 客户端key + * @param restHighLevelClient 客户端 + */ + public static void addRestHighLevelClient(String restHighLevelClientKey, RestHighLevelClient restHighLevelClient) { + REST_HIGH_LEVEL_CLIENT_MAP.put(restHighLevelClientKey, restHighLevelClient); + } + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/annotation/EntityField.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/annotation/EntityField.java new file mode 100644 index 0000000..0421e2d --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/annotation/EntityField.java @@ -0,0 +1,24 @@ +package org.kangspace.messagepush.core.elasticsearch.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * es属性 + * + * @author kango2gler@gmail.com + */ +@Target(ElementType.FIELD) +@Retention(RetentionPolicy.RUNTIME) +public @interface EntityField { + + /** + * es字段名 + * + * @return java.lang.String + */ + String field() default ""; + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/annotation/QueryField.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/annotation/QueryField.java new file mode 100644 index 0000000..550e488 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/annotation/QueryField.java @@ -0,0 +1,49 @@ +package org.kangspace.messagepush.core.elasticsearch.annotation; + + +import org.kangspace.messagepush.core.elasticsearch.enumeration.OccurEnum; +import org.kangspace.messagepush.core.elasticsearch.enumeration.ParseEnum; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * es查询对象辅助注解 + * + * @author kango2gler@gmail.com + */ +@Target(ElementType.FIELD) +@Retention(RetentionPolicy.RUNTIME) +public @interface QueryField { + /** + * 查询处理注解 + * + * @return ParseEnum + */ + ParseEnum value() default ParseEnum.TERM; + + /** + * es字段名 + * + * @return java.lang.String + */ + String field() default ""; + + /** + * 在must、filter、should中哪个位置中块拼装 + * + * @return OccurEnum + */ + OccurEnum occur() default OccurEnum.FILTER; + + /** + * 格式化表达式，主要用于日期格式，QueryUtil中默认格式：yyyy-MM-dd'T'HH:mm:ss + * + * @return java.lang.String + */ + String format() default ""; + + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/annotation/Score.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/annotation/Score.java new file mode 100644 index 0000000..0211afb --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/annotation/Score.java @@ -0,0 +1,16 @@ +package org.kangspace.messagepush.core.elasticsearch.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * ES分数注解 + * + * @author kango2gler@gmail.com + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD, ElementType.METHOD, ElementType.ANNOTATION_TYPE}) +public @interface Score { +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/config/GaussConfig.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/config/GaussConfig.java new file mode 100644 index 0000000..776bf14 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/config/GaussConfig.java @@ -0,0 +1,41 @@ +package org.kangspace.messagepush.core.elasticsearch.config; + +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; + +/** + * 高斯函数变量 + * + * @author kango2gler@gmail.com + */ +@Setter +@Getter +public class GaussConfig implements Serializable { + + /** + * 属性名 + */ + private String fieldName; + /** + * 起始值 + */ + private String origin; + /** + * 补偿系数 + */ + private String offset; + /** + * 级别因子 + */ + private String scale; + /** + * 衰减系数 + */ + private Double decay; + /** + * 权重 + */ + private Float weight; +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/config/ScriptConfig.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/config/ScriptConfig.java new file mode 100644 index 0000000..9ef5e83 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/config/ScriptConfig.java @@ -0,0 +1,25 @@ +package org.kangspace.messagepush.core.elasticsearch.config; + +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; + +/** + * 脚本配置 + * + * @author kango2gler@gmail.com + */ +@Setter +@Getter +public class ScriptConfig implements Serializable { + + /** + * 脚本 + */ + private String script; + /** + * 权重 + */ + private Float weight; +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/CombinatorWord.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/CombinatorWord.java new file mode 100644 index 0000000..38c1bf0 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/CombinatorWord.java @@ -0,0 +1,28 @@ +package org.kangspace.messagepush.core.elasticsearch.domain; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +/** + * @author kango2gler@gmail.com + * @description 逆向组合同义词，暂未用到 + */ +@Setter +@Getter +public class CombinatorWord { + + /** + * 逆向组合同义词 + */ + private Word word; + + /** + * 成员的索引集合 + */ + @JsonProperty(value = "Positions") + private List positionsList; + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/FullTextItem.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/FullTextItem.java new file mode 100644 index 0000000..5b20875 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/FullTextItem.java @@ -0,0 +1,53 @@ +package org.kangspace.messagepush.core.elasticsearch.domain; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.kangspace.messagepush.core.elasticsearch.config.GaussConfig; +import org.kangspace.messagepush.core.elasticsearch.config.ScriptConfig; + +import java.io.Serializable; +import java.util.List; + +/** + * 全文检索查询属性对象 + * + * @author kango2gler@gmail.com + */ +@Setter +@Getter +@NoArgsConstructor +@AllArgsConstructor +public class FullTextItem implements Serializable { + + /** + * 查询属性名 + */ + private String fieldName; + + /** + * 整体权重，影响拼接出来的结果，默认为1 + */ + private Float weight = 1F; + + /** + * 高斯衰减函数列表 + */ + private List gaussConfigs; + + /** + * 脚本函数列表 + */ + private List scriptConfigs; + + public FullTextItem(String fieldName) { + this.fieldName = fieldName; + } + + public FullTextItem(String fieldName, List gaussConfigs, List scriptConfigs) { + this.fieldName = fieldName; + this.gaussConfigs = gaussConfigs; + this.scriptConfigs = scriptConfigs; + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/FullTextQuery.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/FullTextQuery.java new file mode 100644 index 0000000..714166a --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/FullTextQuery.java @@ -0,0 +1,72 @@ +package org.kangspace.messagepush.core.elasticsearch.domain; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.kangspace.messagepush.core.elasticsearch.config.GaussConfig; +import org.kangspace.messagepush.core.elasticsearch.config.ScriptConfig; +import org.kangspace.messagepush.core.util.ListUtil; + +import java.io.Serializable; +import java.util.List; + +/** + * 全文检索查询对象 + * + * @author kango2gler@gmail.com + */ +@Setter +@Getter +@NoArgsConstructor +@AllArgsConstructor +public class FullTextQuery implements Serializable { + /** + * 词库URL + */ + private String ikUrl; + /** + * 搜索关键字，如果提供了关键字，但无分词结果，则不再进行全文检索；如果未提供关键字，则直接进行查询计算 + */ + private String keyword; + /** + * 全文检索项，一个字段一项 + */ + private List fullTextItems; + /** + * 是否需要打分。如果为true(默认值)，则全文检索并打分（分词分、高斯分、脚本分）；如果为false，则仅分词后拼如filter用于过滤查询 + */ + private boolean score = true; + /** + * 分词信息，构建对象时，如果传入则直接使用，如果没有传入则根据ikUrl和keyword调用词库获取。 + */ + private List words; + + /** + * 构建单个属性的需要打分的全文检索对象 + * + * @param ikUrl ikUrl + * @param fieldName fieldName + * @param gaussConfigs gaussConfigs + * @param scriptConfigs scriptConfigs + */ + public FullTextQuery(String ikUrl, String fieldName, String keyword, List gaussConfigs, List scriptConfigs) { + this.ikUrl = ikUrl; + this.keyword = keyword; + this.fullTextItems = ListUtil.getList(new FullTextItem(fieldName, gaussConfigs, scriptConfigs)); + } + + /** + * 构建单个属性的不需要打分的全文检索对象 + * + * @param ikUrl ikUrl + * @param fieldName fieldName + */ + public FullTextQuery(String ikUrl, String fieldName, String keyword) { + this.ikUrl = ikUrl; + this.keyword = keyword; + this.fullTextItems = ListUtil.getList(new FullTextItem(fieldName)); + this.score = false; + } + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/ImportantBizPosEnum.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/ImportantBizPosEnum.java new file mode 100644 index 0000000..b4a81ea --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/ImportantBizPosEnum.java @@ -0,0 +1,88 @@ +package org.kangspace.messagepush.core.elasticsearch.domain; + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.elasticsearch.ElasticSearchConst; + +import java.util.Objects; + +/** + * 重要业务词性以及对用的权重 + * + * @author kango2gler@gmail.com + */ +@Slf4j +public enum ImportantBizPosEnum { + + SW_KNOWLEDGE(131_072L, 1.4f), + + HX_KNOWLEDGE(65_536L, 1.4f), + + WL_KNOWLEDGE(32_768L, 1.4f), + + DL_KNOWLEDGE(16_384L, 1.4f), + + LS_KNOWLEDGE(8192L, 1.4f), + + ZZ_KNOWLEDGE(4096L, 1.4f), + + YY_KNOWLEDGE(2048L, 1.4f), + + SXKnowledge(1024L, 1.4f), + + YW_KNOWLEDGE(512L, 1.4f); + + /** + * 词性权重 + */ + private final Long posBizCode; + /** + * 权重 + */ + private final Float weight; + + ImportantBizPosEnum(Long posBizCode, Float weight) { + this.posBizCode = posBizCode; + this.weight = weight; + } + + public static ImportantBizPosEnum resolve(Long posBizCode) { + ImportantBizPosEnum response = null; + for (ImportantBizPosEnum importantBizPosEnum : ImportantBizPosEnum.values()) { + if (!Objects.equals(importantBizPosEnum.getPosBizCode() & posBizCode, ElasticSearchConst.ZERO)) { + log.debug("重要的业务词性：{}", importantBizPosEnum); + if (Objects.isNull(response) || importantBizPosEnum.weight > response.weight) { + response = importantBizPosEnum; + } + } + } + return response; + } + + public static Boolean judgeIsImportant(Long posBizCode) { + if (posBizCode == null) { + return false; + } + Long judge = 1L; + for (ImportantBizPosEnum importantBizPosEnum : ImportantBizPosEnum.values()) { + judge |= importantBizPosEnum.getPosBizCode(); + } + return !Objects.equals(judge & posBizCode, ElasticSearchConst.ZERO); + } + + public Long getPosBizCode() { + return posBizCode; + } + + public Float getWeight() { + return weight; + } + + @Override + public String toString() { + return "ImportantBizPosEnum{" + + "posBizCode=" + posBizCode + + ", weight=" + weight + + '}'; + } + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/ImportantPosEnum.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/ImportantPosEnum.java new file mode 100644 index 0000000..8df3f17 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/ImportantPosEnum.java @@ -0,0 +1,81 @@ +package org.kangspace.messagepush.core.elasticsearch.domain; + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.elasticsearch.ElasticSearchConst; + +import java.util.Objects; + +/** + * 重要基本词性以及对用的权重 + * + * @author kango2gler@gmail.com + */ +@Slf4j +public enum ImportantPosEnum { + + + /** + * 版本 + */ + POS_C_VS(137_438_953_472L, 1.1f), + + /** + * 文件格式 + */ + POS_C_EXT(274_877_906_944L, 1.1f); + + /** + * 词性编号 + */ + private final Long posCode; + /** + * 权重 + */ + private final Float weight; + + ImportantPosEnum(Long posCode, Float weight) { + this.posCode = posCode; + this.weight = weight; + } + + public static ImportantPosEnum resolve(Long posCode) { + ImportantPosEnum response = null; + for (ImportantPosEnum importantPosEnum : ImportantPosEnum.values()) { + if (!Objects.equals(importantPosEnum.getPosCode() & posCode, ElasticSearchConst.ZERO)) { + log.debug("命中的重要基本词性：{}", importantPosEnum); + if (Objects.isNull(response) || importantPosEnum.weight > response.weight) { + response = importantPosEnum; + } + } + } + return response; + } + + public static Boolean judgeIsImportant(Long posCode) { + if (posCode == null) { + return false; + } + Long judge = 1L; + for (ImportantPosEnum importantPosEnum : ImportantPosEnum.values()) { + judge |= importantPosEnum.getPosCode(); + } + return !Objects.equals(judge & posCode, ElasticSearchConst.ZERO); + } + + public Long getPosCode() { + return posCode; + } + + public Float getWeight() { + return weight; + } + + @Override + public String toString() { + return "ImportantPosEnum{" + + "posCode=" + posCode + + ", weight=" + weight + + '}'; + } + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/Segmentation.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/Segmentation.java new file mode 100644 index 0000000..0b13140 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/Segmentation.java @@ -0,0 +1,52 @@ +package org.kangspace.messagepush.core.elasticsearch.domain; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.util.List; + +/** + * @author kango2gler@gmail.com + * @description 分词结果DTO + */ +@Data +public class Segmentation { + + /** + * 消耗时间 + */ + private Integer elapsed; + + /** + * 是否成功 + */ + private Boolean isValid; + + /** + * 文本信息 + */ + private String message; + + /** + * 原始文本 + */ + private String originalWord; + + /** + * 分词文本 + */ + private String segmentWord; + + /** + * 分词集合 + */ + @JsonProperty(value = "synonymWords") + private List synonymWordList; + + /** + * 组合词集合 + */ + @JsonProperty(value = "combinatorWords") + private List combinatorWordList; + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/SynonymWord.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/SynonymWord.java new file mode 100644 index 0000000..5164f31 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/SynonymWord.java @@ -0,0 +1,24 @@ +package org.kangspace.messagepush.core.elasticsearch.domain; + +import lombok.Data; + +import java.util.List; + +/** + * @author kango2gler@gmail.com + * @description 分词集合 + */ +@Data +public class SynonymWord { + + /** + * 同义词集合，暂未用到 + */ + private List> synonymList; + + /** + * 分出来的词 + */ + private Word word; + +} \ No newline at end of file diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/Word.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/Word.java new file mode 100644 index 0000000..00cf5a5 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/domain/Word.java @@ -0,0 +1,39 @@ +package org.kangspace.messagepush.core.elasticsearch.domain; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.io.Serializable; + +/** + * @author kango2gler@gmail.com + **/ +@Setter +@Getter +@NoArgsConstructor +@AllArgsConstructor +public class Word implements Serializable { + + /** + * 业务词性 + */ + @JsonProperty(value = "businessPos") + private Long businessPos; + /** + * 词性 + */ + private Long pos; + /** + * 分出来的词 + */ + private String word; + + public Word(String word) { + this.word = word; + } + + +} \ No newline at end of file diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/enumeration/OccurEnum.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/enumeration/OccurEnum.java new file mode 100644 index 0000000..7cda554 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/enumeration/OccurEnum.java @@ -0,0 +1,56 @@ +package org.kangspace.messagepush.core.elasticsearch.enumeration; + +import org.kangspace.messagepush.core.elasticsearch.ElasticSearchConst; + +/** + * 查询条件拼装的位置，是在must、filter、should中 + * + * @author kango2gler@gmail.com + */ +public enum OccurEnum { + + /** + * must节点枚举 + */ + MUST(ElasticSearchConst.MUST, "与，算分"), + + /** + * must_not节点枚举 + */ + MUST_NOT(ElasticSearchConst.MUST_NOT, "非，算分"), + /** + * filter节点枚举 + */ + FILTER(ElasticSearchConst.FILTER, "与，不算分"), + + /** + * should节点枚举 + */ + SHOULD(ElasticSearchConst.SHOULD, "或，算分"); + + /** + * 类型值 + */ + private final String occur; + /** + * 描述 + */ + private final String description; + + + OccurEnum(String occur, String description) { + this.occur = occur; + this.description = description; + } + + + public static String getDescription(String occur) { + for (OccurEnum p : OccurEnum.values()) { + if (p.occur.equals(occur)) { + return p.description; + } + } + return null; + } + +} \ No newline at end of file diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/enumeration/ParseEnum.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/enumeration/ParseEnum.java new file mode 100644 index 0000000..e291aaa --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/enumeration/ParseEnum.java @@ -0,0 +1,136 @@ +package org.kangspace.messagepush.core.elasticsearch.enumeration; + +/** + * 通用拼装枚举 + * + * @author kango2gler@gmail.com + */ +public enum ParseEnum { + + /** + * 关键词完全匹配 + */ + TERM(0, "关键词完全匹配"), + + /** + * 关键词完全匹配，且值大于0 + */ + TERM_GT_ZERO(1, "关键词完全匹配，且值大于0"), + + /** + * 关键词完全匹配，且值大于等于0 + */ + TERM_GTE_ZERO(2, "关键词完全匹配，且值大于等于0"), + + /** + * 关键词完全匹配，多项或运算，支持List及String + */ + TERMS(3, "关键词完全匹配，多项或运算，支持List及String"), + + /** + * 关键词完全匹配，多项或运算，且值大于0 + */ + TERMS_GT_ZERO(4, "关键词完全匹配，多项或运算，且值大于0"), + + /** + * 关键词完全匹配，多项或运算，且值大于0 + */ + TERMS_GTE_ZERO(5, "关键词完全匹配，多项或运算，且值大于0"), + + /** + * 关键词完全匹配，多项与运算，支持List及String + */ + TERMS_AND(3, "关键词完全匹配，多项与运算，支持List及String"), + + /** + * 分词匹配 + */ + MATCH(10, "分词匹配"), + /** + * 
+     * 分词匹配-短语匹配
+     * 与MATCH类似,区别:match_phrase的分词结果必须在text字段分词中都包含，而且分词顺序必须相同，必须都是连续的
+     *
+ */ + MATCH_PHRASE(11, "分词匹配-短语匹配"), + /** + * 
+     * 分词匹配-短语匹配-对最后一个分词进行通配符匹配
+     * 与MATCH_PHRASE类似,区别:MATCH_PHRASE_PREFIX会对最后一个分词进行通配符匹配
+     *
+     *
+ */ + MATCH_PHRASE_PREFIX(12, "分词匹配-短语匹配-对最后一个分词进行通配符匹配"), + + /** + * 前缀查询，类似于sql中的like + */ + PREFIX(13, "前缀查询"), + + /** + * 范围查询，大于某个值 + */ + RANGE_GT(20, "范围查询，大于某个值"), + + /** + * 范围查询，大于等于某个值 + */ + RANGE_GTE(21, "范围查询，大于等于某个值"), + + /** + * 范围查询，小于某个值 + */ + RANGE_LT(22, "范围查询，小于某个值"), + + /** + * 范围查询，小于等于某个值 + */ + RANGE_LTE(23, "范围查询，小于等于某个值"), + + /** + * 多项与查询 + */ + MUST_TERM_FROM_STRING(30, "多项与查询"), + + /** + * 多项或查询 + */ + SHOULD_TERM_FROM_LIST(40, "多项或查询"), + + /** + * 半角逗号分割的字符串转多项或查询 + */ + SHOULD_TERM_FROM_STRING(41, "半角逗号分割的字符串转多项或查询"), + + /** + * 全文检索 + */ + FULL_TEXT(90, "全文检索"); + + /** + * 索引 + */ + private final int index; + /** + * 描述 + */ + private final String description; + + + ParseEnum(int index, String description) { + this.index = index; + this.description = description; + } + + + public static String getDescription(int index) { + for (ParseEnum p : ParseEnum.values()) { + if (p.index == index) { + return p.description; + } + } + return null; + } + + +} \ No newline at end of file diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/impl/ElasticsearchManagerImpl.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/impl/ElasticsearchManagerImpl.java new file mode 100644 index 0000000..5e4afbb --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/impl/ElasticsearchManagerImpl.java @@ -0,0 +1,872 @@ +package org.kangspace.messagepush.core.elasticsearch.impl; + +import com.alibaba.fastjson.JSON; +import com.google.common.collect.Maps; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.indices.rollover.RolloverRequest; +import org.elasticsearch.client.indices.rollover.RolloverResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.UpdateByQueryRequest; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; +import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram; +import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms; +import org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.metrics.ParsedSingleValueNumericMetricsAggregation; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.kangspace.messagepush.core.dto.page.PageResponseDto; +import org.kangspace.messagepush.core.elasticsearch.ElasticSearchConst; +import org.kangspace.messagepush.core.elasticsearch.ElasticsearchManager; +import org.kangspace.messagepush.core.elasticsearch.RestHighLevelClientContextHolder; +import org.kangspace.messagepush.core.elasticsearch.annotation.EntityField; +import org.kangspace.messagepush.core.elasticsearch.annotation.Score; +import org.kangspace.messagepush.core.elasticsearch.request.*; +import org.kangspace.messagepush.core.elasticsearch.response.AggregationsResult; +import org.kangspace.messagepush.core.util.*; +import org.springframework.cglib.core.ReflectUtils; +import org.springframework.data.annotation.Id; +import org.springframework.http.HttpMethod; +import org.springframework.util.ReflectionUtils; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.*; +import java.util.function.Consumer; +import java.util.stream.Collectors; + + +/** + * Elasticsearch数据访问对象 + * + * @author kango2gler@gmail.com + */ +@Slf4j +public class ElasticsearchManagerImpl implements ElasticsearchManager { + + /** + * 根据value类型返回script脚本value值。
+ * (字符串时返回带双引号的值)
+ * 如: + * 1. value = "123", 返回 "123" + * 2. value = 123, 返回 123 + * 3. value = ["1","2","3"],返回 "[\"1\",\"2\",\"3\"]" + * 3. value = [1,2,3],返回 "[1, 2, 3]" + * + * @return 返回实际的脚本value字符串 + */ + private static Object toActualTypeValueForScript(T value) { + if (value == null) { + return ""; + } + if (value instanceof String) { +// return new StringBuilder("\"").append(value).append("\"").toString(); + return new StringBuilder("").append(value).append("").toString(); + } + boolean isArray; + if ((isArray = value.getClass().isArray()) || value instanceof List) { + List listObj; + if (isArray) { + listObj = Arrays.asList(ArrayUtil.toBoxed(value)); + } else { + listObj = (List) value; + } + return listObj.stream().map(t -> { + if (t instanceof String) { +// return new StringBuilder("\"").append(t).append("\""); + return new StringBuilder("").append(t).append(""); + } + return t.toString(); + }).collect(Collectors.joining(",")); + } + return value; + } + + @Override + public Boolean existsIndex(String index) throws IOException { + GetIndexRequest request = new GetIndexRequest(); + request.indices(index); + return RestHighLevelClientContextHolder.getRestHighLevelClientByKey().indices().exists(request, RequestOptions.DEFAULT); + } + + @Override + public Boolean createIndex(JsonCreateIndexRequest jsonIndexRequest) throws Exception { + if (existsIndex(jsonIndexRequest.getIndex())) { + log.warn("索引" + jsonIndexRequest.getIndex() + "已存在！"); + return false; + } + String endpoint = "/" + jsonIndexRequest.getIndex(); + Request req = new Request(HttpMethod.PUT.name(), endpoint); + String json = JsonUtil.toJson(jsonIndexRequest.getRootNode()); + req.setJsonEntity(json); + log.debug("ElasticsearchDaoImpl.createIndex:{endpoint:{},json:{}}", endpoint, json); + Response response = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().getLowLevelClient().performRequest(req); + return RestStatus.OK.getStatus() == response.getStatusLine().getStatusCode(); + } + + @Override + public Boolean deleteIndex(String index) throws IOException { + DeleteIndexRequest request = new DeleteIndexRequest(index.split(",")); + AcknowledgedResponse response = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().indices().delete(request, RequestOptions.DEFAULT); + return response.isAcknowledged(); + } + + @Override + public Boolean existsAlias(String alias) throws IOException { + GetAliasesRequest request = new GetAliasesRequest(); + request.aliases(alias); + return RestHighLevelClientContextHolder.getRestHighLevelClientByKey().indices().existsAlias(request, RequestOptions.DEFAULT); + } + + @Override + public Boolean aliasAction(JsonAliasActionsRequest jsonAliasActionsRequest) throws IOException { + String json = JsonUtil.toJson(jsonAliasActionsRequest.getRootNode()); + log.debug("ElasticsearchDaoImpl.aliasAction: json:{}}", json); + IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest(); + jsonAliasActionsRequest.getAliasActions().forEach(aa -> { + IndicesAliasesRequest.AliasActions aliasActions = null; + if (ElasticSearchConst.ALIASES_ACTIONS_ADD.equals(aa.getAction())) { + aliasActions = IndicesAliasesRequest.AliasActions.add(); + } else if (ElasticSearchConst.ALIASES_ACTIONS_REMOVE.equals(aa.getAction())) { + aliasActions = IndicesAliasesRequest.AliasActions.remove(); + } + if (aliasActions != null) { + aliasActions.alias(aa.getAlias()).index(aa.getIndex()).writeIndex(aa.getWriteIndex()); + indicesAliasesRequest.addAliasAction(aliasActions); + } + }); + AcknowledgedResponse response = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().indices() + .updateAliases(indicesAliasesRequest, RequestOptions.DEFAULT); + return response.isAcknowledged(); + } + + @Override + public Boolean rollover(PlainRolloverRequest rolloverRequest) throws IOException { + String alias = rolloverRequest.getAlias(); + String newIndexName = rolloverRequest.getNewIndexName(); + Objects.requireNonNull(rolloverRequest.getAlias(), "alias 别名不能为空"); + + RolloverRequest request = new RolloverRequest(alias, newIndexName); + if (rolloverRequest.getMaxAge() != null) { + request.addMaxIndexAgeCondition(rolloverRequest.getMaxAge()); + } + if (rolloverRequest.getMaxDocs() != null) { + request.addMaxIndexDocsCondition(rolloverRequest.getMaxDocs()); + } + if (rolloverRequest.getMaxSize() != null) { + request.addMaxIndexSizeCondition(rolloverRequest.getMaxSize()); + } + if (rolloverRequest.getDryRun() != null) { + request.dryRun(rolloverRequest.getDryRun()); + } + log.debug("ElasticsearchDaoImpl.rollover: plainRolloverRequest:{}}", rolloverRequest); + RolloverResponse response = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().indices().rollover(request, RequestOptions.DEFAULT); + log.debug("ElasticsearchDaoImpl.rollover: response:{}}", response); + return response.isAcknowledged() || (response.isDryRun() && response.getConditionStatus().values().stream().anyMatch(t -> t)); + } + + @Override + public Boolean insert(String index, T entity) throws Exception { + log.debug("ElasticsearchDaoImpl.insert:{index:{},entity:{}}", index, entity); + IndexRequest indexRequest = getIndexRequest(index, entity); + IndexResponse response = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().index(indexRequest, RequestOptions.DEFAULT); + log.debug("update status:{}", response.status()); + return RestStatus.OK.equals(response.status()) || RestStatus.CREATED.equals(response.status()); + } + + @Override + public Boolean insert(String index, List list) throws Exception { + log.debug("ElasticsearchDaoImpl.insert:{index:{},list:{}}", index, list); + BulkRequest bulkRequest = new BulkRequest(); + for (T item : list) { + bulkRequest.add(getIndexRequest(index, item)); + } + BulkResponse response = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().bulk(bulkRequest, RequestOptions.DEFAULT); + log.debug("update status:{}", response.status()); + return RestStatus.OK.equals(response.status()); + } + + @Override + public List batchInsert(String index, List list) throws Exception { + log.debug("ElasticsearchDaoImpl.bactchInsert:{index:{},list:{}}", index, list); + BulkRequest bulkRequest = new BulkRequest(); + for (T item : list) { + bulkRequest.add(getIndexRequest(index, item)); + } + BulkResponse response = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().bulk(bulkRequest, RequestOptions.DEFAULT); + List failuerMessages = Arrays.asList(response.getItems()) + .stream().filter(p -> StringUtils.isNotBlank(p.getFailureMessage())) + .map(BulkItemResponse::getFailureMessage).collect(Collectors.toList()); + if (!CollectionUtils.isEmpty(failuerMessages)) { + log.info("es bactchInsert error->{}", failuerMessages); + return failuerMessages; + } + log.info("es bactchInsert totalCount：{}", list.size()); + return null; + } + + @Override + public Boolean insert(String index, String idKey, List> list) throws Exception { + BulkRequest bulkRequest = new BulkRequest(); + for (Map map : list) { + IndexRequest indexRequest = new IndexRequest(index); + if (StrUtil.isNotEmpty(idKey) && map.containsKey(idKey)) { + String id = map.get(idKey) + ""; + indexRequest.id(id); + map.remove(idKey); + indexRequest.source(JsonUtil.toJson(map), XContentType.JSON); + map.put(idKey, id); + } else { + indexRequest.source(JsonUtil.toJson(map), XContentType.JSON); + } + bulkRequest.add(indexRequest); + } + BulkResponse response = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().bulk(bulkRequest, RequestOptions.DEFAULT); + log.debug("update status:{}", response.status()); + return RestStatus.OK.equals(response.status()); + } + + @Override + public IndexRequest getIndexRequest(String index, T entity) throws IOException, IllegalAccessException { + IndexRequest indexRequest = new IndexRequest(index); + //设置id + Field idField = getIdField(entity.getClass()); + String id = null; + if (idField != null) { + id = (String) idField.get(entity); + indexRequest.id(id); + idField.set(entity, null); + } + String source = JsonUtil.toJson(entity); + if (idField != null) { + idField.set(entity, id); + } + indexRequest.id(); + indexRequest.source(source, XContentType.JSON); + + return indexRequest; + } + + @Override + public Boolean update(String index, String id, Map map) throws Exception { + UpdateRequest updateRequest = new UpdateRequest(index, null, id); + updateRequest.doc(map); + UpdateResponse response = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().update(updateRequest, RequestOptions.DEFAULT); + log.debug("update status:{}", response.status()); + return RestStatus.OK.equals(response.status()); + } + + @Override + public Boolean update(String index, String id, T entity) throws Exception { + Map map = Maps.newHashMap(); + Field[] fields = entity.getClass().getDeclaredFields(); + for (Field field : fields) { + field.setAccessible(true); + Object value = field.get(entity); + if (value == null) { + continue; + } + EntityField queryEntityField = field.getAnnotation(EntityField.class); + map.put(queryEntityField != null && StrUtil.isNotEmpty(queryEntityField.field()) ? queryEntityField.field() : field.getName(), value); + } + return update(index, id, map); + } + + @Override + public Boolean update(String index, List ids, List> list) throws Exception { + BulkRequest bulkRequest = new BulkRequest(); + if (ids.size() != list.size()) { + throw new RuntimeException("id列表与跟新数据列表size不同"); + } + for (int i = 0; i < list.size(); i++) { + String id = ids.get(i); + Map item = list.get(i); + UpdateRequest updateRequest = new UpdateRequest(index, null, id); + updateRequest.doc(item); + bulkRequest.add(updateRequest); + } + BulkResponse response = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().bulk(bulkRequest, RequestOptions.DEFAULT); + log.debug("update status:{}", response.status()); + return RestStatus.OK.equals(response.status()); + } + + @Override + public List batchUpdate(String index, List ids, List> list) throws Exception { + BulkRequest bulkRequest = new BulkRequest(); + if (ids.size() != list.size()) { + throw new RuntimeException("id列表与跟新数据列表size不同"); + } + for (int i = 0; i < list.size(); i++) { + String id = ids.get(i); + Map item = list.get(i); + UpdateRequest updateRequest = new UpdateRequest(index, null, id); + updateRequest.doc(JSON.toJSONString(item.get(id)), XContentType.JSON); + bulkRequest.add(updateRequest); + } + BulkResponse response = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().bulk(bulkRequest, RequestOptions.DEFAULT); + List failuerMessages = (List) Arrays.asList(response.getItems()).stream().filter((p) -> { + return StringUtils.isNotBlank(p.getFailureMessage()); + }).map(BulkItemResponse::getFailureMessage).collect(Collectors.toList()); + if (!CollectionUtils.isEmpty(failuerMessages)) { + log.info("es batchUpdate error->{}", failuerMessages); + return failuerMessages; + } else { + log.info("es batchUpdate totalCount：{}", list.size()); + return null; + } + } + + @Override + public Boolean delete(String index, String id) throws Exception { + DeleteRequest request = new DeleteRequest(index, null, id); + DeleteResponse response = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().delete(request, RequestOptions.DEFAULT); + return RestStatus.OK.equals(response.status()); + } + + @Override + public Boolean delete(String index, List ids) throws Exception { + BulkRequest bulkRequest = new BulkRequest(); + ids.forEach(id -> bulkRequest.add(new DeleteRequest(index, null, id))); + BulkResponse response = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().bulk(bulkRequest, RequestOptions.DEFAULT); + log.debug("delete status:{}", response.status()); + return RestStatus.OK.equals(response.status()); + } + + @Override + public T get(String index, String id, Class classType) throws Exception { + return get(index, id, null, classType); + } + + @Override + public T get(String index, String id, List fields, Class classType) throws Exception { + GetRequest request = new GetRequest(index, id); + GetResponse response = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().get(request, RequestOptions.DEFAULT); + if (response.getSource() == null) { + return null; + } + + //设置id + Field idField = getIdField(classType); + T t = JsonUtil.toObject(response.getSourceAsString(), classType); + if (idField != null) { + idField.set(t, response.getId()); + } + BeanUtil.setFieldsNull(t, fields); + return t; + } + + @Override + public T get(JsonSearchRequest jsonSearchRequest) throws Exception { + //如果放弃查询，返回为空 + if (Objects.nonNull(jsonSearchRequest.getAbandon()) && jsonSearchRequest.getAbandon()) { + return null; + } + jsonSearchRequest.getRootNode().put(ElasticSearchConst.FROM, 0); + jsonSearchRequest.getRootNode().put(ElasticSearchConst.SIZE, 1); + List list = list(jsonSearchRequest); + return ListUtil.isEmpty(list) ? null : list.get(0); + } + + @Override + public List list(JsonSearchRequest jsonSearchRequest) throws Exception { + //如果放弃查询，返回为空 + if (Objects.nonNull(jsonSearchRequest.getAbandon()) && jsonSearchRequest.getAbandon()) { + return new ArrayList<>(); + } + String json = JsonUtil.toJson(jsonSearchRequest.getRootNode()); + log.debug("ElasticsearchDaoImpl.list:{}", json); + SearchResponse searchResponse = getSearchResponse(jsonSearchRequest.getIndex(), json); + return getResults(searchResponse, jsonSearchRequest); + } + + private SearchResponse getSearchResponse(String index, String query) throws IOException { + String endpoint = "/" + index + "/_search"; + Request request = new Request(HttpMethod.POST.name(), endpoint); + request.setJsonEntity(query); + Response response = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().getLowLevelClient().performRequest(request); + XContentParser parser = getXContentParser(response); + return SearchResponse.fromXContent(parser); + } + + private XContentParser getXContentParser(Response response) throws IOException { + return XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, response.getEntity().getContent()); + } + + @Override + public PageResponseDto page(JsonSearchRequest jsonSearchRequest) throws Exception { + //如果放弃查询，返回为空 + if (Objects.nonNull(jsonSearchRequest.getAbandon()) && jsonSearchRequest.getAbandon()) { + return new PageResponseDto<>(0L, 0L, new ArrayList<>()); + } + String json = JsonUtil.toJson(jsonSearchRequest.getRootNode()); + log.debug("ElasticsearchDaoImpl.page:{}", json); + SearchResponse searchResponse = getSearchResponse(jsonSearchRequest.getIndex(), json); + return getListPageResponseDTO(searchResponse, jsonSearchRequest.getSize(), jsonSearchRequest); + + } + + @Override + public Map map(JsonSearchRequest jsonSearchRequest) throws Exception { + //如果放弃查询，返回为空 + if (Objects.nonNull(jsonSearchRequest.getAbandon()) && jsonSearchRequest.getAbandon()) { + return new HashMap<>(); + } + String json = JsonUtil.toJson(jsonSearchRequest.getRootNode()); + log.debug("ElasticsearchDaoImpl.list:{}", json); + String endpoint = "/" + jsonSearchRequest.getIndex() + "/_search"; + Request request = new Request(HttpMethod.POST.name(), endpoint); + request.setJsonEntity(json); + Response response = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().getLowLevelClient().performRequest(request); + XContentParser parser = getXContentParser(response); + return parser.map(); + } + + private PageResponseDto getListPageResponseDTO(SearchResponse searchResponse, int pageSize, JsonSearchRequest jsonSearchRequest) throws Exception { + List results = getResults(searchResponse, jsonSearchRequest); + long totalCount = searchResponse.getHits().getTotalHits().value; + long totalPages = totalCount % pageSize == 0 ? totalCount / pageSize : totalCount / pageSize + 1; + return new PageResponseDto<>(totalCount, totalPages, results); + } + + @Override + public Long count(JsonSearchRequest jsonSearchRequest) throws Exception { + //如果放弃查询，返回为空 + if (Objects.nonNull(jsonSearchRequest.getAbandon()) && jsonSearchRequest.getAbandon()) { + return NumberUtils.LONG_ZERO; + } + String json = JsonUtil.toJson(jsonSearchRequest.getRootNode()); + log.debug("ElasticsearchDaoImpl.count:{}", json); + SearchResponse searchResponse = getSearchResponse(jsonSearchRequest.getIndex(), json); + jsonSearchRequest.setTook(searchResponse.getTook().getMillis()); + return searchResponse.getHits().getTotalHits().value; + } + + @Override + public Boolean upsert(String index, String id, T entity) throws Exception { + return upsert(index, id, entity, null); + } + + @Override + public Boolean upsert(String index, String id, T entity, Consumer dataHandleOnInsert) throws Exception { + UpdateRequest updateRequest = getUpsertRequest(index, id, entity, dataHandleOnInsert); + UpdateResponse response = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().update(updateRequest, RequestOptions.DEFAULT); + log.debug("upsert status: index:{}, id:{}, status:{}", index, id, response.status()); + return RestStatus.OK.equals(response.status()) || RestStatus.CREATED.equals(response.status()); + } + + @Override + public List batchUpsert(String index, List list) throws Exception { + return batchUpsert(index, list, null); + } + + /** + * 组织UpsetRequest + * + * @param index 索引 + * @param id doc _id + * @param entity 更新的实体 + * @param dataHandleOnInsert 插入时的数据处理 + * @param T + * @return {@link UpdateRequest} + */ + private UpdateRequest getUpsertRequest(String index, String id, T entity, Consumer dataHandleOnInsert) { + UpdateRequest updateRequest = new UpdateRequest(index, id); + String updateVal = JsonUtil.toJson(entity); + updateRequest.doc(updateVal, XContentType.JSON); + String insertVal = updateVal; + if (dataHandleOnInsert != null) { + dataHandleOnInsert.accept(entity); + insertVal = JsonUtil.toJson(entity); + } + IndexRequest insertIndexRequest = new IndexRequest(index).id(id).source(insertVal, XContentType.JSON); + updateRequest.upsert(insertIndexRequest); + return updateRequest; + } + + @Override + public List batchUpsert(String index, List list, Consumer dataHandleOnInsert) throws Exception { + if (CollectionUtils.isEmpty(list)) { + return Collections.emptyList(); + } + BulkRequest bulkRequest = new BulkRequest(); + list.forEach(entity -> { + String id = getIdFieldValue(entity); + UpdateRequest updateRequest = getUpsertRequest(index, id, entity, dataHandleOnInsert); + bulkRequest.add(updateRequest); + }); + BulkResponse response = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().bulk(bulkRequest, RequestOptions.DEFAULT); + if (response.hasFailures()) { + List failureMessages = Arrays.stream(response.getItems()) + .map(BulkItemResponse::getFailureMessage) + .filter(StringUtils::isNotBlank).collect(Collectors.toList()); + log.info("es batchUpsert error, failureMessages:{}", failureMessages); + return failureMessages; + } + log.info("es batchUpsert totalCount：{}", list.size()); + return null; + } + + @Override + public List simpleGroupBy(SimpleGroupByJsonSearchRequest jsonSearchRequest) throws Exception { + String json = JsonUtil.toJson(jsonSearchRequest.getRootNode()); + log.debug("ElasticsearchDaoImpl.groupBy:{}", json); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(new NamedXContentRegistry(searchModule.getNamedXContents()), null, json); + searchSourceBuilder.parseXContent(parser); + SearchRequest request = new SearchRequest(); + request.indices(jsonSearchRequest.getIndex()); + request.source(searchSourceBuilder); + SearchResponse searchResponse = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().search(request, RequestOptions.DEFAULT); + return getSimpleGroupByResults(searchResponse, jsonSearchRequest); + } + + /** + * 分组查询 + * + * @param jsonScript 完整的查询json语句 + * @return {@link AggregationsResult} + * @throws Exception ex + */ + @Override + public AggregationsResult aggs(String[] indexes, String jsonScript) throws Exception { + log.debug("ElasticsearchDaoImpl.aggs:{}", jsonScript); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(new NamedXContentRegistry(searchModule.getNamedXContents()), null, jsonScript); + searchSourceBuilder.parseXContent(parser); + SearchRequest request = new SearchRequest(); + request.indices(indexes); + request.source(searchSourceBuilder); + SearchResponse searchResponse = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().search(request, RequestOptions.DEFAULT); + return toAggregationResult(searchResponse); + } + + /** + * SearchResponse 转换为 AggregationResult + * + * @param searchResponse response + * @return {@link AggregationsResult} + * @throws Exception ex + */ + private AggregationsResult toAggregationResult(SearchResponse searchResponse) throws Exception { + + AggregationsResult aggregationResult = new AggregationsResult(); + + Long totalCount = searchResponse.getHits().getTotalHits().value; + Long took = searchResponse.getTook().millis(); + boolean timeout = searchResponse.isTimedOut(); + + aggregationResult.setTotal(totalCount); + aggregationResult.setTook(took); + aggregationResult.setTimeout(timeout); + + Map aggregationsMap = new HashMap<>(); + aggregationResult.setAggregations(aggregationsMap); + Aggregations aggregations = searchResponse.getAggregations(); + doResolveGroupBucketDetail(aggregationsMap, aggregations); + return aggregationResult; + } + + /** + * 解析Aggregations为{@link AggregationsResult.GroupBuckets} + * + * @param aggregationsMap + * @param aggregations + */ + public void doResolveGroupBucketDetail(Map aggregationsMap, Aggregations aggregations) { + Map aggs = aggregations.asMap(); + /* 解析aggregations, 如: + { + "totalSmsCountPerHourGroup" : { + "buckets" : [ + { + "key_as_string" : "2022-05-10 14", + "key" : 1652191200000, + "doc_count" : 32, + "smsSendResultGroup" : { + "doc_count_error_upper_bound" : 0, + "sum_other_doc_count" : 0, + "buckets" : [{ + "key" : "3", + "doc_count" : 31, + "smsTypeGroup" : { + "doc_count_error_upper_bound" : 0, + "sum_other_doc_count" : 0, + "buckets" : [] + }] + } + } + } + */ + for (Map.Entry it : aggs.entrySet()) { + // 返回结果中的groupBuckets + AggregationsResult.GroupBuckets resultGroupBuckets = new AggregationsResult.GroupBuckets(); + List resultGroupBucketsDetails = new ArrayList<>(); + resultGroupBuckets.setBuckets(resultGroupBucketsDetails); + aggregationsMap.put(it.getKey(), resultGroupBuckets); + Aggregation agg = it.getValue(); + // 各agg分组的bucket数组 + List buckets; + if (agg instanceof ParsedStringTerms) { + // terms分组处理 + buckets = ((ParsedStringTerms) agg).getBuckets(); + resultGroupBuckets.setDocCountErrorUpperBound(((ParsedStringTerms) agg).getDocCountError()); + resultGroupBuckets.setSumOtherDocCount(((ParsedStringTerms) agg).getSumOfOtherDocCounts()); + } else if (agg instanceof ParsedDateHistogram) { + // date_histogram 分组处理 + buckets = ((ParsedDateHistogram) agg).getBuckets(); + } else if (agg instanceof ParsedSingleValueNumericMetricsAggregation) { + // 单值数值型 分组处理2 + ParsedSingleValueNumericMetricsAggregation singleValue = ((ParsedSingleValueNumericMetricsAggregation) agg); + String value = singleValue.getValueAsString(); + resultGroupBuckets.setSingleNumericValue(value); + return; + } else { + // TODO 需要时扩展其他实现分组类型实现 + return; + } + for (Object temp : buckets) { + MultiBucketsAggregation.Bucket bucket = (MultiBucketsAggregation.Bucket) temp; + AggregationsResult.GroupBucketDetail resultDetail = new AggregationsResult.GroupBucketDetail(bucket.getKey(), bucket.getKeyAsString(), bucket.getDocCount()); + resultGroupBucketsDetails.add(resultDetail); + Aggregations subAggs = bucket.getAggregations(); + if (subAggs != null) { + Map subAggregationsMap = new HashMap<>(); + resultDetail.setAggs(subAggregationsMap); + doResolveGroupBucketDetail(subAggregationsMap, subAggs); + } + } + } + } + + @Override + public Boolean updateByQuery(String[] indexes, QueryBuilder query, Map updateMap) throws Exception { + UpdateByQueryRequest request = new UpdateByQueryRequest(indexes); + request.setQuery(query); + request.setConflicts("proceed"); + request.setScript(buildUpdateScript(updateMap)); + log.info("updateByQuery request:{}", request); + BulkByScrollResponse response = RestHighLevelClientContextHolder.getRestHighLevelClientByKey().updateByQuery(request, RequestOptions.DEFAULT); + printFailures(response.getBulkFailures()); + return response.getTotal() > 0; + } + + /** + * 通过要修改的字段构建更新脚本 + * (脚本中使用参数化传值) + * + * @param updateMap 更新的参数, key: 更新的字段(嵌套字段可使用.分隔符), value: 更新的值 + * @return + */ + private Script buildUpdateScript(Map updateMap) { + StringBuilder sourceSb = new StringBuilder(); + Map paramMap = new HashMap<>(); + updateMap.forEach((k, v) -> { + if (v != null) { + sourceSb.append("ctx._source.").append(k).append("=params.").append(k).append(";"); +// toActualTypeValueForScript(v) + paramMap.put(k, v); + } + }); + return new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, sourceSb.toString(), paramMap); + } + + /** + * 打印失败内容 + * + * @param failures {@link BulkItemResponse.Failure} + */ + private List printFailures(List failures) { + if (CollectionUtils.isNotEmpty(failures)) { + List failureMessages = failures.stream() + .map(t -> t.getMessage()) + .filter(StringUtils::isNotBlank).collect(Collectors.toList()); + log.info("es handle error, failureMessages:{}", failureMessages); + return failureMessages; + } + return Collections.emptyList(); + } + + private List getSimpleGroupByResults(SearchResponse searchResponse, SimpleGroupByJsonSearchRequest jsonSearchRequest) throws Exception { + Aggregations aggregations = searchResponse.getAggregations(); + List aggs = aggregations.asList(); + List resultList = new ArrayList<>(); + Class responseClass = jsonSearchRequest.getResponseClass(); + Objects.requireNonNull(responseClass, "responseClass 不能为空"); + List fields = jsonSearchRequest.getGroupByAggFields(); + for (Aggregation aggregation : aggs) { + ParsedStringTerms agg = (ParsedStringTerms) aggregation; + for (int i1 = 0; i1 < agg.getBuckets().size(); i1++) { + Terms.Bucket bucket = agg.getBuckets().get(i1); + T obj = (T) ReflectUtils.newInstance(responseClass); + for (String fieldName : fields) { + if (bucket == null) { + break; + } + Field field = ReflectionUtils.findField(responseClass, fieldName); + if (field != null) { + field.setAccessible(true); + field.set(obj, bucket.getKeyAsString()); + } + if (bucket.getAggregations() != null && bucket.getAggregations().asList().size() > 0) { + List buckets = ((ParsedTerms) (bucket.getAggregations().asList().get(0))).getBuckets(); + if (CollectionUtils.isNotEmpty(buckets)) { + bucket = ((ParsedTerms) (bucket.getAggregations().asList().get(0))).getBuckets().get(0); + } else { + bucket = null; + } + } else { + bucket = null; + } + } + resultList.add(obj); + } + } + return resultList; + } + + /** + * 查询结果转对象列表 + * + * @param searchResponse 查询响应 + * @param jsonSearchRequest json查询请求 + * @param 响应列表项泛型 + * @return 查询列表 + * @throws Exception 异常 + */ + private List getResults(SearchResponse searchResponse, JsonSearchRequest jsonSearchRequest) throws Exception { + + Field idField = getIdField(jsonSearchRequest.getResponseClass()); + Field scoreField = getScoreField(jsonSearchRequest.getResponseClass()); + SearchHit[] searchHits = searchResponse.getHits().getHits(); + List list = new ArrayList<>(); + for (SearchHit searchHit : searchHits) { + T obj = JsonUtil.toObject(searchHit.getSourceAsString(), jsonSearchRequest.getResponseClass()); + if (idField != null) { + idField.set(obj, searchHit.getId()); + } + if (scoreField != null) { + scoreField.set(obj, searchHit.getScore()); + } + list.add(obj); + } + jsonSearchRequest.setTook(searchResponse.getTook().getMillis()); + return list; + } + + /** + * 获取使用Id注解的属性对象 + * + * @param classType 域所属的的类 + * @param 泛型 + * @return 域对象 + */ + private Field getIdField(Class classType) { + Field idField = null; + Field[] fields = classType.getDeclaredFields(); + for (Field field : fields) { + if (field.isAnnotationPresent(Id.class)) { + field.setAccessible(true); + idField = field; + } + } + return idField; + } + + /** + * 获取使用Id注解的属性对象值 + * + * @param 泛型 + * @param t 对象 + * @return ID值 + */ + @SneakyThrows + private V getIdFieldValue(T t) { + Field field = getIdField(t.getClass()); + if (field != null) { + V val = (V) field.get(t); + if (val == null) { + val = getFieldValueByGetMethod(t, field); + } + return val; + } + return null; + } + + /** + * 通过Get方法获取参数值 + * + * @param t 目标对象 + * @param field 目标字段 + * @param 目标对象类型 + * @param 返回值类型 + * @return + */ + private V getFieldValueByGetMethod(T t, Field field) { + Method method = ReflectionUtils.findMethod(t.getClass(), "get" + StringUtils.capitalize(field.getName())); + if (method != null) { + try { + return (V) method.invoke(t, null); + } catch (Exception e) { + log.error("通过Get方法获取参数值错误,方法调用异常,error: {}", e.getMessage(), e); + } + } + return null; + } + + /** + * 获取使用Id注解的属性对象 + * + * @param classType 域所属的的类 + * @param 泛型 + * @return 域对象 + */ + private Field getScoreField(Class classType) { + Field idField = null; + Field[] fields = classType.getDeclaredFields(); + for (Field field : fields) { + if (field.isAnnotationPresent(Score.class)) { + field.setAccessible(true); + idField = field; + } + } + return idField; + } + +} \ No newline at end of file diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/query/CustomQueryBuilder.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/query/CustomQueryBuilder.java new file mode 100644 index 0000000..2044f7b --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/query/CustomQueryBuilder.java @@ -0,0 +1,20 @@ +package org.kangspace.messagepush.core.elasticsearch.query; + + +import org.kangspace.messagepush.core.elasticsearch.request.JsonSearchRequest; + +/** + * 自定义查询语句构建器接口 + * + * @author kango2gler@gmail.com + */ +public interface CustomQueryBuilder { + + /** + * 拼装自定义查询语句 + * + * @param jsonSearchRequest elasticsearch.request.JsonSearchRequest + * @param query query + */ + void handler(JsonSearchRequest jsonSearchRequest, T query); +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/query/FulltextQueryBuilder.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/query/FulltextQueryBuilder.java new file mode 100644 index 0000000..555892c --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/query/FulltextQueryBuilder.java @@ -0,0 +1,303 @@ +package org.kangspace.messagepush.core.elasticsearch.query; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Lists; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.protocol.HTTP; +import org.kangspace.messagepush.core.elasticsearch.ElasticSearchConst; +import org.kangspace.messagepush.core.elasticsearch.domain.*; +import org.kangspace.messagepush.core.elasticsearch.enumeration.OccurEnum; +import org.kangspace.messagepush.core.elasticsearch.request.JsonSearchRequest; +import org.kangspace.messagepush.core.elasticsearch.util.QueryUtil; +import org.kangspace.messagepush.core.http.RestProperties; +import org.kangspace.messagepush.core.http.RestTemplateFactory; +import org.kangspace.messagepush.core.util.JsonUtil; +import org.kangspace.messagepush.core.util.ListUtil; +import org.kangspace.messagepush.core.util.StrUtil; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.client.RestTemplate; + +import javax.annotation.Resource; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * 全文检索构建器 + * + * @author kango2gler@gmail.com + */ +@Slf4j +public class FulltextQueryBuilder { + + private static HttpHeaders DEFAULT_HEADERS; + + static { + DEFAULT_HEADERS = new HttpHeaders(); + DEFAULT_HEADERS.add(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537.36"); + DEFAULT_HEADERS.add(HttpHeaders.ACCEPT_ENCODING, "gzip,deflate"); + DEFAULT_HEADERS.add(HttpHeaders.ACCEPT_LANGUAGE, "zh-CN"); + DEFAULT_HEADERS.add(HttpHeaders.CONTENT_TYPE, "application/x-www-form-urlencoded;charset=utf-8"); + DEFAULT_HEADERS.add(HttpHeaders.CONNECTION, HTTP.CONN_KEEP_ALIVE); + } + + @Resource + private RestTemplateFactory restTemplateFactory; + + public void handler(JsonSearchRequest jsonSearchRequest, FullTextQuery fullTextQuery) { + //获取分词 + segmentation(fullTextQuery); + + //有关键词但无分词结果 则 设置放弃查询为true + if (StrUtil.isNotEmpty(fullTextQuery.getKeyword()) && ListUtil.isEmpty(fullTextQuery.getWords())) { + jsonSearchRequest.setAbandon(true); + return; + } + + //需要打分 + if (fullTextQuery.isScore()) { + //设置分词增强函数 + QueryUtil.addQuery(jsonSearchRequest, OccurEnum.MUST, getFunctionScoreNode(fullTextQuery.getWords(), fullTextQuery.getFullTextItems())); + //增加排序字段 + Optional.ofNullable(jsonSearchRequest.getRootNode().get(ElasticSearchConst.SORT)).ifPresent( + jsonNode -> ((ArrayNode) jsonNode).insert(0, QueryUtil.getNode(ElasticSearchConst.UNDERSCORE_SCORE, ElasticSearchConst.DESC)) + ); + } + //不需要打分 + else if (fullTextQuery.getFullTextItems() != null) { + List words = fullTextQuery.getWords().stream().map(Word::getWord).collect(Collectors.toList()); + fullTextQuery.getFullTextItems().forEach(fullTextItem -> { + QueryUtil.terms(jsonSearchRequest, OccurEnum.FILTER, fullTextItem.getFieldName(), words); + }); + } + } + + /** + * 获取分词信息 + * + * @param fullTextQuery 查询参数 + */ + private void segmentation(FullTextQuery fullTextQuery) { + if (ListUtil.isNotEmpty(fullTextQuery.getWords()) || StrUtil.isEmpty(fullTextQuery.getIkUrl())) { + return; + } + + //发送http请求获取分词结果 + List words = Optional.ofNullable(segmentation(fullTextQuery.getIkUrl(), fullTextQuery.getKeyword())) + //获取分词结果中的同义词集合 + .map(segmentation -> Optional.ofNullable(segmentation.getSynonymWordList()) + //获取同义词 + .map(synonymWordInfos -> synonymWordInfos.stream().map(SynonymWord::getWord).collect(Collectors.toList())) + //如果为空则分会空集合 + .orElse(Lists.newArrayList())) + //如果为空则分会空集合 + .orElse(Lists.newArrayList()); + fullTextQuery.setWords(words); + } + + /** + * 设置增强函数 + * + * @param words 分词结果 + * @param fullTextItems 全文检索项列表结果 + */ + private JsonNode getFunctionScoreNode(List words, List fullTextItems) { + //初始化增强函数参数节点 + ObjectNode functionScoreParamNode = JsonUtil.createObjectNode() + .put(ElasticSearchConst.SCORE_MODE, ElasticSearchConst.SUM) + .put(ElasticSearchConst.BOOST_MODE, ElasticSearchConst.REPLACE); + //设置分词权重、高斯函数、脚本函数 + setFunctions(functionScoreParamNode, words, fullTextItems); + + if (ListUtil.isNotEmpty(words)) { + //设置should参数节点 + ArrayNode shouldParamNode = JsonUtil.createArrayNode(); + fullTextItems.forEach(fullTextItem -> { + words.forEach(word -> { + JsonNode termNode = QueryUtil.getNode(ElasticSearchConst.TERM, QueryUtil.getNode(fullTextItem.getFieldName(), word.getWord())); + shouldParamNode.add(termNode); + }); + }); + //设置should节点 + JsonNode shouldNode = QueryUtil.getNode(ElasticSearchConst.SHOULD, shouldParamNode); + //设置bool节点 + JsonNode boolNode = QueryUtil.getNode(ElasticSearchConst.BOOL, shouldNode); + //设置query节点 + functionScoreParamNode.set(ElasticSearchConst.QUERY, boolNode); + } + + //设置functionScore节点 + JsonNode functionScoreNode = JsonUtil.createObjectNode().set(ElasticSearchConst.FUNCTION_SCORE, functionScoreParamNode); + return functionScoreNode; + } + + /** + * 构建高斯函数 + * + * @param functionScoreParamNode functionScore节点 + */ + private void setFunctions(ObjectNode functionScoreParamNode, List words, List fullTextItems) { + //默认是先构造高斯函数 + ArrayNode functionsNode = JsonUtil.createArrayNode(); + //遍历配置，依次构建分词、高斯、脚本函数 + fullTextItems.forEach(fullTextItem -> { + //根据分词设置filter增强函数 + buildWordFilterFunction(functionsNode, words, fullTextItem); + //获取高斯函数配置 + buildGaussFunction(functionsNode, fullTextItem); + //获取脚本函数配置 + buildScriptFunction(functionsNode, fullTextItem); + }); + + //设置functionScore参数 + functionScoreParamNode.set(ElasticSearchConst.FUNCTIONS, functionsNode); + + } + + + /** + * 根据分词结果设置filter增强函数 + * + * @param functionsNode 增强函数集合 + * @param words 分词结果 + * @param words 分词结果 + */ + private void buildWordFilterFunction(ArrayNode functionsNode, List words, FullTextItem fullTextItem) { + if (ListUtil.isEmpty(words)) { + return; + } + //循环分词结果，设置增强函数 + words.forEach(word -> { + Float wordWeight = getWordWeight(word); + //设置 某个分词的增强函数 + ObjectNode result = getFilterFunctionBuilder(fullTextItem.getFieldName(), word, wordWeight * fullTextItem.getWeight()); + //添加分词分节点 + functionsNode.add(result); + }); + } + + /** + * 获取分词权重 + * + * @param word 分词对象 + * @return + */ + public Float getWordWeight(Word word) { + //获取基础词性 + Long pos = word.getPos(); + //获取业务词性 + Long businessPos = word.getBusinessPos(); + //默认权重为1 + Float wordWeight = (float) 1; + //判断是否是重要基础词性 + if (ImportantPosEnum.judgeIsImportant(pos)) { + //获取词性的权重 + ImportantPosEnum importantPosEnum = ImportantPosEnum.resolve(pos); + //设置增强函数以及权重 + wordWeight = wordWeight.compareTo(importantPosEnum.getWeight()) >= 0 ? wordWeight : importantPosEnum.getWeight(); + //判断是否是重要业务词性 + } + if (ImportantBizPosEnum.judgeIsImportant(businessPos)) { + //获取词性的权重 + ImportantBizPosEnum importantBizPosEnum = ImportantBizPosEnum.resolve(businessPos); + //设置增强函数以及权重 + wordWeight = wordWeight.compareTo(importantBizPosEnum.getWeight()) >= 0 ? wordWeight : importantBizPosEnum.getWeight(); + } + return wordWeight; + } + + + /** + * 获得分词的评分函数 + * + * @param fieldName 字段名 + * @param word 分词 + * @param weight 权重 + * @return 分词的评分函数 + */ + private ObjectNode getFilterFunctionBuilder(String fieldName, Word word, Float weight) { + //创建分词的增强函数节点 + ObjectNode filterNode = (ObjectNode) QueryUtil.getNode(ElasticSearchConst.FILTER, QueryUtil.getNode(ElasticSearchConst.TERM, QueryUtil.getNode(fieldName, word.getWord()))); + //设置权重 + filterNode.put(ElasticSearchConst.WEIGHT, weight); + return filterNode; + } + + /** + * 拼装高斯函数 + * + * @param functionsNode 函数列表节点 + * @param fullTextItem 全文检索对象 + */ + private void buildGaussFunction(ArrayNode functionsNode, FullTextItem fullTextItem) { + if (ListUtil.isEmpty(fullTextItem.getGaussConfigs())) { + return; + } + fullTextItem.getGaussConfigs().forEach(gaussConfig -> { + ObjectNode gaussParamNode = JsonUtil.createObjectNode() + //设置起始值 + .put(ElasticSearchConst.ORIGIN, gaussConfig.getOrigin()) + //设置级别银子 + .put(ElasticSearchConst.SCALE, gaussConfig.getScale()) + //设置补偿系数 + .put(ElasticSearchConst.OFFSET, gaussConfig.getOffset()) + //设置衰减系数 + .put(ElasticSearchConst.DECAY, gaussConfig.getDecay()); + //设置高斯属性节点 + ObjectNode fieldNode = (ObjectNode) JsonUtil.createObjectNode().set(gaussConfig.getFieldName(), gaussParamNode); + //设置高斯节点以及权重 + functionsNode.add(JsonUtil.createObjectNode().put(ElasticSearchConst.WEIGHT, gaussConfig.getWeight() * fullTextItem.getWeight()).set(ElasticSearchConst.GAUSS, fieldNode)); + }); + } + + /** + * 拼装高斯函数 + * + * @param functionsNode 函数列表节点 + * @param fullTextItem 全文检索对象 + */ + private void buildScriptFunction(ArrayNode functionsNode, FullTextItem fullTextItem) { + if (ListUtil.isEmpty(fullTextItem.getScriptConfigs())) { + return; + } + fullTextItem.getScriptConfigs().forEach(scriptConfig -> { + ObjectNode scriptNode = (ObjectNode) QueryUtil.getNode(ElasticSearchConst.SCRIPT_SCORE, QueryUtil.getNode(ElasticSearchConst.SCRIPT, QueryUtil.getNode(ElasticSearchConst.SOURCE, scriptConfig.getScript()))); + scriptNode.put(ElasticSearchConst.WEIGHT, scriptConfig.getWeight()); + functionsNode.add(scriptNode); + }); + } + + /** + * 获取RestTemplate + * + * @return + */ + private synchronized RestTemplate getRestTemplate() { + RestTemplate restTemplate = restTemplateFactory.getRestTemplate(ElasticSearchConst.REST_TEMPLATE_POOL); + if (restTemplate == null) { + restTemplate = restTemplateFactory.registerRestTemplate(ElasticSearchConst.REST_TEMPLATE_POOL, new RestProperties()); + } + return restTemplate; + } + + + /** + * 获取分词信息 + * + * @param ikUrl + * @param keyword + * @return + */ + public Segmentation segmentation(String ikUrl, String keyword) { + //构建分词查询参数 + MultiValueMap paramMap = new LinkedMultiValueMap<>(); + paramMap.add(ElasticSearchConst.KW, keyword); + HttpEntity> httpEntity = new HttpEntity<>(paramMap, DEFAULT_HEADERS); + return getRestTemplate().postForObject(ikUrl, httpEntity, Segmentation.class); + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/AliasAction.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/AliasAction.java new file mode 100644 index 0000000..60a0d77 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/AliasAction.java @@ -0,0 +1,37 @@ +package org.kangspace.messagepush.core.elasticsearch.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 别名操作对象 + * + * @author kango2gler@gmail.com + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class AliasAction { + + /** + * 操作方式 remove 或 add + */ + private String action; + /** + * 索引名 + */ + private String index; + /** + * 别名 + */ + private String alias; + /** + * 是否写索引 + * is_write_index + */ + @JsonProperty("is_write_index") + private Boolean writeIndex; + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/BaseRequest.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/BaseRequest.java new file mode 100644 index 0000000..c607191 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/BaseRequest.java @@ -0,0 +1,31 @@ +package org.kangspace.messagepush.core.elasticsearch.request; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.*; +import lombok.experimental.SuperBuilder; +import org.kangspace.messagepush.core.util.JsonUtil; + +/** + * Es请求基础实体 + * + * @author kango2gler@gmail.com + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +@SuperBuilder +@RequiredArgsConstructor +public class BaseRequest { + + /** + * 索引名，如果是多个索引半角逗号分割 + */ + @NonNull + private String index; + + /** + * 根节点 + */ + private ObjectNode rootNode = JsonUtil.createObjectNode(); + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/JsonAliasActionsRequest.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/JsonAliasActionsRequest.java new file mode 100644 index 0000000..8d6af65 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/JsonAliasActionsRequest.java @@ -0,0 +1,67 @@ +package org.kangspace.messagepush.core.elasticsearch.request; + + +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Data; +import org.kangspace.messagepush.core.elasticsearch.ElasticSearchConst; +import org.kangspace.messagepush.core.util.JsonUtil; + +import java.util.ArrayList; +import java.util.List; + +/** + * es别名操作请求对象，用于创建索引时使用 + * + * @author kango2gler@gmail.com + * @since 2019/10/29 + */ +@Data +public class JsonAliasActionsRequest { + + /** + * 别名操作列表 + */ + private List aliasActions; + + /** + * 根节点 + */ + private ObjectNode rootNode = JsonUtil.createObjectNode(); + + public JsonAliasActionsRequest() { + aliasActions = new ArrayList<>(); + } + + public JsonAliasActionsRequest(AliasAction aliasAction) { + this(); + addAliasAction(aliasAction); + } + + public JsonAliasActionsRequest addAliasAction(AliasAction aliasAction) { + aliasActions.add(aliasAction); + build(); + return this; + } + + /** + * 构建查询语句 + */ + public void build() { + ArrayNode actions = JsonUtil.createArrayNode(); + //别名 + if (aliasActions != null && aliasActions.size() > 0) { + aliasActions.forEach(aliasAction -> { + ObjectNode node = JsonUtil.createObjectNode(); + ObjectNode nodeInner = JsonUtil.createObjectNode(); + nodeInner.put(ElasticSearchConst.INDEX, aliasAction.getIndex()); + nodeInner.put(ElasticSearchConst.ALIAS, aliasAction.getAlias()); + nodeInner.put(ElasticSearchConst.IS_WRITE_INDEX, aliasAction.getWriteIndex()); + node.set(aliasAction.getAction(), nodeInner); + actions.add(node); + }); + } + this.rootNode.set(ElasticSearchConst.ALIASES_ACTIONS, actions); + } + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/JsonCreateIndexRequest.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/JsonCreateIndexRequest.java new file mode 100644 index 0000000..ef74da7 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/JsonCreateIndexRequest.java @@ -0,0 +1,112 @@ +package org.kangspace.messagepush.core.elasticsearch.request; + + +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import org.kangspace.messagepush.core.elasticsearch.ElasticSearchConst; +import org.kangspace.messagepush.core.util.JsonUtil; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * es索引创建对象，用于创建索引时使用 + * + * @author kango2gler@gmail.com + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor +@SuperBuilder +public class JsonCreateIndexRequest extends BaseRequest { + + /** + * 分片数，一般同节点数 + */ + private Integer numberOfShards; + + /** + * 副本数，一般为1 + */ + private Integer numberOfReplicas; + + /** + * 别名 + */ + private List aliases; + + /** + * 映射<类型,<字段名，字段类型数据类型字符串 或 ObjectNode对象>> + */ + private Map mappings; + + + /** + * 构建创建索引对象 + * + * @param index 索引名 + * @param numberOfShards 分片数，一般同节点数 + * @param numberOfReplicas 副本数，一般为1 + * @param mappings 别名，支持多个 + */ + public JsonCreateIndexRequest(String index, Integer numberOfShards, Integer numberOfReplicas, Map mappings) { + super(index); + this.numberOfShards = numberOfShards; + this.numberOfReplicas = numberOfReplicas; + this.mappings = mappings; + build(); + } + + /** + * 构建创建索引对象 + * + * @param index 索引名 + * @param numberOfShards 分片数，一般同节点数 + * @param numberOfReplicas 副本数，一般为1 + * @param mappings 映射<类型,<字段名，字段类型数据类型字符串 或 ObjectNode对象>> + * @param aliases 别名，支持多个 + */ + public JsonCreateIndexRequest(String index, Integer numberOfShards, Integer numberOfReplicas, Map mappings, List aliases) { + super(index); + this.numberOfShards = numberOfShards; + this.numberOfReplicas = numberOfReplicas; + this.mappings = mappings; + this.aliases = aliases; + build(); + } + + /** + * 构建查询语句 + */ + public void build() { + //索引状态为开启 + //super.getRootNode().put(ElasticSearchConst.STATE, ElasticSearchConst.OPEN); + + //分片数和副本数 + super.getRootNode().set(ElasticSearchConst.SETTINGS, JsonUtil.createObjectNode().put(ElasticSearchConst.NUMBER_OF_SHARDS, numberOfShards).put(ElasticSearchConst.NUMBER_OF_REPLICAS, numberOfReplicas)); + + //别名 + if (aliases != null && aliases.size() > 0) { + ObjectNode aliasesNode = super.getRootNode().putObject(ElasticSearchConst.ALIASES); + aliases.forEach(alias -> aliasesNode.set(alias, JsonUtil.createObjectNode())); + } + + //映射 + ObjectNode mappingsNode = super.getRootNode().putObject(ElasticSearchConst.MAPPINGS); + ObjectNode propertiesNode = mappingsNode.putObject(ElasticSearchConst.PROPERTIES); + Optional.ofNullable(mappings) + .ifPresent(v -> v.forEach((fieldName, fieldType) -> { + if (fieldType instanceof String) { + propertiesNode.putObject(fieldName).put(ElasticSearchConst.TYPE, (String) fieldType); + } else if (fieldType instanceof ObjectNode) { + propertiesNode.set(fieldName, (ObjectNode) fieldType); + } + }) + ); + } + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/JsonSearchRequest.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/JsonSearchRequest.java new file mode 100644 index 0000000..031dd13 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/JsonSearchRequest.java @@ -0,0 +1,218 @@ +package org.kangspace.messagepush.core.elasticsearch.request; + + +import com.fasterxml.jackson.databind.node.ArrayNode; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.constant.OrderConstant; +import org.kangspace.messagepush.core.dto.page.Order; +import org.kangspace.messagepush.core.dto.page.PageRequestDto; +import org.kangspace.messagepush.core.elasticsearch.ElasticSearchConst; +import org.kangspace.messagepush.core.elasticsearch.query.CustomQueryBuilder; +import org.kangspace.messagepush.core.elasticsearch.util.QueryUtil; +import org.kangspace.messagepush.core.util.JsonUtil; +import org.kangspace.messagepush.core.util.ListUtil; + +import java.util.List; + +/** + * json查询请求 + * + * @author kango2gler@gmail.com + */ +@Data +@EqualsAndHashCode(callSuper = true) +@Slf4j +@NoArgsConstructor +public class JsonSearchRequest extends BaseRequest { + private String scriptJson; + /** + * 类型，如果多个类型半角逗号分割 + */ + private String type; + + /** + * 查询数量 + */ + private Integer size; + + /** + * 默认排序字段,默认排序字段 + */ + private Order defaultOrder; + + /** + * 返回结果类对象 + */ + private Class responseClass; + + /** + * 自定义查询条件构建器，调用其handler方法构建自定义查询语句 + */ + private CustomQueryBuilder customQueryBuilder; + + /** + * 是否放弃查询，某些特殊情况，如全文检索无可用分词，可设置此值为true，则调用es查询方法时不进行es查询，直接返回空数据 + */ + private Boolean abandon; + + /** + * es查询时间（毫秒） + */ + private Long took; + + + /** + * 构建查询对象 + * + * @param index 索引，多个索引半角逗号分割 + * @param query 查询对象 + * @param responseClass 返回结果类 + * @param 查询对象泛型 + */ + public JsonSearchRequest(String index, T query, Class responseClass) throws Exception { + this(index, query, null, null, responseClass); + } + + /** + * 构建查询对象 + * + * @param index 索引，多个索引半角逗号分割 + * @param query 查询对象 + * @param customQueryBuilder 自定义查询条件构建器 + * @param defaultOrder 默认排序字段 + * @param responseClass 返回结果类 + * @param 查询对象泛型 + */ + public JsonSearchRequest(String index, T query, CustomQueryBuilder customQueryBuilder, Order defaultOrder, Class responseClass) throws Exception { + super(index); + this.defaultOrder = defaultOrder; + this.responseClass = responseClass; + + //显示字段 + setSource(query); + //分页 + setFromAndSize(query); + //排序 + setSort(query, getDefaultOrder()); + //查询 + setQuery(query, customQueryBuilder); + } + + /** + * 构建简版查询对象，适用于count查询 + * + * @param index 索引，多个索引半角逗号分割 + * @param query 查询对象 + * @param customQueryBuilder 自定义查询条件构建器 + * @param 查询对象泛型 + */ + public JsonSearchRequest(String index, T query, CustomQueryBuilder customQueryBuilder) throws Exception { + super(index); + //查询 + setQuery(query, customQueryBuilder); + //设置不需要返回结果集 + super.getRootNode().put(ElasticSearchConst.SIZE, 0); + //获取按条查询出来真实的总数 + super.getRootNode().put(ElasticSearchConst.TRACK_TOTAL_HITS, true); + } + + /** + * 设置查询语句 + * + * @param query + * @param customQueryBuilder + * @param + */ + private void setQuery(T query, CustomQueryBuilder customQueryBuilder) throws Exception { + QueryUtil.entityToQuery(this, query); + if (customQueryBuilder != null) { + customQueryBuilder.handler(this, query); + } + + } + + /** + * 分页处理 + * + * @param query 查询实体类 + */ + public void setFromAndSize(T query) { + if (query == null) { + return; + } + + //设置默认页码 + if (query.getPageNum() == null || query.getPageNum() <= 0) { + log.debug("PageNum={},不符合规范，设置为默认值{}。{}", query.getPageNum(), 1, query); + query.setPageNum(1); + } + + //未分页，查询默认条数 + if (query.getPageSize() == null || query.getPageSize() <= 0) { + log.debug("pageSize={},不符合规范，设置为默认值{}。{}", query.getPageNum(), ElasticSearchConst.DEFAULT_PAGE_SIZE, query.getPageSize()); + query.setPageSize(ElasticSearchConst.DEFAULT_PAGE_SIZE); + } + + //保证性能，默认不超过10000条，调整页码 + if (query.getPageNum() * query.getPageSize() > ElasticSearchConst.MAX_PAGE_SEARCH_COUNT) { + log.warn("保证性能，PageNum*pageSize不得超过{}条，设置PageNum为{}。{}", ElasticSearchConst.MAX_PAGE_SEARCH_COUNT, 1, query); + query.setPageNum(1); + } + + //保证性能，默认不超过10000条，调整页显示记录数 + if (query.getPageNum() * query.getPageSize() > ElasticSearchConst.MAX_PAGE_SEARCH_COUNT) { + log.warn("保证性能，PageNum*pageSize不得超过{}条，设置pageSize为{}。{}", ElasticSearchConst.MAX_PAGE_SEARCH_COUNT, ElasticSearchConst.DEFAULT_PAGE_SIZE, query); + query.setPageSize(ElasticSearchConst.MAX_PAGE_SEARCH_COUNT); + } + + super.getRootNode().put(ElasticSearchConst.FROM, (query.getPageNum() - 1) * query.getPageSize()); + super.getRootNode().put(ElasticSearchConst.SIZE, query.getPageSize()); + setSize(query.getPageSize()); + } + + /** + * 排序处理 + * + * @param query 查询实体类 + * @param defaultOrder 默认排序字段 + */ + public void setSort(T query, Order defaultOrder) { + if (query == null) { + return; + } + List orders = ListUtil.isEmpty(query.getOrders()) && defaultOrder != null ? ListUtil.getList(defaultOrder) : query.getOrders(); + + if (ListUtil.isEmpty(orders)) { + return; + } + ArrayNode sorNode = JsonUtil.createArrayNode(); + orders.forEach(order -> { + if (!"".equals(order.getField())) { + sorNode.add(JsonUtil.createObjectNode().put(order.getField(), OrderConstant.ASC.equals(order.getSequence()) ? ElasticSearchConst.ASC : ElasticSearchConst.DESC)); + } + }); + if (sorNode.size() > 0) { + super.getRootNode().set(ElasticSearchConst.SORT, sorNode); + } + + } + + /** + * 拼装显示字段 + * + * @param query + * @param + */ + public void setSource(T query) { + if (query == null || ListUtil.isEmpty(query.getFields())) { + return; + } + ArrayNode arrayNode = JsonUtil.createArrayNode(); + query.getFields().forEach(field -> arrayNode.add(field)); + super.getRootNode().set(ElasticSearchConst.UNDERSCORE_SOURCE, arrayNode); + } + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/PlainRolloverRequest.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/PlainRolloverRequest.java new file mode 100644 index 0000000..d40c5ba --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/PlainRolloverRequest.java @@ -0,0 +1,44 @@ +package org.kangspace.messagepush.core.elasticsearch.request; + +import lombok.Data; +import lombok.NoArgsConstructor; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; + +/** + * 滚动索引操作请求类 + * + * @author kango2gler@gmail.com + * @since 2021/11/12 + */ +@Data +@NoArgsConstructor +public class PlainRolloverRequest extends BaseRequest { + /** + * 是否实际执行rollover + * true: 不执行,只检查是否需要翻转 + * false: 执行翻转 + */ + private Boolean dryRun; + /** + * 需要翻转的别名 + */ + private String alias; + /** + * 翻转时的新索引名 + */ + private String newIndexName; + /** + * 翻转条件,最大有效期 + */ + private TimeValue maxAge; + /** + * 翻转条件,最大文档数 + */ + private Long maxDocs; + /** + * 翻转跳转,最大索引大小 + */ + private ByteSizeValue maxSize; + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/SimpleGroupByJsonSearchRequest.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/SimpleGroupByJsonSearchRequest.java new file mode 100644 index 0000000..e06f45f --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/request/SimpleGroupByJsonSearchRequest.java @@ -0,0 +1,142 @@ +package org.kangspace.messagepush.core.elasticsearch.request; + + +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.elasticsearch.ElasticSearchConst; +import org.kangspace.messagepush.core.elasticsearch.util.QueryUtil; +import org.kangspace.messagepush.core.util.JsonUtil; + +import java.util.ArrayList; +import java.util.List; + +/** + * json查询请求 + * + * @author kango2gler@gmail.com + */ +@Data +@EqualsAndHashCode(callSuper = true) +@Slf4j +public class SimpleGroupByJsonSearchRequest extends JsonSearchRequest { + + /** + * groupBy Agg分组字段 + */ + private List groupByAggFields = new ArrayList<>(); + + public SimpleGroupByJsonSearchRequest(String index, R query, Class responseClass) throws Exception { + setIndex(index); + setSize(0); + setQuery(query); + setResponseClass(responseClass); + super.getRootNode().put(ElasticSearchConst.SIZE, 0); + } + + public SimpleGroupByJsonSearchRequest(String index, String scriptJson, Class responseClass) throws Exception { + setIndex(index); + setScriptJson(scriptJson); + setResponseClass(responseClass); + } + + /** + * 设置查询语句 + * + * @param query 查询对象 + * @param T + */ + private void setQuery(T query) throws Exception { + QueryUtil.entityToQuery(this, query); + } + + /** + * 添加分组字段 + * + * "aggs":{ + * "groupByIndex":{ + * "terms":{ + * "field":"index", + * "size":1000 + * }, + * "aggs":{ + * "groupByTime":{ + * "terms":{ + * "field":"time", + * "size":1 + * } + * } + * } + * } + * } + * + * + * @param field 分组字段名 + * @return {@link AggSimpleGroupByParam} + */ + public AggSimpleGroupByParam addGroupByField(String field) { + AggSimpleGroupByParam param = new AggSimpleGroupByParam(this).addAggNodeField(field, 10000); + super.getRootNode().setAll(param.getAggNode()); + return param; + } + + /** + * agg 简单分组字段对象 + * + * @author kango2gler@gmail.com + * @since 2022/1/24 + */ + public static class AggSimpleGroupByParam { + private SimpleGroupByJsonSearchRequest searchRequest; + private ObjectNode aggNode; + private ObjectNode node; + + private AggSimpleGroupByParam() { + } + + AggSimpleGroupByParam(SimpleGroupByJsonSearchRequest searchRequest) { + this.searchRequest = searchRequest; + } + + /** + * 添加子 + * + * @param field + * @return + */ + public AggSimpleGroupByParam addField(String field) { + addNodeField(this.node, field, 1); + return this; + } + + AggSimpleGroupByParam addAggNodeField(String field, int size) { + ObjectNode aggNode = JsonUtil.createObjectNode(); + ObjectNode node = addNodeField(aggNode, field, size); + this.aggNode = aggNode; + this.node = node; + return this; + } + + /** + * agg节点下添加group by 字段 + * + * @param aggNode aggNode + * @param field group by 字段 + * @param size 结果集size,默认为1 + */ + private ObjectNode addNodeField(ObjectNode aggNode, String field, int size) { + ObjectNode node = JsonUtil.createObjectNode(); + node.set("terms", JsonUtil.createObjectNode().put("field", field).put("size", size)); + ObjectNode groupByNode = JsonUtil.createObjectNode(); + groupByNode.set("group_by_" + field, node); + aggNode.set(ElasticSearchConst.AGGS, groupByNode); + searchRequest.getGroupByAggFields().add(field); + return node; + } + + ObjectNode getAggNode() { + return aggNode; + } + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/response/AggregationsResult.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/response/AggregationsResult.java new file mode 100644 index 0000000..3abb6ee --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/response/AggregationsResult.java @@ -0,0 +1,247 @@ +package org.kangspace.messagepush.core.elasticsearch.response; + +import lombok.Data; + +import java.util.List; +import java.util.Map; + +/** + * 分组查询结果基础Bean + * 
+ * 查询:
+ *    GET message-gateway.index_operators_info/_search
+ * {
+ *   "size": 0,
+ *   "query": {
+ *     "bool": {
+ *       "filter": [
+ *         {
+ *           "range": {
+ *             "reportTime": {
+ *               "gt": "2022-05-10 14:00:00",
+ *               "lt": "2022-05-10 15:00:00"
+ *             }
+ *           }
+ *         }
+ *       ],
+ *       "must": [],
+ *       "must_not": [],
+ *       "should": []
+ *     }
+ *   },
+ *   "aggs": {
+ *     "totalSmsCountPerHourGroup": {
+ *       "date_histogram": {
+ *         "field": "reportTime",
+ *         "interval": "hour",
+ *         "format": "yyyy-MM-dd HH"
+ *       },
+ *       "aggs": {
+ *         "smsSendResultGroup": {
+ *           "terms": {
+ *             "field": "operatorsSendStatus",
+ *             "size": 10
+ *           },
+ *           "aggs": {
+ *             "smsTypeGroup": {
+ *               "terms": {
+ *                 "field": "smsType",
+ *                 "size": 10
+ *               },
+ *               "aggs": {
+ *                 "serviceProviderGroup": {
+ *                   "terms": {
+ *                     "field": "operatorsType",
+ *                     "size": 10
+ *                   },
+ *                   "aggs": {
+ *                     "smsSizeSum": {
+ *                       "sum": {
+ *                         "field": "smsSize"
+ *                       }
+ *                     }
+ *                   }
+ *                 }
+ *               }
+ *             }
+ *           }
+ *         }
+ *       }
+ *     }
+ *   }
+ * }
+ * 结果样例:
+ *    "aggregations" : {
+ *     "agroup" : {
+ *       "value" : 1.65625
+ *     },
+ *     "totalSmsCountPerHourGroup" : {
+ *       "buckets" : [
+ *         {
+ *           "key_as_string" : "2022-05-10 14",
+ *           "key" : 1652191200000,
+ *           "doc_count" : 32,
+ *           "smsSendResultGroup" : {
+ *             "doc_count_error_upper_bound" : 0,
+ *             "sum_other_doc_count" : 0,
+ *             "buckets" : [
+ *               {
+ *                 "key" : "3",
+ *                 "doc_count" : 31,
+ *                 "smsTypeGroup" : {
+ *                   "doc_count_error_upper_bound" : 0,
+ *                   "sum_other_doc_count" : 0,
+ *                   "buckets" : [
+ *                     {
+ *                       "key" : "2",
+ *                       "doc_count" : 20,
+ *                       "serviceProviderGroup" : {
+ *                         "doc_count_error_upper_bound" : 0,
+ *                         "sum_other_doc_count" : 0,
+ *                         "buckets" : [
+ *                           {
+ *                             "key" : "6",
+ *                             "doc_count" : 20,
+ *                             "smsSizeSum" : {
+ *                               "value" : 41.0
+ *                             }
+ *                           }
+ *                         ]
+ *                       }
+ *                     },
+ *                     {
+ *                       "key" : "1",
+ *                       "doc_count" : 8,
+ *                       "serviceProviderGroup" : {
+ *                         "doc_count_error_upper_bound" : 0,
+ *                         "sum_other_doc_count" : 0,
+ *                         "buckets" : [
+ *                           {
+ *                             "key" : "6",
+ *                             "doc_count" : 8,
+ *                             "smsSizeSum" : {
+ *                               "value" : 8.0
+ *                             }
+ *                           }
+ *                         ]
+ *                       }
+ *                     },
+ *                     {
+ *                       "key" : "3",
+ *                       "doc_count" : 3,
+ *                       "serviceProviderGroup" : {
+ *                         "doc_count_error_upper_bound" : 0,
+ *                         "sum_other_doc_count" : 0,
+ *                         "buckets" : [
+ *                           {
+ *                             "key" : "5",
+ *                             "doc_count" : 3,
+ *                             "smsSizeSum" : {
+ *                               "value" : 3.0
+ *                             }
+ *                           }
+ *                         ]
+ *                       }
+ *                     }
+ *                   ]
+ *                 }
+ *               },
+ *               {
+ *                 "key" : "2",
+ *                 "doc_count" : 1,
+ *                 "smsTypeGroup" : {
+ *                   "doc_count_error_upper_bound" : 0,
+ *                   "sum_other_doc_count" : 0,
+ *                   "buckets" : [
+ *                     {
+ *                       "key" : "1",
+ *                       "doc_count" : 1,
+ *                       "serviceProviderGroup" : {
+ *                         "doc_count_error_upper_bound" : 0,
+ *                         "sum_other_doc_count" : 0,
+ *                         "buckets" : [
+ *                           {
+ *                             "key" : "6",
+ *                             "doc_count" : 1,
+ *                             "smsSizeSum" : {
+ *                               "value" : 1.0
+ *                             }
+ *                           }
+ *                         ]
+ *                       }
+ *                     }
+ *                   ]
+ *                 }
+ *               }
+ *             ]
+ *           }
+ *         }
+ *       ]
+ *     }
+ *   }
+ *
+ *
+ *
+ *
+ * + * @author kango2gler@gmail.com + * @since 2022/5/16 + */ +@Data +public class AggregationsResult extends Result { + /** + * key: group name + * value: {buckets...} + */ + Map aggregations; + + + /** + * 分组结果 + */ + @Data + public static class GroupBuckets { + /** + * buckets列表 + */ + private List buckets; + + private Long docCountErrorUpperBound; + private Long sumOtherDocCount; + /** + * 简单分组函数值(key: 分组名称, value: 分组结果),可能为double类型. + * 如sum, avg分组等 + */ + private Object singleNumericValue; + } + + /** + * 分组结果详情 + */ + @Data + public static class GroupBucketDetail { + /** + * 内部分组 + * key: group name + * value: {buckets...} + */ + Map aggs; + private Object key; + private String keyAsString; + /** + * 文档总数 + */ + private Long docCount; + + public GroupBucketDetail() { + } + + public GroupBucketDetail(Object key, String keyAsString, Long docCount) { + this.key = key; + this.keyAsString = keyAsString; + this.docCount = docCount; + } + + + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/response/Result.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/response/Result.java new file mode 100644 index 0000000..4568d44 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/response/Result.java @@ -0,0 +1,47 @@ +package org.kangspace.messagepush.core.elasticsearch.response; + +import lombok.Data; + +/** + * @author kango2gler@gmail.com + * @since 2022/5/16 + */ +@Data +public class Result { + /** + * 返回结果中的took + */ + private Long took; + /** + * 返回结果中的timeout + */ + private Boolean timeout; + /** + * 返回结果中 hits.total.value + */ + private Long total; + + /** + * 响应码 + */ + private Integer status; + + /** + * 错误信息 + */ + private String error; + + public Result() { + } + + public Result(Integer status, String error) { + this.status = status; + this.error = error; + } + + public Result(Long took, Boolean timeout, Long total) { + this.took = took; + this.timeout = timeout; + this.total = total; + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/util/QueryUtil.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/util/QueryUtil.java new file mode 100644 index 0000000..5e40908 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/elasticsearch/util/QueryUtil.java @@ -0,0 +1,565 @@ +package org.kangspace.messagepush.core.elasticsearch.util; + +import cn.hutool.core.date.DateUtil; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.elasticsearch.ElasticSearchConst; +import org.kangspace.messagepush.core.elasticsearch.annotation.QueryField; +import org.kangspace.messagepush.core.elasticsearch.domain.FullTextQuery; +import org.kangspace.messagepush.core.elasticsearch.enumeration.OccurEnum; +import org.kangspace.messagepush.core.elasticsearch.query.FulltextQueryBuilder; +import org.kangspace.messagepush.core.elasticsearch.request.JsonSearchRequest; +import org.kangspace.messagepush.core.util.JsonUtil; +import org.kangspace.messagepush.core.util.ListUtil; +import org.kangspace.messagepush.core.util.StrUtil; + +import java.beans.PropertyDescriptor; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.math.BigDecimal; +import java.time.LocalDateTime; +import java.util.Date; +import java.util.List; + +/** + * 查询工具类 + * + * @author kango2gler@gmail.com + */ +@Slf4j +public class QueryUtil { + + /** + * Date类型默认格式 + */ + private static final String DEFAULT_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss"; + + /** + * LocalDateTime类型默认格式 + */ + private static final String DEFAULT_LOCAL_DATE_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'+'08:00"; + + /** + * 全文检索构建器 + */ + private static FulltextQueryBuilder fulltextQueryBuilder; + + /** + * 初始化 + * + * @param fulltextQueryBuilder 全文检索构建对象 + */ + public static void init(FulltextQueryBuilder fulltextQueryBuilder) { + QueryUtil.fulltextQueryBuilder = fulltextQueryBuilder; + } + + /** + * 自动拼装 + * + * @param jsonSearchRequest 查询请求对象 + * @param entity 实体 + * @param 泛型 + * @throws Exception 异常 + */ + public static void entityToQuery(JsonSearchRequest jsonSearchRequest, T entity) throws Exception { + + //初始化查询框架 + initQueryNode(jsonSearchRequest); + if (entity == null) { + return; + } + //遍历实体类属性 + for (Field field : entity.getClass().getDeclaredFields()) { + if (!field.isAnnotationPresent(QueryField.class)) { + continue; + } + //属性名 + String fieldName = field.getName(); + QueryField fieldAnnotation = field.getAnnotation(QueryField.class); + if (StrUtil.isNotEmpty(fieldAnnotation.field())) { + fieldName = fieldAnnotation.field().trim(); + } + + //属性值 + Object fieldValue = getMethod(field, entity); + if (fieldValue == null) { + continue; + } + fieldValue = formatValue(fieldValue, fieldAnnotation.format()); + + //按处理方式拼装查询条件 + switch (fieldAnnotation.value()) { + case TERM: + term(jsonSearchRequest, fieldAnnotation.occur(), fieldName, fieldValue); + break; + case TERM_GT_ZERO: + termGtZero(jsonSearchRequest, fieldAnnotation.occur(), fieldName, Integer.parseInt(String.valueOf(fieldValue))); + break; + case TERM_GTE_ZERO: + termGteZero(jsonSearchRequest, fieldAnnotation.occur(), fieldName, Integer.parseInt(String.valueOf(fieldValue))); + break; + case TERMS: + terms(jsonSearchRequest, fieldAnnotation.occur(), fieldName, fieldValue); + break; + case TERMS_GT_ZERO: + termsGtZero(jsonSearchRequest, fieldAnnotation.occur(), fieldName, fieldValue); + break; + case TERMS_GTE_ZERO: + termsGteZero(jsonSearchRequest, fieldAnnotation.occur(), fieldName, fieldValue); + break; + case TERMS_AND: + termsAnd(jsonSearchRequest, fieldAnnotation.occur(), fieldName, fieldValue); + break; + case MATCH: + match(jsonSearchRequest, fieldAnnotation.occur(), fieldName, fieldValue); + break; + case MATCH_PHRASE: + matchPhrase(jsonSearchRequest, fieldAnnotation.occur(), fieldName, fieldValue); + break; + case MATCH_PHRASE_PREFIX: + buildQueryNode(ElasticSearchConst.MATCH_PHRASE_PREFIX, jsonSearchRequest, fieldAnnotation.occur(), fieldName, fieldValue); + break; + case PREFIX: + buildQueryNode(ElasticSearchConst.PREFIX, jsonSearchRequest, fieldAnnotation.occur(), fieldName, fieldValue); + break; + case RANGE_GT: + range(jsonSearchRequest, fieldAnnotation.occur(), fieldName, fieldValue, ElasticSearchConst.GT); + break; + case RANGE_GTE: + range(jsonSearchRequest, fieldAnnotation.occur(), fieldName, fieldValue, ElasticSearchConst.GTE); + break; + case RANGE_LT: + range(jsonSearchRequest, fieldAnnotation.occur(), fieldName, fieldValue, ElasticSearchConst.LT); + break; + case RANGE_LTE: + range(jsonSearchRequest, fieldAnnotation.occur(), fieldName, fieldValue, ElasticSearchConst.LTE); + break; + case FULL_TEXT: + fulltext(jsonSearchRequest, fieldValue); + break; + default: + break; + } + } + } + + /** + * 全文检索 + * + * @param jsonSearchRequest 查询对象 + * @param fieldValue 属性值 + */ + private static void fulltext(JsonSearchRequest jsonSearchRequest, Object fieldValue) { + if (fulltextQueryBuilder == null || !(fieldValue instanceof FullTextQuery)) { + return; + } + fulltextQueryBuilder.handler(jsonSearchRequest, ((FullTextQuery) fieldValue)); + } + + /** + * 初始化查询节点 + * + * @param jsonSearchRequest 查询对象 + */ + private static void initQueryNode(JsonSearchRequest jsonSearchRequest) { + ObjectNode queryNode = getNode(); + ObjectNode boolNode = getNode(); + ArrayNode filterNode = JsonUtil.createArrayNode(); + ArrayNode mustNode = JsonUtil.createArrayNode(); + ArrayNode mustNotNode = JsonUtil.createArrayNode(); + ArrayNode shouldNode = JsonUtil.createArrayNode(); + boolNode.set(ElasticSearchConst.FILTER, filterNode); + boolNode.set(ElasticSearchConst.MUST, mustNode); + boolNode.set(ElasticSearchConst.MUST_NOT, mustNotNode); + boolNode.set(ElasticSearchConst.SHOULD, shouldNode); + queryNode.set(ElasticSearchConst.BOOL, boolNode); + jsonSearchRequest.getRootNode().set(ElasticSearchConst.QUERY, queryNode); + } + + /** + * 得到属性get方法的值 + * + * @param field 字段域 + * @param entity 实体 + * @param 泛型 + * @return 调用get方法返回字段值 + * @throws Exception 异常 + */ + private static Object getMethod(Field field, T entity) throws Exception { + PropertyDescriptor pd = new PropertyDescriptor(field.getName(), entity.getClass()); + Method method = pd.getReadMethod(); + method.setAccessible(true); + return method.invoke(entity); + } + + /** + * 格式化属性值 + * + * @param fieldValue 属性值 + * @param format 格式 + * @return 格式化后值 + */ + private static Object formatValue(Object fieldValue, String format) { + if (fieldValue == null) { + return null; + } + Object value; + if (fieldValue instanceof LocalDateTime) { + if (StrUtil.isEmpty(format)) { + format = DEFAULT_LOCAL_DATE_TIME_FORMAT; + } + value = DateUtil.format(((LocalDateTime) fieldValue), format); + } else if (fieldValue instanceof Date) { + if (StrUtil.isEmpty(format)) { + format = DEFAULT_DATE_FORMAT; + } + value = DateUtil.format(((Date) fieldValue), format); + } else { + value = fieldValue; + } + return value; + } + + + /** + * 等于 + * + * @param jsonSearchRequest 查询对象 + * @param occurEnum 查询条件拼装的位置，是在must、filter、should中 + * @param fieldName 属性名 + * @param fieldValue 属性值 + */ + public static void term(JsonSearchRequest jsonSearchRequest, OccurEnum occurEnum, String fieldName, Object fieldValue) { + if (jsonSearchRequest == null || occurEnum == null || StrUtil.isEmpty(fieldName) || fieldValue == null) { + return; + } + getOccurNode(jsonSearchRequest, occurEnum).add(getNode(ElasticSearchConst.TERM, getNode(fieldName, fieldValue))); + + } + + /** + * 等于，值大于0才拼装 + * + * @param jsonSearchRequest 查询对象 + * @param occurEnum 查询条件拼装的位置，是在must、filter、should中 + * @param fieldName 属性名 + * @param fieldValue 属性值 + */ + public static void termGtZero(JsonSearchRequest jsonSearchRequest, OccurEnum occurEnum, String fieldName, Integer fieldValue) { + if (jsonSearchRequest == null || occurEnum == null || StrUtil.isEmpty(fieldName) || fieldValue == null || fieldValue <= 0) { + return; + } + getOccurNode(jsonSearchRequest, occurEnum).add(getNode(ElasticSearchConst.TERM, getNode(fieldName, fieldValue))); + } + + /** + * 等于，值大于等于0才拼装 + * + * @param jsonSearchRequest 查询对象 + * @param occurEnum 查询条件拼装的位置，是在must、filter、should中 + * @param fieldName 属性名 + * @param fieldValue 属性值 + */ + public static void termGteZero(JsonSearchRequest jsonSearchRequest, OccurEnum occurEnum, String fieldName, Integer fieldValue) { + if (jsonSearchRequest == null || occurEnum == null || StrUtil.isEmpty(fieldName) || fieldValue == null || fieldValue < 0) { + return; + } + getOccurNode(jsonSearchRequest, occurEnum).add(getNode(ElasticSearchConst.TERM, getNode(fieldName, fieldValue))); + } + + /** + * 多项或完全匹配 + * + * @param jsonSearchRequest 查询对象 + * @param occurEnum 查询条件拼装的位置，是在must、filter、should中 + * @param fieldName 属性名 + * @param fieldValue 属性值 + */ + public static void terms(JsonSearchRequest jsonSearchRequest, OccurEnum occurEnum, String fieldName, Object fieldValue) { + List values = getValues(fieldValue); + if (values == null) { + return; + } + baseTerms(jsonSearchRequest, occurEnum, fieldName, values); + } + + /** + * 多项或完全匹配，值为正整数才拼装 + * + * @param jsonSearchRequest 查询对象 + * @param occurEnum 查询条件拼装的位置，是在must、filter、should中 + * @param fieldName 属性名 + * @param fieldValue 属性值 + */ + public static void termsGtZero(JsonSearchRequest jsonSearchRequest, OccurEnum occurEnum, String fieldName, Object fieldValue) { + List values = getValues(fieldValue); + if (values == null) { + return; + } + for (int i = values.size() - 1; i >= 0; i--) { + Object value = values.get(i); + if (value instanceof Integer) { + if ((Integer) value <= 0) { + values.remove(i); + } + } else if (value instanceof Double) { + if ((Double) value <= 0) { + values.remove(i); + } + } else if (value instanceof String) { + if (StrUtil.strToInt((String) value, 0) <= 0) { + values.remove(i); + } + } else { + values.remove(i); + } + } + baseTerms(jsonSearchRequest, occurEnum, fieldName, values); + } + + /** + * 多项或完全匹配，值为自然数才拼装 + * + * @param jsonSearchRequest 查询对象 + * @param occurEnum 查询条件拼装的位置，是在must、filter、should中 + * @param fieldName 属性名 + * @param fieldValue 属性值 + */ + public static void termsGteZero(JsonSearchRequest jsonSearchRequest, OccurEnum occurEnum, String fieldName, Object fieldValue) { + List values = getValues(fieldValue); + if (values == null) { + return; + } + for (int i = values.size() - 1; i >= 0; i--) { + Object value = values.get(i); + if (value instanceof Integer) { + if ((Integer) value < 0) { + values.remove(i); + } + } else if (value instanceof Double) { + if ((Double) value < 0) { + values.remove(i); + } + } else if (value instanceof String) { + if (StrUtil.strToInt((String) value, 0) < 0) { + values.remove(i); + } + } else { + values.remove(i); + } + } + baseTerms(jsonSearchRequest, occurEnum, fieldName, values); + } + + /** + * 多项与完全匹配 + * + * @param jsonSearchRequest 查询对象 + * @param occurEnum 查询条件拼装的位置，是在must、filter、should中 + * @param fieldName 属性名 + * @param fieldValue 属性值 + */ + public static void termsAnd(JsonSearchRequest jsonSearchRequest, OccurEnum occurEnum, String fieldName, Object fieldValue) { + List values = getValues(fieldValue); + if (values == null) { + return; + } + values.forEach(value -> getOccurNode(jsonSearchRequest, occurEnum).add(getNode(ElasticSearchConst.TERM, getNode(fieldName, value)))); + } + + /** + * 基于Object对象获取List + * + * @param fieldValue 属性值 + * @return + */ + private static List getValues(Object fieldValue) { + if (fieldValue == null) { + return null; + } + if (fieldValue instanceof List) { + if (((List) fieldValue).size() == 0) { + return null; + } + return (List) fieldValue; + } else if (fieldValue instanceof String) { + if (StrUtil.isEmpty((String) fieldValue)) { + return null; + } + return StrUtil.strToList((String) fieldValue); + } + return null; + } + + /** + * 基础terms方法 + * + * @param jsonSearchRequest 查询对象 + * @param occurEnum 查询条件拼装的位置，是在must、filter、should中 + * @param fieldName 属性名 + * @param fieldValue 属性值 + */ + private static void baseTerms(JsonSearchRequest jsonSearchRequest, OccurEnum occurEnum, String fieldName, List fieldValue) { + if (jsonSearchRequest == null || occurEnum == null || StrUtil.isEmpty(fieldName) || ListUtil.isEmpty(fieldValue)) { + return; + } + getOccurNode(jsonSearchRequest, occurEnum).add(getNode(ElasticSearchConst.TERMS, getNode(fieldName, fieldValue))); + } + + /** + * 分词匹配查询 + * + * @param jsonSearchRequest 查询对象 + * @param occurEnum 查询条件拼装的位置，是在must、filter、should中 + * @param fieldName 属性名 + * @param fieldValue 属性值 + */ + public static void match(JsonSearchRequest jsonSearchRequest, OccurEnum occurEnum, String fieldName, Object fieldValue) { + if (jsonSearchRequest == null || occurEnum == null || StrUtil.isEmpty(fieldName) || fieldValue == null) { + return; + } + getOccurNode(jsonSearchRequest, occurEnum).add(getNode(ElasticSearchConst.MATCH, getNode(fieldName, fieldValue))); + } + + /** + * 分词匹配-短语匹配查询 + * + * @param jsonSearchRequest 查询对象 + * @param occurEnum 查询条件拼装的位置，是在must、filter、should中 + * @param fieldName 属性名 + * @param fieldValue 属性值 + */ + public static void matchPhrase(JsonSearchRequest jsonSearchRequest, OccurEnum occurEnum, String fieldName, Object fieldValue) { + if (jsonSearchRequest == null || occurEnum == null || StrUtil.isEmpty(fieldName) || fieldValue == null) { + return; + } + getOccurNode(jsonSearchRequest, occurEnum).add(getNode(ElasticSearchConst.MATCH_PHRASE, getNode(fieldName, fieldValue))); + } + + /** + * 构建查询节点 + * + * @param queryType 查询类型 如:{@link ElasticSearchConst#MATCH_PHRASE_PREFIX},{@link ElasticSearchConst#MATCH_PHRASE}等 + * @param jsonSearchRequest jsonSearchRequest + * @param occurEnum 查询条件拼装的位置，是在must、filter、should中 + * @param fieldName 属性名 + * @param fieldValue 属性值 + */ + public static void buildQueryNode(String queryType, JsonSearchRequest jsonSearchRequest, OccurEnum occurEnum, String fieldName, Object fieldValue) { + if (jsonSearchRequest == null || occurEnum == null || StrUtil.isEmpty(fieldName) || fieldValue == null) { + return; + } + getOccurNode(jsonSearchRequest, occurEnum).add(getNode(queryType, getNode(fieldName, fieldValue))); + } + + /** + * 范围查询 + * + * @param jsonSearchRequest 查询对象 + * @param occurEnum 查询条件拼装的位置，是在must、filter、should中 + * @param fieldName 属性名 + * @param fieldValue 属性值 + * @param relation 关系运算符 ElasticSearchConst.GT、ElasticSearchConst.GTE、ElasticSearchConst.LT、ElasticSearchConst.LTE + */ + public static void range(JsonSearchRequest jsonSearchRequest, OccurEnum occurEnum, String fieldName, Object fieldValue, String relation) { + if (jsonSearchRequest == null || occurEnum == null || StrUtil.isEmpty(fieldName) || fieldValue == null) { + return; + } + getOccurNode(jsonSearchRequest, occurEnum).add(getNode(ElasticSearchConst.RANGE, getNode(fieldName, getNode(relation, fieldValue)))); + } + + /** + * 增加查询条件 + * + * @param jsonSearchRequest 查询对象 + * @param occurEnum 查询条件拼装的位置，是在must、filter、should中 + * @param jsonNode + */ + public static void addQuery(JsonSearchRequest jsonSearchRequest, OccurEnum occurEnum, JsonNode jsonNode) { + if (jsonSearchRequest == null || occurEnum == null || jsonNode == null) { + return; + } + getOccurNode(jsonSearchRequest, occurEnum).add(jsonNode); + } + + /** + * 获取一个空JsonNode + * + * @return + */ + private static ObjectNode getNode() { + return JsonUtil.createObjectNode(); + } + + /** + * 获取一个节点jsonNode + * + * @param fieldName 属性名 + * @param fieldValue 属性值 + * @return + */ + public static JsonNode getNode(String fieldName, Object fieldValue) { + if (fieldValue instanceof JsonNode) { + return JsonUtil.createObjectNode().set(fieldName, (JsonNode) fieldValue); + } else if (fieldValue instanceof List) { + ArrayNode arrayNode = JsonUtil.createArrayNode(); + ((List) fieldValue).forEach(value -> { + if (value instanceof String) { + arrayNode.add((String) value); + } else { + arrayNode.addPOJO(value); + } + }); + return JsonUtil.createObjectNode().set(fieldName, arrayNode); + } else if (fieldValue instanceof String) { + return JsonUtil.createObjectNode().put(fieldName, (String) fieldValue); + } else if (fieldValue instanceof Integer) { + return JsonUtil.createObjectNode().put(fieldName, (Integer) fieldValue); + } else if (fieldValue instanceof Long) { + return JsonUtil.createObjectNode().put(fieldName, (Long) fieldValue); + } else if (fieldValue instanceof Short) { + return JsonUtil.createObjectNode().put(fieldName, (Short) fieldValue); + } else if (fieldValue instanceof Float) { + return JsonUtil.createObjectNode().put(fieldName, (Float) fieldValue); + } else if (fieldValue instanceof Double) { + return JsonUtil.createObjectNode().put(fieldName, (Double) fieldValue); + } else if (fieldValue instanceof Boolean) { + return JsonUtil.createObjectNode().put(fieldName, (Boolean) fieldValue); + } else if (fieldValue instanceof BigDecimal) { + return JsonUtil.createObjectNode().put(fieldName, (BigDecimal) fieldValue); + } else { + return JsonUtil.createObjectNode().putPOJO(fieldName, fieldValue); + } + } + + /** + * 获取查询条件拼装的位置 + * + * @param jsonSearchRequest 查询对象 + * @param occurEnum 查询条件拼装的位置，是在must、filter、should中 + * @return 查询条件拼装的位置 + */ + public static ArrayNode getOccurNode(JsonSearchRequest jsonSearchRequest, OccurEnum occurEnum) { + ArrayNode occurNode = null; + switch (occurEnum) { + case FILTER: + occurNode = ((ArrayNode) jsonSearchRequest.getRootNode().get(ElasticSearchConst.QUERY).get(ElasticSearchConst.BOOL).get(ElasticSearchConst.FILTER)); + break; + case MUST: + occurNode = ((ArrayNode) jsonSearchRequest.getRootNode().get(ElasticSearchConst.QUERY).get(ElasticSearchConst.BOOL).get(ElasticSearchConst.MUST)); + break; + case MUST_NOT: + occurNode = ((ArrayNode) jsonSearchRequest.getRootNode().get(ElasticSearchConst.QUERY).get(ElasticSearchConst.BOOL).get(ElasticSearchConst.MUST_NOT)); + break; + case SHOULD: + occurNode = ((ArrayNode) jsonSearchRequest.getRootNode().get(ElasticSearchConst.QUERY).get(ElasticSearchConst.BOOL).get(ElasticSearchConst.SHOULD)); + break; + default: + break; + } + return occurNode; + } + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/enums/ResponseEnum.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/enums/ResponseEnum.java new file mode 100644 index 0000000..70eece4 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/enums/ResponseEnum.java @@ -0,0 +1,160 @@ +package org.kangspace.messagepush.core.enums; + +import org.springframework.lang.Nullable; + +import java.util.Objects; + +/** + * @author kango2gler@gmail.com + * @date 2024/7/13 + * @since + */ +public enum ResponseEnum { + CONTINUE(100, "Continue"), + SWITCHING_PROTOCOLS(101, "Switching Protocols"), + PROCESSING(102, "Processing"), + CHECKPOINT(103, "Checkpoint"), + OK(1, "OK"), + CREATED(201, "Created"), + ACCEPTED(202, "Accepted"), + NON_AUTHORITATIVE_INFORMATION(203, "Non-Authoritative Information"), + NO_CONTENT(204, "No Content"), + RESET_CONTENT(205, "Reset Content"), + PARTIAL_CONTENT(206, "Partial Content"), + MULTI_STATUS(207, "Multi-Status"), + ALREADY_REPORTED(208, "Already Reported"), + IM_USED(226, "IM Used"), + MULTIPLE_CHOICES(300, "Multiple Choices"), + MOVED_PERMANENTLY(301, "Moved Permanently"), + FOUND(302, "Found"), + /** + * @deprecated + */ + @Deprecated + MOVED_TEMPORARILY(302, "Moved Temporarily"), + SEE_OTHER(303, "See Other"), + NOT_MODIFIED(304, "Not Modified"), + /** + * @deprecated + */ + @Deprecated + USE_PROXY(305, "Use Proxy"), + TEMPORARY_REDIRECT(307, "Temporary Redirect"), + PERMANENT_REDIRECT(308, "Permanent Redirect"), + BAD_REQUEST(400, "Bad Request"), + UNAUTHORIZED(401, "Unauthorized"), + PAYMENT_REQUIRED(402, "Payment Required"), + FORBIDDEN(403, "Forbidden"), + NOT_FOUND(404, "Not Found"), + METHOD_NOT_ALLOWED(405, "Method Not Allowed"), + NOT_ACCEPTABLE(406, "Not Acceptable"), + PROXY_AUTHENTICATION_REQUIRED(407, "Proxy Authentication Required"), + REQUEST_TIMEOUT(408, "Request Timeout"), + CONFLICT(409, "Conflict"), + GONE(410, "Gone"), + LENGTH_REQUIRED(411, "Length Required"), + PRECONDITION_FAILED(412, "Precondition Failed"), + PAYLOAD_TOO_LARGE(413, "Payload Too Large"), + /** + * @deprecated + */ + @Deprecated + REQUEST_ENTITY_TOO_LARGE(413, "Request Entity Too Large"), + URI_TOO_LONG(414, "URI Too Long"), + /** + * @deprecated + */ + @Deprecated + REQUEST_URI_TOO_LONG(414, "Request-URI Too Long"), + UNSUPPORTED_MEDIA_TYPE(415, "Unsupported Media Type"), + REQUESTED_RANGE_NOT_SATISFIABLE(416, "Requested range not satisfiable"), + EXPECTATION_FAILED(417, "Expectation Failed"), + I_AM_A_TEAPOT(418, "I'm a teapot"), + /** + * @deprecated + */ + @Deprecated + INSUFFICIENT_SPACE_ON_RESOURCE(419, "Insufficient Space On Resource"), + /** + * @deprecated + */ + @Deprecated + METHOD_FAILURE(420, "Method Failure"), + /** + * @deprecated + */ + @Deprecated + DESTINATION_LOCKED(421, "Destination Locked"), + UNPROCESSABLE_ENTITY(422, "Unprocessable Entity"), + LOCKED(423, "Locked"), + FAILED_DEPENDENCY(424, "Failed Dependency"), + UPGRADE_REQUIRED(426, "Upgrade Required"), + PRECONDITION_REQUIRED(428, "Precondition Required"), + TOO_MANY_REQUESTS(429, "Too Many Requests"), + REQUEST_HEADER_FIELDS_TOO_LARGE(431, "Request Header Fields Too Large"), + UNAVAILABLE_FOR_LEGAL_REASONS(451, "Unavailable For Legal Reasons"), + INTERNAL_SERVER_ERROR(0, "Internal Server Error"), + NOT_IMPLEMENTED(501, "Not Implemented"), + BAD_GATEWAY(502, "Bad Gateway"), + SERVICE_UNAVAILABLE(503, "Service Unavailable"), + GATEWAY_TIMEOUT(504, "Gateway Timeout"), + HTTP_VERSION_NOT_SUPPORTED(505, "HTTP Version not supported"), + VARIANT_ALSO_NEGOTIATES(506, "Variant Also Negotiates"), + INSUFFICIENT_STORAGE(507, "Insufficient Storage"), + LOOP_DETECTED(508, "Loop Detected"), + BANDWIDTH_LIMIT_EXCEEDED(509, "Bandwidth Limit Exceeded"), + NOT_EXTENDED(510, "Not Extended"), + NETWORK_AUTHENTICATION_REQUIRED(511, "Network Authentication Required"), + BCL_CLIENT_FEIGN_ERROR(700, "Feign异常"), + BCL_CLIENT_HYSTRIX_CIRCUIT_ERROR(701, "触发熔断"), + BCL_CLIENT_HYSTRIX_TIME_OUT_ERROR(702, "HYSTRIX超时"), + BCL_CLIENT_GATEWAY_NET_ERROR(703, "网关网络异常"), + BCL_GATEWAY_AUTH_ERROR(800, "鉴权异常"), + BCL_GATEWAY_SIGN_ERROR(801, "无效签名"), + BCL_GATEWAY_APP_ERROR(802, "无效系统"), + BCL_GATEWAY_API_ERROR(803, "无效接口"), + BCL_GATEWAY_PERM_ERROR(804, "无权限"), + BCL_GATEWAY_TS_ERROR(805, "无效TS"), + BCL_GATEWAY_TS_TIME_OUT_ERROR(806, "TS超时"), + BCL_GATEWAY_ERROR(807, "网关代码异常"), + BCL_GATEWAY_SERVER_NET_ERROR(808, "服务端网络异常"), + BCL_GATEWAY_SERVER_CODE_ERROR(809, "服务端代码异常"), + BCL_GATEWAY_HYSTRIX_CIRCUIT_ERROR(810, "触发熔断"), + BCL_DB_ERROR(900, "数据库异常"), + BCL_REDIS_ERROR(901, "Redis异常"), + BCL_ES_ERROR(902, "ES异常"), + BCL_BUSINESS_ERROR(903, "业务异常"), + BCL_ILLEGAL_ARGUMENT_ERROR(904, "参数异常"), + BCL_UNKNOWN_ERROR(999, "未知异常"); + + private final int value; + private final String reasonPhrase; + + private ResponseEnum(int value, String reasonPhrase) { + this.value = value; + this.reasonPhrase = reasonPhrase; + } + + @Nullable + public static ResponseEnum resolve(int statusCode) { + ResponseEnum[] var1 = values(); + int var2 = var1.length; + + for (int var3 = 0; var3 < var2; ++var3) { + ResponseEnum status = var1[var3]; + if (Objects.equals(status.value, statusCode)) { + return status; + } + } + + return null; + } + + public int getValue() { + return this.value; + } + + public String getReasonPhrase() { + return this.reasonPhrase; + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/event/NacosServiceUpInfo.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/event/NacosServiceUpInfo.java new file mode 100644 index 0000000..8578488 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/event/NacosServiceUpInfo.java @@ -0,0 +1,23 @@ +package org.kangspace.messagepush.core.event; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.hash.ConsistencyHashing; + +import java.io.Serializable; + +/** + * 服务上线信息(用于Rehash时客户端剔除下线) + * + * @author kango2gler@gmail.com + * @since 2021/11/2 + */ +@Slf4j +@Data +@NoArgsConstructor +@AllArgsConstructor +public class NacosServiceUpInfo implements Serializable { + private ConsistencyHashing consistencyHashing; +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/event/NacosServiceUpdateEvent.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/event/NacosServiceUpdateEvent.java new file mode 100644 index 0000000..97297d4 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/event/NacosServiceUpdateEvent.java @@ -0,0 +1,15 @@ +package org.kangspace.messagepush.core.event; + +import org.springframework.context.ApplicationEvent; + +/** + * Nacos服务更新事件 + * + * @author kango2gler@gmail.com + * @since 2021/11/2 + */ +public class NacosServiceUpdateEvent extends ApplicationEvent { + public NacosServiceUpdateEvent(T source) { + super(source); + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/event/NacosServiceUpdateInfo.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/event/NacosServiceUpdateInfo.java new file mode 100644 index 0000000..311ed29 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/event/NacosServiceUpdateInfo.java @@ -0,0 +1,52 @@ +package org.kangspace.messagepush.core.event; + +import com.alibaba.nacos.api.naming.pojo.Instance; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.util.MD5Util; +import org.springframework.util.CollectionUtils; + +import java.io.Serializable; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +/** + * 服务变化信息 + * + * @author kango2gler@gmail.com + * @since 2021/11/2 + */ +@Slf4j +@Data +@NoArgsConstructor +@AllArgsConstructor +public class NacosServiceUpdateInfo implements Serializable { + /** + * 服务ID + */ + private String serviceId; + /** + * 实例服务信息 + */ + private List instances; + + /** + * 获取服务列表摘要字符串 + * 摘要逻辑: 1. instances 按 "ip:端口" Hash排序 + * 2. 转换为按,分割的字符串 + * 3. 对字符串取MD5 + * + * @return + * @see MD5Util#hashDigest(Collection) + */ + public String getInstancesDigest() { + if (CollectionUtils.isEmpty(instances)) { + return null; + } + return MD5Util.hashDigest(instances.stream().map(t -> t.getIp() + ":" + t.getPort()) + .collect(Collectors.toList())); + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/ConsistencyHashing.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/ConsistencyHashing.java new file mode 100644 index 0000000..a4b588f --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/ConsistencyHashing.java @@ -0,0 +1,239 @@ +package org.kangspace.messagepush.core.hash; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.hash.algoithm.HashAlgorithm; +import org.kangspace.messagepush.core.hash.algoithm.KetamaHashAlgorithm; +import org.kangspace.messagepush.core.util.MD5Util; +import org.springframework.util.CollectionUtils; + +import java.util.*; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; + +/** + * 一致性hash实现 + * 基于Ketama 一致性Hash算法: + * 1. 取node的md5 + * 2. 再将md5值每4字节计算一个Hash Key存到Hash环中,即每个node会有4个hash节点 + * 3. 为node的所有虚拟节点做2的处理,并将结果存到Hash环中 + * 4. 每个物理节点的虚拟节点在hash环上最好分配100-200个点来抑制分布不均匀，最大限度地减小服务器增减时的缓存重新分布 + * + * @author kango2gler@gmail.com + * @since 2021/10/22 + */ +@Data +@Slf4j +public class ConsistencyHashing { + /** + * 虚拟节点分割符 + */ + private static final String VIRTUAL_DELIMITER = "#VN"; + /** + * hash 分组大小 + */ + private static final int HASH_GROUP_SIZE = 4; + /** + * 并发锁 + */ + private final ReentrantReadWriteLock rwl = new ReentrantReadWriteLock(); + private final Lock r = rwl.readLock(); + private final Lock w = rwl.writeLock(); + /** + * 
+     * 每个物理节点的虚拟节点数
+     * 每个物理节点的虚拟节点在hash环上最好分配100-200个点来抑制分布不均匀，最大限度地减小服务器增减时的缓存重新分布
+     * 计算虚拟节点总数时需考虑:numberOfVirtualNode中的每个节点会创建4个虚拟节点
+     *
+ */ + private Integer numberOfVirtualNode; + /** + * Hash算法 + */ + private HashAlgorithm hashAlgorithm = new KetamaHashAlgorithm(); + /** + * 物理节点的摘要 + * + * @see MD5Util#hashDigest(Collection) + */ + private String physicalNodesDigest; + /** + * 物理节点列表 + */ + private List> physicalNodes = new ArrayList<>(); + /** + * Hash环 + * key: hash值 + * value: 虚拟节点 + */ + private TreeMap> ring = new TreeMap<>(); + + public ConsistencyHashing(int numberOfVirtualNode, List physicalNodes) { + this.numberOfVirtualNode = numberOfVirtualNode; + if (!CollectionUtils.isEmpty(physicalNodes)) { + List> pNodes = physicalNodes.stream().distinct() + .map(node -> new PhysicalNode(hashAlgorithm.hashing(node), node)) + .collect(Collectors.toList()); + pNodes.forEach(node -> addVirtualNode(node, numberOfVirtualNode)); + setPhysicalNodes(pNodes); + } + } + + public ConsistencyHashing(List> physicalNodes, int numberOfVirtualNode) { + this.numberOfVirtualNode = numberOfVirtualNode; + if (!CollectionUtils.isEmpty(physicalNodes)) { + physicalNodes.forEach(node -> addVirtualNode(node, numberOfVirtualNode)); + setPhysicalNodes(physicalNodes); + } + } + + public ConsistencyHashing(Map> fromRing, int numberOfVirtualNode) { + if (!CollectionUtils.isEmpty(fromRing)) { + fromRing.forEach((k, v) -> this.ring.put(Long.valueOf(k), v)); + List> physicalNodes = fromRing.values().stream().map(t -> t.getPhysicalNode()).collect(Collectors.toList()); + List> distinctPhysicalNodes = new ArrayList<>(physicalNodes.stream().collect(Collectors.toMap(k -> k.getNode(), v -> v, (v1, v2) -> v1)).values()); + int totalVirtualCount = fromRing.size(); + long physicalNodeCount = distinctPhysicalNodes.size(); + this.numberOfVirtualNode = Math.toIntExact(physicalNodeCount > 0 ? totalVirtualCount / 4 / physicalNodeCount : numberOfVirtualNode); + setPhysicalNodes(distinctPhysicalNodes); + } + } + + private void setPhysicalNodes(List> physicalNodes) { + this.physicalNodes = physicalNodes; + List nodeIpPorts = physicalNodes.stream().map(t -> t.getNode()).distinct().collect(Collectors.toList()); + this.physicalNodesDigest = MD5Util.hashDigest(nodeIpPorts); + } + + private void addPhysicalNodes(PhysicalNode physicalNode) { + this.physicalNodes.add(physicalNode); + List nodeIpPorts = this.physicalNodes.stream().map(t -> t.getNode()).collect(Collectors.toList()); + this.physicalNodesDigest = MD5Util.hashDigest(nodeIpPorts); + } + + private void removePhysicalNodes(PhysicalNode physicalNode) { + this.physicalNodes = this.physicalNodes.stream().filter(t -> !t.getNode().equals(physicalNode.getNode())).collect(Collectors.toList()); + List nodeIpPorts = this.physicalNodes.stream().map(t -> t.getNode()).collect(Collectors.toList()); + this.physicalNodesDigest = MD5Util.hashDigest(nodeIpPorts); + } + + /** + * 获取节点hash + * + * @param node node + * @return hash结果 + */ + public Long getNodeHash(String node) { + return this.hashAlgorithm.hashing(node); + } + + + /** + * 获取data所在环的虚拟节点 + * + * @param data 数据节点 + * @return virtualNode + */ + public VirtualNode getVirtualNode(String data) { + if (ring.isEmpty()) { + return null; + } + Long hash = hashAlgorithm.hashing(data); + r.lock(); + try { + if (!ring.containsKey(hash)) { + SortedMap> tailMap = ring.tailMap(hash); + hash = tailMap.isEmpty() ? ring.firstKey() : tailMap.firstKey(); + } + return ring.get(hash); + } finally { + r.unlock(); + } + } + + /** + * 添加虚拟节点(使用Ketama一致性HASH算法) + * + * @param physicalNode 物理节点 + * @param numberOfVirtualNode 每个物理节点的虚拟节点数 + */ + private void addVirtualNode(Node physicalNode, int numberOfVirtualNode) { + w.lock(); + try { + // / HASH_GROUP_SIZE + for (int i = 0; i < numberOfVirtualNode; i++) { + String virtualNode = getVirtualNode(physicalNode, i); + byte[] digest = hashAlgorithm.md5(virtualNode); + for (int j = 0; j < HASH_GROUP_SIZE; j++) { + Long hash = hashAlgorithm.hashing(digest, j); + ring.put(hash, new VirtualNode(hash, virtualNode, (PhysicalNode) physicalNode)); + } + } + } finally { + w.unlock(); + } + } + + + /** + * 删除一个物理节点 + * + * @param physicalNode 物理节点 + */ + public void removeNode(PhysicalNode physicalNode) { + if (ring.isEmpty()) { + return; + } + w.lock(); + try { + // 实现注意遍历删除可能存在的并发修改异常 + Iterator iterator = ring.keySet().iterator(); + while (iterator.hasNext()) { + Long nodeHashKey = iterator.next(); + VirtualNode virtualNode = ring.get(nodeHashKey); + if (virtualNode.isVirtualOf(physicalNode)) { + iterator.remove(); + } + } + removePhysicalNodes(physicalNode); + } finally { + w.unlock(); + } + } + + /** + * 添加一个物理节点 + * + * @param physicalNode 物理节点 + */ + public void addNode(PhysicalNode physicalNode) { + w.lock(); + try { + addPhysicalNodes(physicalNode); + addVirtualNode(physicalNode, this.numberOfVirtualNode); + } finally { + w.unlock(); + } + } + + /** + * 获取虚拟节点节点node值 + * + * @param physicalNode 物理节点 + * @param number 下表 + * @return 新的node字符串 + */ + private String getVirtualNode(Node physicalNode, int number) { + return physicalNode.getNode() + VIRTUAL_DELIMITER + number; + } + + /** + * 获取虚拟节点数量 + * + * @return 虚拟节点数量 + */ + public int getVirtualNodeCount() { + return ring.size(); + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/Node.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/Node.java new file mode 100644 index 0000000..39c7eb8 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/Node.java @@ -0,0 +1,38 @@ +package org.kangspace.messagepush.core.hash; + +/** + * 基本的服务节点(hash环上的节点) + * + * @author kango2gler@gmail.com + * @since 2021/10/22 + */ +public interface Node { + /** + * 获取节点Key(一般为HASH值) + * + * @return key + */ + Long getKey(); + + /** + * 获取node原始值 + * + * @return String + */ + String getNode(); + + /** + * 是否物理节点 + * + * @return boolean + */ + boolean isPhysicalNode(); + + /** + * 获取节点数据 + * + * @return T + */ + T getData(); + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/PhysicalNode.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/PhysicalNode.java new file mode 100644 index 0000000..f013d3c --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/PhysicalNode.java @@ -0,0 +1,46 @@ +package org.kangspace.messagepush.core.hash; + +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Objects; + +/** + * 物理服务节点 + * + * @author kango2gler@gmail.com + * @since 2021/10/22 + */ +@Data +@NoArgsConstructor +public class PhysicalNode implements Node { + /** + * 节点数据 + */ + private String node; + /** + * 节点key(HASH) + */ + private Long key; + + /** + * 节点数据 + */ + private T data; + + public PhysicalNode(Long key, String node) { + this(key, node, null); + } + + public PhysicalNode(Long key, String node, T data) { + Objects.requireNonNull(node, "PhysicalServiceNode [node] must be not null!"); + this.key = key; + this.node = node; + this.data = data; + } + + @Override + public boolean isPhysicalNode() { + return true; + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/VirtualNode.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/VirtualNode.java new file mode 100644 index 0000000..61886c7 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/VirtualNode.java @@ -0,0 +1,62 @@ +package org.kangspace.messagepush.core.hash; + +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Objects; + +/** + * 物理服务节点 + * + * @author kango2gler@gmail.com + * @since 2021/10/22 + */ +@Data +@NoArgsConstructor +public class VirtualNode implements Node { + + /** + * 物理节点 + */ + private PhysicalNode physicalNode; + /** + * 节点数据 + */ + private String node; + /** + * 节点key(HASH) + */ + private Long key; + + /** + * 节点数据 + */ + private T data; + + public VirtualNode(Long key, String node, PhysicalNode physicalNode) { + this(key, node, physicalNode, null); + } + + public VirtualNode(Long key, String node, PhysicalNode physicalNode, T data) { + Objects.requireNonNull(node, "PhysicalNode [node] must be not null!"); + this.key = key; + this.node = node; + this.physicalNode = physicalNode; + this.data = data; + } + + @Override + public boolean isPhysicalNode() { + return false; + } + + /** + * 检查是否指定物理节点的虚拟节点 + * + * @param physicalNode 物理节点 + * @return boolean + */ + public boolean isVirtualOf(PhysicalNode physicalNode) { + return Objects.equals(physicalNode.getNode(), this.getPhysicalNode().getNode()); + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/algoithm/HashAlgorithm.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/algoithm/HashAlgorithm.java new file mode 100644 index 0000000..4825353 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/algoithm/HashAlgorithm.java @@ -0,0 +1,46 @@ +package org.kangspace.messagepush.core.hash.algoithm; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; + +/** + * key Hash算法(32位) + * + * @author kango2gler@gmail.com + * @since 2021/10/22 + */ +public interface HashAlgorithm { + /** + * key hash 算法 + * + * @param node 待hash字符串 + * @return hash + */ + Long hashing(String node); + + /** + * Ketama key hash 算法 + * + * @param digest md5后的byte[] + * @param number 虚拟节点索引 + * @return hash + */ + Long hashing(byte[] digest, int number); + + /** + * MD5 + * + * @param str 待计算的字符串 + * @return byte[] + */ + default byte[] md5(String str) { + String algorithm = "MD5"; + try { + MessageDigest md = MessageDigest.getInstance(algorithm); + return md.digest(str.getBytes(StandardCharsets.UTF_8)); + } catch (Exception e) { + return str.getBytes(); + } + + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/algoithm/KetamaHashAlgorithm.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/algoithm/KetamaHashAlgorithm.java new file mode 100644 index 0000000..3652631 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/algoithm/KetamaHashAlgorithm.java @@ -0,0 +1,45 @@ +package org.kangspace.messagepush.core.hash.algoithm; + +/** + * Ketama key Hash算法(32位) + * + * @author kango2gler@gmail.com + * @since 2021/10/22 + */ +public class KetamaHashAlgorithm implements HashAlgorithm { + /** + * Ketama key hash 算法 + * + * @param node 待hash字符串 + * @return hash值 + */ + @Override + public Long hashing(String node) { + byte[] digest = md5(node); + return hash(digest, 0); + } + + + @Override + public Long hashing(byte[] digest, int number) { + return hash(digest, number); + } + + + /** + * Ketama Hash + * + * @param digest MD5 digest + * @param number 序号 + * @return hash + */ + private Long hash(byte[] digest, int number) { + return (((long) (digest[3 + number * 4] & 0xFF) << 24) + | ((long) (digest[2 + number * 4] & 0xFF) << 16) + | ((long) (digest[1 + number * 4] & 0xFF) << 8) + | (digest[number * 4] & 0xFF)) + & 0xFFFFFFFFL; + } + + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/repository/HashRouterRepository.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/repository/HashRouterRepository.java new file mode 100644 index 0000000..f1e5ccb --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/hash/repository/HashRouterRepository.java @@ -0,0 +1,61 @@ +package org.kangspace.messagepush.core.hash.repository; + +import org.kangspace.messagepush.core.hash.ConsistencyHashing; + +import java.util.List; + +/** + * 一致性Hash路由数据接口 + * + * @author kango2gler@gmail.com + * @since 2021/10/28 + */ +public interface HashRouterRepository { + + /** + * 初始化一致性Hash数据 + * + * @return {@link ConsistencyHashing} + */ + ConsistencyHashing init(); + + /** + * 获取一致性Hash数据 + * + * @return + */ + ConsistencyHashing get(); + + /** + * 保存一致性Hash数据 + * + * @param hashRouter ConsistencyHashing + * @return boolean + */ + boolean store(ConsistencyHashing hashRouter); + + /** + * rehash + * + * @param actualServices 最新的服务列表(ip:端口列表) + * @return ConsistencyHashing + */ + ConsistencyHashing rehash(List actualServices); + + /** + * 检查Hash环数据是否和当前Server列表一致 + * + * @param hashRouter 需要对比的一致性Hash环数据 + * @param actualServices 最新的服务列表(ip:端口列表) + * @return Hash数据是否一致, 是否需要rehash, true:需要rehash,false:不需要 + */ + boolean compareHashData(ConsistencyHashing hashRouter, List actualServices); + + /** + * 检查Hash换数据是否和当前Server列表一致, + * 若不一致则rehash + * + * @return + */ + ConsistencyHashing compareHashDataAndRehash(ConsistencyHashing hashRouter, List actualServices); +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/http/RestMapProperties.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/http/RestMapProperties.java new file mode 100644 index 0000000..ab1a906 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/http/RestMapProperties.java @@ -0,0 +1,30 @@ +package org.kangspace.messagepush.core.http; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.Map; + +/** + * RestTemplate 连接池配置类 + * + * @author kango2gler@gmail.com + */ +@Data +@Component +@ConfigurationProperties("rest") +public class RestMapProperties { + + /** + * 是否开启rest组件 + */ + private Boolean restEnable; + + /** + * 多连接池支持配置Map<连接池名称,连接池配置信息> + */ + private Map rest = new HashMap<>(); + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/http/RestProperties.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/http/RestProperties.java new file mode 100644 index 0000000..b61c0a6 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/http/RestProperties.java @@ -0,0 +1,58 @@ +package org.kangspace.messagepush.core.http; + +import com.google.common.collect.Maps; +import lombok.Getter; +import lombok.Setter; + +import java.nio.charset.StandardCharsets; +import java.util.Map; + +/** + * @author kango2gler@gmail.com + * @description rest配置类，需全部由默认值 + **/ +@Setter +@Getter +public class RestProperties { + + /** + * 连接池的最大连接数 + */ + private Integer maxTotalConnect = 50; + /** + * 同路由的并发数 + */ + private Integer maxConnectPerRoute = 200; + + /** + * 客户端和服务器建立连接超时，默认2s + */ + private Integer connectTimeout = 2000; + /** + * 指客户端从服务器读取数据包的间隔超时时间,不是总读取时间，默认30s + */ + private Integer readTimeout = 10000; + + /** + * 编码格式 + */ + private String charset = StandardCharsets.UTF_8.name(); + /** + * 重试次数,默认2次 + */ + private Integer retryTimes = 1; + /** + * 从连接池获取连接的超时时间,不宜过长,单位ms + */ + private Integer connectionRequestTimeout = 200; + /** + * 针对不同的地址,特别设置不同的长连接保持时间 + */ + private Map keepAliveTargetHost = Maps.newHashMap(); + /** + * 针对不同的地址,特别设置不同的长连接保持时间,单位 s，若defaultKeepAliveTime为0且keepAliveTargetHost大小为空，则不启用长连接策略 + */ + private Integer defaultKeepAliveTime = 60; + + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/http/RestTemplateFactory.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/http/RestTemplateFactory.java new file mode 100644 index 0000000..1137ca2 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/http/RestTemplateFactory.java @@ -0,0 +1,163 @@ +package org.kangspace.messagepush.core.http; + +import com.google.common.collect.Lists; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HeaderElement; +import org.apache.http.HeaderElementIterator; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpHost; +import org.apache.http.client.HttpClient; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.conn.ConnectionKeepAliveStrategy; +import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicHeaderElementIterator; +import org.apache.http.protocol.HTTP; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.http.MediaType; +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.http.converter.FormHttpMessageConverter; +import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.http.converter.StringHttpMessageConverter; +import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; +import org.springframework.web.client.RestTemplate; + +import javax.annotation.Resource; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +/** + * RestTemplateFactory工厂，初始化管理连接 + * + * @author kango2gler@gmail.com + */ +@Slf4j +public class RestTemplateFactory implements InitializingBean { + public static final String DEFAULT = "default"; + /** + * rest模板 + */ + private static final Map REST_TEMPLATE_MAP = new ConcurrentHashMap<>(); + @Resource + private RestMapProperties restConfig; + + /** + * 获取RestTemplate + * + * @return + */ + public RestTemplate getRestTemplate() { + return REST_TEMPLATE_MAP.get(DEFAULT); + } + + /** + * 获取RestTemplate + * + * @return + */ + public RestTemplate getRestTemplate(String name) { + return REST_TEMPLATE_MAP.get(name); + } + + /** + * 注册restTemplate + * + * @param name + * @param restProperties + */ + public synchronized RestTemplate registerRestTemplate(String name, RestProperties restProperties) { + RestTemplate restTemplate = getRestTemplate(restProperties); + REST_TEMPLATE_MAP.put(name, restTemplate); + return restTemplate; + } + + @Override + public void afterPropertiesSet() { + log.info("rest连接池初始化开始，配置参数：" + restConfig.getRest()); + restConfig.getRest().forEach((name, restProperties) -> REST_TEMPLATE_MAP.put(name, getRestTemplate(restProperties))); + if (!REST_TEMPLATE_MAP.containsKey(DEFAULT)) { + REST_TEMPLATE_MAP.put(DEFAULT, getRestTemplate(new RestProperties())); + } + log.info("rest连接池初始化完成，连接池名称：" + REST_TEMPLATE_MAP.keySet()); + } + + + private RestTemplate getRestTemplate(RestProperties restProperties) { + HttpComponentsClientHttpRequestFactory clientHttpRequestFactory = new HttpComponentsClientHttpRequestFactory(httpClient(restProperties)); + clientHttpRequestFactory.setConnectTimeout(restProperties.getConnectTimeout()); + clientHttpRequestFactory.setReadTimeout(restProperties.getReadTimeout()); + RestTemplate restTemplate = new RestTemplate(clientHttpRequestFactory); + modifyDefaultCharset(restTemplate); + return restTemplate; + } + + private HttpClient httpClient(RestProperties restProperties) { + HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); + //使用Httpclient连接池的方式配置(推荐)，同时支持netty，okHttp以及其他http框架 + PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = new PoolingHttpClientConnectionManager(); + // 最大连接数 + poolingHttpClientConnectionManager.setMaxTotal(restProperties.getMaxTotalConnect()); + // 同路由并发数 + poolingHttpClientConnectionManager.setDefaultMaxPerRoute(restProperties.getMaxConnectPerRoute()); + //配置连接池 + httpClientBuilder.setConnectionManager(poolingHttpClientConnectionManager); + // 重试次数 + httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(restProperties.getRetryTimes(), Boolean.TRUE)); + //设置长连接保持策略 + if (restProperties.getKeepAliveTargetHost().size() > 0 || restProperties.getDefaultKeepAliveTime() > 0) { + httpClientBuilder.setKeepAliveStrategy(connectionKeepAliveStrategy(restProperties)); + } + return httpClientBuilder.build(); + } + + + /** + * 配置长连接保持策略 + * + * @return 长连接策略 + */ + private ConnectionKeepAliveStrategy connectionKeepAliveStrategy(RestProperties restProperties) { + return (response, context) -> { + // 设置长连接头 + HeaderElementIterator headerElementIterator = new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE)); + while (headerElementIterator.hasNext()) { + HeaderElement headerElement = headerElementIterator.nextElement(); + String param = headerElement.getName(); + String value = headerElement.getValue(); + if (StringUtils.isNumericSpace(value) && StringUtils.endsWithIgnoreCase(HttpHeaders.TIMEOUT, param)) { + return Duration.ofSeconds(Long.parseLong(value)).toMillis(); + } + } + HttpHost target = (HttpHost) context.getAttribute(HttpClientContext.HTTP_TARGET_HOST); + //如果请求目标地址,单独配置了长连接保持时间,使用该配置 + Optional> targetMap = restProperties.getKeepAliveTargetHost().entrySet().stream().filter( + e -> StringUtils.endsWithIgnoreCase(e.getKey(), target.getHostName())).findAny(); + //否则使用默认长连接保持时间 + return targetMap.map(targetTimeOut -> Duration.ofSeconds(targetTimeOut.getValue()).toMillis()).orElse(Duration.ofSeconds(restProperties.getDefaultKeepAliveTime()).toMillis()); + }; + } + + /** + * 修改默认的字符集类型为utf-8 + * + * @param restTemplate 请求客户端 + */ + private void modifyDefaultCharset(RestTemplate restTemplate) { + List> messageConverters = new ArrayList<>(); + messageConverters.add(new StringHttpMessageConverter(StandardCharsets.UTF_8)); + messageConverters.add(new FormHttpMessageConverter()); + MappingJackson2HttpMessageConverter mappingJackson2HttpMessageConverter = new MappingJackson2HttpMessageConverter(); + List supportedMediaTypes = Lists.newArrayList(mappingJackson2HttpMessageConverter.getSupportedMediaTypes()); + supportedMediaTypes.add(MediaType.TEXT_HTML); + mappingJackson2HttpMessageConverter.setSupportedMediaTypes(supportedMediaTypes); + messageConverters.add(mappingJackson2HttpMessageConverter); + restTemplate.setMessageConverters(messageConverters); + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/redis/RedisService.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/redis/RedisService.java new file mode 100644 index 0000000..ed8197e --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/redis/RedisService.java @@ -0,0 +1,138 @@ +package org.kangspace.messagepush.core.redis; + + +import cn.hutool.core.map.MapUtil; +import lombok.Getter; +import org.kangspace.messagepush.core.util.JsonUtil; +import org.kangspace.messagepush.core.util.StrUtil; +import org.springframework.data.redis.core.RedisTemplate; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +/** + * Redis相关操作Service + * + * @author kango2gler@gmail.com + * @since 2021/8/9 + */ +public class RedisService { + + @Getter + private RedisTemplate redisTemplate; + + /** + * 
+     * 获取Redis缓存对象;
+     * 若fetchData数据为空时,返回null,反之若缓存不存在时,重新进行缓存。
+     *
+ * + * @param key redis key + * @param clazz redis 缓存的对象类型 + * @param time 缓存超时时间 + * @param fetchData 重新获取对象的方法 + * @param 目标对象 + * @return T or null, 当 fetchData 为空时,返回null + */ + public T getAndCache(String key, Class clazz, long time, Supplier fetchData) { + T cache = this.get(key, clazz); + if (cache == null) { + synchronized (this) { + cache = this.get(key, clazz); + if (cache == null) { + cache = fetchData.get(); + if (cache != null) { + //重新缓存 + this.setEX(key, cache, time); + } + } + } + } + return cache; + } + + /** + * 分布式锁 + * + * @param key redis key + * @param ttl 超时时间 + * @param callback 获取到锁后的执行函数 + * @return boolean true:已获取到锁,false:未获取到锁 + */ + public boolean lock(String key, long ttl, Runnable callback) { + String flag = System.currentTimeMillis() + ""; + if (this.setNX(key, flag, ttl)) { + try { + callback.run(); + } finally { + String value = this.get(key); + if (value != null && flag.equals(value)) { + this.del(key); + } + } + return true; + } + return false; + } + + public Map hGetAll(String key, Class type) { + if (!StrUtil.isEmpty(key) && type != null) { + Map map = this.getRedisTemplate().opsForHash().entries(key); + Map result = new HashMap(); + Iterator var5 = map.entrySet().iterator(); + + while (var5.hasNext()) { + Map.Entry obj = (Map.Entry) var5.next(); + result.put((String) obj.getKey(), JsonUtil.toObject((String) obj.getValue(), type)); + } + + return result; + } else { + return null; + } + } + + public Boolean setNX(String key, T value) { + return !StrUtil.isEmpty(key) && value != null ? this.getRedisTemplate().opsForValue().setIfAbsent(key, JsonUtil.toJson(value)) : false; + } + + public Boolean setNX(String key, T value, long time) { + return !StrUtil.isEmpty(key) && value != null ? this.getRedisTemplate().opsForValue().setIfAbsent(key, JsonUtil.toJson(value), time, TimeUnit.SECONDS) : false; + } + + public void setEX(String key, T value, long time) { + if (!StrUtil.isEmpty(key) && value != null) { + this.getRedisTemplate().opsForValue().set(key, JsonUtil.toJson(value), time, TimeUnit.SECONDS); + } + } + + public String get(String key) { + return StrUtil.isEmpty(key) ? null : (String) this.getRedisTemplate().opsForValue().get(key); + } + + public T get(String key, Class type) { + return !StrUtil.isEmpty(key) && type != null ? JsonUtil.toObject((String) this.getRedisTemplate().opsForValue().get(key), type) : null; + } + + public Boolean del(String key) { + return StrUtil.isEmpty(key) ? false : this.getRedisTemplate().delete(key); + } + + public void hMSet(String key, Map map) { + if (!StrUtil.isEmpty(key) && !MapUtil.isEmpty(map)) { + Map saveMap = new HashMap(); + Iterator var4 = map.entrySet().iterator(); + + while (var4.hasNext()) { + Map.Entry obj = (Map.Entry) var4.next(); + saveMap.put(obj.getKey(), JsonUtil.toJson(obj.getValue())); + } + + this.getRedisTemplate().opsForHash().putAll(key, saveMap); + } + } + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/util/AppGenerator.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/AppGenerator.java new file mode 100644 index 0000000..3343ab9 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/AppGenerator.java @@ -0,0 +1,83 @@ +package org.kangspace.messagepush.core.util; + +import cn.hutool.core.lang.UUID; +import cn.hutool.crypto.digest.MD5; +import lombok.Data; +import org.springframework.util.StringUtils; + +import java.nio.charset.StandardCharsets; + +/** + * 
+ * 应用信息生成器
+ * appId: 32位UUID
+ * appSecret: md5("messagepush"+{appId})
+ *
+ * + * @author kango2gler@gmail.com + * @since 2021/10/26 + */ +public class AppGenerator { + /** + * AppSecret生成的加密因子 + */ + private static final byte[] APP_SECRET_SEEDS = "messagepush".getBytes(StandardCharsets.UTF_8); + + /** + * 生成应用信息 + * + * @return {@link AppInfo} + */ + public static AppInfo generate() { + String appKey = UUID.fastUUID().toString(true); + String appSecret = generateAppSecret(appKey); + return new AppInfo(appKey, appSecret); + } + + /** + * 通过AppKey生成AppSecret + * + * @param appKey appKey + * @return AppSecret + */ + private static String generateAppSecret(String appKey) { + return new MD5(APP_SECRET_SEEDS).digestHex16(appKey); + } + + /** + * 验证应用信息 + * + * @param appKey appKey + * @param appSecret appSecret + * @return boolean + */ + public static boolean validAppInfo(String appKey, String appSecret) { + if (StringUtils.hasText(appKey) && StringUtils.hasText(appSecret)) { + String correctSecret = generateAppSecret(appKey); + return correctSecret.equals(appSecret); + } + return false; + } + + public static void main(String[] args) { + AppInfo appInfo = AppGenerator.generate(); + System.out.println(appInfo); + } + + /** + * Http Basic认证头 + */ + @Data + public static class AppInfo { + private String appKey; + private String appSecret; + + public AppInfo() { + } + + public AppInfo(String appKey, String appSecret) { + this.appKey = appKey; + this.appSecret = appSecret; + } + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/util/ArrayUtil.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/ArrayUtil.java new file mode 100644 index 0000000..f4d2163 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/ArrayUtil.java @@ -0,0 +1,60 @@ +package org.kangspace.messagepush.core.util; + +import org.apache.commons.lang3.ArrayUtils; + +/** + * 数组工具类 + * + * @author kango2gler@gmail.com + * @since 2022/5/13 + */ +public class ArrayUtil { + + /** + * 基本类型数组转换为包装类型数据 + * + * @param arr 数组 + * @return 包装类型数据 + */ + public static T[] toBoxed(Object arr) { + T[] result; + if (arr.getClass().isArray()) { + switch (arr.getClass().getComponentType().getName()) { + case "boolean": + result = (T[]) ArrayUtils.toObject((boolean[]) arr); + break; + case "byte": + result = (T[]) ArrayUtils.toObject((byte[]) arr); + break; + case "char": + result = (T[]) ArrayUtils.toObject((char[]) arr); + break; + case "int": + result = (T[]) ArrayUtils.toObject((int[]) arr); + break; + case "short": + result = (T[]) ArrayUtils.toObject((short[]) arr); + break; + case "float": + result = (T[]) ArrayUtils.toObject((float[]) arr); + break; + case "double": + result = (T[]) ArrayUtils.toObject((double[]) arr); + break; + case "long": + result = (T[]) ArrayUtils.toObject((long[]) arr); + break; + default: + result = (T[]) arr; + } + return result; + } + return null; + } + + + public static void main(String[] args) { + System.out.println(toBoxed(new int[]{1, 2, 3})); + System.out.println(toBoxed(new Integer[]{1, 2, 3})); + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/util/BeanUtil.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/BeanUtil.java new file mode 100644 index 0000000..8d637a5 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/BeanUtil.java @@ -0,0 +1,82 @@ +package org.kangspace.messagepush.core.util; + + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.lang.reflect.Field; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.List; +import java.util.stream.Collectors; + +/** + * bean工具类 + * + * @author kango2gler@gmail.com + * @since 2021-04-23 + */ +public class BeanUtil { + + /** + * 设置bean对象中不再fields中的属性为null(兼容大小写和_) + * + * @param bean 对象 + * @param keepFields 保留的属性名 + * @param 对象泛型 + * @throws IllegalAccessException + */ + public static void setFieldsNull(T bean, List keepFields) throws IllegalAccessException { + if (ListUtil.isEmpty(keepFields)) { + return; + } + //小写的保留字段名 + List lowerKeepFields = keepFields.stream().map(f -> f.toLowerCase()).collect(Collectors.toList()); + //遍历对象属性 + for (Field field : bean.getClass().getDeclaredFields()) { + field.setAccessible(true); + if (field.get(bean) == null) { + continue; + } + JsonProperty jsonProperty = field.getAnnotation(JsonProperty.class); + //存在JsonProperty 且 value不为空 则 使用jsonProperty.value()作为属性名判断 + if (jsonProperty != null && StrUtil.isNotEmpty(jsonProperty.value())) { + //fields不包含该属性名则置空该属性 + if (!lowerKeepFields.contains(jsonProperty.value().toLowerCase())) { + field.set(bean, null); + } + //存在JsonProperty 或 JsonProperty.value不为空 则 使用原始属性名判断，fields不包含该属性名则置空该属性 + } else if (!lowerKeepFields.contains(field.getName().toLowerCase())) { + field.set(bean, null); + } + } + } + + /** + * 返回实现的接口或继承的父类的第一个泛型（优先取父类中的泛型） + * + * @param o + * @return + */ + public static Class getGenericsClass(Object o) { + List parentClasses = ListUtil.arrayToList(o.getClass().getGenericInterfaces()); + parentClasses.add(o.getClass().getGenericSuperclass()); + if (o.getClass().getSuperclass() != null) { + parentClasses.addAll(ListUtil.arrayToList(o.getClass().getSuperclass().getGenericInterfaces())); + } + for (Type type : parentClasses) { + if (!(type instanceof ParameterizedType)) { + continue; + } + Type[] types = ((ParameterizedType) type).getActualTypeArguments(); + if (types == null || types.length == 0) { + continue; + } + if (!(types[0] instanceof ParameterizedType)) { + return ((Class) types[0]); + } + return (Class) ((ParameterizedType) types[0]).getRawType(); + } + return String.class; + } + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/util/ExchangeRequestUtils.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/ExchangeRequestUtils.java new file mode 100644 index 0000000..48b6e0f --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/ExchangeRequestUtils.java @@ -0,0 +1,46 @@ +package org.kangspace.messagepush.core.util; + +import org.springframework.http.HttpHeaders; +import org.springframework.web.server.ServerWebExchange; + +/** + * Exchange请求工具类 + * + * @author kango2gler@gmail.com + * @since 2021/10/27 + */ +public class ExchangeRequestUtils { + private final static String HTTP = "http"; + private final static String HTTPS = "http"; + + /** + * 
+     * 是否为Websocket请求
+     * 协议为Http且Request Header: [Connection: Upgrade ,Upgrade: WebSocket]
+     *
+ * + * @param exchange {@link ServerWebExchange} + * @return boolean + */ + public static boolean isWebsocketRequest(ServerWebExchange exchange) { + String scheme = exchange.getRequest().getURI().getScheme().toLowerCase(); + String connection = exchange.getRequest().getHeaders().getConnection().stream().findFirst().orElse(null); + String upgrade = exchange.getRequest().getHeaders().getUpgrade(); + boolean isWebSocketRequest = "WebSocket".equalsIgnoreCase(upgrade) + && HttpHeaders.UPGRADE.equalsIgnoreCase(connection) + && (HTTP.equals(scheme) || HTTPS.equals(scheme)); + return isWebSocketRequest; + } + + /** + * 
+     * 获取客户端Ip地址
+     *
+ * + * @param exchange {@link ServerWebExchange} + * @return ip + */ + public static String getIP(ServerWebExchange exchange) { + return IpUtils.getClientIp(exchange.getRequest()); + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/util/HttpUtils.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/HttpUtils.java new file mode 100644 index 0000000..9c07af3 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/HttpUtils.java @@ -0,0 +1,188 @@ +package org.kangspace.messagepush.core.util; + +import lombok.Data; +import org.springframework.core.io.buffer.DataBuffer; +import org.springframework.core.io.buffer.DataBufferUtils; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.server.reactive.ServerHttpRequest; +import org.springframework.util.Base64Utils; +import org.springframework.util.StringUtils; +import reactor.core.publisher.Flux; + +import javax.servlet.http.HttpServletRequest; +import java.nio.CharBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Http相关工具类 + * + * @author kango2gler@gmail.com + * @since 2021/10/26 + */ +public class HttpUtils { + /** + * Http Basic认证元素个数(即username,password) + */ + public static final int HTTP_BASIC_AUTH_ELE_NUM = 2; + /** + * Http Basic认证头的值前缀 + */ + public static final String HTTP_BASIC_AUTH_VALUE_PREFIX = "Basic "; + /** + * Http Bearer认证头的值前缀 + */ + public static final String HTTP_BEARER_TOKEN_VALUE_PREFIX = "Bearer "; + public static final String HTTP_SCHEMA = "http://"; + public static final String HTTPS_SCHEMA = "https://"; + /** + * multipart 媒体类型 + */ + public static List MULTI_PART_MEDIA_TYPES = Arrays.asList(MediaType.MULTIPART_FORM_DATA_VALUE, + MediaType.MULTIPART_MIXED_VALUE, + MediaType.MULTIPART_RELATED_VALUE); + + /** + * 获取请求头中的Basic认证信息 + * + * @param request {@link HttpServletRequest} + * @return {@link HttpBasicAuth} + */ + public static HttpBasicAuth getBasicAuth(HttpServletRequest request) { + String authorization = request.getHeader(HttpHeaders.AUTHORIZATION); + if (StringUtils.hasText(authorization) && authorization.startsWith(HTTP_BASIC_AUTH_VALUE_PREFIX)) { + try { + byte[] base64Dec = Base64Utils.decode(authorization.substring(HTTP_BASIC_AUTH_VALUE_PREFIX.length()).getBytes(StandardCharsets.UTF_8)); + String credentialsString = new String(base64Dec, StandardCharsets.UTF_8); + String[] credentials = credentialsString.split(":"); + if (credentials.length == HTTP_BASIC_AUTH_ELE_NUM) { + return new HttpBasicAuth(credentials[0], credentials[1]); + } + } catch (Exception ignore) { + } + } + return null; + } + + /** + * 获取BearerToken + * + * @param request HttpServletRequest + * @return Bearer Token + */ + public static String getHttpBearerToken(ServerHttpRequest request) { + String authorization = request.getHeaders().getFirst(HttpHeaders.AUTHORIZATION); + if (StringUtils.hasText(authorization) && authorization.startsWith(HTTP_BEARER_TOKEN_VALUE_PREFIX)) { + return authorization.substring(HTTP_BEARER_TOKEN_VALUE_PREFIX.length()); + } + return authorization; + } + + /** + * 是否是文件上传类型 + * + * @param contentType contentTYpe + * @return boolean + */ + public static boolean isMultipartContent(String contentType) { + if (StringUtils.hasText(contentType)) { + return MULTI_PART_MEDIA_TYPES.stream().anyMatch(contentType::contains); + } + return false; + } + + + /** + * 获取请求头 + * + * @param request {@link ServerHttpRequest} + * @return headers + */ + public static String requestHeader(ServerHttpRequest request) { + return request.getHeaders().toString(); + } + + /** + * 获取请求参数 + * + * @param request {@link ServerHttpRequest} + * @return headers + */ + public static String requestParams(ServerHttpRequest request) { + return request.getQueryParams().toString(); + } + + /** + * 获取请求体RequestBody(除文件上传外) + * + * @return request body + */ + public static String resolveBody(ServerHttpRequest request) { + String contentType = request.getHeaders().getFirst(HttpHeaders.CONTENT_TYPE); + if (!isMultipartContent(contentType)) { + //获取请求体 + Flux body = request.getBody(); + AtomicReference bodyRef = new AtomicReference<>(); + body.subscribe(buffer -> { + CharBuffer charBuffer = StandardCharsets.UTF_8.decode(buffer.asByteBuffer()); + DataBufferUtils.release(buffer); + bodyRef.set(charBuffer.toString()); + }); + //获取request body + return bodyRef.get(); + } + return ""; + } + + /** + * 适配schema,自动填充http:// + * + * @param uri uri + * @return 带 http://的uri + */ + public static String fitSchema(String uri) { + if (!StringUtils.hasText(uri)) { + return uri; + } + String lowerUri = uri.toLowerCase(); + if (!lowerUri.startsWith(HTTP_SCHEMA) && !lowerUri.startsWith(HTTPS_SCHEMA)) { + return HTTP_SCHEMA + uri; + } + return uri; + } + + + /** + * 获取BearerToken + * + * @param headers getHttpBearerToken + * @return Bearer Token + */ + public static String getHttpBearerToken(HttpHeaders headers) { + String authorization = headers.getFirst(HttpHeaders.AUTHORIZATION); + if (StringUtils.hasText(authorization) && authorization.startsWith(HTTP_BEARER_TOKEN_VALUE_PREFIX)) { + return authorization.substring(HTTP_BEARER_TOKEN_VALUE_PREFIX.length()); + } + return null; + } + + /** + * Http Basic认证头 + */ + @Data + public static class HttpBasicAuth { + private String username; + private String password; + + public HttpBasicAuth() { + } + + public HttpBasicAuth(String username, String password) { + this.username = username; + this.password = password; + } + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/util/IpUtils.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/IpUtils.java new file mode 100644 index 0000000..c7f0f3e --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/IpUtils.java @@ -0,0 +1,72 @@ +package org.kangspace.messagepush.core.util; + +import org.springframework.http.server.reactive.ServerHttpRequest; + +import java.util.function.Supplier; + +/** + * Ip相关工具类 + * + * @author kango2gler@gmail.com + * @since 2021/8/7 + */ +public class IpUtils { + + public static final String UNKNOWN = "unknown"; + public static final String COMMA = ","; + public static final String HEADER_X_FORWARDED_FOR = "x-forwarded-for"; + public static final String HEADER_PROXY_CLIENT_IP = "Proxy-Client-IP"; + public static final String HEADER_WL_PROXY_CLIENT_IP = "WL-Proxy-Client-IP"; + public static final String HEADER_HTTP_CLIENT_IP = "HTTP_CLIENT_IP"; + public static final String HEADER_HTTP_X_FORWARDED_FOR = "HTTP_X_FORWARDED_FOR"; + public static final String HEADER_X_REAL_IP = "X-Real-IP"; + + /** + * 获取客户端Ip + * + * @return string + */ + public static String getClientIp(ServerHttpRequest request) { + String ip = request.getHeaders().getFirst(HEADER_X_FORWARDED_FOR); + if (ip != null && ip.length() != 0 && !UNKNOWN.equalsIgnoreCase(ip)) { + // 多次反向代理后会有多个ip值，第一个ip才是真实ip + if (ip.contains(COMMA)) { + ip = ip.split(",")[0]; + } + } + ip = getIp(request, ip, HEADER_PROXY_CLIENT_IP, HEADER_WL_PROXY_CLIENT_IP); + ip = getIp(request, ip, HEADER_HTTP_CLIENT_IP, HEADER_HTTP_X_FORWARDED_FOR); + ip = getIp(ip, () -> request.getHeaders().getFirst(HEADER_X_REAL_IP)); + ip = getIp(ip, () -> request.getRemoteAddress() != null ? request.getRemoteAddress().getHostString() : ""); + return ip; + } + + /** + * 获取HEADER_WL_PROXY_CLIENT_IP,HEADER_HTTP_X_FORWARDED_FOR类型的IP + * + * @param request request + * @param ip 当前IP + * @param headerProxyClientIp {@link #HEADER_WL_PROXY_CLIENT_IP} + * @param headerWlProxyClientIp {@link #HEADER_HTTP_X_FORWARDED_FOR} + * @return 新IP + */ + private static String getIp(ServerHttpRequest request, String ip, String headerProxyClientIp, String headerWlProxyClientIp) { + ip = getIp(ip, () -> request.getHeaders().getFirst(headerProxyClientIp)); + ip = getIp(ip, () -> request.getHeaders().getFirst(headerWlProxyClientIp)); + return ip; + } + + /** + * 获取IP + * + * @param ip 当前IP + * @param supplier 提供获取IP的方法 + * @return 新IP值 + */ + private static String getIp(String ip, Supplier supplier) { + if (ip == null || ip.length() == 0 || UNKNOWN.equalsIgnoreCase(ip)) { + return supplier.get(); + } + return ip; + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/util/JsonUtil.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/JsonUtil.java new file mode 100644 index 0000000..7650c67 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/JsonUtil.java @@ -0,0 +1,360 @@ +package org.kangspace.messagepush.core.util; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * JSON工具类 + * + * @author kango2gler@gmail.com + * @since 2021-04-29 + */ +public final class JsonUtil { + + /** + * 默认MAPPER + */ + private static final ObjectMapper OBJECT_MAPPER; + + /** + * 自定义属性类型的MAPPER + */ + private static final Map OBJECT_MAPPER_MAP = new ConcurrentHashMap<>(); + + /** + * 缓存自定义ObjectMapper，可在系统启动时使用此方法缓存自定义ObjectMapper + * + * @param key 自定义ObjectMapper标识主键 + * @param objectMapper ObjectMapper + */ + public static void setObjectMapper(String key, ObjectMapper objectMapper) { + if (StrUtil.isEmpty(key) || objectMapper == null) { + return; + } + OBJECT_MAPPER_MAP.put(key, objectMapper); + } + + /** + * 获取默认ObjectMapper + */ + public static ObjectMapper getObjectMapper() { + return OBJECT_MAPPER; + } + + /** + * 获取缓存的自定义ObjectMapper + * + * @param key 自定义ObjectMapper标识主键 + */ + public static ObjectMapper getObjectMapper(String key) { + if (StrUtil.isEmpty(key)) { + return null; + } + return OBJECT_MAPPER_MAP.get(key); + } + + /** + * 基于指定的ObjectMapper进行对象转JSON字符串（ObjectMapper创建开销较大，务必缓存） + * + * @param src 对象 + * @param objectMapper ObjectMapper + * @param 对象泛型 + * @return JSON字符串 + */ + public static String toJson(T src, ObjectMapper objectMapper) { + if (src == null || objectMapper == null) { + return null; + } + try { + return src instanceof String ? (String) src : objectMapper.writeValueAsString(src); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + /** + * 对象转JSON字符串 + * + * @param src 对象 + * @param 对象泛型 + * @return JSON字符串 + */ + public static String toJson(T src) { + return toJson(src, OBJECT_MAPPER); + } + + /** + * 对象转换为格式化后的JSON字符串 + * + * @param src 对象 + * @param objectMapper 指定ObjectMapper + * @param 对象泛型 + * @return 格式化后的JSON字符串 + */ + public static String toFormatJson(T src, ObjectMapper objectMapper) { + if (src == null || objectMapper == null) { + return null; + } + try{ + if (src instanceof String) { + if (StrUtil.isBlank((String) src)) { + return ""; + } + Object obj = objectMapper.readValue((String) src, Object.class); + return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(obj); + } + return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(src); + }catch (IOException e){ + throw new RuntimeException(e); + } + + } + + /** + * 对象转换为格式化后的JSON字符串 + * + * @param src 对象 + * @param 对象泛型 + * @return 格式化后的JSON字符串 + */ + public static String toFormatJson(T src) { + return toFormatJson(src, OBJECT_MAPPER); + } + + /** + * JSON字符串转换为对象 + * + * @param json JSON字符串 + * @param objectMapper 指定ObjectMapper + * @param valueType 对象类 + * @param 类泛型 + * @return 对象 + */ + public static T toObject(String json, Class valueType, ObjectMapper objectMapper) { + if (StrUtil.isBlank(json) || objectMapper == null) { + return null; + } + try { + return objectMapper.readValue(json, valueType); + } catch (IOException e) { + // 抛出转换异常 + throw new RuntimeException(e); + } + } + + /** + * JSON字符串转换为对象 + * + * @param json JSON字符串 + * @param valueType 对象类 + * @param 类泛型 + * @return 对象 + */ + public static T toObject(String json, Class valueType) { + if (String.class.equals(valueType)) { + return (T) json; + } + return toObject(json, valueType, OBJECT_MAPPER); + } + + /** + * JSON字符串转换为对象 + * + * @param json JSON字符串 + * @param typeReference 返回对象泛型 + * @param 类泛型 + * @return 对象 + */ + public static T toObject(String json, TypeReference typeReference) { + if (StrUtil.isBlank(json) || typeReference == null) { + return null; + } + + try { + return OBJECT_MAPPER.readValue(json, typeReference); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * 返回List对象 + * + * @param json JSON字符串 + * @param objectMapper 指定ObjectMapper + * @param 泛型 + * @return List对象 + */ + public static Set toSet(String json, ObjectMapper objectMapper) { + if (StrUtil.isBlank(json) || objectMapper == null) { + return null; + } + try{ + return OBJECT_MAPPER.readValue(json, new TypeReference>() { + }); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + + /** + * 返回List对象 + * + * @param json JSON字符串 + * @param objectMapper 指定ObjectMapper + * @param 泛型 + * @return List对象 + */ + public static List toList(String json, ObjectMapper objectMapper) { + if (StrUtil.isBlank(json) || objectMapper == null) { + return null; + } + try { + return OBJECT_MAPPER.readValue(json, new TypeReference>() { + }); + }catch (IOException e){ + throw new RuntimeException(e); + } + } + + /** + * 返回List对象 + * + * @param json JSON字符串 + * @param 泛型 + * @return List对象 + */ + public static List toList(String json) { + return toList(json, OBJECT_MAPPER); + } + + /** + * 返回List对象 + * + * @param json JSON字符串 + * @param 泛型 + * @return List对象 + */ + public static Set toSet(String json) { + return toSet(json, OBJECT_MAPPER); + } + + /** + * 返回Map对象 + * + * @param json JSON字符串 + * @param objectMapper 指定ObjectMapper + * @param Map的key泛型 + * @param Map的value泛型 + * @return Map对象 + */ + public static Map toMap(String json, ObjectMapper objectMapper) { + if (StrUtil.isBlank(json) || objectMapper == null) { + return null; + } + try{ + return objectMapper.readValue(json, new TypeReference>() { + }); + }catch (IOException e){ + throw new RuntimeException(e); + } + } + + /** + * 返回Map对象 + * + * @param json JSON字符串 + * @param Map的key泛型 + * @param Map的value泛型 + * @return Map对象 + */ + public static Map toMap(String json) { + return toMap(json, OBJECT_MAPPER); + } + + /** + * 创建一个json对象节点 + * + * @return + */ + public static ObjectNode createObjectNode() { + return OBJECT_MAPPER.createObjectNode(); + } + + /** + * 创建一个json数组节点 + * + * @return + */ + public static ArrayNode createArrayNode() { + return OBJECT_MAPPER.createArrayNode(); + } + + /** + * 克隆数据，大数据量复杂对象克隆慎用 + * + * @param t + * @param valueType + * @param + * @return + */ + public static T clone(T t, Class valueType) { + return toObject(toJson(t), valueType); + } + + /** + * 判断字符串是否为json + * @param json + * @param valueType + * @param objectMapper + * @return + */ + public static boolean isJson(String json, Class valueType, ObjectMapper objectMapper){ + if (StrUtil.isBlank(json) || objectMapper == null) { + return false; + } + try { + objectMapper.readValue(json, valueType); + return true; + } catch (IOException e) { + // 抛出转换异常 + return false; + } + } + + + + /** + * 初始化默认ObjectMapper + */ + static { + ObjectMapper customMapper = new ObjectMapper(); + //对象属性为null时不输出JSON字符串 + customMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + //忽略 JSON字符串中存在但Java对象无响应属性 时的报错 + customMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + //禁止 使用int代表Enum的order()來反序列化Enum + customMapper.configure(DeserializationFeature.FAIL_ON_NUMBERS_FOR_ENUMS, true); + //忽略 属性大小写 + customMapper.configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true); + customMapper.findAndRegisterModules(); + OBJECT_MAPPER = customMapper; + } + + + +} + diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/util/ListUtil.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/ListUtil.java new file mode 100644 index 0000000..822febf --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/ListUtil.java @@ -0,0 +1,148 @@ +package org.kangspace.messagepush.core.util; + + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; + +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * 列表工具类 + * + * @author kango2gler@gmail.com + * @since 2021-04-29 + */ +@Slf4j +public class ListUtil { + + /** + * 列表是否为空 + * + * @param list + * @return + */ + public static boolean isEmpty(List list) { + return list == null || list.isEmpty(); + } + + /** + * 列表是否不为空 + * + * @param list + * @return + */ + public static boolean isNotEmpty(List list) { + return !isEmpty(list); + } + + /** + * 对象数组转对象列表 + * + * @param objs + * @param + * @return + */ + public static List getList(T... objs) { + if (objs == null || objs.length == 0) { + return new ArrayList(); + } + return new ArrayList(Arrays.asList(objs)); + } + + /** + * 逗号分割的字符串转字符串列表 + * + * @param str 逗号分割的字符串 + * @return 字符串列表 + */ + public static List strToList(String str) { + if (StrUtil.isEmpty(str)) { + return new ArrayList<>(); + } + return new ArrayList<>(Arrays.asList(str.split(","))); + } + + /** + * 数组转列表 + * + * @param array + * @param + * @return + */ + public static List arrayToList(T[] array) { + if (array == null) { + return null; + } + return new ArrayList<>(Arrays.asList(array)); + } + + /** + * 列表转数组 + * + * @param list + * @param + * @return + */ + public static T[] listToArray(List list, Class classType) { + if (list == null) { + return null; + } + return list.toArray((T[]) Array.newInstance(classType, list.size())); + } + + /** + * 拆分对象列表 + * + * @param list + * @param size + * @return + */ + public static List> splitList(List list, int size) { + List> res = new ArrayList<>(); + List temp = new ArrayList<>(); + res.add(temp); + int index = 1; + for (T map : list) { + if (index > size) { + temp = new ArrayList<>(); + res.add(temp); + index = 1; + } + temp.add(map); + index++; + } + return res; + } + + /** + * 拷贝列表（仅支持浅拷贝）效果同BeanUtils.copyProperties + * + * @param sourceList 源列表 + * @param targetClass 目标列表项Class + * @param 目标列表项泛型 + * @return 返回目标列表 + */ + public static List copyList(List sourceList, Class targetClass) { + List result = new ArrayList<>(); + if (ListUtil.isNotEmpty(sourceList)) { + for (Object item : sourceList) { + T data = null; + try { + data = targetClass.newInstance(); + } catch (InstantiationException e) { + log.warn("拷贝列表异常-不可实例化异常，目标类：[" + targetClass.getName() + "]"); + return result; + } catch (IllegalAccessException e) { + log.warn("拷贝列表异常-反射异常，目标类：[" + targetClass.getName() + "]"); + return result; + } + BeanUtils.copyProperties(item, data); + result.add(data); + } + } + return result; + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/util/MD5Util.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/MD5Util.java new file mode 100644 index 0000000..120be23 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/MD5Util.java @@ -0,0 +1,36 @@ +package org.kangspace.messagepush.core.util; + +import cn.hutool.crypto.digest.MD5; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; + +import java.util.Collection; +import java.util.Comparator; +import java.util.stream.Collectors; + +/** + * Md5相关工具类 + * + * @author kango2gler@gmail.com + * @since 2021/11/2 + */ +@Slf4j +public class MD5Util { + + /** + * 获取列表Hash摘要字符串 + * 摘要逻辑: 1. 输入list元素按Hash排序 + * 2. 转换为,分割的字符串 + * 3. 对字符串取MD5 + * + * @return + */ + public static String hashDigest(Collection list) { + if (CollectionUtils.isEmpty(list)) { + return ""; + } + String hashSort = list.stream().sorted(Comparator.comparingInt(String::hashCode)) + .collect(Collectors.joining(",")); + return MD5.create().digestHex(hashSort); + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/util/ObjectUtil.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/ObjectUtil.java new file mode 100644 index 0000000..a8943d8 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/ObjectUtil.java @@ -0,0 +1,42 @@ +package org.kangspace.messagepush.core.util; + + +import org.apache.commons.lang3.StringUtils; +import org.springframework.util.ReflectionUtils; + +import javax.el.MethodNotFoundException; +import java.lang.reflect.Method; + +/** + * 对象工具类 + * + * @author kango2gler@gmail.com + * @since 2021/10/25 + */ +public class ObjectUtil { + /** + * 对象字段设置默认值 + * + * @param obj 对象 + * @param field 字段名 + * @param setDefault 设置默认的条件 + * @param defaultValue 默认值 + */ + public static void defaultFieldValue(Object obj, String field, boolean setDefault, Object defaultValue) { + if (obj != null && StringUtils.isNotBlank(field)) { + Class clazz = obj.getClass(); + if (setDefault) { + Method setMethod = ReflectionUtils.findMethod(clazz, "set" + field); + if (setMethod == null) { + throw new MethodNotFoundException("class:[" + clazz.getName() + "] field:[" + field + "] method:[set]"); + } + try { + setMethod.setAccessible(true); + setMethod.invoke(obj, defaultValue); + } catch (Exception e) { + throw new RuntimeException(e.getMessage(), e); + } + } + } + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/util/PathVariableResolver.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/PathVariableResolver.java new file mode 100644 index 0000000..75d3530 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/PathVariableResolver.java @@ -0,0 +1,111 @@ +package org.kangspace.messagepush.core.util; + +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; + +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * 
+ * 路径参数解析器:
+ * 将路径中的花括号包裹的变量替换为实际值,如:
+ * /user/{id}/info 替换为/user/1/info
+ * /user/type/{type}/ 替换为 /user/type/1
+ * 其中 id=1,type=1。
+ *
+ * + * @author kango2gler@gmail.com + * @since 2021/10/10 + */ +public class PathVariableResolver { + /** + * 路径参数正则 + */ + public static Pattern PATH_VARIABLES_PATTERN = Pattern.compile("(\\{.*?\\})", Pattern.MULTILINE); + /** + * 待处理的url + */ + private String url; + /** + * 参数map + */ + private Map variableMap = new HashMap<>(1); + + private PathVariableResolver() { + } + + public PathVariableResolver(String url, Map variableMap) { + Objects.requireNonNull(url, "url不能为空"); + this.url = url; + if (variableMap != null) { + this.variableMap = variableMap; + } + } + + /** + * 将URL中的变量处理为具体值,并返回最终URL + * + * @return 替换变量后的URL + */ + public String resolve() { + String url = this.url; + if (!StringUtils.hasText(url)) { + return url; + } + List pathVariables = this.findPathVariables(); + if (CollectionUtils.isEmpty(pathVariables)) { + return url; + } + Map variableMap = this.variableMap; + for (String pv : pathVariables) { + url = url.replaceAll(toVariableReplacePattern(pv), nullToEmpty(variableMap.get(pv))); + } + return url; + } + + + /** + * 查找所有的路径变量 + * + * @return all pathVariables + */ + public List findPathVariables() { + String url = this.url; + Matcher matcher = PATH_VARIABLES_PATTERN.matcher(url); + List pathVariables = new ArrayList<>(4); + while (matcher.find()) { + pathVariables.add(removeBraces(matcher.group())); + } + return pathVariables.stream().distinct().collect(Collectors.toList()); + } + + /** + * 移除大括号 + * + * @param str 待处理的字符串 + * @return 新字符串 + */ + private String removeBraces(String str) { + if (!StringUtils.hasText(str)) { + return str; + } + return str.replace("{", "").replace("}", ""); + } + + /** + * 转换为变量替换的pattern + * + * @param var 待处理的字符串 + * @return 带{}的字符串 + */ + private String toVariableReplacePattern(String var) { + return "\\{" + var + "\\}"; + } + + private String nullToEmpty(Object str) { + return str != null ? str.toString() : ""; + } +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/util/StrUtil.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/StrUtil.java new file mode 100644 index 0000000..571ba01 --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/util/StrUtil.java @@ -0,0 +1,289 @@ +package org.kangspace.messagepush.core.util; + +import java.math.BigDecimal; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; + +/** + * 字符串工具类 + * + * @author kango2gler@gmail.com + * @since 2021-04-29 + */ +public class StrUtil { + + /** + * SimpleDateFormat是线程不安全的，此处用ThreadLocal来设置线程的SimpleDateFormat对象 + */ + private static final ThreadLocal SIMPLE_DATE_FORMAT_THREAD_LOCAL = new ThreadLocal<>(); + + /** + * 是否为空 + * + * @param str + * @return + */ + public static boolean isEmpty(String str) { + return str == null || str.length() == 0; + } + + /** + * 是否不为空 + * + * @param str + * @return + */ + public static boolean isNotEmpty(String str) { + return !isEmpty(str); + } + + /** + * 是否为空 + * + * @param str + * @return + */ + public static boolean isBlank(String str) { + int strLen; + if (str != null && (strLen = str.length()) != 0) { + for (int i = 0; i < strLen; ++i) { + // 判断字符是否为空格、制表符、tab + if (!Character.isWhitespace(str.charAt(i))) { + return false; + } + } + return true; + } else { + return true; + } + } + + /** + * 是否不为空 + * + * @param str + * @return + */ + public static boolean isNotBlank(String str) { + return !isBlank(str); + } + + /** + * @param str + * @param defaultValue 默认值 + * @return + * @description 字符串转int + * @date 2014-1-3 + */ + public static int strToInt(String str, int defaultValue) { + if (isBlank(str)) { + return defaultValue; + } + try { + return Integer.parseInt(str); + } catch (Exception ignored) { + return defaultValue; + } + } + + /** + * @param str + * @param defaultValue 默认值 + * @return + * @description 字符串转long + */ + public static long strToLong(String str, long defaultValue) { + if (isBlank(str)) { + return defaultValue; + } + try { + return Long.parseLong(str); + } catch (Exception ignored) { + return defaultValue; + } + } + + /** + * @param str + * @param defaultValue 默认值 + * @return + * @description 字符串转double + */ + public static double strToDouble(String str, Double defaultValue) { + if (isBlank(str)) { + return defaultValue; + } + try { + return Double.parseDouble(str); + } catch (Exception ignored) { + return defaultValue; + } + } + + /** + * @param str + * @param defaultValue 默认值 + * @return + * @description 字符串转long + */ + public static BigDecimal strToBigDecimal(String str, BigDecimal defaultValue) { + if (isBlank(str)) { + return defaultValue; + } + try { + return new BigDecimal(str); + } catch (Exception ignored) { + return defaultValue; + } + } + + /** + * @param str + * @return + * @description 字符串转日期，格式：yyyy-MM-dd HH:mm:ss + * @date 2014-1-3 + */ + public static Date strToDate(String str) { + if (isBlank(str)) { + return null; + } + try { + return getSimpleDateFormat().parse(str); + } catch (Exception e) { + return null; + } + + } + + /** + * 日期转字符串，格式：yyyy-MM-dd HH:mm:ss + * + * @param date + * @return + */ + public static String dateToStr(Date date) { + if (date == null) { + return null; + } + return getSimpleDateFormat().format(date); + } + + /** + * 列表转字符串，半角逗号分割 + * + * @param list + * @return + */ + public static String listToStr(List list) { + return listToStr(list, ",", "", ""); + } + + /** + * 列表转字符串，自定义分割方式 + * + * @param list + * @param separator 分隔符 + * @param prefix 附加前缀 + * @param suffix 附加后缀 + * @return + */ + public static String listToStr(List list, String separator, String prefix, String suffix) { + if (list == null || list.size() == 0) { + return ""; + } + if (separator == null) { + separator = ""; + } + if (prefix == null) { + prefix = ""; + } + if (suffix == null) { + suffix = ""; + } + StringBuffer s = new StringBuffer(""); + for (int i = 0, len = list.size(); i < len; i++) { + if (isNotEmpty(list.get(i))) { + s.append(prefix + list.get(i) + suffix + separator); + } + } + if (s.length() > 0) { + s.delete(s.length() - separator.length(), s.length()); + } + return s.toString(); + } + + /** + * @param array + * @return + * @description 数组转字符串 + * @date 2012-12-5 + */ + public static String arrayToStr(String[] array) { + if (array == null || array.length == 0) { + return ""; + } + StringBuffer s = new StringBuffer(""); + for (int i = 0, len = array.length; i < len; i++) { + if (isNotBlank(array[i])) { + s.append(array[i] + ","); + } + } + if (s.length() > 0) { + s.delete(s.length() - 1, s.length()); + } + return s.toString(); + } + + + public static List strToList(String str) { + return strToList(str, ","); + } + + public static List strToList(String str, String separator) { + ArrayList list = new ArrayList(); + if (isBlank(str)) { + return list; + } + return new ArrayList(Arrays.asList(str.split(separator))); + } + + /** + * @param strs + * @return + * @description 字符串数组转字符串列表 + * @date Feb 21, 2012 + */ + public static List getList(String... strs) { + if (strs == null || strs.length == 0) { + return new ArrayList(); + } + return new ArrayList(Arrays.asList(strs)); + } + + + /** + * SimpleDateFormat是线程不安全的，此处用ThreadLocal来设置线程的SimpleDateFormat对象 + * + * @return + */ + private static synchronized SimpleDateFormat getSimpleDateFormat() { + if (SIMPLE_DATE_FORMAT_THREAD_LOCAL.get() == null) { + SIMPLE_DATE_FORMAT_THREAD_LOCAL.set(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")); + } + return SIMPLE_DATE_FORMAT_THREAD_LOCAL.get(); + } + + /** + * 打印 + * + * @param o + */ + public static void p(Object o) { + System.out.println(o); + } + + +} diff --git a/message-push-common/src/main/java/org/kangspace/messagepush/core/websocket/WebSocketSessionManager.java b/message-push-common/src/main/java/org/kangspace/messagepush/core/websocket/WebSocketSessionManager.java new file mode 100644 index 0000000..da06b5a --- /dev/null +++ b/message-push-common/src/main/java/org/kangspace/messagepush/core/websocket/WebSocketSessionManager.java @@ -0,0 +1,56 @@ +package org.kangspace.messagepush.core.websocket; + +import java.util.List; +import java.util.Map; + +/** + * WebSocket用户Session管理器 + * + * @author kango2gler@gmail.com + * @since 2021/10/28 + */ +public interface WebSocketSessionManager { + /** + * 添加Session + * + * @param key session关联的推送对象 + * @param sessionHolder sessionHolder + * @return SessionHolder + */ + T addSession(String key, T sessionHolder); + + /** + * 删除Session + * + * @param key session关联的推送对象 + * @param socketSession WebSocketSession + * @return boolean + */ + boolean removeSession(String key, T socketSession); + + /** + * 为应用添加用户Session关系 + * + * @param appKey 应用ID + * @param key session关联的推送对象 + * @param sessionHolder sessionHolder + * @return SessionHolder + */ + T addSession(String appKey, String key, T sessionHolder); + + /** + * 检查Session是否已添加 + * + * @param session WebSocketSession + * @return boolean + */ + boolean sessionCheck(T session); + + /** + * 通过appKey获取所有用户Session信息 + * + * @param appKey appKey + * @return 用户SessionMap + */ + Map> getKeySessions(String appKey); +} diff --git a/message-push-common/src/main/resources/META-INF/spring.factories b/message-push-common/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000..e69de29 diff --git a/message-push-common/src/test/java/org/kangspace/messagepush/common/MD5UtilTest.java b/message-push-common/src/test/java/org/kangspace/messagepush/common/MD5UtilTest.java new file mode 100644 index 0000000..31410bb --- /dev/null +++ b/message-push-common/src/test/java/org/kangspace/messagepush/common/MD5UtilTest.java @@ -0,0 +1,58 @@ +package org.kangspace.messagepush.common; + +import cn.hutool.crypto.digest.MD5; +import lombok.extern.slf4j.Slf4j; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.kangspace.messagepush.core.util.MD5Util; + +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Md5相关工具类 + * + * @author kango2gler@gmail.com + * @since 2021/11/2 + */ +@Slf4j +@RunWith(JUnit4.class) +public class MD5UtilTest { + + /** + * 获取列表Hash摘要字符串 + * 摘要逻辑: 1. 输入list元素按Hash排序 + * 2. 转换为,分割的字符串 + * 3. 对字符串取MD5 + * + * @return + */ + @Test + public void testHashDigest() { + List serverList = Arrays.asList( + "192.168.0.9:8080", + "192.168.0.6:8080", + "192.168.0.1:8080", + "192.168.0.2:8080", + "192.168.0.4:8080", + "192.168.0.3:8080", + "192.168.0.8:8080", + "192.168.0.7:8080" + ); + String utilResult = MD5Util.hashDigest(serverList); + log.info("utilResult:{}", utilResult); + Assert.assertTrue("验证失败,hash排序生成失败", utilResult != null); + } + + public String listHash(List list) { + Set set = list.stream().collect(Collectors.toSet()); + String hashSort = set.stream().collect(Collectors.joining(",")); + String result = MD5.create().digestHex(hashSort); + log.info("listHash by hash set, list:[{}], hashSort:[{}]", list, hashSort); + return result; + } +} diff --git a/message-push-common/src/test/java/org/kangspace/messagepush/common/PathVariableResolverTest.java b/message-push-common/src/test/java/org/kangspace/messagepush/common/PathVariableResolverTest.java new file mode 100644 index 0000000..8bd2697 --- /dev/null +++ b/message-push-common/src/test/java/org/kangspace/messagepush/common/PathVariableResolverTest.java @@ -0,0 +1,59 @@ +package org.kangspace.messagepush.common; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.kangspace.messagepush.core.util.PathVariableResolver; + +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; + +/** + * 路径替换处理器测试类 + * + * @author kango2gler@gmail.com + * @since 2021/10/10 + */ +@RunWith(JUnit4.class) +public class PathVariableResolverTest { + + /** + * 路径正则测试 + */ + @Test + public void pathPatternTest() { + String test = "/user/{id}/{name}/{id}"; + Matcher matcher = PathVariableResolver.PATH_VARIABLES_PATTERN.matcher(test); + while (matcher.find()) { + int count = matcher.groupCount(); + System.out.println(matcher.group()); + } + } + + /** + * 获取路径参数测试 + */ + @Test + public void findPathVariablesTest() { + String test = "/user/{id}/{name}/{id}"; + PathVariableResolver pvr = new PathVariableResolver(test, null); + System.out.println(pvr.findPathVariables()); + } + + + /** + * 路径替换测试 + */ + @Test + public void pathVariablesResolveTest() { + String test = "/user/{id}/{name}/{id}"; + Map variableMap = new HashMap<>(2); + variableMap.put("id", "1"); + variableMap.put("name", "will"); + PathVariableResolver pvr = new PathVariableResolver(test, variableMap); + System.out.println(pvr.resolve()); + } + + +} diff --git a/message-push-consumer/.gitignore b/message-push-consumer/.gitignore new file mode 100644 index 0000000..51a2ad8 --- /dev/null +++ b/message-push-consumer/.gitignore @@ -0,0 +1,13 @@ +.idea +/message-push-consumer*/target +/message-push-consumer-*/target +/message-push-consumer*/target/* +/message-push-consumer-*/target/* + +.DS_Store +*.iml + +/.idea/* +/target/* + +!.mvn/wrapper/maven-wrapper.jar \ No newline at end of file diff --git a/message-push-consumer/README.md b/message-push-consumer/README.md new file mode 100644 index 0000000..4cb20c2 --- /dev/null +++ b/message-push-consumer/README.md @@ -0,0 +1,19 @@ +# message-push-consumer + +消息推送消费者服务 + +## 项目提供服务 + +1. 消费Kafka消息数据 +2. 推送消息到message-push-ws服务 + +## 项目涉及中间件 + + nacos(namespace): + dev: kangspace_dev + Kafka: + topic: message_push_single_topic + group: message-topic-default-consumer + consumer.concurrency: 1 + ElasticSearch: + index: message_push_single_topic_index \ No newline at end of file diff --git a/message-push-consumer/message-push-consumer-core/pom.xml b/message-push-consumer/message-push-consumer-core/pom.xml new file mode 100644 index 0000000..5c2744a --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/pom.xml @@ -0,0 +1,77 @@ + + + + org.kangspace.messagepush + message-push-consumer + ${revision} + + 4.0.0 + + message-push-consumer-core + ${revision} + + + 8 + 8 + + + + + org.kangspace.messagepush + message-push-rest-api + + + + org.kangspace.messagepush + message-push-common + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-tomcat + + + + + + org.springframework.boot + spring-boot-starter-undertow + + + + org.springframework.cloud + spring-cloud-stream + + + + org.springframework.cloud + spring-cloud-stream-binder-kafka + + + + org.springframework.cloud + spring-cloud-starter-openfeign + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.2 + + true + UTF-8 + + + + + + \ No newline at end of file diff --git a/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/config/ControllerAccessConfig.java b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/config/ControllerAccessConfig.java new file mode 100644 index 0000000..8f7319a --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/config/ControllerAccessConfig.java @@ -0,0 +1,133 @@ +package org.kangspace.messagepush.consumer.core.config; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.aspectj.lang.annotation.Aspect; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import org.springframework.util.StopWatch; +import org.springframework.web.filter.OncePerRequestFilter; +import org.springframework.web.util.ContentCachingRequestWrapper; +import org.springframework.web.util.ContentCachingResponseWrapper; +import org.springframework.web.util.WebUtils; + +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Enumeration; + +/** + * Controller 请求日志 + * + * @author kango2gler@gmail.com + * @since 2021/8/17 + */ +@Component +@Order +@Aspect +@Slf4j +public class ControllerAccessConfig extends OncePerRequestFilter implements Ordered { + + /** + * 获取请求参数 + * + * @param request 请求 + * @return params: a=b,c=d + */ + public static String getRequestParams(HttpServletRequest request) { + StringBuilder sb = new StringBuilder(); + Enumeration enu = request.getParameterNames(); + //获取请求参数 + while (enu.hasMoreElements()) { + String name = enu.nextElement(); + sb.append(name).append("=").append(request.getParameter(name)); + if (enu.hasMoreElements()) { + sb.append(","); + } + } + return sb.toString(); + } + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) throws ServletException, IOException { + boolean isSkipUrl = skipUrl(request); + if (isSkipUrl) { + return; + } + StopWatch sw = new StopWatch(); + sw.start(); + String requestURL = request.getRequestURL().toString(); + String requestMethod = request.getMethod(); + String queryString = request.getQueryString(); + requestURL += StringUtils.isNotBlank(queryString) ? ("?" + queryString) : ""; + ContentCachingRequestWrapper requestWrapper = new ContentCachingRequestWrapper(request); + ContentCachingResponseWrapper responseWrapper = new ContentCachingResponseWrapper(response); + filterChain.doFilter(requestWrapper, responseWrapper); + String body = getRequestBody(requestWrapper); + String responseBody = getResponseBody(responseWrapper); + responseWrapper.copyBodyToResponse(); + sw.stop(); + double costTime = sw.getTotalTimeSeconds(); + log.info("RECEIVE<== " + requestMethod + " " + requestURL + "" + + " Request Params：" + getRequestParams(request) + " " + + " Request Body：" + body + "\n" + + "<== 请求耗时：" + costTime + "s\n" + + "<== 响应内容：" + responseBody); + } + + /** + * 忽略url + * + * @param request + * @return + */ + private boolean skipUrl(HttpServletRequest request) { + String url = request.getRequestURI(); + return url.indexOf("/swagger") > -1; + } + + /** + * 获取请求体 + * + * @param request request + * @return request body + */ + private String getRequestBody(ContentCachingRequestWrapper request) { + ContentCachingRequestWrapper wrapper = WebUtils.getNativeRequest(request, ContentCachingRequestWrapper.class); + if (wrapper != null) { + byte[] buf = wrapper.getContentAsByteArray(); + if (buf.length > 0) { + String payload = new String(buf, 0, buf.length, StandardCharsets.UTF_8); + return payload.replaceAll("\\n", ""); + } + } + return ""; + } + + /** + * 获取响应体 + * + * @param response response + * @return response body + */ + private String getResponseBody(ContentCachingResponseWrapper response) { + ContentCachingResponseWrapper wrapper = WebUtils.getNativeResponse(response, ContentCachingResponseWrapper.class); + if (wrapper != null) { + byte[] buf = wrapper.getContentAsByteArray(); + if (buf.length > 0) { + String payload = new String(buf, 0, buf.length, StandardCharsets.UTF_8); + return payload.replaceAll("\\n", ""); + } + } + return ""; + } + + @Override + public int getOrder() { + return Ordered.LOWEST_PRECEDENCE - 8; + } +} diff --git a/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/config/ElasticSearchIndexInitial.java b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/config/ElasticSearchIndexInitial.java new file mode 100644 index 0000000..aae92f5 --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/config/ElasticSearchIndexInitial.java @@ -0,0 +1,162 @@ +package org.kangspace.messagepush.consumer.core.config; + +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.kangspace.messagepush.consumer.core.service.ElasticSearchService; +import org.kangspace.messagepush.core.elasticsearch.ElasticSearchConst; +import org.kangspace.messagepush.core.elasticsearch.request.AliasAction; +import org.kangspace.messagepush.core.elasticsearch.request.JsonAliasActionsRequest; +import org.kangspace.messagepush.core.elasticsearch.request.JsonCreateIndexRequest; +import org.kangspace.messagepush.core.elasticsearch.request.PlainRolloverRequest; +import org.kangspace.messagepush.core.util.JsonUtil; +import org.springframework.beans.BeansException; +import org.springframework.boot.context.event.ApplicationStartedEvent; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.context.ApplicationListener; +import org.springframework.context.annotation.Configuration; + +import javax.annotation.Resource; +import java.util.HashMap; +import java.util.Map; + +/** + * ES索引初始化 + * + * @author kango2gler@gmail.com + * @since 2021/8/9 + */ +@Slf4j +@Setter +@Getter +@Configuration +@ConfigurationProperties("messagepush.elasticsearch") +public class ElasticSearchIndexInitial implements ApplicationListener, ApplicationContextAware { + /** + * 索引别名对应的第一个索引名 + */ + private static final String DEFAULT_FIRST_INDEX_SUFFIX = "-000001"; + /** + * 索引别名关联的索引字符串(模糊索引) + */ + private static final String DEFAULT_ALIAS_INDIES_SUFFIX = "-*"; + private ApplicationContext applicationContext; + private Boolean isLoaded = false; + + + @Resource + private ElasticSearchService elasticSearchService; + + /** + * 消息推送记录索引 + * (当前名称会作为索引记录的别名,实际索引名为-000001格式) + * 如: + * messagePushSingleTopicIndex = "messagePushSingleTopicIndex" + * 则: + * 1. 创建名称为 messagePushSingleTopicIndex-000001 的索引 + * 2. 创建别名为 messagePushSingleTopicIndex 的别名索引, 别名对应的索引为: messagePushSingleTopicIndex-*,同时设置可写 + * 3. 检查 messagePushSingleTopicIndex 别名是否满足滚动(_rollover)索引条件(2000w条), + * 若满足滚动索引条件,则调用滚动API,将自动生成名称为 messagePushSingleTopicIndex-000002 的索引,并将名 messagePushSingleTopicIndex-000002 + * 设置为可写索引 + */ + private String messagePushSingleTopicIndex; + /** + * 消息推送记录索引别名滚动最大日志数(默认2000W) + */ + private Long messagePushSingleTopicIndexRolloverMaxDoc = 2000_0000L; + + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.applicationContext = applicationContext; + } + + @Override + public void onApplicationEvent(ApplicationStartedEvent event) { + if (this.isLoaded) { + return; + } + log.info("ElasticSearch 索引初始化: start, ElasticSearchIndexInitial: messagepush.elasticsearch.message-push-single-topic-index: [{}]", messagePushSingleTopicIndex); + if (StringUtils.isBlank(messagePushSingleTopicIndex)) { + log.error("ElasticSearch 索引初始化:: ElasticSearchIndexInitial error, config [{}] not found", "messagepush.elasticsearch.message-push-single-topic-index"); + return; + } + initElasticSearchIndexAlias(); + log.info("ElasticSearch 索引初始化: end"); + this.isLoaded = true; + } + + /** + * 初始化ElasticSearch 索引 + * (包括设置滚动索引,别名) + * 1. 检查别名是否存在, + * a. 别名不存在->[检查别名对应的第一个索引是否存在,第一个索引不存在则自动创建索引]->创建别名 + * b. 别名存在-> 检查别名索引是否需要滚动,需要则调用滚动API + */ + public void initElasticSearchIndexAlias() { + // 当前索引名作为别名,新索引名: 在当前索引名后添加-000001 + String indexAlias = messagePushSingleTopicIndex; + String defaultFirstIndex = indexAlias + DEFAULT_FIRST_INDEX_SUFFIX; + // 检查索引别名是否存在 + boolean aliasExists = elasticSearchService.existsAlias(indexAlias); + // 别名不存在则检查索引是否存在 + if (!aliasExists) { + log.info("ElasticSearch 索引初始化: 别名不存在: 别名:{}", indexAlias); + boolean firstIndexExists = elasticSearchService.existsIndex(defaultFirstIndex); + // 检查默认索引是否存在,即索引名 + if (!firstIndexExists) { + log.info("ElasticSearch 索引初始化: index:[{}] 不存在,开始创建!", defaultFirstIndex); + boolean result = elasticSearchService.createIndex(defaultJsonCreateIndexRequest(defaultFirstIndex)); + log.info("ElasticSearch 索引初始化: index: [{}] 创建结束, result: [{}]", defaultFirstIndex, result); + } + String aliasIndies = indexAlias + DEFAULT_ALIAS_INDIES_SUFFIX; + // 创建索引别名 + AliasAction action = new AliasAction("add", aliasIndies, indexAlias, true); + JsonAliasActionsRequest actionsRequest = new JsonAliasActionsRequest(action); + boolean createdAlias = elasticSearchService.aliasAction(actionsRequest); + log.info("ElasticSearch 索引初始化: 创建别名:{} ,创建结果:{}", actionsRequest, createdAlias); + } else { + PlainRolloverRequest plainRolloverRequest = new PlainRolloverRequest(); + plainRolloverRequest.setAlias(indexAlias); + plainRolloverRequest.setDryRun(true); + plainRolloverRequest.setMaxDocs(messagePushSingleTopicIndexRolloverMaxDoc); + log.info("ElasticSearch 索引初始化: 别名存在,检查别名是否需要滚动, 别名:{}, plainRolloverRequest:{}", indexAlias, plainRolloverRequest); + if (elasticSearchService.rollover(plainRolloverRequest)) { + log.info("ElasticSearch 索引初始化: 别名需要滚动,alias:{}", indexAlias); + plainRolloverRequest.setDryRun(false); + boolean rollover = elasticSearchService.rollover(plainRolloverRequest); + log.info("ElasticSearch 索引初始化: 别名需要滚动结果,alias:{},结果:{}", indexAlias, rollover); + } else { + log.info("ElasticSearch 索引初始化: 别名无需滚动, alias:{}", indexAlias); + } + } + } + + /** + * 创建ElasticSearch 创建索引请求对象 + * + * @param indexName 索引名称 + * @return {@link JsonCreateIndexRequest} + */ + public JsonCreateIndexRequest defaultJsonCreateIndexRequest(String indexName) { + return new JsonCreateIndexRequest(indexName, 2, 1, getDefaultModelMapping(), null); + } + + + /** + * 获取默认实体映射(设置c_time类型) + * + * @return map + */ + private Map getDefaultModelMapping() { + Map defaultMap = new HashMap<>(); + defaultMap.put("c_time", JsonUtil.createObjectNode().put(ElasticSearchConst.TYPE, ElasticSearchConst.DATE) + .put("format", "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||" + ElasticSearchConst.DATE_FORMAT) + ); + return defaultMap; + } + +} diff --git a/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/config/PushConsumerConfiguration.java b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/config/PushConsumerConfiguration.java new file mode 100644 index 0000000..8479a32 --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/config/PushConsumerConfiguration.java @@ -0,0 +1,28 @@ +package org.kangspace.messagepush.consumer.core.config; + +import org.kangspace.messagepush.consumer.core.hash.HashRouterLoader; +import org.kangspace.messagepush.consumer.core.redis.RedisService; +import org.kangspace.messagepush.core.constant.RedisConstants; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * 消费者公共配置加载类 + * + * @author kango2gler@gmail.com + * @since 2021/11/3 + */ +@Configuration +public class PushConsumerConfiguration { + + /** + * HashRouter加载 + * + * @return HashRouterLoader + */ + @Bean + public HashRouterLoader hashRouterLoader(RedisService redisService) { + return new HashRouterLoader(redisService, RedisConstants.MESSAGE_PUSH_HASH_RING_KEY); + } + +} diff --git a/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/config/TaskExecutorConfig.java b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/config/TaskExecutorConfig.java new file mode 100644 index 0000000..8c96eb6 --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/config/TaskExecutorConfig.java @@ -0,0 +1,32 @@ +package org.kangspace.messagepush.consumer.core.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; + +/** + * 任务线程池配置 + * + * @author kango2gler@gmail.com + * @since 2021/10/26 + */ +@Configuration +public class TaskExecutorConfig { + + + @Bean("asyncTaskExecutor") + public Executor taskExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(10); + executor.setMaxPoolSize(20); + executor.setQueueCapacity(200); + executor.setThreadNamePrefix("asyncTaskExecutor-"); + executor.setWaitForTasksToCompleteOnShutdown(true); + executor.setAwaitTerminationSeconds(60); + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); + return executor; + } +} diff --git a/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/config/WebMvcConfig.java b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/config/WebMvcConfig.java new file mode 100644 index 0000000..4623fab --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/config/WebMvcConfig.java @@ -0,0 +1,114 @@ +package org.kangspace.messagepush.consumer.core.config; + + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.dto.response.ApiResponse; +import org.kangspace.messagepush.core.enums.ResponseEnum; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.HttpStatus; +import org.springframework.http.converter.HttpMessageConversionException; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.validation.BindingResult; +import org.springframework.validation.FieldError; +import org.springframework.validation.ObjectError; +import org.springframework.web.HttpRequestMethodNotSupportedException; +import org.springframework.web.bind.MethodArgumentNotValidException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.servlet.config.annotation.CorsRegistry; +import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurationSupport; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.List; +import java.util.stream.Collectors; + +/** + * @author kango2gler@gmail.com + * @since 2021/10/26 + */ +@Slf4j +@Configuration +public class WebMvcConfig extends WebMvcConfigurationSupport { + + /** + * 设置允许跨域 + * + * @param registry registry + */ + @Override + public void addCorsMappings(CorsRegistry registry) { + super.addCorsMappings(registry); + registry.addMapping("/**") + .allowedHeaders("*") + .allowedMethods("POST", "GET", "OPTIONS", "PUT", "PATCH", "DELETE") + .allowedOrigins("*"); + } + + @Override + public void addResourceHandlers(ResourceHandlerRegistry registry) { + registry.addResourceHandler("/**").addResourceLocations( + "classpath:/static/"); + registry.addResourceHandler("swagger-ui.html").addResourceLocations( + "classpath:/META-INF/resources/"); + registry.addResourceHandler("/webjars/**").addResourceLocations( + "classpath:/META-INF/resources/webjars/"); + super.addResourceHandlers(registry); + } + + /** + * 接口异常处理 + */ + @RestControllerAdvice + public static class ControllerExceptionHandleAdvice { + @ExceptionHandler + public ApiResponse handler(HttpServletRequest request, HttpServletResponse response, Exception e) { + log.error("Controller 处理异常,url:{},错误信息:{}", request.getRequestURL(), e.getMessage(), e); + ApiResponse responseDTO; + int responseStatus = HttpStatus.INTERNAL_SERVER_ERROR.value(); + if (e instanceof HttpRequestMethodNotSupportedException) { + responseDTO = new ApiResponse(ResponseEnum.METHOD_NOT_ALLOWED); + responseStatus = HttpStatus.BAD_REQUEST.value(); + } else if (e instanceof HttpMessageNotReadableException) { + responseDTO = new ApiResponse(ResponseEnum.BAD_REQUEST); + responseDTO.setMsg(responseDTO.getMsg() + ":输入参数(JSON)格式错误,请确认后重试."); + responseStatus = HttpStatus.BAD_REQUEST.value(); + } else if (e instanceof HttpMessageConversionException) { + responseDTO = new ApiResponse(ResponseEnum.BAD_REQUEST); + responseStatus = HttpStatus.BAD_REQUEST.value(); + } else { + responseDTO = new ApiResponse(ResponseEnum.INTERNAL_SERVER_ERROR.getValue(), ResponseEnum.INTERNAL_SERVER_ERROR.getReasonPhrase()); + } + response.setStatus(responseStatus); + return responseDTO; + } + + /** + * 参数错误处理 + * + * @param exception exception + * @return ApiResponse + */ + @ResponseBody + @ResponseStatus(HttpStatus.BAD_REQUEST) + @ExceptionHandler(MethodArgumentNotValidException.class) + public ApiResponse exceptionHandler(MethodArgumentNotValidException exception) { + BindingResult result = exception.getBindingResult(); + StringBuilder sb = new StringBuilder("参数错误:"); + if (result.hasErrors()) { + List errors = result.getAllErrors(); + if (errors != null) { + sb.append(errors.stream().map(p -> { + FieldError fieldError = (FieldError) p; + log.warn("Bad Request Parameters: dto entity [{}],field [{}],message [{}]", fieldError.getObjectName(), fieldError.getField(), fieldError.getDefaultMessage()); + return fieldError.getDefaultMessage(); + }).collect(Collectors.joining(","))); + } + } + return new ApiResponse(ResponseEnum.BAD_REQUEST.getValue(), sb.toString()); + } + } +} diff --git a/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/domain/dto/request/MessagePushRequestDto.java b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/domain/dto/request/MessagePushRequestDto.java new file mode 100644 index 0000000..ce38002 --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/domain/dto/request/MessagePushRequestDto.java @@ -0,0 +1,33 @@ +package org.kangspace.messagepush.consumer.core.domain.dto.request; + + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.kangspace.messagepush.core.dto.page.PageRequestDto; +import org.kangspace.messagepush.core.elasticsearch.annotation.QueryField; + +import java.io.Serializable; + +/** + * 消息推送数据ElasticSearch Query对象 + * + * @author kango2gler@gmail.com + * @since 2021/10/26 + */ +@Getter +@Setter +@NoArgsConstructor +@ApiModel("消息推送数据ElasticSearch Query对象") +public class MessagePushRequestDto extends PageRequestDto implements Serializable { + private static final long serialVersionUID = 1L; + @ApiModelProperty("messageId") + @QueryField(field = "message_id") + private String messageId; + + private String emptyToNull(String str) { + return str == null || "".equals(str) || str.trim().length() == 0 ? null : str; + } +} \ No newline at end of file diff --git a/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/feign/DefaultFeignFallBack.java b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/feign/DefaultFeignFallBack.java new file mode 100644 index 0000000..d33523d --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/feign/DefaultFeignFallBack.java @@ -0,0 +1,14 @@ +package org.kangspace.messagepush.consumer.core.feign; + +import org.springframework.stereotype.Service; + +/** + * 默认feign降级处理 + * + * @author kango2gler@gmail.com + * @since 2021/8/9 + */ +@Service +public class DefaultFeignFallBack implements TempFeignAPi { + +} diff --git a/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/feign/MessagePushWsApi.java b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/feign/MessagePushWsApi.java new file mode 100644 index 0000000..5d22239 --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/feign/MessagePushWsApi.java @@ -0,0 +1,32 @@ +package org.kangspace.messagepush.consumer.core.feign; + + +import org.kangspace.messagepush.consumer.core.feign.config.MessagePushWsApiFeignConfig; +import org.kangspace.messagepush.core.dto.response.ApiResponse; +import org.kangspace.messagepush.rest.api.dto.request.MessagePushRequestTimeDTO; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; + +/** + * 消息websocket推送服务 + * + * @author kango2gler@gmail.com + * @since 2021/8/9 + */ +@FeignClient(value = "message-push-ws-microservice", path = "/", + url = "dynamicUrl", + configuration = MessagePushWsApiFeignConfig.class, + fallback = MessagePushWsApiFeignFallBack.class) +public interface MessagePushWsApi { + /** + * 消息推送PUSH接口 + * + * @param messagePushRequestDto + * @return ApiResponse + */ + @PostMapping("/v1/inner/push") + ApiResponse messagePush(@RequestBody MessagePushRequestTimeDTO messagePushRequestDto, + @RequestHeader(MessagePushWsApiFeignConfig.DEFAULT_SERVICE_NODE_HEADER_KEY) String serviceNode); +} diff --git a/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/feign/MessagePushWsApiFeignFallBack.java b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/feign/MessagePushWsApiFeignFallBack.java new file mode 100644 index 0000000..d098a95 --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/feign/MessagePushWsApiFeignFallBack.java @@ -0,0 +1,24 @@ +package org.kangspace.messagepush.consumer.core.feign; + + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.dto.response.ApiResponse; +import org.kangspace.messagepush.rest.api.dto.request.MessagePushRequestTimeDTO; +import org.springframework.stereotype.Service; + +/** + * 默认feign降级处理 + * + * @author kango2gler@gmail.com + * @since 2021/8/9 + */ +@Slf4j +@Service +public class MessagePushWsApiFeignFallBack implements MessagePushWsApi { + + @Override + public ApiResponse messagePush(MessagePushRequestTimeDTO messagePushRequestDto, String serviceNode) { + log.error("messagePush接口访问失败,进入FallBack. args: serviceNode:[{}],dto: [{}]", serviceNode, messagePushRequestDto); + return null; + } +} diff --git a/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/feign/TempFeignAPi.java b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/feign/TempFeignAPi.java new file mode 100644 index 0000000..82822a0 --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/feign/TempFeignAPi.java @@ -0,0 +1,12 @@ +package org.kangspace.messagepush.consumer.core.feign; + +import org.springframework.cloud.openfeign.FeignClient; + +/** + * @author kango2gler@gmail.com + * @since 2021/8/9 + */ +@FeignClient(value = "temp", path = "/", fallback = DefaultFeignFallBack.class) +public interface TempFeignAPi { + +} diff --git a/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/feign/config/MessagePushWsApiFeignConfig.java b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/feign/config/MessagePushWsApiFeignConfig.java new file mode 100644 index 0000000..d56458b --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/feign/config/MessagePushWsApiFeignConfig.java @@ -0,0 +1,42 @@ +package org.kangspace.messagepush.consumer.core.feign.config; + +import feign.RequestInterceptor; +import org.springframework.cloud.openfeign.FeignClientsConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +/** + * 消息Websocket推送接口Feign配置 + * + * @author kango2gler@gmail.com + * @since 2021/11/6 + */ +@Configuration +@Import(FeignClientsConfiguration.class) +public class MessagePushWsApiFeignConfig { + /** + * 默认服务节点请求头Key + */ + public static final String DEFAULT_SERVICE_NODE_HEADER_KEY = "serviceNode"; + /** + * 默认访问协议 + */ + private final String DEFAULT_HTTP_SCHEMA_PROTOCOL = "http://"; + + /** + * 动态URL路径处理 + * + * @return + */ + @Bean + public RequestInterceptor dynamicUrlInterceptor() { + return template -> { + String serviceNode = template.request().headers().get(DEFAULT_SERVICE_NODE_HEADER_KEY).stream().findFirst().orElse(null); + if (serviceNode != null) { + template.target(DEFAULT_HTTP_SCHEMA_PROTOCOL + serviceNode); + } + }; + } + +} diff --git a/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/hash/HashRouterLoader.java b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/hash/HashRouterLoader.java new file mode 100644 index 0000000..a8882bb --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/hash/HashRouterLoader.java @@ -0,0 +1,78 @@ +package org.kangspace.messagepush.consumer.core.hash; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.consumer.core.redis.RedisService; +import org.kangspace.messagepush.core.constant.MessagePushConstants; +import org.kangspace.messagepush.core.hash.ConsistencyHashing; +import org.kangspace.messagepush.core.hash.VirtualNode; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; + +import java.util.Map; + +/** + * 一致性Hash数据加载类 + * + * @author kango2gler@gmail.com + * @since 2021/11/3 + */ +@Slf4j +@Data +public class HashRouterLoader { + private final RedisService redisService; + /** + * 一致性Hash缓存Key + */ + private final String redisKey; + /** + * 单线程定时线程池 + */ + private ThreadPoolTaskScheduler scheduler; + /** + * 一致性Hash数据 + */ + private ConsistencyHashing hashRouter; + + public HashRouterLoader(RedisService redisService, String redisKey) { + this.redisService = redisService; + this.redisKey = redisKey; + this.scheduler = new ThreadPoolTaskScheduler(); + this.scheduler.setPoolSize(1); + this.scheduler.initialize(); + start(); + } + + private void setHashRouter(ConsistencyHashing hashRouter) { + this.hashRouter = hashRouter; + } + + /** + * 加载一致性Hash数据 + */ + private void load() { + final Map ring = redisService.hGetAll(this.redisKey, VirtualNode.class); + ConsistencyHashing hashRouter = new ConsistencyHashing(ring, MessagePushConstants.NUMBER_OF_VIRTUAL_NODE); + this.setHashRouter(hashRouter); + if (log.isDebugEnabled()) { + log.debug("一致性Hash数据加载: 加载成功, 物理节点:[{}]", hashRouter.getPhysicalNodes()); + } + } + + /** + * 开始定时job + */ + private void start() { + log.debug("一致性Hash数据加载: 定时任务开始!"); + this.scheduler.scheduleAtFixedRate(() -> load(), 3000L); + } + + /** + * 获取物理节点node值 + * + * @param key 需要hash的key + * @return 目标node值 + */ + public String getPhysicalNode(String key) { + return this.getHashRouter().getVirtualNode(key).getPhysicalNode().getNode(); + } +} diff --git a/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/mq/kafka/KafkaBindingConfig.java b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/mq/kafka/KafkaBindingConfig.java new file mode 100644 index 0000000..b678dc1 --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/mq/kafka/KafkaBindingConfig.java @@ -0,0 +1,15 @@ +package org.kangspace.messagepush.consumer.core.mq.kafka; + +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.context.annotation.Configuration; + +/** + * Kafka 通道绑定 + * + * @author kango2gler@gmail.com + * @since 2021/09/03 + */ +@Configuration +@EnableBinding(value = MessagePushChannel.class) +public class KafkaBindingConfig { +} diff --git a/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/mq/kafka/MessagePushChannel.java b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/mq/kafka/MessagePushChannel.java new file mode 100644 index 0000000..b8333e6 --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/mq/kafka/MessagePushChannel.java @@ -0,0 +1,28 @@ +package org.kangspace.messagepush.consumer.core.mq.kafka; + +import org.springframework.cloud.stream.annotation.Input; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.SubscribableChannel; + +/** + * 消息推送Topic通道 + * + * @author kango2gler@gmail.com + */ +public interface MessagePushChannel { + + /** + * 消息推送Kafka数据通道 + */ + String INPUT_MESSAGE_PUSH_SINGLE_TOPIC = "message_push_single_topic"; + + + /** + * 消息推送Kafka数据通道订阅 + * + * @return {@link MessageChannel} + */ + @Input(INPUT_MESSAGE_PUSH_SINGLE_TOPIC) + SubscribableChannel inputMessagePushSingle(); + +} diff --git a/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/mq/kafka/MessagePushConsumer.java b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/mq/kafka/MessagePushConsumer.java new file mode 100644 index 0000000..ea1a797 --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/mq/kafka/MessagePushConsumer.java @@ -0,0 +1,39 @@ +package org.kangspace.messagepush.consumer.core.mq.kafka; + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.consumer.core.service.MessagePushConsumerService; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.messaging.handler.annotation.Payload; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; + +/** + * 消息推送数据消费 + * + * @author kango2gler@gmail.com + * @since 2021/10/26 + */ +@Slf4j +@Component +public class MessagePushConsumer { + @Resource + private MessagePushConsumerService messagePushConsumerService; + + /** + * 订阅kafka消息处理 + * + * @param message Kafka消息 + */ + @StreamListener(target = MessagePushChannel.INPUT_MESSAGE_PUSH_SINGLE_TOPIC) + public void handle(@Payload String message) { + log.info("消费Kafka消息: received a message from [{}]: {}", MessagePushChannel.INPUT_MESSAGE_PUSH_SINGLE_TOPIC, message); + boolean result = false; + try { + result = messagePushConsumerService.messageHandle(message); + } catch (Exception e) { + log.error("消费Kafka消息: consumer message error:{}", e.getMessage(), e); + } + log.info("消费Kafka消息: consumer message result :[{}], message:[{}]", result, message); + } +} diff --git a/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/redis/RedisService.java b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/redis/RedisService.java new file mode 100644 index 0000000..6d5cb2d --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/redis/RedisService.java @@ -0,0 +1,69 @@ +package org.kangspace.messagepush.consumer.core.redis; + +import org.springframework.stereotype.Service; + +import java.util.function.Supplier; + +/** + * Redis相关操作Service + * + * @author kango2gler@gmail.com + * @since 2021/8/9 + */ +@Service +public class RedisService extends org.kangspace.messagepush.core.redis.RedisService { + + /** + * 
+     * 获取Redis缓存对象;
+     * 若fetchData数据为空时,返回null,反之若缓存不存在时,重新进行缓存。
+     *
+ * + * @param key redis key + * @param clazz redis 缓存的对象类型 + * @param time 缓存超时时间 + * @param fetchData 重新获取对象的方法 + * @param 目标对象 + * @return T or null, 当 fetchData 为空时,返回null + */ + public T getAndCache(String key, Class clazz, long time, Supplier fetchData) { + T cache = super.get(key, clazz); + if (cache == null) { + synchronized (this) { + cache = super.get(key, clazz); + if (cache == null) { + cache = fetchData.get(); + if (cache != null) { + //重新缓存 + super.setEX(key, cache, time); + } + } + } + } + return cache; + } + + /** + * 分布式锁 + * + * @param key redis key + * @param ttl 超时时间 + * @param callback 获取到锁后的执行函数 + * @return boolean true:已获取到锁,false:未获取到锁 + */ + public boolean lock(String key, long ttl, Runnable callback) { + String flag = System.currentTimeMillis() + ""; + if (super.setNX(key, flag, ttl)) { + try { + callback.run(); + } finally { + String value = super.get(key); + if (value != null && flag.equals(value)) { + super.del(key); + } + } + return true; + } + return false; + } +} diff --git a/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/service/BaseService.java b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/service/BaseService.java new file mode 100644 index 0000000..b34973a --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/service/BaseService.java @@ -0,0 +1,10 @@ +package org.kangspace.messagepush.consumer.core.service; + +/** + * 基础Service + * + * @author kango2gler@gmail.com + * @since 2021/10/25 + */ +public class BaseService { +} diff --git a/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/service/ElasticSearchService.java b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/service/ElasticSearchService.java new file mode 100644 index 0000000..2db50cd --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/service/ElasticSearchService.java @@ -0,0 +1,186 @@ +package org.kangspace.messagepush.consumer.core.service; + +import lombok.extern.slf4j.Slf4j; +import org.elasticsearch.ElasticsearchException; +import org.kangspace.messagepush.core.dto.page.Order; +import org.kangspace.messagepush.core.dto.page.PageRequestDto; +import org.kangspace.messagepush.core.dto.page.PageResponseDto; +import org.kangspace.messagepush.core.elasticsearch.ElasticsearchManager; +import org.kangspace.messagepush.core.elasticsearch.request.JsonAliasActionsRequest; +import org.kangspace.messagepush.core.elasticsearch.request.JsonCreateIndexRequest; +import org.kangspace.messagepush.core.elasticsearch.request.JsonSearchRequest; +import org.kangspace.messagepush.core.elasticsearch.request.PlainRolloverRequest; +import org.kangspace.messagepush.core.util.JsonUtil; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + +/** + * ElasticSearch处理Service + * + * @author kango2gler@gmail.com + * @since 2021/8/9 + */ +@Slf4j +@Service +public class ElasticSearchService { + @Resource + private ElasticsearchManager elasticsearchManager; + + /** + * 创建索引 + * + * @param request {@link JsonCreateIndexRequest} + * @return true/false + */ + public Boolean createIndex(JsonCreateIndexRequest request) { + Boolean created = false; + try { + created = elasticsearchManager.createIndex(request); + } catch (Exception e) { + log.error("::: ElasticSearchService createIndex error,request:{} ,error:{}", request, e.getMessage(), e); + } + return created; + } + + /** + * 是否存在索引 + * + * @param index 索引名称 + * @return true/false + */ + public Boolean existsIndex(String index) { + Boolean exists = false; + try { + exists = elasticsearchManager.existsIndex(index); + } catch (Exception e) { + log.error("::: ElasticSearchService existsIndex error,index:{} ,error:{}", index, e.getMessage(), e); + } + return exists; + } + + /** + * 判断指定别名是否存在 + * + * @param alias 别名 + * @return true: 存在, false: 不存在 + */ + public Boolean existsAlias(String alias) { + Boolean exists = false; + try { + exists = elasticsearchManager.existsAlias(alias); + } catch (Exception e) { + log.error("::: ElasticSearchService existsAlias error,alias:{} ,error:{}", alias, e.getMessage(), e); + } + return exists; + } + + /** + * 别名操作 + * + * @param jsonAliasActionsRequest 别名操作对象 + * @return true:成功, false:失败 + */ + public Boolean aliasAction(JsonAliasActionsRequest jsonAliasActionsRequest) { + Boolean action = false; + try { + action = elasticsearchManager.aliasAction(jsonAliasActionsRequest); + } catch (Exception e) { + log.error("::: ElasticSearchService aliasAction error,jsonAliasActionsRequest:{} ,error:{}", jsonAliasActionsRequest, e.getMessage(), e); + } + return action; + } + + /** + * 别名滚动索引 + * + * @param rolloverRequest 滚动请求 + * @return true: 需要滚动/滚动成功,false: 无需滚动 + */ + public Boolean rollover(PlainRolloverRequest rolloverRequest) { + Boolean rollover = false; + try { + rollover = elasticsearchManager.rollover(rolloverRequest); + } catch (Exception e) { + log.error("::: ElasticSearchService rollover error,rolloverRequest:{} ,error:{}", rolloverRequest, e.getMessage(), e); + } + return rollover; + } + + + /** + * 插入数据 + * + * @param index 索引名称 + * @param data 索引数据 + * @return true/false + */ + public Boolean insert(String index, T data) { + Boolean inserted = false; + try { + inserted = elasticsearchManager.insert(index, data); + } catch (Exception e) { + log.error("::: ElasticSearchService insert error, index:{}, data:{},error:{}", index, data, e.getMessage(), e); + } + return inserted; + } + + /** + * ES分页查询 + * + * @param jsonSearchRequest 查询对象 + * @return {@link PageResponseDto} + */ + public PageResponseDto page(JsonSearchRequest jsonSearchRequest) { + String index = jsonSearchRequest.getIndex(); + if (log.isDebugEnabled()) { + log.debug("::: ElasticSearchService page,分页查询语句为[{}]", JsonUtil.toFormatJson(jsonSearchRequest.getRootNode())); + } + PageResponseDto page = null; + try { + page = elasticsearchManager.page(jsonSearchRequest); + } catch (Exception e) { + log.error("::: ElasticSearchService page error, index:{}, jsonSearchRequest:{},error:{}", index, jsonSearchRequest, e.getMessage(), e); + } + return page; + } + + /** + * ES分页查询 + * + * @param index 索引 + * @param query 查询对象,继承于{@link PageResponseDto} + * @param defaultOrder 默认排序 + * @param clazz 返回对象类型 + * @return {@link PageResponseDto} + */ + public PageResponseDto page(String index, QUERY query, Order defaultOrder, Class clazz) { + JsonSearchRequest jsonSearchRequest; + try { + jsonSearchRequest = new JsonSearchRequest<>(index, query, null, defaultOrder, clazz); + } catch (Exception e) { + log.error("::: ElasticSearchService page new JsonSearchRequest() error, index:{}, query:{},order:{},clazz:{},error:{}", + index, query, defaultOrder, clazz, e.getMessage(), e); + throw new ElasticsearchException(e); + } + return page(jsonSearchRequest); + } + + /** + * 异步写ElasticSearch + * + * @param index 索引 + * @param data 数据 + */ + @Async("asyncTaskExecutor") + public void writeElasticSearch(String index, Object data) { + // 消息异步写ES + boolean inserted = insert(index, data); + if (log.isDebugEnabled()) { + log.debug("ElasticSearch 操作: insert [{}],index:[{}],data:[{}]", inserted, index, data.toString()); + } else { + log.info("ElasticSearch 操作: insert [{}],index:[{}]", inserted, index); + } + } +} diff --git a/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/service/MessagePushConsumerService.java b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/service/MessagePushConsumerService.java new file mode 100644 index 0000000..0ca7bcc --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/service/MessagePushConsumerService.java @@ -0,0 +1,17 @@ +package org.kangspace.messagepush.consumer.core.service; + +/** + * 消息推送Service + * + * @author kango2gler@gmail.com + * @since 2021/10/25 + */ +public interface MessagePushConsumerService { + /** + * 消息处理 + * + * @param message kafka的消息内容 + * @return boolean + */ + boolean messageHandle(String message); +} diff --git a/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/service/impl/MessagePushServiceImpl.java b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/service/impl/MessagePushServiceImpl.java new file mode 100644 index 0000000..8eba56f --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/main/java/org/kangspace/messagepush/consumer/core/service/impl/MessagePushServiceImpl.java @@ -0,0 +1,99 @@ +package org.kangspace.messagepush.consumer.core.service.impl; + + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.consumer.core.config.ElasticSearchIndexInitial; +import org.kangspace.messagepush.consumer.core.feign.MessagePushWsApi; +import org.kangspace.messagepush.consumer.core.hash.HashRouterLoader; +import org.kangspace.messagepush.consumer.core.service.BaseService; +import org.kangspace.messagepush.consumer.core.service.ElasticSearchService; +import org.kangspace.messagepush.consumer.core.service.MessagePushConsumerService; +import org.kangspace.messagepush.core.util.JsonUtil; +import org.kangspace.messagepush.rest.api.dto.request.MessagePushRequestTimeDTO; +import org.springframework.beans.BeanUtils; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +/** + * 消息推送Service + * + * @author kango2gler@gmail.com + * @since 2021/10/25 + */ +@Slf4j +@Service +public class MessagePushServiceImpl extends BaseService implements MessagePushConsumerService { + + @Resource + private ElasticSearchIndexInitial elasticSearchIndexInitial; + + @Resource + private ElasticSearchService elasticSearchService; + + @Resource + private MessagePushWsApi messagePushWsApi; + + @Resource + private HashRouterLoader hashRouterLoader; + + @Override + public boolean messageHandle(String message) { + log.info("推送消息消费数据处理: start,message:[{}]", message); + // 消息处理 + MessagePushRequestTimeDTO messagePushRequestTimeDto = JsonUtil.toObject(message, MessagePushRequestTimeDTO.class); + // 保存消息到ES + messageStore(messagePushRequestTimeDto); + // 消息分发到ws + messagePushToWs(messagePushRequestTimeDto); + log.info("推送消息消费数据处理: end,message:[{}]", messagePushRequestTimeDto.getMessageId()); + return true; + } + + /** + * 消息推送到Websocket处理 + * + * @param messageDto MessagePushRequestTimeDTO + */ + private void messagePushToWs(MessagePushRequestTimeDTO messageDto) { + List targetUIds = messageDto.getAudience().getUids(); + // 计算各用户所在的服务节点 + Map> nodeUIdsMap = targetUIds.stream() + .map(uid -> new String[]{hashRouterLoader.getPhysicalNode(uid), uid}) + .collect(Collectors.toMap(arr -> arr[0], arr -> { + List list = new ArrayList<>(); + list.add(arr[1]); + return list; + }, (v1, v2) -> { + v1.addAll(v2); + return v1; + })); + if (CollectionUtils.isEmpty(nodeUIdsMap)) { + log.warn("推送消息到Websocket处理,查询服务节点为空,message:[{}]", messageDto); + return; + } + AtomicInteger pushCount = new AtomicInteger(0); + nodeUIdsMap.forEach((node, uIds) -> { + MessagePushRequestTimeDTO dto = new MessagePushRequestTimeDTO(); + BeanUtils.copyProperties(messageDto, dto); + dto.getAudience().setUids(uIds); + //node 负载处理 + messagePushWsApi.messagePush(dto, node); + pushCount.addAndGet(1); + }); + log.warn("推送消息到Websocket处理,结束, 服务调用次数:[{}]", pushCount.get()); + } + + public void messageStore(MessagePushRequestTimeDTO messagePushRequestTimeDto) { + String messagePushSingleTopicIndex = elasticSearchIndexInitial.getMessagePushSingleTopicIndex(); + // 写ElasticSearch + elasticSearchService.writeElasticSearch(messagePushSingleTopicIndex, messagePushRequestTimeDto); + } + +} diff --git a/message-push-consumer/message-push-consumer-core/src/test/java/org/kangspace/messagepush/consumer/core/Test.java b/message-push-consumer/message-push-consumer-core/src/test/java/org/kangspace/messagepush/consumer/core/Test.java new file mode 100644 index 0000000..d399f75 --- /dev/null +++ b/message-push-consumer/message-push-consumer-core/src/test/java/org/kangspace/messagepush/consumer/core/Test.java @@ -0,0 +1,16 @@ +package org.kangspace.messagepush.consumer.core; + +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + + +/** + * 公共测试类型 + * + * @author kango2gler@gmail.com + * @since 2021/10/26 + */ +@RunWith(JUnit4.class) +public class Test { + +} diff --git a/message-push-consumer/message-push-consumer-microservice/pom.xml b/message-push-consumer/message-push-consumer-microservice/pom.xml new file mode 100644 index 0000000..e5c9150 --- /dev/null +++ b/message-push-consumer/message-push-consumer-microservice/pom.xml @@ -0,0 +1,113 @@ + + + + org.kangspace.messagepush + message-push-consumer + ${revision} + + 4.0.0 + + message-push-consumer-microservice + ${revision} + + + 8 + 8 + + + + + org.kangspace.messagepush + message-push-consumer-core + compile + + + org.kangspace.messagepush + message-push-rest-api + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-tomcat + + + + + + org.springframework.boot + spring-boot-starter-undertow + + + + org.springframework.boot + spring-boot-starter-test + test + + + + org.springframework.cloud + spring-cloud-starter-openfeign + + + + com.spring4all + swagger-spring-boot-starter + + + io.springfox + springfox-swagger-ui + + + io.springfox + springfox-swagger2 + + + + + + message-push-consumer-microservice + + + org.springframework.boot + spring-boot-maven-plugin + + + + repackage + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.2 + + true + UTF-8 + + + + + + + src/main/resources + true + + + + + \ No newline at end of file diff --git a/message-push-consumer/message-push-consumer-microservice/src/main/java/org/kangspace/messagepush/consumer/MessagePushConsumerApplication.java b/message-push-consumer/message-push-consumer-microservice/src/main/java/org/kangspace/messagepush/consumer/MessagePushConsumerApplication.java new file mode 100644 index 0000000..a8902c4 --- /dev/null +++ b/message-push-consumer/message-push-consumer-microservice/src/main/java/org/kangspace/messagepush/consumer/MessagePushConsumerApplication.java @@ -0,0 +1,29 @@ +package org.kangspace.messagepush.consumer; + + +import com.spring4all.swagger.EnableSwagger2Doc; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.cloud.openfeign.EnableFeignClients; +import org.springframework.scheduling.annotation.EnableAsync; + +/** + * 服务主入口 + * + * @author kango2gler@gmail.com + * @since 2021/10/25 + */ +@Slf4j +@EnableAsync +@EnableFeignClients +@EnableSwagger2Doc +@SpringBootApplication(exclude = DataSourceAutoConfiguration.class) +public class MessagePushConsumerApplication { + + public static void main(String[] args) { + SpringApplication.run(MessagePushConsumerApplication.class, args); + } + +} diff --git a/message-push-consumer/message-push-consumer-microservice/src/main/resources/bootstrap.yml b/message-push-consumer/message-push-consumer-microservice/src/main/resources/bootstrap.yml new file mode 100644 index 0000000..51eaa93 --- /dev/null +++ b/message-push-consumer/message-push-consumer-microservice/src/main/resources/bootstrap.yml @@ -0,0 +1,17 @@ +spring: + application: + name: @artifactId@ + cloud: + nacos: + discovery: + server-addr: ${SERVICE_DISCOVERY_ADDR:discory.kangspace.org:8443} + namespace: ${SERVICE_DISCOVERY_NAMESPACE:kangspace_dev} + metadata: + version: @project.version@ + config: + server-addr: ${spring.cloud.nacos.discovery.server-addr} + namespace: ${spring.cloud.nacos.discovery.namespace} + file-extension: yaml + shared-configs: + - application.${spring.cloud.nacos.config.file-extension} + - message-push-consumer-microservice.${spring.cloud.nacos.config.file-extension} \ No newline at end of file diff --git a/message-push-consumer/message-push-consumer-microservice/src/test/java/org/kangspace/messagepush/consumer/MessagePushServiceTest.java b/message-push-consumer/message-push-consumer-microservice/src/test/java/org/kangspace/messagepush/consumer/MessagePushServiceTest.java new file mode 100644 index 0000000..85025a8 --- /dev/null +++ b/message-push-consumer/message-push-consumer-microservice/src/test/java/org/kangspace/messagepush/consumer/MessagePushServiceTest.java @@ -0,0 +1,117 @@ +package org.kangspace.messagepush.consumer; + + +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import org.joda.time.LocalDateTime; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.kangspace.messagepush.consumer.core.config.ElasticSearchIndexInitial; +import org.kangspace.messagepush.consumer.core.domain.dto.request.MessagePushRequestDto; +import org.kangspace.messagepush.consumer.core.service.ElasticSearchService; +import org.kangspace.messagepush.consumer.core.service.MessagePushConsumerService; +import org.kangspace.messagepush.rest.api.dto.request.MessagePushRequestTimeDTO; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.Optional; +import java.util.UUID; + +/** + * @author kango2gler@gmail.com + * @since 2021/8/10 + */ +@Slf4j +@RunWith(SpringJUnit4ClassRunner.class) +@SpringBootTest(classes = MessagePushConsumerApplication.class) +public class MessagePushServiceTest { + @Resource + private ElasticSearchIndexInitial elasticSearchIndexInitial; + @Resource + private MessagePushConsumerService messagePushConsumerService; + @Resource + private ElasticSearchService elasticSearchService; + + @Test + public void temp() { + } + + + /** + * ES插入测试 + */ + @Test + public void esInsert() { + log.info("::: ES插入测试:"); + String index = elasticSearchIndexInitial.getMessagePushSingleTopicIndex(); + MessagePushRequestTimeDTO dto = newMessagePushRequestTimeDTO(); + boolean inserted = elasticSearchService.insert(index, dto); + Assert.assertTrue("ES插入测试,数据插入失败", inserted); + //查询日志信息 + MessagePushRequestTimeDTO responseDto = queryEsData(dto.getMessageId()); + Assert.assertTrue("ES插入测试,数据查询失败,数据为空", responseDto != null); + } + + + /** + * 消息消费测试 + */ + @Test + public void messageHandleTest() { + log.info("::: 消息消费测试:"); + MessagePushRequestTimeDTO dto = newMessagePushRequestTimeDTO(); + String message = JSONObject.toJSONString(dto); + messagePushConsumerService.messageHandle(message); + //查询日志信息 + try { + Thread.sleep(10000L); + } catch (InterruptedException e) { + e.printStackTrace(); + } + //查询日志信息 + MessagePushRequestTimeDTO responseDto = queryEsData(dto.getMessageId()); + Assert.assertTrue("ES插入测试,数据查询失败,数据为空", responseDto != null); + } + + /** + * 获取ES查询结果 + */ + @Test + public void queryEsDataTest() { + String messageId = "f58889e5fce64b8fa2de1da26f69ce68"; + MessagePushRequestTimeDTO dto = queryEsData(messageId); + System.out.println(dto); + } + + /** + * 获取ES查询结果 + * + * @param messageId 消息ID + * @return {@link MessagePushRequestTimeDTO} + */ + public MessagePushRequestTimeDTO queryEsData(String messageId) { + String index = elasticSearchIndexInitial.getMessagePushSingleTopicIndex(); + Order defaultOrder = Order.builder().field("c_time").sequence(-1).build(); + MessagePushRequestDto requestDto = new MessagePushRequestDto(); + requestDto.setMessageId(messageId); + return Optional.ofNullable( + elasticSearchService.page(index, requestDto, defaultOrder, MessagePushRequestTimeDTO.class) + ).map(t -> t.getList()).orElseGet(ArrayList::new).stream().findFirst().orElse(null); + } + + /** + * 获取测试的newMessagePushRequestTimeDTO + * + * @return + */ + public MessagePushRequestTimeDTO newMessagePushRequestTimeDTO() { + String message = "{\"push_method\":1,\"platform\":\"\",\"audience\":{\"alias\":[\"alias1\",\"alias2\"]},\"message\":{\"title\":\"title1\",\"content\":\"content1\",\"content_type\":\"type1\",\"extras\":\"\"}}"; + MessagePushRequestTimeDTO dto = JSONObject.parseObject(message, MessagePushRequestTimeDTO.class); + dto.setMessageId(UUID.randomUUID().toString().replace("-", "")); + dto.setCTime(new LocalDateTime().toDate()); + return dto; + } +} diff --git a/message-push-consumer/message-push-consumer-microservice/src/test/java/org/kangspace/messagepush/consumer/Test.java b/message-push-consumer/message-push-consumer-microservice/src/test/java/org/kangspace/messagepush/consumer/Test.java new file mode 100644 index 0000000..019968e --- /dev/null +++ b/message-push-consumer/message-push-consumer-microservice/src/test/java/org/kangspace/messagepush/consumer/Test.java @@ -0,0 +1,13 @@ +package org.kangspace.messagepush.consumer; + +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * @author kango2gler@gmail.com + * @since 2021/8/9 + */ +@RunWith(JUnit4.class) +public class Test { + +} diff --git a/message-push-consumer/pom.xml b/message-push-consumer/pom.xml new file mode 100644 index 0000000..9ff57ef --- /dev/null +++ b/message-push-consumer/pom.xml @@ -0,0 +1,50 @@ + + + 4.0.0 + + + org.kangspace.messagepush + message-push + ${revision} + + + message-push-consumer + ${revision} + pom + + + message-push-consumer-core + message-push-consumer-microservice + + + + 8 + 8 + + + + + + org.kangspace.messagepush + message-push-consumer-core + ${revision} + + + + org.kangspace.messagepush + message-push-rest-api + ${revision} + + + + org.kangspace.messagepush + message-push-common + ${revision} + + + + + + \ No newline at end of file diff --git a/message-push-rest/.gitignore b/message-push-rest/.gitignore new file mode 100644 index 0000000..332017e --- /dev/null +++ b/message-push-rest/.gitignore @@ -0,0 +1,13 @@ +.idea +/message-push-rest*/target +/message-push-rest-*/target +/message-push-rest*/target/* +/message-push-rest-*/target/* + +.DS_Store +*.iml + +/.idea/* +/target/* + +!.mvn/wrapper/maven-wrapper.jar \ No newline at end of file diff --git a/message-push-rest/README.md b/message-push-rest/README.md new file mode 100644 index 0000000..9e758be --- /dev/null +++ b/message-push-rest/README.md @@ -0,0 +1,14 @@ +# message-push-rest + +消息推送REST API服务 + +## 项目提供服务 + +1. 提供对外消息推送的REST API接口,并将数据发送到Kafka。 + +## 项目涉及中间件 + + nacos(namespace): + dev: kangspace_dev + Kafka: + topic: message_push_single_topic \ No newline at end of file diff --git a/message-push-rest/message-push-rest-api/pom.xml b/message-push-rest/message-push-rest-api/pom.xml new file mode 100644 index 0000000..27d2d7b --- /dev/null +++ b/message-push-rest/message-push-rest-api/pom.xml @@ -0,0 +1,43 @@ + + + + org.kangspace.messagepush + message-push-rest + ${revision} + + 4.0.0 + + message-push-rest-api + ${revision} + + + 8 + 8 + 1.8 + + + + + jakarta.validation + jakarta.validation-api + + + + com.spring4all + swagger-spring-boot-starter + + + io.springfox + springfox-swagger-ui + + + io.springfox + springfox-swagger2 + + + + + + \ No newline at end of file diff --git a/message-push-rest/message-push-rest-api/src/main/java/org/kangspace/messagepush/rest/api/dto/ApiBaseDTO.java b/message-push-rest/message-push-rest-api/src/main/java/org/kangspace/messagepush/rest/api/dto/ApiBaseDTO.java new file mode 100644 index 0000000..31c7fea --- /dev/null +++ b/message-push-rest/message-push-rest-api/src/main/java/org/kangspace/messagepush/rest/api/dto/ApiBaseDTO.java @@ -0,0 +1,15 @@ +package org.kangspace.messagepush.rest.api.dto; + +import io.swagger.annotations.ApiModel; + +import java.io.Serializable; + +/** + * API 层基础DTO + * + * @author kango2gler@gmail.com + * @since 2021/10/25 + */ +@ApiModel("API 层基础DTO") +public class ApiBaseDTO implements Serializable { +} diff --git a/message-push-rest/message-push-rest-api/src/main/java/org/kangspace/messagepush/rest/api/dto/request/MessagePushRequestDTO.java b/message-push-rest/message-push-rest-api/src/main/java/org/kangspace/messagepush/rest/api/dto/request/MessagePushRequestDTO.java new file mode 100644 index 0000000..0ce7d75 --- /dev/null +++ b/message-push-rest/message-push-rest-api/src/main/java/org/kangspace/messagepush/rest/api/dto/request/MessagePushRequestDTO.java @@ -0,0 +1,94 @@ +package org.kangspace.messagepush.rest.api.dto.request; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.kangspace.messagepush.rest.api.dto.ApiBaseDTO; + +import javax.validation.Valid; +import javax.validation.constraints.*; +import java.util.List; + +/** + * 消息推送请求DTO + * + * @author kango2gler@gmail.com + * @since 2021/10/25 + */ +@ApiModel("消息推送请求DTO") +@Data +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class MessagePushRequestDTO extends ApiBaseDTO { + /** + * 推送方式；1: websocket方式 + * 必填 + */ + @JsonProperty("push_method") + @NotNull(message = "push_method:推送方式不能为空") + @Min(value = 1, message = "push_method:值错误,取值范围: 1:websocket") + @Max(value = 1, message = "push_method:值错误,取值范围: 1:websocket") + private Integer pushMethod; + /** + * 目标平台:all,h5,android,ios + * 非必填 + */ + @Pattern(regexp = "^(?i)(()|(all)|(h5)|(android)|(ios))$", message = "platform:值错误,取值范围: all,h5,android,ios,默认all") + private String platform = "all"; + /** + * 推送目标 + * 必填 + */ + @NotNull(message = "audience:推送目标不能为空") + @Valid + private Audience audience; + /** + * 消息 + * 必填 + */ + @NotNull(message = "message:消息内容不能为空") + @Valid + private Message message; + + @Data + @NoArgsConstructor + public static class Message { + /** + * 消息标题 + * 非必填 + */ + private String title; + /** + * 消息内容 + * 必填 + */ + @NotEmpty(message = "content:消息内容不能为空") + private String content; + /** + * 消息类型，由调用方自定义内容类型 + * 非必填 + */ + @JsonProperty("content_type") + private String contentType; + /** + * 扩展内容，由调用方自定义扩展 + * 非必填 + */ + private String extras; + } + + /** + * + */ + @Data + @NoArgsConstructor + public static class Audience { + /** + * 用户uid数组,最多1000个 + */ + @Size(min = 1, max = 1000, message = "uids:一次最少1个,最多1000") + private List uids; + } +} diff --git a/message-push-rest/message-push-rest-api/src/main/java/org/kangspace/messagepush/rest/api/dto/request/MessagePushRequestTimeDTO.java b/message-push-rest/message-push-rest-api/src/main/java/org/kangspace/messagepush/rest/api/dto/request/MessagePushRequestTimeDTO.java new file mode 100644 index 0000000..57397a7 --- /dev/null +++ b/message-push-rest/message-push-rest-api/src/main/java/org/kangspace/messagepush/rest/api/dto/request/MessagePushRequestTimeDTO.java @@ -0,0 +1,67 @@ +package org.kangspace.messagepush.rest.api.dto.request; + +import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.springframework.beans.BeanUtils; + +import javax.validation.constraints.NotNull; +import java.util.Date; +import java.util.UUID; + +/** + * 消息推送请求DTO + * + * @author kango2gler@gmail.com + * @since 2021/10/25 + */ +@ApiModel("消息推送请求DTO(含时间)") +@Data +@ToString(callSuper = true) +@NoArgsConstructor +public class MessagePushRequestTimeDTO extends MessagePushRequestDTO { + /** + * 消息ID,32位UUID + */ + @ApiModelProperty("消息ID") + @JsonProperty("message_id") + @NotNull(message = "message_id:不能为空") + private String messageId; + /** + * 消息ID,32位UUID + */ + @ApiModelProperty("消息ID") + @JsonProperty("app_key") + @NotNull(message = "app_key:不能为空") + private String appKey; + /** + * 创建时间 + */ + @ApiModelProperty("创建时间") + @JsonProperty("c_time") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + @NotNull(message = "c_time:不能为空") + private Date cTime; + + + /** + * 通过{@link MessagePushRequestDTO}创建{@link MessagePushRequestTimeDTO} + * + * @param dto MessagePushRequestDto 对象 + * @return MessagePushRequestTimeDto 对象 + */ + public static MessagePushRequestTimeDTO build(MessagePushRequestDTO dto) { + if (dto == null) { + return null; + } + MessagePushRequestTimeDTO requestTimeDto = new MessagePushRequestTimeDTO(); + requestTimeDto.setMessageId(UUID.randomUUID().toString().replace("-", "")); + requestTimeDto.setCTime(new Date()); + BeanUtils.copyProperties(dto, requestTimeDto); + return requestTimeDto; + } +} diff --git a/message-push-rest/message-push-rest-core/pom.xml b/message-push-rest/message-push-rest-core/pom.xml new file mode 100644 index 0000000..ac9c5ae --- /dev/null +++ b/message-push-rest/message-push-rest-core/pom.xml @@ -0,0 +1,84 @@ + + + + org.kangspace.messagepush + message-push-rest + ${revision} + + 4.0.0 + + message-push-rest-core + ${revision} + + + 8 + 8 + + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-tomcat + + + + + + org.springframework.boot + spring-boot-starter-undertow + + + + org.springframework.cloud + spring-cloud-stream + + + + org.springframework.cloud + spring-cloud-stream-binder-kafka + + + + org.springframework.cloud + spring-cloud-starter-openfeign + + + + org.kangspace.messagepush + message-push-rest-api + + + + org.kangspace.messagepush + message-push-common + + + + org.aspectj + aspectjweaver + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.2 + + true + UTF-8 + + + + + + \ No newline at end of file diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/auth/ApiAuthentication.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/auth/ApiAuthentication.java new file mode 100644 index 0000000..c6c3121 --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/auth/ApiAuthentication.java @@ -0,0 +1,15 @@ +package org.kangspace.messagepush.rest.core.auth; + +import java.lang.annotation.*; + +/** + * API请求认证注解 + * + * @author kango2gler@gmail.com + * @since 2021/10/25 + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +public @interface ApiAuthentication { +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/auth/AuthRequestAop.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/auth/AuthRequestAop.java new file mode 100644 index 0000000..f4310c6 --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/auth/AuthRequestAop.java @@ -0,0 +1,96 @@ +package org.kangspace.messagepush.rest.core.auth; + + +import com.alibaba.fastjson.JSONObject; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.reflect.MethodSignature; +import org.kangspace.messagepush.core.dto.response.ApiResponse; +import org.kangspace.messagepush.core.enums.ResponseEnum; +import org.kangspace.messagepush.core.util.HttpUtils; +import org.kangspace.messagepush.rest.core.constant.AppThreadLocal; +import org.kangspace.messagepush.rest.core.utils.AppGenerator; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Component; +import org.springframework.web.context.request.RequestContextHolder; +import org.springframework.web.context.request.ServletRequestAttributes; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.lang.annotation.Annotation; +import java.nio.charset.StandardCharsets; + +/** + * 接口请求过滤器 + * + * @author kango2gler@gmail.com + * @since 2021/10/25 + */ +@Component +@Aspect +public class AuthRequestAop { + + /** + * 检查认证信息 + * 拦截所有Controller请求,校验包含{@link ApiAuthentication}的方法 + */ + @Around("execution(public * org.kangspace.messagepush.rest.controller.*.*(..))") + public Object checkAuthentication(ProceedingJoinPoint pjp) throws Throwable { + // 获取当前方法 + MethodSignature methodSignature = (MethodSignature) pjp.getSignature(); + Annotation apiAuthentication = methodSignature.getMethod().getAnnotation(ApiAuthentication.class); + if (apiAuthentication != null) { + // 获取当前请求 + ServletRequestAttributes servlet = ((ServletRequestAttributes) RequestContextHolder.currentRequestAttributes()); + HttpServletRequest request = servlet.getRequest(); + HttpServletResponse response = servlet.getResponse(); + // 判断请求头 + if (!validHttpBasic(request)) { + // 截断请求 + httpBasicValidFailed(response); + return null; + } + // 请求继续 + AppThreadLocal.setAppKey(HttpUtils.getBasicAuth(request).getUsername()); + } + Object result = pjp.proceed(); + AppThreadLocal.reset(); + return result; + } + + /** + * 验证HttpBasic + * (此处验证自定义AppKey,AppSecret) + * + * @param request HttpServletRequest + * @return boolean 验证成功/失败 + * @see AppGenerator + */ + public boolean validHttpBasic(HttpServletRequest request) { + HttpUtils.HttpBasicAuth basicAuth = HttpUtils.getBasicAuth(request); + if (basicAuth != null) { + String username = basicAuth.getUsername(); + String password = basicAuth.getPassword(); + return AppGenerator.validAppInfo(username, password); + } + return false; + } + + /** + * HttpBasic 鉴权失败响应 + * + * @param response HttpServletResponse + * @throws IOException IOException + */ + private void httpBasicValidFailed(HttpServletResponse response) throws IOException { + response.setStatus(HttpStatus.FORBIDDEN.value()); + response.setContentType(MediaType.APPLICATION_JSON_VALUE); + response.setCharacterEncoding(StandardCharsets.UTF_8.toString()); + ApiResponse result = new ApiResponse(ResponseEnum.FORBIDDEN.getValue(), "invalid Authorization"); + response.getWriter().print(JSONObject.toJSONString(result)); + } + +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/config/ControllerAccessConfig.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/config/ControllerAccessConfig.java new file mode 100644 index 0000000..cb6355a --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/config/ControllerAccessConfig.java @@ -0,0 +1,133 @@ +package org.kangspace.messagepush.rest.core.config; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.aspectj.lang.annotation.Aspect; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import org.springframework.util.StopWatch; +import org.springframework.web.filter.OncePerRequestFilter; +import org.springframework.web.util.ContentCachingRequestWrapper; +import org.springframework.web.util.ContentCachingResponseWrapper; +import org.springframework.web.util.WebUtils; + +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Enumeration; + +/** + * Controller 请求日志 + * + * @author kango2gler@gmail.com + * @since 2021/8/17 + */ +@Component +@Order +@Aspect +@Slf4j +public class ControllerAccessConfig extends OncePerRequestFilter implements Ordered { + + /** + * 获取请求参数 + * + * @param request 请求 + * @return params: a=b,c=d + */ + public static String getRequestParams(HttpServletRequest request) { + StringBuilder sb = new StringBuilder(); + Enumeration enu = request.getParameterNames(); + //获取请求参数 + while (enu.hasMoreElements()) { + String name = enu.nextElement(); + sb.append(name).append("=").append(request.getParameter(name)); + if (enu.hasMoreElements()) { + sb.append(","); + } + } + return sb.toString(); + } + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) throws ServletException, IOException { + boolean isSkipUrl = skipUrl(request); + if (isSkipUrl) { + return; + } + StopWatch sw = new StopWatch(); + sw.start(); + String requestURL = request.getRequestURL().toString(); + String requestMethod = request.getMethod(); + String queryString = request.getQueryString(); + requestURL += StringUtils.isNotBlank(queryString) ? ("?" + queryString) : ""; + ContentCachingRequestWrapper requestWrapper = new ContentCachingRequestWrapper(request); + ContentCachingResponseWrapper responseWrapper = new ContentCachingResponseWrapper(response); + filterChain.doFilter(requestWrapper, responseWrapper); + String body = getRequestBody(requestWrapper); + String responseBody = getResponseBody(responseWrapper); + responseWrapper.copyBodyToResponse(); + sw.stop(); + double costTime = sw.getTotalTimeSeconds(); + log.info("RECEIVE<== " + requestMethod + " " + requestURL + "" + + " Request Params：" + getRequestParams(request) + " " + + " Request Body：" + body + "\n" + + "<== 请求耗时：" + costTime + "s\n" + + "<== 响应内容：" + responseBody); + } + + /** + * 忽略url + * + * @param request + * @return + */ + private boolean skipUrl(HttpServletRequest request) { + String url = request.getRequestURI(); + return url.indexOf("/swagger") > -1; + } + + /** + * 获取请求体 + * + * @param request request + * @return request body + */ + private String getRequestBody(ContentCachingRequestWrapper request) { + ContentCachingRequestWrapper wrapper = WebUtils.getNativeRequest(request, ContentCachingRequestWrapper.class); + if (wrapper != null) { + byte[] buf = wrapper.getContentAsByteArray(); + if (buf.length > 0) { + String payload = new String(buf, 0, buf.length, StandardCharsets.UTF_8); + return payload.replaceAll("\\n", ""); + } + } + return ""; + } + + /** + * 获取响应体 + * + * @param response response + * @return response body + */ + private String getResponseBody(ContentCachingResponseWrapper response) { + ContentCachingResponseWrapper wrapper = WebUtils.getNativeResponse(response, ContentCachingResponseWrapper.class); + if (wrapper != null) { + byte[] buf = wrapper.getContentAsByteArray(); + if (buf.length > 0) { + String payload = new String(buf, 0, buf.length, StandardCharsets.UTF_8); + return payload.replaceAll("\\n", ""); + } + } + return ""; + } + + @Override + public int getOrder() { + return Ordered.LOWEST_PRECEDENCE - 8; + } +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/config/KafkaBindingConfig.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/config/KafkaBindingConfig.java new file mode 100644 index 0000000..6ef3c0e --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/config/KafkaBindingConfig.java @@ -0,0 +1,14 @@ +package org.kangspace.messagepush.rest.core.config; + +import org.kangspace.messagepush.rest.core.mq.kafka.MessagePushChannel; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.context.annotation.Configuration; + +/** + * @author kango2gler@gmail.com + * @since 2021/8/7 + */ +@Configuration +@EnableBinding(value = MessagePushChannel.class) +public class KafkaBindingConfig { +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/config/WebMvcConfig.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/config/WebMvcConfig.java new file mode 100644 index 0000000..4a08537 --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/config/WebMvcConfig.java @@ -0,0 +1,114 @@ +package org.kangspace.messagepush.rest.core.config; + + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.dto.response.ApiResponse; +import org.kangspace.messagepush.core.enums.ResponseEnum; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.HttpStatus; +import org.springframework.http.converter.HttpMessageConversionException; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.validation.BindingResult; +import org.springframework.validation.FieldError; +import org.springframework.validation.ObjectError; +import org.springframework.web.HttpRequestMethodNotSupportedException; +import org.springframework.web.bind.MethodArgumentNotValidException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.servlet.config.annotation.CorsRegistry; +import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurationSupport; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.List; +import java.util.stream.Collectors; + +/** + * @author kango2gler@gmail.com + * @since 2021/8/7 + */ +@Slf4j +@Configuration +public class WebMvcConfig extends WebMvcConfigurationSupport { + + /** + * 设置允许跨域 + * + * @param registry registry + */ + @Override + public void addCorsMappings(CorsRegistry registry) { + super.addCorsMappings(registry); + registry.addMapping("/**") + .allowedHeaders("*") + .allowedMethods("POST", "GET", "OPTIONS", "PUT", "PATCH", "DELETE") + .allowedOrigins("*"); + } + + @Override + public void addResourceHandlers(ResourceHandlerRegistry registry) { + registry.addResourceHandler("/**").addResourceLocations( + "classpath:/static/"); + registry.addResourceHandler("swagger-ui.html").addResourceLocations( + "classpath:/META-INF/resources/"); + registry.addResourceHandler("/webjars/**").addResourceLocations( + "classpath:/META-INF/resources/webjars/"); + super.addResourceHandlers(registry); + } + + /** + * 接口异常处理 + */ + @RestControllerAdvice + public static class ControllerExceptionHandleAdvice { + @ExceptionHandler + public ApiResponse handler(HttpServletRequest request, HttpServletResponse response, Exception e) { + log.error("Controller 处理异常,url:{},错误信息:{}", request.getRequestURL(), e.getMessage(), e); + ApiResponse responseDTO; + int responseStatus = HttpStatus.INTERNAL_SERVER_ERROR.value(); + if (e instanceof HttpRequestMethodNotSupportedException) { + responseDTO = new ApiResponse(ResponseEnum.METHOD_NOT_ALLOWED); + responseStatus = HttpStatus.BAD_REQUEST.value(); + } else if (e instanceof HttpMessageNotReadableException) { + responseDTO = new ApiResponse(ResponseEnum.BAD_REQUEST); + responseDTO.setMsg(responseDTO.getMsg() + ":输入参数(JSON)格式错误,请确认后重试."); + responseStatus = HttpStatus.BAD_REQUEST.value(); + } else if (e instanceof HttpMessageConversionException) { + responseDTO = new ApiResponse(ResponseEnum.BAD_REQUEST); + responseStatus = HttpStatus.BAD_REQUEST.value(); + } else { + responseDTO = new ApiResponse(ResponseEnum.INTERNAL_SERVER_ERROR.getValue(), ResponseEnum.INTERNAL_SERVER_ERROR.getReasonPhrase()); + } + response.setStatus(responseStatus); + return responseDTO; + } + + /** + * 参数错误处理 + * + * @param exception + * @return + */ + @ResponseBody + @ResponseStatus(HttpStatus.BAD_REQUEST) + @ExceptionHandler(MethodArgumentNotValidException.class) + public ApiResponse exceptionHandler(MethodArgumentNotValidException exception) { + BindingResult result = exception.getBindingResult(); + StringBuilder sb = new StringBuilder("参数错误:"); + if (result.hasErrors()) { + List errors = result.getAllErrors(); + if (errors != null) { + sb.append(errors.stream().map(p -> { + FieldError fieldError = (FieldError) p; + log.warn("Bad Request Parameters: dto entity [{}],field [{}],message [{}]", fieldError.getObjectName(), fieldError.getField(), fieldError.getDefaultMessage()); + return fieldError.getDefaultMessage(); + }).collect(Collectors.joining(","))); + } + } + return new ApiResponse(ResponseEnum.BAD_REQUEST.getValue(), sb.toString()); + } + } +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/constant/AppThreadLocal.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/constant/AppThreadLocal.java new file mode 100644 index 0000000..8b4418c --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/constant/AppThreadLocal.java @@ -0,0 +1,39 @@ +package org.kangspace.messagepush.rest.core.constant; + +/** + * @author kango2gler@gmail.com + * @since 2021/11/1 + */ +public class AppThreadLocal { + private static ThreadLocal threadLocal = new ThreadLocal(); + + private AppThreadLocal() { + } + + /** + * 设置AppKey + * + * @param appKey appKey + * @return appKey + */ + public static String setAppKey(String appKey) { + threadLocal.set(appKey); + return appKey; + } + + /** + * 获取AppKey + * + * @return appKey + */ + public static String getAppKey() { + return (String) threadLocal.get(); + } + + /** + * 重置AppThreadLocal + */ + public static void reset() { + threadLocal.remove(); + } +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/constant/MessagePushConstants.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/constant/MessagePushConstants.java new file mode 100644 index 0000000..9d546ea --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/constant/MessagePushConstants.java @@ -0,0 +1,24 @@ +package org.kangspace.messagepush.rest.core.constant; + +/** + * 常量类 + * + * @author kango2gler@gmail.com + * @since 2021/8/7 + */ +public interface MessagePushConstants { + /** + * 推送目标平台 + */ + enum PUSH_PLATFORM { + ALL, + H5, + Android, + iOS; + + @Override + public String toString() { + return super.toString().toLowerCase(); + } + } +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/domain/entity/TemplateEntity.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/domain/entity/TemplateEntity.java new file mode 100644 index 0000000..f649760 --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/domain/entity/TemplateEntity.java @@ -0,0 +1,5 @@ +package org.kangspace.messagepush.rest.core.domain.entity; + + +public class TemplateEntity { +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/feign/DefaultFeignFallBack.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/feign/DefaultFeignFallBack.java new file mode 100644 index 0000000..cecd332 --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/feign/DefaultFeignFallBack.java @@ -0,0 +1,14 @@ +package org.kangspace.messagepush.rest.core.feign; + +import org.springframework.stereotype.Service; + +/** + * 默认feign降级处理 + * + * @author kango2gler@gmail.com + * @since 2021/8/9 + */ +@Service +public class DefaultFeignFallBack implements TempFeignAPi { + +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/feign/TempFeignAPi.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/feign/TempFeignAPi.java new file mode 100644 index 0000000..0557154 --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/feign/TempFeignAPi.java @@ -0,0 +1,12 @@ +package org.kangspace.messagepush.rest.core.feign; + +import org.springframework.cloud.openfeign.FeignClient; + +/** + * @author kango2gler@gmail.com + * @since 2021/8/9 + */ +@FeignClient(value = "temp", path = "/", fallback = DefaultFeignFallBack.class) +public interface TempFeignAPi { + +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/manager/TemplateManager.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/manager/TemplateManager.java new file mode 100644 index 0000000..c8907c5 --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/manager/TemplateManager.java @@ -0,0 +1,10 @@ +package org.kangspace.messagepush.rest.core.manager; + + +/** + * @author kango2gler@gmail.com + */ +public interface TemplateManager { + + +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/manager/TemplateManagerImpl.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/manager/TemplateManagerImpl.java new file mode 100644 index 0000000..0b07468 --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/manager/TemplateManagerImpl.java @@ -0,0 +1,10 @@ +package org.kangspace.messagepush.rest.core.manager; + +import org.springframework.stereotype.Service; + +/** + * @author kango2gler@gmail.com + */ +@Service +public class TemplateManagerImpl implements TemplateManager { +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/mapper/TemplateMapper.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/mapper/TemplateMapper.java new file mode 100644 index 0000000..80170aa --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/mapper/TemplateMapper.java @@ -0,0 +1,11 @@ +package org.kangspace.messagepush.rest.core.mapper; + + +/** + * 模板mapper + * + * @author kango2gler@gmail.com + */ +public interface TemplateMapper { + +} \ No newline at end of file diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/mq/kafka/KafkaMqSender.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/mq/kafka/KafkaMqSender.java new file mode 100644 index 0000000..8786235 --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/mq/kafka/KafkaMqSender.java @@ -0,0 +1,46 @@ +package org.kangspace.messagepush.rest.core.mq.kafka; + + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.util.JsonUtil; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; + +/** + * kafka消息生产者 + * + * @author kango2gler@gmail.com + * @see KafkaMqTopicChannelMapping + */ +@Slf4j +@Component +public class KafkaMqSender { + @Resource + private KafkaMqTopicChannelMapping kafkaMqTopicChannelMapping; + + /** + * 发送kafka消息到topic + * + * @param message message + * @see KafkaMqTopicChannelMapping + */ + public boolean send(String topic, Object message) { + String msg = JsonUtil.toJson(message); + log.info("发送Kafka消息: kafka send msg [begin], topic:{}, msg:[{}]", topic, msg); + Message messageBuild = MessageBuilder.withPayload(msg).build(); + MessageChannel messageChannel = kafkaMqTopicChannelMapping.getMessageChannel(topic); + boolean isSendSucceed = false; + if (messageChannel != null) { + isSendSucceed = messageChannel.send(messageBuild); + } else { + log.error("发送Kafka消息: topic [{}] <=> MessageChannel is not exist in KafkaMqTopicChannelMapping !", topic); + } + log.info("发送Kafka消息: kafka send msg [{}], topic:[{}], msg:[{}]", isSendSucceed, topic, msg); + return isSendSucceed; + } + +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/mq/kafka/KafkaMqTopicChannelMapping.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/mq/kafka/KafkaMqTopicChannelMapping.java new file mode 100644 index 0000000..50531cf --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/mq/kafka/KafkaMqTopicChannelMapping.java @@ -0,0 +1,43 @@ +package org.kangspace.messagepush.rest.core.mq.kafka; + +import org.springframework.messaging.MessageChannel; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; +import java.util.HashMap; +import java.util.Map; + +/** + * Topic和Channel映射表 + * + * @author kango2gler@gmail.com + * @since 2021/10/25 + */ +@Component +public class KafkaMqTopicChannelMapping { + /** + * Topic和MessageChannel映射 + */ + public Map topicMessageChannelMap = new HashMap<>(); + @Resource + private MessagePushChannel messagePushChannel; + + /** + * 注册Topic和MessageChannel映射关系 + */ + @PostConstruct + public void init() { + topicMessageChannelMap.put(MessagePushChannel.OUTPUT_MESSAGE_PUSH_SINGLE_TOPIC, messagePushChannel.outputMessagePushSingle()); + } + + /** + * 通过Topic获取MessageChannel + * + * @param topic + * @return + */ + public MessageChannel getMessageChannel(String topic) { + return topicMessageChannelMap.get(topic); + } +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/mq/kafka/MessagePushChannel.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/mq/kafka/MessagePushChannel.java new file mode 100644 index 0000000..f5f2266 --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/mq/kafka/MessagePushChannel.java @@ -0,0 +1,36 @@ +package org.kangspace.messagepush.rest.core.mq.kafka; + +import org.springframework.cloud.stream.annotation.Output; +import org.springframework.messaging.MessageChannel; + +/** + * 
+ * 消息推送输出通道
+ * 添加新的Kafka输出通道需要以下3个步骤:
+ * 1. 在Nacos spring.cloud.stream.bindings下配置新的channel,如:
+ *       message_push_single_topic:
+ *                 destination: message_push_single_topic
+ *                 content-type: text/plain
+ *    
+ * 2. 在{@link MessagePushChannel}中添加 Topic和@Output配置,如{@link #OUTPUT_MESSAGE_PUSH_SINGLE_TOPIC}和 {@link #outputMessagePushSingle()}
+ * 3. 在{@link KafkaMqTopicChannelMapping#init()} 中添加Topic和MessageChannel映射
+ *
+ * + * @author kango2gler@gmail.com + */ +public interface MessagePushChannel { + + /** + * 消息推送Kafka数据通道 + */ + String OUTPUT_MESSAGE_PUSH_SINGLE_TOPIC = "message_push_single_topic"; + + /** + * 消息推送Kafka数据通道 + * + * @return {@link org.springframework.messaging.MessageChannel} + */ + @Output(OUTPUT_MESSAGE_PUSH_SINGLE_TOPIC) + MessageChannel outputMessagePushSingle(); + +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/properties/TemplateProperties.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/properties/TemplateProperties.java new file mode 100644 index 0000000..144111a --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/properties/TemplateProperties.java @@ -0,0 +1,18 @@ +package org.kangspace.messagepush.rest.core.properties; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +/** + * 配置属性 + * + * @author kango2gler@gmail.com + */ +@Setter +@Getter +@Configuration +@ConfigurationProperties(prefix = "template") +public class TemplateProperties { +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/service/BaseService.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/service/BaseService.java new file mode 100644 index 0000000..5da3c34 --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/service/BaseService.java @@ -0,0 +1,10 @@ +package org.kangspace.messagepush.rest.core.service; + +/** + * 基础Service + * + * @author kango2gler@gmail.com + * @since 2021/10/25 + */ +public class BaseService { +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/service/MessagePushService.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/service/MessagePushService.java new file mode 100644 index 0000000..25fb442 --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/service/MessagePushService.java @@ -0,0 +1,24 @@ +package org.kangspace.messagepush.rest.core.service; + + +import org.kangspace.messagepush.core.dto.response.ApiResponse; +import org.kangspace.messagepush.rest.api.dto.request.MessagePushRequestDTO; + +import javax.servlet.http.HttpServletRequest; + +/** + * 消息推送Service + * + * @author kango2gler@gmail.com + * @since 2021/10/25 + */ +public interface MessagePushService { + /** + * 消息处理 + * + * @param messagePushRequestDto 消息请求Dto + * @param request HttpServletRequest + * @return ApiResponse + */ + ApiResponse messageHandle(MessagePushRequestDTO messagePushRequestDto, HttpServletRequest request); +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/service/impl/MessagePushServiceImpl.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/service/impl/MessagePushServiceImpl.java new file mode 100644 index 0000000..44b01f0 --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/service/impl/MessagePushServiceImpl.java @@ -0,0 +1,40 @@ +package org.kangspace.messagepush.rest.core.service.impl; + + +import org.kangspace.messagepush.core.dto.response.ApiResponse; +import org.kangspace.messagepush.core.enums.ResponseEnum; +import org.kangspace.messagepush.rest.api.dto.request.MessagePushRequestDTO; +import org.kangspace.messagepush.rest.api.dto.request.MessagePushRequestTimeDTO; +import org.kangspace.messagepush.rest.core.constant.AppThreadLocal; +import org.kangspace.messagepush.rest.core.mq.kafka.KafkaMqSender; +import org.kangspace.messagepush.rest.core.mq.kafka.MessagePushChannel; +import org.kangspace.messagepush.rest.core.service.BaseService; +import org.kangspace.messagepush.rest.core.service.MessagePushService; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; +import javax.servlet.http.HttpServletRequest; + +/** + * 消息推送Service + * + * @author kango2gler@gmail.com + * @since 2021/10/25 + */ +@Service +public class MessagePushServiceImpl extends BaseService implements MessagePushService { + @Resource + private KafkaMqSender kafkaMqSender; + + @Override + public ApiResponse messageHandle(MessagePushRequestDTO messagePushRequestDto, HttpServletRequest request) { + // 构建带时间的消息对象 + MessagePushRequestTimeDTO messagePushRequestTimeDto = MessagePushRequestTimeDTO.build(messagePushRequestDto); + messagePushRequestTimeDto.setAppKey(AppThreadLocal.getAppKey()); + // 发送Kafka + boolean isSendSucceed = kafkaMqSender.send(MessagePushChannel.OUTPUT_MESSAGE_PUSH_SINGLE_TOPIC, messagePushRequestTimeDto); + // 返回处理 + ApiResponse response = new ApiResponse(isSendSucceed ? ResponseEnum.OK : ResponseEnum.INTERNAL_SERVER_ERROR); + return response; + } +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/utils/AppGenerator.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/utils/AppGenerator.java new file mode 100644 index 0000000..3e704ef --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/utils/AppGenerator.java @@ -0,0 +1,83 @@ +package org.kangspace.messagepush.rest.core.utils; + +import cn.hutool.core.lang.UUID; +import cn.hutool.crypto.digest.MD5; +import lombok.Data; +import org.springframework.util.StringUtils; + +import java.nio.charset.StandardCharsets; + +/** + * 
+ * 应用信息生成器
+ * appId: 32位UUID
+ * appSecret: md5("messagepush"+{appId})
+ *
+ * + * @author kango2gler@gmail.com + * @since 2021/10/26 + */ +public class AppGenerator { + /** + * AppSecret生成的加密因子 + */ + private static final byte[] APP_SECRET_SEEDS = "messagepush".getBytes(StandardCharsets.UTF_8); + + /** + * 生成应用信息 + * + * @return {@link AppInfo} + */ + public static AppInfo generate() { + String appKey = UUID.fastUUID().toString(true); + String appSecret = generateAppSecret(appKey); + return new AppInfo(appKey, appSecret); + } + + /** + * 通过AppKey生成AppSecret + * + * @param appKey appKey + * @return AppSecret + */ + private static String generateAppSecret(String appKey) { + return new MD5(APP_SECRET_SEEDS).digestHex16(appKey); + } + + /** + * 验证应用信息 + * + * @param appKey appKey + * @param appSecret appSecret + * @return boolean + */ + public static boolean validAppInfo(String appKey, String appSecret) { + if (StringUtils.hasText(appKey) && StringUtils.hasText(appSecret)) { + String correctSecret = generateAppSecret(appKey); + return correctSecret.equals(appSecret); + } + return false; + } + + public static void main(String[] args) { + AppInfo appInfo = AppGenerator.generate(); + System.out.println(appInfo); + } + + /** + * Http Basic认证头 + */ + @Data + public static class AppInfo { + private String appKey; + private String appSecret; + + public AppInfo() { + } + + public AppInfo(String appKey, String appSecret) { + this.appKey = appKey; + this.appSecret = appSecret; + } + } +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/utils/IpUtil.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/utils/IpUtil.java new file mode 100644 index 0000000..ea3afcc --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/utils/IpUtil.java @@ -0,0 +1,49 @@ +package org.kangspace.messagepush.rest.core.utils; + +import javax.servlet.http.HttpServletRequest; + +/** + * Ip相关工具类 + * + * @author kango2gler@gmail.com + * @since 2021/8/7 + */ +public class IpUtil { + + public static String UNKNOWN = "unknown"; + public static String COMMA = ","; + + /** + * 获取客户端Ip + * + * @return + */ + public static String getClientIp(HttpServletRequest request) { + String ip = request.getHeader("x-forwarded-for"); + if (ip != null && ip.length() != 0 && !UNKNOWN.equalsIgnoreCase(ip)) { + // 多次反向代理后会有多个ip值，第一个ip才是真实ip + if (ip.indexOf(COMMA) != -1) { + ip = ip.split(",")[0]; + } + } + if (ip == null || ip.length() == 0 || UNKNOWN.equalsIgnoreCase(ip)) { + ip = request.getHeader("Proxy-Client-IP"); + } + if (ip == null || ip.length() == 0 || UNKNOWN.equalsIgnoreCase(ip)) { + ip = request.getHeader("WL-Proxy-Client-IP"); + } + if (ip == null || ip.length() == 0 || UNKNOWN.equalsIgnoreCase(ip)) { + ip = request.getHeader("HTTP_CLIENT_IP"); + } + if (ip == null || ip.length() == 0 || UNKNOWN.equalsIgnoreCase(ip)) { + ip = request.getHeader("HTTP_X_FORWARDED_FOR"); + } + if (ip == null || ip.length() == 0 || UNKNOWN.equalsIgnoreCase(ip)) { + ip = request.getHeader("X-Real-IP"); + } + if (ip == null || ip.length() == 0 || UNKNOWN.equalsIgnoreCase(ip)) { + ip = request.getRemoteAddr(); + } + return ip; + } +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/utils/UidCoder.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/utils/UidCoder.java new file mode 100644 index 0000000..4ff1dcc --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/utils/UidCoder.java @@ -0,0 +1,72 @@ +package org.kangspace.messagepush.rest.core.utils; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +/** + * uid编码解码器 + * + * @author kango2gler@gmail.com + * @since 2021/10/19 + */ +@Slf4j +public class UidCoder { + /** + * 
+     * 解码UID
+     * figui混淆加密解密算法
+     *
+ * + * @param uidStr 已加密的uid + * @return 解密的uid + */ + public static String decode(String uidStr) { + if (uidStr == null || uidStr.trim().length() == 0) { + return null; + } + uidStr = uidStr.replace("A", "0") + .replaceAll("(\\D)", ""); + return StringUtils.reverse(uidStr); + } + + /** + * 将加密的uid转换为解密的Long + * + * @param uidStr 加密的uid + * @return 解密的uid + */ + public static Long decodeToLong(String uidStr) { + Long uidLong = null; + String uid = decode(uidStr); + if (uid != null && uid.length() > 0) { + uidLong = Long.valueOf(uid); + } + return uidLong; + } + + + public static void main(String[] args) { + System.out.println("123 reverse => " + StringUtils.reverse("123")); + String uidStr = "AwSjG569syGq26A2"; + System.out.println(uidStr.replaceAll("(\\D)", "")); + System.out.println(uidStr + " => " + decode(uidStr) + " src:20629650"); + uidStr = null; + System.out.println(uidStr + " => " + decode(uidStr) + " src:null"); + // 38437120 + uidStr = "AaCISHQ2nY173483"; + System.out.println(uidStr + " => " + decode(uidStr) + " src:38437120"); + // '334552042' + uidStr = "MUqu24oWA255O433"; + System.out.println(uidStr + " => " + decode(uidStr) + " src:334552042"); + // 234234232 + uidStr = "b23SN2D4i32L4i32"; + System.out.println(uidStr + " => " + decode(uidStr) + " src:234234232"); + + Long uid = decodeToLong(uidStr); + System.out.println(uidStr + " => " + uid + " src:234234232"); + + uidStr = "中国"; + System.out.println(uidStr + " => " + decode(uidStr) + " src:中国"); + + } +} diff --git a/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/utils/VariableUtils.java b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/utils/VariableUtils.java new file mode 100644 index 0000000..e1490a9 --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/main/java/org/kangspace/messagepush/rest/core/utils/VariableUtils.java @@ -0,0 +1,33 @@ +package org.kangspace.messagepush.rest.core.utils; + +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; + +import java.util.Map; + +/** + * 变量处理工具类 + * + * @author kango2gler@gmail.com + * @since 2021/8/18 + */ +public class VariableUtils { + /** + * 变量替换 + * + * @param str 输入字符串 + * @param varReplaceMap 变量-替换值 map(key:变量,value:替换值) + * @return String + */ + public static String variableSwap(String str, Map varReplaceMap) { + if (!StringUtils.hasText(str) || CollectionUtils.isEmpty(varReplaceMap)) { + return str; + } + for (String v : varReplaceMap.keySet()) { + if (str.contains(v)) { + str = str.replace(v, varReplaceMap.get(v)); + } + } + return str; + } +} diff --git a/message-push-rest/message-push-rest-core/src/test/java/org/kangspace/messagepush/rest/core/Test.java b/message-push-rest/message-push-rest-core/src/test/java/org/kangspace/messagepush/rest/core/Test.java new file mode 100644 index 0000000..d162f01 --- /dev/null +++ b/message-push-rest/message-push-rest-core/src/test/java/org/kangspace/messagepush/rest/core/Test.java @@ -0,0 +1,81 @@ +package org.kangspace.messagepush.rest.core; + +import cn.hutool.core.util.IdUtil; +import com.google.common.collect.ImmutableMap; +import org.junit.Assert; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.kangspace.messagepush.rest.core.utils.VariableUtils; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + + +/** + * 公共测试类型 + * + * @author kango2gler@gmail.com + * @since 2021/8/7 + */ +@RunWith(JUnit4.class) +public class Test { + Pattern levelPattern = Pattern.compile("(?i)^((info)|(warn)|(debug)|(error)|(fatal))$"); + + @org.junit.Test + public void levelPatternTest() { + List rightLevels = Arrays.asList("info", "warn", "debug", "Error", "FATAL"); + List errorLevels = Arrays.asList("info1", "1warn", "consumer", "123"); + for (String rightLevel : rightLevels) { + Assert.assertTrue("rightLevel:" + rightLevel + "校验错误!", levelPattern.matcher(rightLevel).find()); + } + for (String errorLevel : errorLevels) { + Assert.assertFalse("errorLevel:" + errorLevel + "校验错误!", levelPattern.matcher(errorLevel).find()); + } + } + + @org.junit.Test + public void uuid() { + System.out.println(IdUtil.fastSimpleUUID()); + } + + + @org.junit.Test + public void variableSwapTest() { + String appName = "APP"; + String logId = "5cafe88ae6f64faeb356155fda0cefad"; + Map varReplaceMap = ImmutableMap.of( + "{{LOG_ID}}", logId, + "{{APP_NAME}}", appName + ); + List testStrs = Arrays.asList( + null, + "这是应用：{{APP_NAME}}，数据ID为：{{LOG_ID}}", + "应用：{{APP_NAME}}", + "数据ID为：{{LOG_ID}}", + "一个三四五", + "上山打老虎OK", + null + ); + System.out.println("源数据:"); + System.out.println(String.join("\n", testStrs)); + System.out.println("\n 替换后的数据:"); + testStrs.forEach(t -> { + String newStr = VariableUtils.variableSwap(t, varReplaceMap); + System.out.println(newStr); + }); + } + + /** + * 1000个别名 + */ + @org.junit.Test + public void alias1000Test() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 1001; i++) { + sb.append("\"alias" + i + "\","); + } + System.out.println(sb.toString()); + } +} diff --git a/message-push-rest/message-push-rest-microservice/pom.xml b/message-push-rest/message-push-rest-microservice/pom.xml new file mode 100644 index 0000000..811fd3e --- /dev/null +++ b/message-push-rest/message-push-rest-microservice/pom.xml @@ -0,0 +1,114 @@ + + + + org.kangspace.messagepush + message-push-rest + ${revision} + + 4.0.0 + + message-push-rest-microservice + ${revision} + + + 8 + 8 + + + + + org.kangspace.messagepush + message-push-rest-core + + + + org.kangspace.messagepush + message-push-rest-api + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-tomcat + + + + + + org.springframework.boot + spring-boot-starter-undertow + + + + org.springframework.cloud + spring-cloud-starter-openfeign + + + + org.springframework.boot + spring-boot-starter-test + test + + + + com.spring4all + swagger-spring-boot-starter + + + io.springfox + springfox-swagger-ui + + + io.springfox + springfox-swagger2 + + + + + + + message-push-rest-microservice + + + org.springframework.boot + spring-boot-maven-plugin + + + + repackage + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.2 + + true + UTF-8 + + + + + + + src/main/resources + true + + + + + \ No newline at end of file diff --git a/message-push-rest/message-push-rest-microservice/src/main/java/org/kangspace/messagepush/rest/MessagePushRestApplication.java b/message-push-rest/message-push-rest-microservice/src/main/java/org/kangspace/messagepush/rest/MessagePushRestApplication.java new file mode 100644 index 0000000..84fe135 --- /dev/null +++ b/message-push-rest/message-push-rest-microservice/src/main/java/org/kangspace/messagepush/rest/MessagePushRestApplication.java @@ -0,0 +1,29 @@ +package org.kangspace.messagepush.rest; + + +import com.spring4all.swagger.EnableSwagger2Doc; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.cloud.openfeign.EnableFeignClients; +import org.springframework.context.annotation.EnableAspectJAutoProxy; + +/** + * 服务主入口 + * + * @author kango2gler@gmail.com + * @since 2021/10/25 + */ +@Slf4j +@EnableAspectJAutoProxy +@EnableFeignClients +@EnableSwagger2Doc +@SpringBootApplication(exclude = DataSourceAutoConfiguration.class) +public class MessagePushRestApplication { + + public static void main(String[] args) { + SpringApplication.run(MessagePushRestApplication.class, args); + } + +} diff --git a/message-push-rest/message-push-rest-microservice/src/main/java/org/kangspace/messagepush/rest/controller/MessagePushRestController.java b/message-push-rest/message-push-rest-microservice/src/main/java/org/kangspace/messagepush/rest/controller/MessagePushRestController.java new file mode 100644 index 0000000..46f8ae5 --- /dev/null +++ b/message-push-rest/message-push-rest-microservice/src/main/java/org/kangspace/messagepush/rest/controller/MessagePushRestController.java @@ -0,0 +1,54 @@ +package org.kangspace.messagepush.rest.controller; + + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.dto.response.ApiResponse; +import org.kangspace.messagepush.core.util.ObjectUtil; +import org.kangspace.messagepush.rest.api.dto.request.MessagePushRequestDTO; +import org.kangspace.messagepush.rest.core.auth.ApiAuthentication; +import org.kangspace.messagepush.rest.core.constant.MessagePushConstants; +import org.kangspace.messagepush.rest.core.service.MessagePushService; +import org.springframework.util.StringUtils; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import javax.annotation.Resource; +import javax.servlet.http.HttpServletRequest; + +/** + * 消息推送REST API接口 + * + * @author kango2gler@gmail.com + */ +@Slf4j +@Validated +@RestController +@Api(value = "消息推送REST API接口", tags = {"消息推送REST API"}) +@RequestMapping(value = "v1/push") +public class MessagePushRestController { + + @Resource + private MessagePushService messagePushService; + + /** + * 消息推送PUSH接口 + * + * @param messagePushRequestDto + * @param request + * @return + */ + @ApiAuthentication + @ApiOperation(value = "消息推送PUSH接口", produces = "application/json") + @PostMapping("") + public ApiResponse messagePush(@Validated @RequestBody MessagePushRequestDTO messagePushRequestDto, + HttpServletRequest request) { + ObjectUtil.defaultFieldValue(messagePushRequestDto, "platform", !StringUtils.hasText(messagePushRequestDto.getPlatform()), + MessagePushConstants.PUSH_PLATFORM.ALL.toString()); + return messagePushService.messageHandle(messagePushRequestDto, request); + } +} diff --git a/message-push-rest/message-push-rest-microservice/src/main/resources/bootstrap.yml b/message-push-rest/message-push-rest-microservice/src/main/resources/bootstrap.yml new file mode 100644 index 0000000..52ad300 --- /dev/null +++ b/message-push-rest/message-push-rest-microservice/src/main/resources/bootstrap.yml @@ -0,0 +1,19 @@ +spring: + application: + name: @artifactId@ + cloud: + nacos: + discovery: + server-addr: ${SERVICE_DISCOVERY_ADDR:discory.kangspace.org:8443} + namespace: ${SERVICE_DISCOVERY_NAMESPACE:kangspace_dev} + metadata: + version: @project.version@ + config: + server-addr: ${spring.cloud.nacos.discovery.server-addr} + namespace: ${spring.cloud.nacos.discovery.namespace} + file-extension: yaml + shared-configs: + - application.${spring.cloud.nacos.config.file-extension} + - message-push-rest-microservice.${spring.cloud.nacos.config.file-extension} +logging: + level: debug \ No newline at end of file diff --git a/message-push-rest/message-push-rest-microservice/src/test/java/org/kangspace/messagepush/rest/Test.java b/message-push-rest/message-push-rest-microservice/src/test/java/org/kangspace/messagepush/rest/Test.java new file mode 100644 index 0000000..d61d0f4 --- /dev/null +++ b/message-push-rest/message-push-rest-microservice/src/test/java/org/kangspace/messagepush/rest/Test.java @@ -0,0 +1,13 @@ +package org.kangspace.messagepush.rest; + +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * @author kango2gler@gmail.com + * @since 2021/8/9 + */ +@RunWith(JUnit4.class) +public class Test { + +} diff --git a/message-push-rest/pom.xml b/message-push-rest/pom.xml new file mode 100644 index 0000000..0e81bf2 --- /dev/null +++ b/message-push-rest/pom.xml @@ -0,0 +1,51 @@ + + + 4.0.0 + + + org.kangspace.messagepush + message-push + ${revision} + + + message-push-rest + ${revision} + pom + + + message-push-rest-api + message-push-rest-core + message-push-rest-microservice + + + + 8 + 8 + + + + + + org.kangspace.messagepush + message-push-rest-api + ${revision} + + + + org.kangspace.messagepush + message-push-rest-core + ${revision} + + + + org.kangspace.messagepush + message-push-common + ${revision} + + + + + + \ No newline at end of file diff --git a/message-push-ws-gateway/.gitignore b/message-push-ws-gateway/.gitignore new file mode 100644 index 0000000..d4a9d54 --- /dev/null +++ b/message-push-ws-gateway/.gitignore @@ -0,0 +1,13 @@ +.idea +/message-push-ws*/target +/message-push-ws-*/target +/message-push-ws*/target/* +/message-push-ws-*/target/* + +.DS_Store +*.iml + +/.idea/* +/target/* + +!.mvn/wrapper/maven-wrapper.jar \ No newline at end of file diff --git a/message-push-ws-gateway/README.md b/message-push-ws-gateway/README.md new file mode 100644 index 0000000..2fae8f5 --- /dev/null +++ b/message-push-ws-gateway/README.md @@ -0,0 +1,25 @@ +# message-push-ws-gateway + +消息推送websocket网关服务 + +## 项目提供服务 + +1. 对外提供Websocket服务,路由转发websocket到message-push-ws服务。 + +实现说明: + +``` +1. 网关启动时,从Redis 拉取 message-push-ws 服务列表的一致性Hash信息 +2. 监听Nacos服务变更,若1中缓存的hash一致性数据不存在,则网关触发rehash(设置本地锁+Redis分布式锁), +3. 通过uid计算负载(使用一致性Hash,每个物理节点指定40个虚拟节点,每个物理节点最终产生160个虚拟节点) +4. 路由转发成功以后,将用户和物理节点关系保存到Redis + +``` + +## 项目涉及中间件 + + Spring-Gateway: + websocket转发 + + nacos(namespace): + dev: kangspace_dev \ No newline at end of file diff --git a/message-push-ws-gateway/pom.xml b/message-push-ws-gateway/pom.xml new file mode 100644 index 0000000..181c5db --- /dev/null +++ b/message-push-ws-gateway/pom.xml @@ -0,0 +1,167 @@ + + + 4.0.0 + + + org.kangspace.messagepush + message-push + ${revision} + + + message-push-ws-gateway + ${revision} + + + 1.8 + 8 + 8 + + + + + org.kangspace.messagepush + message-push-common + ${revision} + + + + + org.springframework.boot + spring-boot-starter-webflux + + + + + org.springframework.cloud + spring-cloud-starter-gateway + + + + org.springframework.cloud + spring-cloud-gateway-webflux + + + + org.springframework.cloud + spring-cloud-netflix-ribbon + + + + org.springframework.cloud + spring-cloud-loadbalancer + + + + org.springframework.cloud + spring-cloud-starter-openfeign + + + + org.springframework.boot + spring-boot-starter-test + test + + + + com.alibaba.cloud + spring-cloud-starter-alibaba-nacos-discovery + true + + + + + org.springframework.boot + spring-boot-starter-data-redis + + + + + com.google.guava + guava + + + + org.apache.commons + commons-lang3 + + + + commons-codec + commons-codec + ${commons-codec.version} + + + + commons-collections + commons-collections + + + + com.alibaba + fastjson + + + + org.projectlombok + lombok + provided + + + javax.servlet + javax.servlet-api + provided + + + + + + + + ${project.artifactId} + + + src/main/resources + true + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + repackage + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.12.4 + + true + + + + + + + + spring-milestones + Spring Milestones + https://repo.spring.io/milestone + + + maven2 + maven2 + https://repo1.maven.org/maven2 + + + + + \ No newline at end of file diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/MessagePushWsGatewayApplication.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/MessagePushWsGatewayApplication.java new file mode 100644 index 0000000..30341d6 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/MessagePushWsGatewayApplication.java @@ -0,0 +1,37 @@ +package org.kangspace.messagepush.ws.gateway; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.openfeign.EnableFeignClients; + +/** + * 服务启动类 + * + * @author kango2gler@gmail.com + * @date 2021/10/28 + */ +@EnableFeignClients +@SpringBootApplication +public class MessagePushWsGatewayApplication { + + public static void main(String[] args) { + setInitProperties(); + SpringApplication.run(MessagePushWsGatewayApplication.class, args); + } + + /** + * 设置初始化配置 + * + * @see reactor.netty.resources.PooledConnectionProvider + * @see reactor.netty.resources.ConnectionProvider + * @see org.springframework.cloud.gateway.config.GatewayAutoConfiguration.NettyConfiguration#gatewayHttpClient + */ + public static void setInitProperties() { + // reactor.netty.pool.leasingStrategy : netty线程池获取线程策略,默认 fifo; + // fifo: 取最早释放到连接池的连接(若连接池中的连接长时间未使用,则取出来的连接可能是已经被重置) + // lifo: 取最近释放到连接池的连接. + System.setProperty("reactor.netty.pool.leasingStrategy", "lifo"); + } + +} + diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/config/EventListenerConfiguration.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/config/EventListenerConfiguration.java new file mode 100644 index 0000000..910c8f3 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/config/EventListenerConfiguration.java @@ -0,0 +1,28 @@ +package org.kangspace.messagepush.ws.gateway.config; + +import com.alibaba.cloud.nacos.NacosDiscoveryProperties; +import org.kangspace.messagepush.core.constant.MessagePushConstants; +import org.kangspace.messagepush.ws.gateway.nacos.NacosDynamicServerListListener; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * 事件监听配置 + * + * @author kango2gler@gmail.com + * @since 2021/10/28 + */ +@Configuration +public class EventListenerConfiguration { + + /** + * 服务上下线监听 + * + * @param properties + * @return NacosDynamicServerListListener + */ + @Bean + public NacosDynamicServerListListener nacosDynamicServerListListener(NacosDiscoveryProperties properties) { + return new NacosDynamicServerListListener(properties, MessagePushConstants.MESSAGE_WS_SERVICE_ID); + } +} \ No newline at end of file diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/config/MessagePushWsGatewayAutoConfiguration.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/config/MessagePushWsGatewayAutoConfiguration.java new file mode 100644 index 0000000..f858813 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/config/MessagePushWsGatewayAutoConfiguration.java @@ -0,0 +1,21 @@ +package org.kangspace.messagepush.ws.gateway.config; + +import org.springframework.boot.autoconfigure.AutoConfigureBefore; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.gateway.config.GatewayLoadBalancerClientAutoConfiguration; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +/** + * 自动配置类 + * + * @author kango2gler@gmail.com + * @since 2021/10/27 + */ +@Configuration +@Import(MessagePushWsGatewayConfiguration.class) +@AutoConfigureBefore(GatewayLoadBalancerClientAutoConfiguration.class) +@EnableConfigurationProperties(MessagePushWsGatewayProperties.class) +public class MessagePushWsGatewayAutoConfiguration { + +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/config/MessagePushWsGatewayConfiguration.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/config/MessagePushWsGatewayConfiguration.java new file mode 100644 index 0000000..3fd7733 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/config/MessagePushWsGatewayConfiguration.java @@ -0,0 +1,220 @@ +package org.kangspace.messagepush.ws.gateway.config; + +import com.alibaba.cloud.nacos.discovery.NacosServiceDiscovery; +import com.alibaba.fastjson.serializer.SerializerFeature; +import com.alibaba.fastjson.support.config.FastJsonConfig; +import com.alibaba.fastjson.support.spring.FastJsonHttpMessageConverter; +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.hash.ConsistencyHashing; +import org.kangspace.messagepush.core.hash.repository.HashRouterRepository; +import org.kangspace.messagepush.core.redis.RedisService; +import org.kangspace.messagepush.ws.gateway.filter.GatewayWebsocketRoutingFilter; +import org.kangspace.messagepush.ws.gateway.filter.RequestValidateFilter; +import org.kangspace.messagepush.ws.gateway.filter.WebsocketReactiveLoadBalancerClientFilter; +import org.kangspace.messagepush.ws.gateway.filter.balancer.LbServiceInstanceChooser; +import org.kangspace.messagepush.ws.gateway.filter.balancer.UIDServiceInstanceChooser; +import org.kangspace.messagepush.ws.gateway.filter.session.WebSocketUserSessionManager; +import org.kangspace.messagepush.ws.gateway.hash.DebugPrintHashingScheduler; +import org.kangspace.messagepush.ws.gateway.hash.RedisHashRouterRepository; +import org.kangspace.messagepush.ws.gateway.nacos.NacosNamingService; +import org.kangspace.messagepush.ws.gateway.validation.PassportSessionValidator; +import org.kangspace.messagepush.ws.gateway.validation.TokenValidator; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.autoconfigure.http.HttpMessageConverters; +import org.springframework.boot.autoconfigure.web.reactive.error.ErrorWebFluxAutoConfiguration; +import org.springframework.cloud.client.ServiceInstance; +import org.springframework.cloud.gateway.config.LoadBalancerProperties; +import org.springframework.cloud.gateway.filter.ReactiveLoadBalancerClientFilter; +import org.springframework.cloud.gateway.filter.headers.HttpHeadersFilter; +import org.springframework.cloud.loadbalancer.config.LoadBalancerAutoConfiguration; +import org.springframework.cloud.loadbalancer.support.LoadBalancerClientFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Primary; +import org.springframework.http.MediaType; +import org.springframework.web.cors.CorsConfiguration; +import org.springframework.web.cors.reactive.CorsWebFilter; +import org.springframework.web.cors.reactive.UrlBasedCorsConfigurationSource; +import org.springframework.web.reactive.socket.client.WebSocketClient; +import org.springframework.web.reactive.socket.server.WebSocketService; + +import java.util.ArrayList; +import java.util.List; + + +/** + * 网关配置类 + * + * @author kango2gler@gmail.com + * @date 2021/10/28 + * @see ErrorWebFluxAutoConfiguration ErrorWebFluxAutoConfiguration错误处理配置 + */ +@Slf4j +public class MessagePushWsGatewayConfiguration { + + /** + * PassportSession校验器 + * + * @return + */ + public PassportSessionValidator tokenValidator() { + return new PassportSessionValidator(); + } + + /** + * 验证过滤器 + * + * @return {@link RequestValidateFilter} + */ + @Bean + public RequestValidateFilter validateFilter(TokenValidator tokenValidator) { + return new RequestValidateFilter(tokenValidator); + } + + /** + * WebSocket负载均衡过滤器 + * + * @param clientFactory {@link LoadBalancerClientFactory} + * @param properties {@link LoadBalancerProperties} + * @return {@link WebsocketReactiveLoadBalancerClientFilter} + * @see LoadBalancerAutoConfiguration + * @see org.springframework.cloud.gateway.filter.ReactiveLoadBalancerClientFilter + */ + @Primary + @Bean + public ReactiveLoadBalancerClientFilter websocketReactiveLoadBalancerClientFilter(LbServiceInstanceChooser lbServiceInstanceChooser, + LoadBalancerClientFactory clientFactory, + LoadBalancerProperties properties) { + return new WebsocketReactiveLoadBalancerClientFilter(lbServiceInstanceChooser, clientFactory, properties); + } + + /** + * 用户Session管理Bean + * + * @return {@link WebSocketUserSessionManager} + */ + @Bean + public WebSocketUserSessionManager webSocketUserSessionManager() { + return new WebSocketUserSessionManager(); + } + + /** + * Websocket路由过滤器 + * + * @param webSocketClient {@link WebSocketClient} + * @param webSocketService {@link WebSocketService} + * @param headersFilters {@link ObjectProvider} + * @return {@link GatewayWebsocketRoutingFilter} + */ + @Bean + public GatewayWebsocketRoutingFilter gatewayWebsocketRoutingFilter(WebSocketClient webSocketClient, + WebSocketService webSocketService, ObjectProvider> headersFilters, + WebSocketUserSessionManager webSocketUserSessionManager) { + return new GatewayWebsocketRoutingFilter(webSocketClient, webSocketService, headersFilters, webSocketUserSessionManager); + } + + /** + * NacosNamingService Instance + * + * @param nacosServiceDiscovery {@link NacosServiceDiscovery} + * @return {@link NacosNamingService} + */ + @Bean + public NacosNamingService nacosNamingService(NacosServiceDiscovery nacosServiceDiscovery) { + return new NacosNamingService(nacosServiceDiscovery); + } + + /** + * Redis相关操作Bean + * + * @return {@link RedisService} + */ + @Bean + public RedisService redisService() { + return new RedisService(); + } + + /** + * 一致性Hash数据持久化类 + * + * @param redisService {@link RedisService} + * @param nacosNamingService {@link NacosNamingService} + * @return {@link HashRouterRepository + */ + @Bean + public HashRouterRepository hashRouterRepository(RedisService redisService, NacosNamingService nacosNamingService) { + return new RedisHashRouterRepository(redisService, nacosNamingService); + } + + /** + * Server一致性Hash对象 + * 1. 服务启动时从Redis中获取 + * 2. 监听Nacos心跳获取服务列表,若有变化则Rehash + * + * @param hashRouterRepository {@link HashRouterRepository} + * @return {@link ConsistencyHashing} + */ + @Bean + public ConsistencyHashing hashRouter(HashRouterRepository hashRouterRepository) { + return hashRouterRepository.get(); + } + + /** + * 负载均衡选择器 + * + * @param hashRouter {@link ConsistencyHashing} + * @return {@link LbServiceInstanceChooser} + */ + @Bean + public LbServiceInstanceChooser lbServiceInstanceChooser(ConsistencyHashing hashRouter) { + return new UIDServiceInstanceChooser(hashRouter); + } + + /** + * 一致性Hash数据定时打印Bean + * + * @return {@link DebugPrintHashingScheduler} + */ + @Bean + public DebugPrintHashingScheduler DebugPrintHashingScheduler(ConsistencyHashing hashRouter) { + return new DebugPrintHashingScheduler(hashRouter); + } + + + /** + * 使用fastjson代替jackson + * + * @return org.springframework.boot.autoconfigure.http.HttpMessageConverters + */ + @Bean + public HttpMessageConverters fastJsonConfigure() { + FastJsonHttpMessageConverter converter = new FastJsonHttpMessageConverter(); + FastJsonConfig fastJsonConfig = new FastJsonConfig(); + fastJsonConfig.setSerializerFeatures(SerializerFeature.WriteNullStringAsEmpty, SerializerFeature.WriteMapNullValue); + // 日期格式化 + fastJsonConfig.setDateFormat("yyyy-MM-dd HH:mm:ss"); + converter.setFastJsonConfig(fastJsonConfig); + List mediaTypes = new ArrayList<>(6); + mediaTypes.add(MediaType.APPLICATION_ATOM_XML); + mediaTypes.add(MediaType.APPLICATION_CBOR); + mediaTypes.add(MediaType.APPLICATION_FORM_URLENCODED); + mediaTypes.add(MediaType.APPLICATION_JSON); + mediaTypes.add(MediaType.APPLICATION_OCTET_STREAM); + converter.setSupportedMediaTypes(mediaTypes); + return new HttpMessageConverters(converter); + } + + /** + * 跨域过滤器 + * gateway采用react形式,需要使用reactive包下的UrlBasedCorsConfigurationSource + */ + @Bean + public CorsWebFilter corsWebFilter() { + UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(); + CorsConfiguration corsConfiguration = new CorsConfiguration(); + corsConfiguration.addAllowedOrigin("*"); + corsConfiguration.addAllowedHeader("*"); + corsConfiguration.addAllowedMethod("*"); + source.registerCorsConfiguration("/**", corsConfiguration); + return new CorsWebFilter(source); + } +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/config/MessagePushWsGatewayProperties.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/config/MessagePushWsGatewayProperties.java new file mode 100644 index 0000000..93e5b86 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/config/MessagePushWsGatewayProperties.java @@ -0,0 +1,26 @@ +package org.kangspace.messagepush.ws.gateway.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +/** + * 网关相关配置 + * + * @author kango2gler@gmail.com + * @since 2021/10/28 + */ +@Data +@Configuration +@ConfigurationProperties(prefix = "message-push.gateway") +public class MessagePushWsGatewayProperties { + /** + * websocket服务名 + */ + private String wsService; + /** + * passport session center接口 + */ + private String passportSessionUrl; + +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/constant/MessagePushWsConstants.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/constant/MessagePushWsConstants.java new file mode 100644 index 0000000..59ddb44 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/constant/MessagePushWsConstants.java @@ -0,0 +1,17 @@ +package org.kangspace.messagepush.ws.gateway.constant; + +/** + * 常量类 + * + * @author kango2gler@gmail.com + * @date 2021/10/28 + */ +public interface MessagePushWsConstants { + + /** + * 模拟UID请求头 + * 用于压测情况;(正常业务情况下建立连接需要传认证token,考虑到压测时无法模拟大批量token) + */ + String MOCK_UID_HEADER = "x-mock-uid"; + +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/debug/DebugWebsocketSessionCountScheduler.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/debug/DebugWebsocketSessionCountScheduler.java new file mode 100644 index 0000000..92a5863 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/debug/DebugWebsocketSessionCountScheduler.java @@ -0,0 +1,55 @@ +package org.kangspace.messagepush.ws.gateway.debug; + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.ws.gateway.filter.session.WebSocketUserSessionManager; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; +import org.springframework.stereotype.Service; + +/** + * websocket session连接数打印 + * + * @author kango2gler@gmail.com + * @since 2021/11/18 + */ +@Slf4j +@Service +public class DebugWebsocketSessionCountScheduler { + /** + * 一致性Hash处理Bean + */ + private final WebSocketUserSessionManager webSocketUserSessionManager; + + private ThreadPoolTaskScheduler scheduler; + + public DebugWebsocketSessionCountScheduler(WebSocketUserSessionManager webSocketUserSessionManager) { + this.webSocketUserSessionManager = webSocketUserSessionManager; + if (!log.isDebugEnabled()) { + return; + } + this.scheduler = new ThreadPoolTaskScheduler(); + this.scheduler.setPoolSize(1); + this.scheduler.initialize(); + startPrint(); + } + + /** + * 开始日志打印任务 + */ + private void startPrint() { + log.debug("WebsocketSession数打印: 定时任务开始!"); + this.scheduler.scheduleAtFixedRate(() -> print(), 1000L); + + } + + /** + * 打印日志 + */ + private void print() { + // 用户数 + int userConnectCnt = webSocketUserSessionManager.getUserSessionsMap().size(); + // 总连接数 + long sessionConnectCnt = webSocketUserSessionManager.getUserSessionsMap().values() + .stream().filter(t -> t != null).flatMap(t -> t.stream()).count(); + log.info("当前Session连接数汇总: 用户数:{} ,总连接数:{}", userConnectCnt, sessionConnectCnt); + } +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/domain/dto/request/SessionCenterUserInfoParamDTO.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/domain/dto/request/SessionCenterUserInfoParamDTO.java new file mode 100644 index 0000000..90d4bef --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/domain/dto/request/SessionCenterUserInfoParamDTO.java @@ -0,0 +1,29 @@ +package org.kangspace.messagepush.ws.gateway.domain.dto.request; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * SessionCenter 用户信息请求参数DTO + * + * @author kango2gler@gmail.com + * @since 2021/11/5 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class SessionCenterUserInfoParamDTO { + /** + * AccessToken + */ + private String accessToken; + /** + * 平台 + */ + private String platForm; + /** + * 应用ID + */ + private String appId; +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/domain/dto/response/SessionCenterUserInfoDTO.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/domain/dto/response/SessionCenterUserInfoDTO.java new file mode 100644 index 0000000..4e2b615 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/domain/dto/response/SessionCenterUserInfoDTO.java @@ -0,0 +1,37 @@ +package org.kangspace.messagepush.ws.gateway.domain.dto.response; + +import lombok.Data; + +/** + * SessionCenter用户信息 + * + * @author kango2gler@gmail.com + * @since 2021/11/5 + */ +@Data +public class SessionCenterUserInfoDTO { + /** + * 头像 + */ + private String avatar; + /** + * 性别 + */ + private String gender; + /** + * 用户ID + */ + private String uid; + /** + * 真实姓名 + */ + private String realname; + /** + * 昵称 + */ + private String nickname; + /** + * 手机号 + */ + private String mobile; +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/exception/GatewayExceptionHandler.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/exception/GatewayExceptionHandler.java new file mode 100644 index 0000000..6a77ded --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/exception/GatewayExceptionHandler.java @@ -0,0 +1,124 @@ +package org.kangspace.messagepush.ws.gateway.exception; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.web.reactive.error.ErrorWebExceptionHandler; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.codec.HttpMessageReader; +import org.springframework.http.codec.HttpMessageWriter; +import org.springframework.util.Assert; +import org.springframework.web.reactive.function.BodyInserters; +import org.springframework.web.reactive.function.server.ServerResponse; +import org.springframework.web.reactive.result.view.ViewResolver; +import org.springframework.web.server.ServerWebExchange; +import reactor.core.publisher.Mono; + +import java.util.Collections; +import java.util.List; + +/** + * 统一异常处理 + * + * @author kango2gler@gmail.com + * @date 2021/10/28 + */ +public class GatewayExceptionHandler implements ErrorWebExceptionHandler { + + private static final Logger log = LoggerFactory.getLogger(GatewayExceptionHandler.class); + + /** + * MessageReader + */ + private List> messageReaders = Collections.emptyList(); + + /** + * MessageWriter + */ + private List> messageWriters = Collections.emptyList(); + + /** + * ViewResolvers + */ + private List viewResolvers = Collections.emptyList(); + + @Override + public Mono handle(ServerWebExchange exchange, Throwable ex) { + return null; + } + + /** + * 参考AbstractErrorWebExceptionHandler + * + * @param messageReaders messageReaders + */ + public void setMessageReaders(List> messageReaders) { + Assert.notNull(messageReaders, "'messageReaders' must not be null"); + this.messageReaders = messageReaders; + } + + /** + * 参考AbstractErrorWebExceptionHandler + * + * @param viewResolvers viewResolvers + */ + public void setViewResolvers(List viewResolvers) { + this.viewResolvers = viewResolvers; + } + + /** + * 参考AbstractErrorWebExceptionHandler + * + * @param messageWriters messageWriters + */ + public void setMessageWriters(List> messageWriters) { + Assert.notNull(messageWriters, "'messageWriters' must not be null"); + this.messageWriters = messageWriters; + } + + + /** + * 参考DefaultErrorWebExceptionHandler + * + * @param result 返回结果 + * @return 返回mono + */ + protected Mono renderErrorResponse(String result) { + return ServerResponse + .status(HttpStatus.OK) + .contentType(MediaType.APPLICATION_JSON) + .body(BodyInserters.fromObject(result)); + } + + /** + * 参考AbstractErrorWebExceptionHandler + * + * @param exchange exchange + * @param response response + * @return 返回Mono + */ + private Mono write(ServerWebExchange exchange, + ServerResponse response) { + exchange.getResponse().getHeaders() + .setContentType(response.headers().getContentType()); + return response.writeTo(exchange, new ResponseContext()); + } + + /** + * 参考AbstractErrorWebExceptionHandler + */ + private class ResponseContext implements ServerResponse.Context { + + @Override + public List> messageWriters() { + return GatewayExceptionHandler.this.messageWriters; + } + + @Override + public List viewResolvers() { + return GatewayExceptionHandler.this.viewResolvers; + } + + } + +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/exception/TokenValidateException.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/exception/TokenValidateException.java new file mode 100644 index 0000000..9b38aa7 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/exception/TokenValidateException.java @@ -0,0 +1,43 @@ +package org.kangspace.messagepush.ws.gateway.exception; + +import lombok.Data; + +/** + * @author kango2gler@gmail.com + * @since 2021/11/5 + */ +@Data +public class TokenValidateException extends RuntimeException { + private ExceptionType exceptionType; + + public TokenValidateException() { + } + + public TokenValidateException(ExceptionType exceptionType, String message) { + super(message); + this.exceptionType = exceptionType; + } + + public TokenValidateException(ExceptionType exceptionType, String message, Throwable cause) { + super(message, cause); + this.exceptionType = exceptionType; + } + + /** + * 异常类型 + */ + public enum ExceptionType { + /** + * 参数错误 + */ + INVALID_PARAM, + /** + * token不存在 + */ + ACCESS_TOKEN_NOT_FOUND, + /** + * 获取用户信息错误 + */ + SERVER_ERROR + } +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/feign/DefaultFeignFallBack.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/feign/DefaultFeignFallBack.java new file mode 100644 index 0000000..fe5e3ad --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/feign/DefaultFeignFallBack.java @@ -0,0 +1,14 @@ +package org.kangspace.messagepush.ws.gateway.feign; + +import org.springframework.stereotype.Service; + +/** + * 默认feign降级处理 + * + * @author kango2gler@gmail.com + * @since 2021/8/9 + */ +@Service +public class DefaultFeignFallBack implements TempFeignAPi { + +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/feign/PassportSessionFeignApi.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/feign/PassportSessionFeignApi.java new file mode 100644 index 0000000..9cd6550 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/feign/PassportSessionFeignApi.java @@ -0,0 +1,29 @@ +package org.kangspace.messagepush.ws.gateway.feign; + + +import org.kangspace.messagepush.core.dto.response.ApiResponse; +import org.kangspace.messagepush.ws.gateway.domain.dto.request.SessionCenterUserInfoParamDTO; +import org.kangspace.messagepush.ws.gateway.domain.dto.response.SessionCenterUserInfoDTO; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; + +/** + * Passport SessionCenter相关接口 + * + * @author kango2gler@gmail.com + * @since 2021/11/05 + */ +@FeignClient(value = "passport-session-center", path = "/", url = "${message-push.gateway.passport-session-url}", fallback = DefaultFeignFallBack.class) +public interface PassportSessionFeignApi { + + /** + * 通过AccessToken获取用户信息 + * + * @param paramDto + * @return + */ + @PostMapping() + ApiResponse userLoginInfo(@RequestBody SessionCenterUserInfoParamDTO paramDto); + +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/feign/PassportSessionFeignFallBack.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/feign/PassportSessionFeignFallBack.java new file mode 100644 index 0000000..78cb7e2 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/feign/PassportSessionFeignFallBack.java @@ -0,0 +1,25 @@ +package org.kangspace.messagepush.ws.gateway.feign; + + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.dto.response.ApiResponse; +import org.kangspace.messagepush.ws.gateway.domain.dto.request.SessionCenterUserInfoParamDTO; +import org.kangspace.messagepush.ws.gateway.domain.dto.response.SessionCenterUserInfoDTO; +import org.springframework.stereotype.Service; + +/** + * Passport SessionCenter 相关接口feign降级处理 + * + * @author kango2gler@gmail.com + * @since 2021/8/9 + */ +@Slf4j +@Service +public class PassportSessionFeignFallBack implements PassportSessionFeignApi { + + @Override + public ApiResponse userLoginInfo(SessionCenterUserInfoParamDTO paramDto) { + log.error("通过AccessToken获取Passport SessionCenter用户信息失败,参数:[{}]", paramDto); + return null; + } +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/feign/TempFeignAPi.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/feign/TempFeignAPi.java new file mode 100644 index 0000000..2da3e06 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/feign/TempFeignAPi.java @@ -0,0 +1,12 @@ +package org.kangspace.messagepush.ws.gateway.feign; + +import org.springframework.cloud.openfeign.FeignClient; + +/** + * @author kango2gler@gmail.com + * @since 2021/8/9 + */ +@FeignClient(value = "temp", path = "/", fallback = DefaultFeignFallBack.class) +public interface TempFeignAPi { + +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/BaseFilter.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/BaseFilter.java new file mode 100644 index 0000000..6d93f7f --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/BaseFilter.java @@ -0,0 +1,73 @@ +package org.kangspace.messagepush.ws.gateway.filter; + + +import com.alibaba.fastjson.JSONObject; +import org.kangspace.messagepush.core.dto.response.ApiResponse; +import org.kangspace.messagepush.core.enums.ResponseEnum; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.server.reactive.ServerHttpResponse; +import org.springframework.util.StringUtils; +import org.springframework.web.server.ServerWebExchange; +import reactor.core.publisher.Mono; + +/** + * 基础过滤器 + * + * @author kango2gler@gmail.com + * @since 2021/10/26 + */ +public class BaseFilter { + + /** + * 拒绝请求处理,返回apiDto + * + * @param exchange {@link ServerWebExchange} + * @param apiDto {@link ApiResponse} + * @return Mono + */ + protected Mono reject(ServerWebExchange exchange, ApiResponse apiDto) { + ServerHttpResponse response = exchange.getResponse(); + HttpStatus responseStatusCode = apiDto.getCode() == 0 ? HttpStatus.OK : HttpStatus.resolve(apiDto.getCode()); + response.setStatusCode(responseStatusCode); + response.getHeaders().setContentType(MediaType.APPLICATION_JSON); + return response.writeWith(Mono.just(exchange.getResponse().bufferFactory().wrap(JSONObject.toJSONBytes(apiDto)))); + } + + /** + * 拒绝请求,返回apiDto + * + * @param exchange {@link ServerWebExchange} + * @param responseEnum {@link ResponseEnum} + * @param message 消息内容 + * @return Mono + */ + protected Mono reject(ServerWebExchange exchange, ResponseEnum responseEnum, String message) { + if (responseEnum == null) { + return rejectBy500(exchange, message); + } + ApiResponse apiDto = new ApiResponse(responseEnum); + if (StringUtils.hasText(message)) { + apiDto.setMsg(message); + } + return reject(exchange, apiDto); + } + + /** + * 拒绝请求,返回500 apiDto + * + * @param exchange {@link ServerWebExchange} + * @param message 消息内容 + * @return Mono + */ + protected Mono rejectBy500(ServerWebExchange exchange, String message) { + ApiResponse apiDto = new ApiResponse(); + apiDto.setCode(HttpStatus.INTERNAL_SERVER_ERROR.value()); + apiDto.setMsg(HttpStatus.INTERNAL_SERVER_ERROR.getReasonPhrase()); + if (StringUtils.hasText(message)) { + apiDto.setMsg(message); + } + return reject(exchange, apiDto); + } + +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/FilterOrders.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/FilterOrders.java new file mode 100644 index 0000000..07503f3 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/FilterOrders.java @@ -0,0 +1,33 @@ +package org.kangspace.messagepush.ws.gateway.filter; + +/** + * 过滤器Order常量类 + * + * @author kango2gler@gmail.com + * @date 2021/10/28 + */ +public class FilterOrders { + + /** + * 初始过滤器Order + */ + public static final int DEFAULT_FILTER_ORDER = 10000; + + /** + * {@link org.springframework.cloud.gateway.filter.WebsocketRoutingFilter} Order + */ + public static final int WEBSOCKET_ROUTING_FILTER_ORDER = 2147483644; + + /** + * 验证拦截器order + */ + public static final int VALIDATE_FILTER_ORDER = DEFAULT_FILTER_ORDER + 1; + + /** + * 请求路由过滤器order + * + * @see org.springframework.cloud.gateway.filter.ReactiveLoadBalancerClientFilter + */ + public static final int WEBSOCKET_REACTIVE_LOADBALANCER_CLIENT_FILTER_ORDER = DEFAULT_FILTER_ORDER + 101; + +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/GatewayWebsocketRoutingFilter.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/GatewayWebsocketRoutingFilter.java new file mode 100644 index 0000000..82edbef --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/GatewayWebsocketRoutingFilter.java @@ -0,0 +1,225 @@ +package org.kangspace.messagepush.ws.gateway.filter; + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.constant.MessagePushConstants; +import org.kangspace.messagepush.ws.gateway.filter.session.SessionProxyHolder; +import org.kangspace.messagepush.ws.gateway.filter.session.WebSocketUserSessionManager; +import org.kangspace.messagepush.ws.gateway.model.MessageRequestParam; +import org.kangspace.messagepush.ws.gateway.util.ExchangeRequestUtils; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.cloud.gateway.filter.GatewayFilterChain; +import org.springframework.cloud.gateway.filter.GlobalFilter; +import org.springframework.cloud.gateway.filter.WebsocketRoutingFilter; +import org.springframework.cloud.gateway.filter.headers.HttpHeadersFilter; +import org.springframework.cloud.gateway.support.ServerWebExchangeUtils; +import org.springframework.core.Ordered; +import org.springframework.http.HttpHeaders; +import org.springframework.util.StringUtils; +import org.springframework.web.reactive.socket.WebSocketHandler; +import org.springframework.web.reactive.socket.WebSocketMessage; +import org.springframework.web.reactive.socket.WebSocketSession; +import org.springframework.web.reactive.socket.client.WebSocketClient; +import org.springframework.web.reactive.socket.server.WebSocketService; +import org.springframework.web.server.ServerWebExchange; +import org.springframework.web.util.UriComponentsBuilder; +import reactor.core.publisher.Mono; + +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * 自定义WebsocketRoutingFilter + * + * @author kango2gler@gmail.com + * @see WebsocketRoutingFilter + * @since 2021/11/4 + */ +@Slf4j +public class GatewayWebsocketRoutingFilter extends BaseFilter implements GlobalFilter, Ordered { + public static final String SEC_WEBSOCKET_PROTOCOL = "Sec-WebSocket-Protocol"; + private static final String HTTP_SCHEMA = "http"; + private static final String HTTPS_SCHEMA = "https"; + private static final String WS_SCHEMA = "ws"; + private static final String WSS_SCHEMA = "wss"; + private static final String WEBSOCKET_UPGRADE = "Websocket"; + private final WebSocketClient webSocketClient; + private final WebSocketService webSocketService; + private final ObjectProvider> headersFiltersProvider; + private final WebSocketUserSessionManager webSocketUserSessionManager; + private volatile List headersFilters; + + public GatewayWebsocketRoutingFilter(WebSocketClient webSocketClient, WebSocketService webSocketService, + ObjectProvider> headersFiltersProvider, + WebSocketUserSessionManager webSocketUserSessionManager) { + this.webSocketClient = webSocketClient; + this.webSocketService = webSocketService; + this.headersFiltersProvider = headersFiltersProvider; + this.webSocketUserSessionManager = webSocketUserSessionManager; + } + + /** + * 协议转换 + * + * @param scheme 请求的Scheme + * @return websocket协议 + */ + static String convertHttpToWs(String scheme) { + scheme = scheme.toLowerCase(); + return HTTP_SCHEMA.equals(scheme) ? WS_SCHEMA : (HTTPS_SCHEMA.equals(scheme) ? WSS_SCHEMA : scheme); + } + + @Override + public int getOrder() { + return FilterOrders.WEBSOCKET_ROUTING_FILTER_ORDER; + } + + @Override + public Mono filter(ServerWebExchange exchange, GatewayFilterChain chain) { + this.changeSchemeIfIsWebSocketUpgrade(exchange); + URI requestUrl = (URI) exchange.getRequiredAttribute(ServerWebExchangeUtils.GATEWAY_REQUEST_URL_ATTR); + String scheme = requestUrl.getScheme(); + if (!ServerWebExchangeUtils.isAlreadyRouted(exchange) && (WS_SCHEMA.equals(scheme) || WSS_SCHEMA.equals(scheme))) { + ServerWebExchangeUtils.setAlreadyRouted(exchange); + HttpHeaders headers = exchange.getRequest().getHeaders(); + HttpHeaders filtered = HttpHeadersFilter.filterRequest(this.getHeadersFilters(), exchange); + List protocols = headers.get(SEC_WEBSOCKET_PROTOCOL); + if (protocols != null) { + protocols = (List) headers.get(SEC_WEBSOCKET_PROTOCOL).stream() + .flatMap((header) -> Arrays.stream(StringUtils.commaDelimitedListToStringArray(header))) + .map(String::trim).collect(Collectors.toList()); + } + return this.webSocketService.handleRequest(exchange, + new GatewayWebsocketRoutingFilter.ProxyWebSocketHandler(exchange, requestUrl, this.webSocketClient, + filtered, protocols, webSocketUserSessionManager)); + } else { + return chain.filter(exchange); + } + } + + /** + * 获取请求头过滤器 + * + * @return + */ + private List getHeadersFilters() { + if (this.headersFilters == null) { + this.headersFilters = (List) this.headersFiltersProvider.getIfAvailable(ArrayList::new); + this.headersFilters.add((headers, exchange) -> { + HttpHeaders filtered = new HttpHeaders(); + headers.entrySet().stream().filter((entry) -> !entry.getKey().toLowerCase().startsWith("sec-websocket")) + .forEach((header) -> filtered.addAll(header.getKey(), header.getValue())); + return filtered; + }); + } + + return this.headersFilters; + } + + /** + * Websocket协议升级 + * + * @param exchange + */ + private void changeSchemeIfIsWebSocketUpgrade(ServerWebExchange exchange) { + URI requestUrl = (URI) exchange.getRequiredAttribute(ServerWebExchangeUtils.GATEWAY_REQUEST_URL_ATTR); + String scheme = requestUrl.getScheme().toLowerCase(); + String upgrade = exchange.getRequest().getHeaders().getUpgrade(); + if (WEBSOCKET_UPGRADE.equalsIgnoreCase(upgrade) && (HTTP_SCHEMA.equals(scheme) || HTTPS_SCHEMA.equals(scheme))) { + String wsScheme = convertHttpToWs(scheme); + URI wsRequestUrl = UriComponentsBuilder.fromUri(requestUrl).scheme(wsScheme).build().toUri(); + exchange.getAttributes().put(ServerWebExchangeUtils.GATEWAY_REQUEST_URL_ATTR, wsRequestUrl); + if (log.isTraceEnabled()) { + log.trace("changeSchemeTo:[" + wsRequestUrl + "]"); + } + } + + } + + /** + * 网关代理Websocket处理类 + * 与后端服务请求交互的核心类 + */ + private static class ProxyWebSocketHandler implements WebSocketHandler { + private final ServerWebExchange exchange; + private final WebSocketClient client; + private final URI url; + private final HttpHeaders headers; + private final List subProtocols; + private final WebSocketUserSessionManager webSocketUserSessionManager; + + ProxyWebSocketHandler(ServerWebExchange exchange, URI url, WebSocketClient client, HttpHeaders headers, + List protocols, WebSocketUserSessionManager webSocketUserSessionManager) { + this.exchange = exchange; + this.client = client; + this.url = url; + this.headers = headers; + this.webSocketUserSessionManager = webSocketUserSessionManager; + if (protocols != null) { + this.subProtocols = protocols; + } else { + this.subProtocols = Collections.emptyList(); + } + } + + @Override + public List getSubProtocols() { + return this.subProtocols; + } + + @Override + public Mono handle(WebSocketSession session) { + return this.client.execute(this.url, this.headers, new WebSocketHandler() { + @Override + public Mono handle(WebSocketSession proxySession) { + Mono proxySessionSend = proxySession.send(session.receive().doOnNext(WebSocketMessage::retain)); + Mono serverSessionSend = session.send(proxySession.receive().doOnNext(WebSocketMessage::retain)); + // 绑定用户Session关系 + addUserSession(proxySession, session); + return Mono.zip(proxySessionSend, serverSessionSend).then(); + } + + @Override + public List getSubProtocols() { + return GatewayWebsocketRoutingFilter.ProxyWebSocketHandler.this.subProtocols; + } + }).doOnTerminate(() -> { + // 移除用户Session关系 + removeUserSession(); + }); + } + + /** + * 添加用户Session + * + * @param proxySession + * @param session + */ + private void addUserSession(WebSocketSession proxySession, WebSocketSession session) { + MessageRequestParam param = ExchangeRequestUtils.getMessageRequestParam(exchange); + String uid = param.getUid(); + String serviceNode = exchange.getAttribute(MessagePushConstants.WEBSOCKET_TARGET_SERVICE_NODE_ATTR_KEY); + SessionProxyHolder sessionProxyHolder = new SessionProxyHolder(proxySession, session, serviceNode); + exchange.getAttributes().put(MessagePushConstants.USER_SESSION_HOLDER_EXCHANGE_ATTR_KEY, sessionProxyHolder); + // 绑定用户Session和双向Session信息 + webSocketUserSessionManager.addSession(uid, sessionProxyHolder); + log.info("Websocket路由: 用户已连接, uid:[{}],session:[{}],目标服务:[{}]", uid, session.getId(), serviceNode); + } + + /** + * 删除用户Session信息 + */ + private void removeUserSession() { + MessageRequestParam param = ExchangeRequestUtils.getMessageRequestParam(exchange); + String uid = param.getUid(); + SessionProxyHolder sessionProxyHolder = exchange.getAttribute(MessagePushConstants.USER_SESSION_HOLDER_EXCHANGE_ATTR_KEY); + if (sessionProxyHolder != null) { + webSocketUserSessionManager.removeSession(uid, sessionProxyHolder.getSession()); + log.info("Websocket路由: 用户已断开, uid:[{}],session:[{}],目标服务:[{}]", uid, sessionProxyHolder.getSession(), sessionProxyHolder.getProxySession()); + } + } + } +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/RequestValidateFilter.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/RequestValidateFilter.java new file mode 100644 index 0000000..d21d8e7 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/RequestValidateFilter.java @@ -0,0 +1,146 @@ +package org.kangspace.messagepush.ws.gateway.filter; + + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.constant.ErrorMessageConstants; +import org.kangspace.messagepush.core.constant.MessagePushConstants; +import org.kangspace.messagepush.core.enums.ResponseEnum; +import org.kangspace.messagepush.ws.gateway.constant.MessagePushWsConstants; +import org.kangspace.messagepush.ws.gateway.domain.dto.response.SessionCenterUserInfoDTO; +import org.kangspace.messagepush.ws.gateway.exception.TokenValidateException; +import org.kangspace.messagepush.ws.gateway.model.MessageRequestParam; +import org.kangspace.messagepush.ws.gateway.util.ExchangeRequestUtils; +import org.kangspace.messagepush.ws.gateway.validation.TokenValidator; +import org.springframework.cloud.gateway.filter.GatewayFilterChain; +import org.springframework.cloud.gateway.filter.GlobalFilter; +import org.springframework.cloud.gateway.support.ServerWebExchangeUtils; +import org.springframework.core.Ordered; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpRequest; +import org.springframework.http.server.reactive.ServerHttpRequest; +import org.springframework.util.StringUtils; +import org.springframework.web.server.ServerWebExchange; +import org.springframework.web.util.UriComponentsBuilder; +import reactor.core.publisher.Mono; + +import java.net.URI; +import java.util.Arrays; +import java.util.Optional; + +/** + * 
+ * 请求验证过滤器
+ *
+ * + * @author kango2gler@gmail.com + * @since 2021/10/26 + */ +@Slf4j +public class RequestValidateFilter extends BaseFilter implements GlobalFilter, Ordered { + /** + * Token验证器 + */ + private final TokenValidator tokenValidator; + + public RequestValidateFilter(TokenValidator tokenValidator) { + this.tokenValidator = tokenValidator; + } + + @Override + public Mono filter(ServerWebExchange exchange, GatewayFilterChain chain) { + String requestPath = exchange.getRequest().getURI().toString(); + MessageRequestParam requestParam = ExchangeRequestUtils.getMessageRequestParam(exchange); + log.info("请求验证过滤器: ValidateFilter handle start, url: [{}], requestParam: [{}]", requestPath, requestParam); + // 1. 验证请求,只允许websocket请求访问 + Optional> isWebsocket = isWebsocketRequest(exchange); + if (isWebsocket.isPresent()) { + log.error("请求验证过滤器: 非Websocket请求,拒绝访问, url: [{}], requestParam: [{}],method: [{}], Connection Header:[{}]", + requestPath, requestParam, exchange.getRequest().getMethodValue(), + exchange.getRequest().getHeaders().getFirst(HttpHeaders.CONNECTION)); + return isWebsocket.get(); + } + // 2. 验证请求参数 + Optional> validRequestParam = validRequestParam(exchange, requestParam); + if (validRequestParam.isPresent()) { + log.error("请求验证过滤器: 参数校验不通过,拒绝访问, url: [{}], requestParam: [{}].", requestPath, requestParam); + return validRequestParam.get(); + } + URI uri = exchange.getRequest().getURI(); + URI requestUrl = UriComponentsBuilder.fromUri(uri).build(ServerWebExchangeUtils.containsEncodedParts(uri)).toUri(); + ServerHttpRequest.Builder builder = exchange.getRequest().mutate().uri(requestUrl) + .header(MessagePushConstants.HTTP_HEADER_UID_KEY, requestParam.getUid()); + exchange = exchange.mutate().request(builder.build()).build(); + // 3. 设置用户信息到exchange attr + ExchangeRequestUtils.setMessageRequestParam(exchange, requestParam); + log.info("请求验证过滤器: ValidateFilter handle end, url: [{}], requestParam: [{}].", requestPath, requestParam); + return chain.filter(exchange); + } + + /** + * 验证请求参数 + * + * @param exchange {@link ServerWebExchange} + * @param requestParam {@link MessageRequestParam} + * @return {@link Optional} + */ + private Optional> validRequestParam(ServerWebExchange exchange, MessageRequestParam requestParam) { + Optional> result = Optional.empty(); + // 参数非空校验 + if (requestParam == null || !StringUtils.hasText(requestParam.getToken()) || !StringUtils.hasText(requestParam.getTokenAppId())) { + // 模拟用户ID处理 + String mockUId = exchange.getRequest().getHeaders().getFirst(MessagePushWsConstants.MOCK_UID_HEADER); + // 若存在模拟UID时使用模拟UID + if (requestParam != null && StringUtils.hasText(mockUId)) { + requestParam.setUid(mockUId); + } else { + String errorMsg = String.format(ErrorMessageConstants.INVALID_REQUEST_PARAM_MSG, "Authorization Bearer, auth-app-id", "null"); + result = Optional.ofNullable(reject(exchange, ResponseEnum.BAD_REQUEST, errorMsg)); + } + } else { + try { + // 通过token验证用户信息 + tokenValidator.valid(requestParam.getTokenAppId(), requestParam.getToken(), ((t) -> requestParam.setUid(t.getUid()))); + } catch (TokenValidateException exception) { + ResponseEnum responseEnum = null; + if (TokenValidateException.ExceptionType.INVALID_PARAM.equals(exception.getExceptionType())) { + responseEnum = ResponseEnum.BAD_REQUEST; + } else if (TokenValidateException.ExceptionType.ACCESS_TOKEN_NOT_FOUND.equals(exception.getExceptionType())) { + responseEnum = ResponseEnum.UNAUTHORIZED; + } + String errorMsg = exception.getMessage(); + log.error("校验参数失败,错误信息:[{}]", errorMsg); + result = Optional.ofNullable(reject(exchange, responseEnum, errorMsg)); + } + + } + return result; + } + + + /** + * 
+     * Websocket访问限制过滤器
+     * 只允许Http Connection: upgrade的连接通过
+     *
+ * + * @author kango2gler@gmail.com + * @since 2021/10/27 + */ + private Optional> isWebsocketRequest(ServerWebExchange exchange) { + Optional> result = Optional.empty(); + //只允许Websocket:Http Upgrade的请求 + if (!ExchangeRequestUtils.isWebsocketRequest(exchange)) { + HttpRequest request = exchange.getRequest(); + result = Optional.ofNullable(reject(exchange, ResponseEnum.BAD_REQUEST, + String.format(ErrorMessageConstants.INVALID_PROTOCOL_TYPE_MSG, + Arrays.toString(MessagePushConstants.WEBSOCKET_PROTOCOLS), + request.getURI().getScheme()))); + } + return result; + } + + @Override + public int getOrder() { + return FilterOrders.VALIDATE_FILTER_ORDER; + } +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/WebsocketReactiveLoadBalancerClientFilter.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/WebsocketReactiveLoadBalancerClientFilter.java new file mode 100644 index 0000000..c24766c --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/WebsocketReactiveLoadBalancerClientFilter.java @@ -0,0 +1,135 @@ +package org.kangspace.messagepush.ws.gateway.filter; + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.constant.MessagePushConstants; +import org.kangspace.messagepush.ws.gateway.filter.balancer.LbServiceInstanceChooser; +import org.kangspace.messagepush.ws.gateway.filter.balancer.WebSocketLoadBalancer; +import org.kangspace.messagepush.ws.gateway.model.MessageRequestParam; +import org.kangspace.messagepush.ws.gateway.util.ExchangeRequestUtils; +import org.springframework.beans.factory.config.BeanPostProcessor; +import org.springframework.cloud.client.ServiceInstance; +import org.springframework.cloud.client.loadbalancer.LoadBalancerUriTools; +import org.springframework.cloud.client.loadbalancer.reactive.DefaultRequest; +import org.springframework.cloud.client.loadbalancer.reactive.Response; +import org.springframework.cloud.gateway.config.LoadBalancerProperties; +import org.springframework.cloud.gateway.filter.GatewayFilterChain; +import org.springframework.cloud.gateway.filter.GlobalFilter; +import org.springframework.cloud.gateway.filter.ReactiveLoadBalancerClientFilter; +import org.springframework.cloud.gateway.support.DelegatingServiceInstance; +import org.springframework.cloud.gateway.support.NotFoundException; +import org.springframework.cloud.gateway.support.ServerWebExchangeUtils; +import org.springframework.cloud.loadbalancer.core.ServiceInstanceListSupplier; +import org.springframework.cloud.loadbalancer.support.LoadBalancerClientFactory; +import org.springframework.core.Ordered; +import org.springframework.web.server.ServerWebExchange; +import reactor.core.publisher.Mono; + +import java.net.URI; +import java.util.Objects; + +/** + * 
+ * WebSocket路由负载均衡过滤器
+ *
+ * + * @author kango2gler@gmail.com + * @since 2021/10/26 + */ +@Slf4j +public class WebsocketReactiveLoadBalancerClientFilter extends ReactiveLoadBalancerClientFilter implements GlobalFilter, Ordered, BeanPostProcessor { + + private final LbServiceInstanceChooser lbServiceInstanceChooser; + + private final LoadBalancerClientFactory clientFactory; + + private final LoadBalancerProperties properties; + + public WebsocketReactiveLoadBalancerClientFilter(LbServiceInstanceChooser lbServiceInstanceChooser, + LoadBalancerClientFactory clientFactory, + LoadBalancerProperties properties) { + super(clientFactory, properties); + this.lbServiceInstanceChooser = lbServiceInstanceChooser; + this.clientFactory = clientFactory; + this.properties = properties; + } + + @Override + public Mono filter(ServerWebExchange exchange, GatewayFilterChain chain) { + String requestPath = exchange.getRequest().getPath().toString(); + MessageRequestParam requestParam = ExchangeRequestUtils.getMessageRequestParam(exchange); + log.info("WebSocket路由负载均衡过滤器: WebsocketReactiveLoadBalancerClientFilter handle start, url: [{}], requestParam: [{}]", + requestPath, requestParam); + URI url = exchange.getAttribute(ServerWebExchangeUtils.GATEWAY_REQUEST_URL_ATTR); + String schemePrefix = exchange.getAttribute(ServerWebExchangeUtils.GATEWAY_SCHEME_PREFIX_ATTR); + Mono result; + boolean isLbUrl = url != null && ("lb".equals(url.getScheme()) || "lb".equals(schemePrefix)); + if (isLbUrl) { + ServerWebExchangeUtils.addOriginalRequestUrl(exchange, url); + if (log.isTraceEnabled()) { + log.trace(ReactiveLoadBalancerClientFilter.class.getSimpleName() + " url before: " + url); + } + + result = this.choose(exchange).doOnNext((response) -> { + if (!response.hasServer()) { + throw NotFoundException.create(this.properties.isUse404(), "Unable to find instance for " + url.getHost()); + } else { + URI uri = exchange.getRequest().getURI(); + String overrideScheme = null; + if (schemePrefix != null) { + overrideScheme = url.getScheme(); + } + + DelegatingServiceInstance serviceInstance = new DelegatingServiceInstance(response.getServer(), overrideScheme); + URI requestUrl = this.reconstructURI(serviceInstance, uri); + if (log.isTraceEnabled()) { + log.trace("LoadBalancerClientFilter url chosen: " + requestUrl); + } + String serviceNode = serviceInstance.getHost() + ":" + serviceInstance.getPort(); + exchange.getAttributes().put(ServerWebExchangeUtils.GATEWAY_REQUEST_URL_ATTR, requestUrl); + exchange.getAttributes().put(MessagePushConstants.WEBSOCKET_TARGET_SERVICE_NODE_ATTR_KEY, serviceNode); + log.info("WebSocket路由负载均衡过滤器: WebsocketReactiveLoadBalancerClientFilter handle end, 目标服务:[{}], url: [{}], requestParam: [{}]", serviceNode, requestPath, requestParam); + } + }).then(chain.filter(exchange)); + } else { + result = chain.filter(exchange); + log.info("WebSocket路由负载均衡过滤器: 非负载请求,直连服务"); + } + + return result; + } + + /** + * 重新设置URI的service info + * + * @param serviceInstance 目标服务信息 + * @param original 源请求URL + * @return {@link URI} + */ + @Override + protected URI reconstructURI(ServiceInstance serviceInstance, URI original) { + return LoadBalancerUriTools.reconstructURI(serviceInstance, original); + } + + /** + * 选择路由Server实例 + * + * @param exchange {@link ServerWebExchange} + * @return {@link Mono} + */ + private Mono> choose(ServerWebExchange exchange) { + URI uri = exchange.getAttribute(ServerWebExchangeUtils.GATEWAY_REQUEST_URL_ATTR); + Objects.requireNonNull(uri, "ServerWebExchangeUtils.GATEWAY_REQUEST_URL_ATTR attr not found!"); + String serviceId = exchange.getRequest().getPath().contextPath().toString(); + WebSocketLoadBalancer loadBalancer = new WebSocketLoadBalancer( + serviceId, + clientFactory.getLazyProvider(uri.getHost(), ServiceInstanceListSupplier.class), + this.lbServiceInstanceChooser); + return loadBalancer.choose(new DefaultRequest<>(exchange)); + } + + @Override + public int getOrder() { + return FilterOrders.WEBSOCKET_REACTIVE_LOADBALANCER_CLIENT_FILTER_ORDER; + } + +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/balancer/LbServiceInstanceChooser.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/balancer/LbServiceInstanceChooser.java new file mode 100644 index 0000000..ae73db7 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/balancer/LbServiceInstanceChooser.java @@ -0,0 +1,27 @@ +package org.kangspace.messagepush.ws.gateway.filter.balancer; + +import org.springframework.cloud.client.ServiceInstance; +import org.springframework.web.server.ServerWebExchange; + +import java.util.List; + +/** + * 服务实例选择器 + * + * @author kango2gler@gmail.com + * @see UIDServiceInstanceChooser + * @see RoundRibbonServiceInstanceChooser + * @since 2021/10/27 + */ +public interface LbServiceInstanceChooser { + + /** + * 服务实例选择 + * + * @param serviceId 服务ID + * @param exchange {@link ServerWebExchange} + * @param instances 实例列表 + * @return ServiceInstance + */ + ServiceInstance choose(String serviceId, ServerWebExchange exchange, List instances); +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/balancer/RoundRibbonServiceInstanceChooser.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/balancer/RoundRibbonServiceInstanceChooser.java new file mode 100644 index 0000000..eeceb18 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/balancer/RoundRibbonServiceInstanceChooser.java @@ -0,0 +1,42 @@ +package org.kangspace.messagepush.ws.gateway.filter.balancer; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.cloud.client.ServiceInstance; +import org.springframework.util.CollectionUtils; +import org.springframework.web.server.ServerWebExchange; + +import java.util.List; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * 轮训选择服务实例 + * + * @author kango2gler@gmail.com + * @since 2021/10/27 + */ +@Slf4j +public class RoundRibbonServiceInstanceChooser implements LbServiceInstanceChooser { + /** + * 轮训计数器 + */ + private final AtomicInteger position; + + public RoundRibbonServiceInstanceChooser() { + this(new Random().nextInt(1000)); + } + + public RoundRibbonServiceInstanceChooser(int seedPosition) { + this.position = new AtomicInteger(seedPosition); + } + + @Override + public ServiceInstance choose(String serviceId, ServerWebExchange exchange, List instances) { + if (CollectionUtils.isEmpty(instances)) { + log.warn("轮训服务实例选择: No servers available for service: " + serviceId); + return null; + } + int pos = Math.abs(this.position.incrementAndGet()); + return instances.get(pos % instances.size()); + } +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/balancer/UIDServiceInstanceChooser.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/balancer/UIDServiceInstanceChooser.java new file mode 100644 index 0000000..ea32ae7 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/balancer/UIDServiceInstanceChooser.java @@ -0,0 +1,60 @@ +package org.kangspace.messagepush.ws.gateway.filter.balancer; + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.constant.MessagePushConstants; +import org.kangspace.messagepush.core.hash.ConsistencyHashing; +import org.kangspace.messagepush.core.hash.VirtualNode; +import org.kangspace.messagepush.ws.gateway.model.MessageRequestParam; +import org.springframework.cloud.client.ServiceInstance; +import org.springframework.util.CollectionUtils; +import org.springframework.web.server.ServerWebExchange; + +import java.util.List; + +/** + * 根据用户ID选择服务实例 + * + * @author kango2gler@gmail.com + * @since 2021/10/27 + */ +@Slf4j +public class UIDServiceInstanceChooser implements LbServiceInstanceChooser { + /** + * 一致性hash路由 + */ + private ConsistencyHashing hashRouter; + + public UIDServiceInstanceChooser(ConsistencyHashing hashRouter) { + this.hashRouter = hashRouter; + } + + @Override + public ServiceInstance choose(String serviceId, ServerWebExchange exchange, List instances) { + if (CollectionUtils.isEmpty(instances)) { + log.warn("用户ID服务实例选择: No servers available for service: " + serviceId); + return null; + } + // 获取必要的请求参数 + MessageRequestParam requestParam = exchange.getAttribute(MessagePushConstants.EXCHANGE_ATTR_REQUEST_PARAM); + String uid = requestParam != null ? requestParam.getUid() : ""; + // 通过uid获取目标Service节点 + VirtualNode node = hashRouter.getVirtualNode(uid); + if (node == null) { + log.warn("用户ID服务实例选择: 当前Hash环无可用服务实例; serviceId: [{}],uid: [{}], hashNode:[{}] serviceInstances:[{}]", + serviceId, uid, node, instances); + return null; + } + String expectNode = node.getPhysicalNode().getNode(); + // 筛选出目标Service服务 + ServiceInstance expectServiceInstance = instances.stream() + .filter(t -> expectNode.equals(t.getHost() + ":" + t.getPort())) + .findFirst().orElse(null); + boolean matched = expectServiceInstance != null; + if (!matched) { + log.warn("用户ID服务实例选择: 通过用户uid获取服务失败,没有匹配的服务实例; serviceId: [{}],uid: [{}], hashNode:[{}] serviceInstances:[{}]", + serviceId, uid, node, instances); + return null; + } + return expectServiceInstance; + } +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/balancer/WebSocketLoadBalancer.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/balancer/WebSocketLoadBalancer.java new file mode 100644 index 0000000..9b68eba --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/balancer/WebSocketLoadBalancer.java @@ -0,0 +1,72 @@ +package org.kangspace.messagepush.ws.gateway.filter.balancer; + + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.cloud.client.ServiceInstance; +import org.springframework.cloud.client.loadbalancer.reactive.DefaultResponse; +import org.springframework.cloud.client.loadbalancer.reactive.EmptyResponse; +import org.springframework.cloud.client.loadbalancer.reactive.Request; +import org.springframework.cloud.client.loadbalancer.reactive.Response; +import org.springframework.cloud.loadbalancer.core.NoopServiceInstanceListSupplier; +import org.springframework.cloud.loadbalancer.core.ReactorServiceInstanceLoadBalancer; +import org.springframework.cloud.loadbalancer.core.RoundRobinLoadBalancer; +import org.springframework.cloud.loadbalancer.core.ServiceInstanceListSupplier; +import org.springframework.web.server.ServerWebExchange; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.util.List; + +/** + * WebSocket服务负载均衡器 + * + * @author kango2gler@gmail.com + * @see RoundRobinLoadBalancer + * @see LbServiceInstanceChooser + * @since 2021/10/27 + */ +@Slf4j +public class WebSocketLoadBalancer implements ReactorServiceInstanceLoadBalancer { + private final String serviceId; + /** + * 服务实例列表提供者 + */ + private ObjectProvider serviceInstanceListSupplierProvider; + /** + * 服务实例选择器 + */ + private LbServiceInstanceChooser instanceChooser; + + public WebSocketLoadBalancer(String serviceId, + ObjectProvider serviceInstanceListSupplierProvider, + LbServiceInstanceChooser instanceChooser) { + this.serviceId = serviceId; + this.serviceInstanceListSupplierProvider = serviceInstanceListSupplierProvider; + this.instanceChooser = instanceChooser; + } + + /** + * 路由服务选择 + * + * @param request {@link Request} + * @return {@link Mono>} + */ + @Override + public Mono> choose(Request request) { + ServiceInstanceListSupplier supplier = this.serviceInstanceListSupplierProvider.getIfAvailable(NoopServiceInstanceListSupplier::new); + return ((Flux) supplier.get()).next().map(instances -> this.getInstanceResponse((List) instances, request)); + } + + /** + * 从服务列表中选择目标服务 + * + * @param instances 服务列表 + * @param request request + * @return {@link Response} + */ + private Response getInstanceResponse(List instances, Request request) { + ServiceInstance instance = this.instanceChooser.choose(this.serviceId, (ServerWebExchange) request.getContext(), instances); + return instance != null ? new DefaultResponse(instance) : new EmptyResponse(); + } +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/session/SessionProxyHolder.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/session/SessionProxyHolder.java new file mode 100644 index 0000000..91ba180 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/session/SessionProxyHolder.java @@ -0,0 +1,30 @@ +package org.kangspace.messagepush.ws.gateway.filter.session; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.web.reactive.socket.WebSocketSession; + +/** + * 网关Websocket双向session持有对象 + * + * @author kango2gler@gmail.com + * @since 2021/11/4 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class SessionProxyHolder { + /** + * 后端服务Session + */ + private WebSocketSession proxySession; + /** + * 当前服务Session + */ + private WebSocketSession session; + /** + * 服务节点,值为: HOST:IP + */ + private String serviceNode; +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/session/WebSocketUserSessionManager.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/session/WebSocketUserSessionManager.java new file mode 100644 index 0000000..4d43687 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/filter/session/WebSocketUserSessionManager.java @@ -0,0 +1,167 @@ +package org.kangspace.messagepush.ws.gateway.filter.session; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.event.NacosServiceUpInfo; +import org.kangspace.messagepush.core.event.NacosServiceUpdateEvent; +import org.kangspace.messagepush.core.hash.ConsistencyHashing; +import org.kangspace.messagepush.core.hash.VirtualNode; +import org.springframework.context.ApplicationEvent; +import org.springframework.context.event.SmartApplicationListener; +import org.springframework.util.CollectionUtils; +import org.springframework.web.reactive.socket.CloseStatus; +import org.springframework.web.reactive.socket.WebSocketSession; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; + +/** + * WebSocket用户Session管理器 + * + * @author kango2gler@gmail.com + * @since 2021/10/28 + */ +@Slf4j +@Getter +public class WebSocketUserSessionManager implements SmartApplicationListener { + /** + * 同步操作锁 + */ + private ReentrantLock lock = new ReentrantLock(); + + /** + * 用户和Session关系 + */ + private ConcurrentHashMap> userSessionsMap; + + + public WebSocketUserSessionManager() { + this.userSessionsMap = new ConcurrentHashMap<>(16); + } + + + /** + * 添加Session信息 + * + * @param uid + * @param sessionHolder {@link SessionProxyHolder} + * @return {@link SessionProxyHolder} + */ + public SessionProxyHolder addSession(String uid, SessionProxyHolder sessionHolder) { + lock.lock(); + try { + List sessionHolders = this.userSessionsMap.getOrDefault(uid, new ArrayList<>()); + boolean isNewKey = CollectionUtils.isEmpty(sessionHolders); + sessionHolders.add(sessionHolder); + if (isNewKey) { + this.userSessionsMap.put(uid, sessionHolders); + } + } finally { + lock.unlock(); + } + return sessionHolder; + } + + /** + * 删除Session + * + * @param uid + * @param session + * @return + */ + public boolean removeSession(String uid, WebSocketSession session) { + lock.lock(); + try { + List sessionHolders = this.userSessionsMap.getOrDefault(uid, new ArrayList<>()); + boolean del = removeUserSessionMapSession(sessionHolders, session); + this.userSessionsMap.forEach((k, v) -> { + if (CollectionUtils.isEmpty(v)) { + this.userSessionsMap.remove(k); + } + }); + return del; + } finally { + lock.unlock(); + } + } + + /** + * 删除用户sessionMap中的Session + * + * @param sessionHolders {@link SessionProxyHolder} 列表 + * @param session 当前网关与客户端建立的session + * @return boolean + */ + private boolean removeUserSessionMapSession(Collection sessionHolders, WebSocketSession session) { + boolean del = false; + if (!CollectionUtils.isEmpty(sessionHolders)) { + for (Iterator iterator = sessionHolders.iterator(); iterator.hasNext(); ) { + SessionProxyHolder sessionHolder = iterator.next(); + if (sessionHolder.getSession().equals(session)) { + iterator.remove(); + del = true; + } + } + } + return del; + } + + /** + * 用户Session下线处理 + * + * @param consistencyHashing {@link ConsistencyHashing} + */ + public void offlineUserSessionHandle(ConsistencyHashing consistencyHashing) { + lock.lock(); + try { + if (!CollectionUtils.isEmpty(this.userSessionsMap) && consistencyHashing != null) { + List removeKeys = new ArrayList<>(); + AtomicInteger closedCount = new AtomicInteger(0); + userSessionsMap.forEach((uid, sessionHolders) -> { + if (!CollectionUtils.isEmpty(sessionHolders)) { + VirtualNode vn = consistencyHashing.getVirtualNode(uid); + if (vn != null) { + String newNode = vn.getPhysicalNode().getNode(); + String oldNode = sessionHolders.get(0).getServiceNode(); + if (!newNode.equals(oldNode)) { + sessionHolders.forEach(sessionProxyHolder -> { + sessionProxyHolder.getSession().close(new CloseStatus(CloseStatus.SERVICE_RESTARTED.getCode(), "新服务上线,服务重平衡!")).toProcessor(); + sessionProxyHolder.getProxySession().close(new CloseStatus(CloseStatus.SERVICE_RESTARTED.getCode(), "新服务上线,服务重平衡!")).toProcessor(); + closedCount.getAndAdd(1); + }); + removeKeys.add(uid); + } + } + } + }); + removeKeys.forEach(userSessionsMap::remove); + log.info("新服务上线,服务重平衡处理: 结束, 剔除连接数:[{}]", closedCount.get()); + } + } finally { + lock.unlock(); + } + } + + @Override + public void onApplicationEvent(ApplicationEvent event) { + if (event != null && event.getSource() != null) { + ConsistencyHashing consistencyHashing = ((NacosServiceUpInfo) event.getSource()).getConsistencyHashing(); + offlineUserSessionHandle(consistencyHashing); + } + } + + @Override + public boolean supportsEventType(Class aClass) { + return NacosServiceUpdateEvent.class.isAssignableFrom(aClass); + } + + @Override + public boolean supportsSourceType(Class sourceType) { + return NacosServiceUpInfo.class.isAssignableFrom(sourceType); + } +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/hash/DebugPrintHashingScheduler.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/hash/DebugPrintHashingScheduler.java new file mode 100644 index 0000000..fb5ac8d --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/hash/DebugPrintHashingScheduler.java @@ -0,0 +1,50 @@ +package org.kangspace.messagepush.ws.gateway.hash; + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.hash.ConsistencyHashing; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; + +/** + * 打印一致性Hash相关数据定时器(Debug级别打印) + * + * @author kango2gler@gmail.com + * @since 2021/11/4 + */ +@Slf4j +public class DebugPrintHashingScheduler { + /** + * 一致性Hash处理Bean + */ + private final ConsistencyHashing consistencyHashing; + + private ThreadPoolTaskScheduler scheduler; + + public DebugPrintHashingScheduler(ConsistencyHashing consistencyHashing) { + this.consistencyHashing = consistencyHashing; + if (!log.isDebugEnabled()) { + return; + } + this.scheduler = new ThreadPoolTaskScheduler(); + this.scheduler.setPoolSize(1); + this.scheduler.initialize(); + startPrint(); + } + + /** + * 开始日志打印任务 + */ + private void startPrint() { + log.debug("一致性Hash数据加载: 定时任务开始!"); + this.scheduler.scheduleAtFixedRate(() -> print(), 3000L); + + } + + /** + * 打印日志 + */ + private void print() { + log.info("当前物理节点数:[{}] ,当前numberOfVNode:[{}]", this.consistencyHashing.getPhysicalNodes(), + this.consistencyHashing.getNumberOfVirtualNode()); + } + +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/hash/RedisHashRouterRepository.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/hash/RedisHashRouterRepository.java new file mode 100644 index 0000000..da9cb10 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/hash/RedisHashRouterRepository.java @@ -0,0 +1,262 @@ +package org.kangspace.messagepush.ws.gateway.hash; + +import com.alibaba.nacos.api.naming.pojo.Instance; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.constant.MessagePushConstants; +import org.kangspace.messagepush.core.constant.RedisConstants; +import org.kangspace.messagepush.core.event.NacosServiceUpInfo; +import org.kangspace.messagepush.core.event.NacosServiceUpdateEvent; +import org.kangspace.messagepush.core.event.NacosServiceUpdateInfo; +import org.kangspace.messagepush.core.hash.ConsistencyHashing; +import org.kangspace.messagepush.core.hash.PhysicalNode; +import org.kangspace.messagepush.core.hash.VirtualNode; +import org.kangspace.messagepush.core.hash.repository.HashRouterRepository; +import org.kangspace.messagepush.core.redis.RedisService; +import org.kangspace.messagepush.core.util.MD5Util; +import org.kangspace.messagepush.ws.gateway.nacos.NacosNamingService; +import org.springframework.beans.BeansException; +import org.springframework.cloud.client.ServiceInstance; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.context.ApplicationEvent; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.event.SmartApplicationListener; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StopWatch; + +import javax.annotation.PostConstruct; +import java.util.*; +import java.util.stream.Collectors; + +/** + * 
+ * Redis一致性Hash路由数据处理类
+ * Gateway网关处理逻辑:
+ * 1. Gateway网关启动时,从Redis拉取Hash环数据,
+ * 拉取Hash环境数据后,校验Hash环数据是否正常(即是否时最新服务列表的Hash数据),
+ * 正常: 继续
+ * 不正常: 说明Redis数据是旧的(也就是说当前Gateway网关是第一个启动的网关实例),则根据最新服务列表更新Hash环数据,并更新Redis
+ * 2. 监听Nacos message-push-ws服务变更事件,服务变更时,所有Gateway网关实例处理各自实例内Hash环中节点的上线,下线操作,并更新本地Hash环数据,
+ * 2.1 本地更新成功后,由其中的一个Gateway网关实例更新Redis Hash环数据(并更新本地Hash环摘要)
+ * Gateway网关更新Redis Hash环数据时先检查摘要是否一致,不一致再更新
+ * 2.2 节点下线时,只更新Hash环数据
+ * 节点上线时,断开部分rehash用户连接(通过步骤3操作)
+ * 3. Gateway网关与message-push-ws建立连接后，各Gateway网关缓存用户与message-push-ws机器连接的关系,
+ * 当2中节点上线时,计算需要rehash的用户,并断开相关连接
+ *
+ * message-push-ws 服务,应用,UID,Session关系
+ * 1. 用户登录后服务本地维护当前实例和用户的关系列表
+ *
+ * message-push-consumer服务
+ * 1. 定时更新Gateway网关保存的Hash环数据,
+ * 2. 收到数据后计算Hash环数据找到服务节点,推送数据
+ *
+ * + * @author kango2gler@gmail.com + * @since 2021/10/28 + */ +@Slf4j +@Data +public class RedisHashRouterRepository implements HashRouterRepository, SmartApplicationListener, + ApplicationContextAware, ApplicationEventPublisher { + private final RedisService redisService; + private ApplicationContext applicationContext; + private ConsistencyHashing consistencyHashing; + private NacosNamingService nacosNamingService; + + public RedisHashRouterRepository(RedisService redisService, NacosNamingService nacosNamingService) { + this.redisService = redisService; + this.nacosNamingService = nacosNamingService; + } + + @Override + public ConsistencyHashing get() { + return consistencyHashing; + } + + @PostConstruct + @Override + public ConsistencyHashing init() { + log.info("Redis Hash环数据初始化: 开始."); + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + setConsistencyHashing(new ConsistencyHashing<>(MessagePushConstants.NUMBER_OF_VIRTUAL_NODE, Collections.emptyList())); + // 1. 从Redis中获取 Hash环数据 + final Map ring = redisService.hGetAll(RedisConstants.MESSAGE_PUSH_HASH_RING_KEY, VirtualNode.class); + // 获取实际服务列表 + List actualServices = getAllInstances(); + ConsistencyHashing hashRouter; + // 2. 初始化到 ConsistencyHashing 中 + if (CollectionUtils.isEmpty(ring)) { + log.warn("Redis Hash环数据初始化: 一致性HASH数据不存在,进行rehashing"); + hashRouter = rehash(actualServices); + } else { + // 检查Hash环数据是否正确,不正确,则Rehash + hashRouter = compareHashDataAndRehash(new ConsistencyHashing(ring, MessagePushConstants.NUMBER_OF_VIRTUAL_NODE), actualServices); + } + stopWatch.stop(); + log.info("Redis Hash环数据初始化: 结束. 新Hash环虚拟节点数:[{}],耗时:{}ms", hashRouter.getVirtualNodeCount(), stopWatch.getTotalTimeMillis()); + setConsistencyHashing(hashRouter); + return hashRouter; + } + + @Override + public boolean store(ConsistencyHashing hashRouter) { + String redisKey = RedisConstants.MESSAGE_PUSH_HASH_RING_KEY; + String lockKey = RedisConstants.MESSAGE_PUSH_HASH_RING_STORE_LOCK_KEY; + long lockExpireSec = RedisConstants.MESSAGE_PUSH_HASH_RING_STORE_LOCK_EXPIRE_SEC; + // 同步锁,同一时间只有1个网关实例更新Hash数据 + return redisService.lock(lockKey, lockExpireSec, () -> { + redisService.del(redisKey); + if (!hashRouter.getRing().isEmpty()) { + NavigableMap> tempMap = hashRouter.getRing().descendingMap(); + Map storeMap = new HashMap<>(tempMap.size()); + tempMap.entrySet().forEach(t -> storeMap.put(t.getKey().toString(), t.getValue())); + redisService.hMSet(redisKey, storeMap); + } + log.info("Redis Hash环数据维护: 更新Redis Hash环数据成功!"); + }); + } + + /** + * Rehash 处理服务在当前网关实例的上下线 + * + * @param services + * @return + */ + @Override + public ConsistencyHashing rehash(List services) { + log.info("Redis Hash环数据维护: rehash 开始"); + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + List> currPhysicalNodes = this.getConsistencyHashing().getPhysicalNodes(); + List physicalNodeNodes = currPhysicalNodes.stream().map(t -> t.getNode()).collect(Collectors.toList()); + List upServices = services.stream().filter(t -> !physicalNodeNodes.contains(t)).collect(Collectors.toList()); + List downServices = physicalNodeNodes.stream().filter(t -> !services.contains(t)).collect(Collectors.toList()); + log.info("Redis Hash环数据维护: rehash,上线的服务:[{}],下线的服务:[{}]", upServices, downServices); + // 存在上线的服务,找出需要断开连接的用户 + if (!CollectionUtils.isEmpty(upServices)) { + upServices.forEach(node -> { + this.getConsistencyHashing().addNode(new PhysicalNode<>(this.getConsistencyHashing().getNodeHash(node), node)); + }); + // 触发服务上线事件: + publishServerUpEvent(); + } + // 存在下线的服务,则直接删除物理节点 + if (!CollectionUtils.isEmpty(downServices) && !CollectionUtils.isEmpty(currPhysicalNodes)) { + downServices.forEach(node -> { + this.getConsistencyHashing().removeNode(new PhysicalNode<>(this.getConsistencyHashing().getNodeHash(node), node)); + }); + } + // 保存rehash结果 + store(this.getConsistencyHashing()); + stopWatch.stop(); + log.info("Redis Hash环数据维护: rehash 结束,耗时:{}ms", stopWatch.getTotalTimeMillis()); + return this.getConsistencyHashing(); + } + + /** + * 发布服务上线事件 + */ + public void publishServerUpEvent() { + log.info("Nacos服务动态监听: 发布服务上线事件!"); + NacosServiceUpdateEvent serviceUpEvent = + new NacosServiceUpdateEvent<>(new NacosServiceUpInfo(this.getConsistencyHashing())); + this.publishEvent((Object) serviceUpEvent); + } + + @Override + public boolean compareHashData(ConsistencyHashing hashRouter, List actualServices) { + String oldHashRoute = hashRouter.getPhysicalNodesDigest(); + String newHashRoute = MD5Util.hashDigest(actualServices); + return !newHashRoute.equals(oldHashRoute) || (!hashRouter.getRing().isEmpty() && CollectionUtils.isEmpty(actualServices)); + } + + @Override + public ConsistencyHashing compareHashDataAndRehash(ConsistencyHashing hashRouter, List actualServices) { + if (compareHashData(hashRouter, actualServices)) { + return rehash(actualServices); + } + return hashRouter; + } + + + /** + * 获取所有实例IP:端口 + * + * @return [ip:端口]列表 + */ + private List getAllInstances() { + return getIpPortListByServiceInstances(this.nacosNamingService.getAllInstances(MessagePushConstants.MESSAGE_WS_SERVICE_ID)); + } + + /** + * 通过{@link ServiceInstance}列表获取IP:端口列表 + * + * @param instances + * @return + */ + private List getIpPortListByServiceInstances(List instances) { + return instances.stream().map(t -> t.getHost() + ":" + t.getPort()).collect(Collectors.toList()); + } + + /** + * 通过{@link Instance}列表获取IP:端口列表 + * + * @param instances + * @return + */ + private List getIpPortListByInstances(List instances) { + return instances.stream().map(t -> t.getIp() + ":" + t.getPort()).collect(Collectors.toList()); + } + + /** + * 设定支持的事件类型 + * + * @param aClass ApplicationEvent + * @return boolean + */ + @Override + public boolean supportsEventType(Class aClass) { + return NacosServiceUpdateEvent.class.isAssignableFrom(aClass); + } + + /** + * 设定事件对象为NacosServerUpdateInfo + * + * @param sourceType NacosServerUpdateInfo + * @return boolean + */ + @Override + public boolean supportsSourceType(Class sourceType) { + return NacosServiceUpdateInfo.class.isAssignableFrom(sourceType); + } + + /** + * 服务变化事件 + * + * @param event ApplicationEvent + */ + @Override + public void onApplicationEvent(ApplicationEvent event) { + log.info("Redis Hash环数据维护: Nacos服务更新事件,处理开始. event:{}", event); + NacosServiceUpdateEvent serviceUpdateEvent = (NacosServiceUpdateEvent) event; + if (serviceUpdateEvent != null && serviceUpdateEvent.getSource() != null) { + List actualServices = getIpPortListByInstances(((NacosServiceUpdateInfo) (serviceUpdateEvent.getSource())).getInstances()); + // 检查服务是否需要Rehash + this.compareHashDataAndRehash(this.consistencyHashing, actualServices); + } + log.info("Redis Hash环数据维护: Nacos服务更新事件,处理结束, 当前服务数:[{}]", this.consistencyHashing.getPhysicalNodes().size()); + } + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.applicationContext = applicationContext; + } + + @Override + public void publishEvent(Object event) { + applicationContext.publishEvent(event); + } +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/model/MessageRequestParam.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/model/MessageRequestParam.java new file mode 100644 index 0000000..ad44148 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/model/MessageRequestParam.java @@ -0,0 +1,58 @@ +package org.kangspace.messagepush.ws.gateway.model; + +import lombok.Data; + +/** + * 消息推送请求参数 + * + * @author kango2gler@gmail.com + * @since 2021/10/27 + */ +@Data +public class MessageRequestParam { + /** + * 参数来源 + */ + private ParamFrom paramFrom = ParamFrom.NULL; + /** + * 用户Token + * 从请求头 Authorization Bearer Token中获取 + */ + private String token; + /** + * 生成passport token所使用的应用ID + */ + private String tokenAppId; + + /** + * 通过token获取用户信息中获取uid + */ + private String uid; + + public MessageRequestParam() { + } + + public MessageRequestParam(String token, String tokenAppId, ParamFrom paramFrom) { + this.token = token; + this.tokenAppId = tokenAppId; + this.paramFrom = paramFrom; + } + + /** + * 参数来源 + */ + public enum ParamFrom { + /** + * 参数在URL中 + */ + URL, + /** + * 参数在请求头中 + */ + HEADER, + /** + * 参数不存在 + */ + NULL + } +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/nacos/NacosDynamicServerListListener.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/nacos/NacosDynamicServerListListener.java new file mode 100644 index 0000000..77acd15 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/nacos/NacosDynamicServerListListener.java @@ -0,0 +1,79 @@ +package org.kangspace.messagepush.ws.gateway.nacos; + +import com.alibaba.cloud.nacos.NacosDiscoveryProperties; +import com.alibaba.nacos.api.exception.NacosException; +import com.alibaba.nacos.api.naming.listener.NamingEvent; +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.event.NacosServiceUpdateEvent; +import org.kangspace.messagepush.core.event.NacosServiceUpdateInfo; +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.context.ApplicationEventPublisher; + +import java.util.Arrays; + +/** + * Nacos动态服务列表监听 + * + * @author kango2gler@gmail.com + * @since 2021/11/1 + */ +@Slf4j +public class NacosDynamicServerListListener implements ApplicationContextAware, ApplicationEventPublisher { + private final NacosDiscoveryProperties properties; + private ApplicationContext applicationContext; + private String serviceId; + + public NacosDynamicServerListListener(NacosDiscoveryProperties properties, String serviceId) { + this.properties = properties; + this.serviceId = serviceId; + startListen(); + } + + /** + * 开始监听 + */ + public void startListen() { + log.info("Nacos服务动态监听: 开始监听,服务名:[{}]", serviceId); + try { + this.properties.namingServiceInstance().subscribe(serviceId, Arrays.asList(properties.getClusterName()), event -> { + log.info("Nacos服务动态监听: 服务变更事件,服务名:[{}],事件:[{}]", serviceId, event); + if (event instanceof NamingEvent) { + NamingEvent namingEvent = (NamingEvent) event; + this.publishEvent(new NacosServiceUpdateEvent<>(new NacosServiceUpdateInfo(serviceId, namingEvent.getInstances()))); + } + }); + } catch (NacosException e) { + log.error("Nacos服务动态监听: 订阅服务变更事件异常, 错误信息:[{}]", e.getMessage(), e); + } + } + + /** + * 发布服务变化事件 + * + * @param serviceChangeEvent 服务变化事件 + */ + public void publishEvent(NacosServiceUpdateEvent serviceChangeEvent) { + if (serviceChangeEvent != null) { + log.info("Nacos服务动态监听: 发布服务更新事件, instanceDigest: [{}]", + ((NacosServiceUpdateInfo) serviceChangeEvent.getSource()).getInstancesDigest()); + this.publishEvent((Object) serviceChangeEvent); + } + } + + /** + * 发布服务变化事件 + * + * @param event 事件对象 + */ + @Override + public void publishEvent(Object event) { + applicationContext.publishEvent(event); + } + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.applicationContext = applicationContext; + } +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/nacos/NacosNamingService.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/nacos/NacosNamingService.java new file mode 100644 index 0000000..a0806cd --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/nacos/NacosNamingService.java @@ -0,0 +1,41 @@ +package org.kangspace.messagepush.ws.gateway.nacos; + +import com.alibaba.cloud.nacos.discovery.NacosServiceDiscovery; +import com.alibaba.nacos.api.exception.NacosException; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cloud.client.ServiceInstance; + +import java.util.Collections; +import java.util.List; + +/** + * NacosNamingService 实例 + * + * @author kango2gler@gmail.com + * @since 2021/11/2 + */ +@Data +@Slf4j +public class NacosNamingService { + private final NacosServiceDiscovery nacosServiceDiscovery; + + public NacosNamingService(NacosServiceDiscovery nacosServiceDiscovery) { + this.nacosServiceDiscovery = nacosServiceDiscovery; + } + + /** + * 获取所有服务实例 + * + * @return List + */ + public List getAllInstances(String serviceId) { + try { + return this.nacosServiceDiscovery.getInstances(serviceId); + } catch (NacosException e) { + log.error("Nacos服务动态监听: 获取所有实例错误, 错误信息:[{}]", e.getMessage(), e); + } + return Collections.emptyList(); + } + +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/util/ExchangeRequestUtils.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/util/ExchangeRequestUtils.java new file mode 100644 index 0000000..acb5988 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/util/ExchangeRequestUtils.java @@ -0,0 +1,111 @@ +package org.kangspace.messagepush.ws.gateway.util; + +import io.netty.handler.codec.http.HttpScheme; +import org.kangspace.messagepush.core.constant.MessagePushConstants; +import org.kangspace.messagepush.core.util.HttpUtils; +import org.kangspace.messagepush.core.util.IpUtils; +import org.kangspace.messagepush.ws.gateway.model.MessageRequestParam; +import org.springframework.http.HttpHeaders; +import org.springframework.util.StringUtils; +import org.springframework.web.server.ServerWebExchange; + +/** + * Exchange请求工具类 + * + * @author kango2gler@gmail.com + * @since 2021/10/27 + */ +public class ExchangeRequestUtils { + + /** + * 
+     * 是否为Websocket请求
+     * 协议为Http且Request Header: [Connection: Upgrade ,Upgrade: WebSocket]
+     *
+ * + * @param exchange {@link ServerWebExchange} + * @return boolean + */ + public static boolean isWebsocketRequest(ServerWebExchange exchange) { + String scheme = exchange.getRequest().getURI().getScheme().toLowerCase(); + String connection = exchange.getRequest().getHeaders().getConnection().stream().findFirst().orElse(null); + String upgrade = exchange.getRequest().getHeaders().getUpgrade(); + boolean isWebSocketRequest = "WebSocket".equalsIgnoreCase(upgrade) + && HttpHeaders.UPGRADE.equalsIgnoreCase(connection) + && (HttpScheme.HTTP.toString().equals(scheme) || HttpScheme.HTTPS.toString().equals(scheme)); + return isWebSocketRequest; + } + + /** + * 
+     * 获取客户端Ip地址
+     *
+ * + * @param exchange {@link ServerWebExchange} + * @return ip + */ + public static String getIP(ServerWebExchange exchange) { + return IpUtils.getClientIp(exchange.getRequest()); + } + + /** + * 获取消息请求参数 + * 参数: {@link MessagePushConstants#HTTP_HEADER_AUTH_APP_ID_KEY},{@link HttpHeaders#AUTHORIZATION} + * 1. 从请求头中获取 + * 2. 从URL中获取 + * + * @param exchange {@link ServerWebExchange} + * @return {@link MessageRequestParam} + */ + public static MessageRequestParam getMessageRequestParam(ServerWebExchange exchange) { + MessageRequestParam requestParam = exchange.getAttribute(MessagePushConstants.EXCHANGE_ATTR_REQUEST_PARAM); + // 从请求头中获取参数 + if (requestParam == null) { + requestParam = getMessageRequestParamFromHeader(exchange); + } + // 从URL中获取参数 + if (requestParam == null) { + requestParam = getMessageRequestParamFromUrl(exchange); + } + return requestParam != null ? requestParam : new MessageRequestParam(); + } + + /** + * 从Url中获取认证信息 + * + * @param exchange {@link ServerWebExchange} + * @return @link MessageRequestParam} + */ + public static MessageRequestParam getMessageRequestParamFromUrl(ServerWebExchange exchange) { + String token = exchange.getRequest().getQueryParams().getFirst(HttpHeaders.AUTHORIZATION); + boolean hasToken = StringUtils.hasText(token); + if (hasToken) { + token = token.startsWith(HttpUtils.HTTP_BEARER_TOKEN_VALUE_PREFIX) ? token.substring(HttpUtils.HTTP_BEARER_TOKEN_VALUE_PREFIX.length()) : token; + } + String authAppId = exchange.getRequest().getQueryParams().getFirst(MessagePushConstants.HTTP_HEADER_AUTH_APP_ID_KEY); + return hasToken && StringUtils.hasText(authAppId) ? new MessageRequestParam(token, authAppId, MessageRequestParam.ParamFrom.URL) : null; + } + + /** + * 从请求头中获取认证信息 + * + * @param exchange + * @return + */ + public static MessageRequestParam getMessageRequestParamFromHeader(ServerWebExchange exchange) { + String token = HttpUtils.getHttpBearerToken(exchange.getRequest()); + String authAppId = exchange.getRequest().getHeaders().getFirst(MessagePushConstants.HTTP_HEADER_AUTH_APP_ID_KEY); + return StringUtils.hasText(token) && StringUtils.hasText(authAppId) ? new MessageRequestParam(token, authAppId, MessageRequestParam.ParamFrom.HEADER) : null; + } + + /** + * 设置消息请求参数 + * + * @param exchange {@link ServerWebExchange} + * @return {@link MessageRequestParam} + */ + public static void setMessageRequestParam(ServerWebExchange exchange, MessageRequestParam param) { + exchange.getAttributes().put(MessagePushConstants.EXCHANGE_ATTR_REQUEST_PARAM, param); + } + +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/validation/PassportSessionValidator.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/validation/PassportSessionValidator.java new file mode 100644 index 0000000..ba34154 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/validation/PassportSessionValidator.java @@ -0,0 +1,93 @@ +package org.kangspace.messagepush.ws.gateway.validation; + + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.dto.response.ApiResponse; +import org.kangspace.messagepush.core.enums.ResponseEnum; +import org.kangspace.messagepush.ws.gateway.domain.dto.request.SessionCenterUserInfoParamDTO; +import org.kangspace.messagepush.ws.gateway.domain.dto.response.SessionCenterUserInfoDTO; +import org.kangspace.messagepush.ws.gateway.exception.TokenValidateException; +import org.kangspace.messagepush.ws.gateway.feign.PassportSessionFeignApi; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; +import java.util.function.Consumer; + +/** + * Passport session验证器 + * + * @author kango2gler@gmail.com + * @since 2021/11/5 + */ +@Slf4j +@Service +public class PassportSessionValidator implements TokenValidator { + /** + * token 元素个数 + */ + private final int TOKEN_ELEMENT_LEN = 2; + /** + * token 元素分隔符 + * 格式: pc:xxxxxx, + */ + private final String TOKEN_ELEMENT_DELIMITER = ":"; + /** + * 移动端token 元素分割符 + * 格式: md5|mobile:xxxxx + */ + private final String MOBILE_TOKEN_ELEMENT_DELIMITER = "\\|"; + + @Resource + private PassportSessionFeignApi passportSessionFeignApi; + + @Override + public SessionCenterUserInfoDTO valid(String authAppId, String token, Consumer callback) { + if (isMobileToken(token)) { + token = getAccessTokenFromMobileToken(token); + } + String[] platformAndToken = token.split(TOKEN_ELEMENT_DELIMITER); + if (TOKEN_ELEMENT_LEN != platformAndToken.length) { + String errorMsg = "token格式错误,应该为[platform:accessToken]格式,token:" + token; + log.error(errorMsg); + throw new TokenValidateException(TokenValidateException.ExceptionType.INVALID_PARAM, errorMsg); + } + String platform = platformAndToken[0]; + String accessToken = platformAndToken[1]; + SessionCenterUserInfoParamDTO paramDTO = new SessionCenterUserInfoParamDTO(accessToken, platform, authAppId); + ApiResponse userLoginInfoResp = passportSessionFeignApi.userLoginInfo(paramDTO); + if (userLoginInfoResp != null) { + if (ResponseEnum.OK.getValue() == userLoginInfoResp.getCode()) { + SessionCenterUserInfoDTO dto = userLoginInfoResp.getData(); + if (dto != null && callback != null) { + callback.accept(dto); + } + return dto; + } + String errorMsg = "获取PassportSession信息不存在,response:" + userLoginInfoResp; + log.error(errorMsg); + throw new TokenValidateException(TokenValidateException.ExceptionType.ACCESS_TOKEN_NOT_FOUND, errorMsg); + } else { + String errorMsg = "获取PassportSession信息请求失败,response:" + userLoginInfoResp; + log.error(errorMsg); + throw new TokenValidateException(TokenValidateException.ExceptionType.SERVER_ERROR, errorMsg); + } + } + + /** + * 是否移动端格式token + * + * @return boolean + */ + private boolean isMobileToken(String token) { + return token.split(MOBILE_TOKEN_ELEMENT_DELIMITER).length == 2; + } + + /** + * 获取移动端token中的AccessToken + * + * @return access_token + */ + private String getAccessTokenFromMobileToken(String token) { + return token.split(MOBILE_TOKEN_ELEMENT_DELIMITER)[1]; + } +} diff --git a/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/validation/TokenValidator.java b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/validation/TokenValidator.java new file mode 100644 index 0000000..5285fb8 --- /dev/null +++ b/message-push-ws-gateway/src/main/java/org/kangspace/messagepush/ws/gateway/validation/TokenValidator.java @@ -0,0 +1,21 @@ +package org.kangspace.messagepush.ws.gateway.validation; + +import java.util.function.Consumer; + +/** + * Token验证接口 + * + * @author kango2gler@gmail.com + * @since 2021/11/5 + */ +public interface TokenValidator { + + /** + * 验证token,并返回token获取的用户信息 + * + * @param token token + * @param callback token获取成功的回调 + * @return token验证成功获取的用户信息 + */ + T valid(String authAppId, String token, Consumer callback); +} diff --git a/message-push-ws-gateway/src/main/resources/META-INF/spring.factories b/message-push-ws-gateway/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000..4f45eba --- /dev/null +++ b/message-push-ws-gateway/src/main/resources/META-INF/spring.factories @@ -0,0 +1 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=org.kangspace.messagepush.ws.gateway.config.MessagePushWsGatewayAutoConfiguration \ No newline at end of file diff --git a/message-push-ws-gateway/src/main/resources/bootstrap.yml b/message-push-ws-gateway/src/main/resources/bootstrap.yml new file mode 100644 index 0000000..2fa8750 --- /dev/null +++ b/message-push-ws-gateway/src/main/resources/bootstrap.yml @@ -0,0 +1,24 @@ +spring: + application: + name: @artifactId@ + cloud: + config: + # 如果想要远程配置优先级高，那么allowOverride设置为false，如果想要本地配置优先级高那么allowOverride设置为true + allow-override: true + # overrideNone为true时本地配置优先级高，包括系统环境变量、本地配置文件等 + override-none: true + # 只有系统环境变量或者系统属性才能覆盖远程配置文件的配置，本地配置文件中配置优先级低于远程配置 + override-system-properties: true + nacos: + discovery: + server-addr: ${SERVICE_DISCOVERY_ADDR:discory.kangspace.org:8443} + namespace: ${SERVICE_DISCOVERY_NAMESPACE:kangspace_dev} + metadata: + version: @project.version@ + config: + server-addr: ${spring.cloud.nacos.discovery.server-addr} + namespace: ${spring.cloud.nacos.discovery.namespace} + file-extension: yaml + shared-configs: + - application.${spring.cloud.nacos.config.file-extension} + - message-push-ws-gateway.${spring.cloud.nacos.config.file-extension} diff --git a/message-push-ws-gateway/src/test/java/org/kangspace/messagepush/ws/gateway/RedisHashRouterRepositoryTest.java b/message-push-ws-gateway/src/test/java/org/kangspace/messagepush/ws/gateway/RedisHashRouterRepositoryTest.java new file mode 100644 index 0000000..1d6e2b8 --- /dev/null +++ b/message-push-ws-gateway/src/test/java/org/kangspace/messagepush/ws/gateway/RedisHashRouterRepositoryTest.java @@ -0,0 +1,56 @@ +package org.kangspace.messagepush.ws.gateway; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.kangspace.messagepush.core.constant.MessagePushConstants; +import org.kangspace.messagepush.core.hash.ConsistencyHashing; +import org.kangspace.messagepush.core.hash.repository.HashRouterRepository; +import org.kangspace.messagepush.core.redis.RedisService; +import org.kangspace.messagepush.ws.gateway.hash.RedisHashRouterRepository; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import javax.annotation.Resource; +import java.util.Arrays; +import java.util.List; + +/** + * Redis一致性Hash路由数据处理测试 + * + * @author kango2gler@gmail.com + * @since 2021/10/28 + */ +@RunWith(SpringJUnit4ClassRunner.class) +@SpringBootTest(classes = MessagePushWsGatewayApplication.class) +public class RedisHashRouterRepositoryTest { + @Resource + private RedisService redisService; + + private HashRouterRepository hashRouterRepository; + + @Before + public void init() { + this.hashRouterRepository = new RedisHashRouterRepository(redisService, null); + } + + @Test + public void storeTest() { + List servers = Arrays.asList( + "node1", "node2", "node3", "node4" + ); + ConsistencyHashing hashRouter = new ConsistencyHashing(MessagePushConstants.NUMBER_OF_VIRTUAL_NODE, servers); + hashRouterRepository.store(hashRouter); + System.out.println(hashRouter); + initTest(); + System.out.println("save end"); + } + + @Test + public void initTest() { + ConsistencyHashing hashRouter = hashRouterRepository.init(); + System.out.println(hashRouter); + Assert.assertTrue("一致性Hash环数据为空", hashRouter != null); + } +} diff --git a/message-push-ws-gateway/src/test/java/org/kangspace/messagepush/ws/gateway/WebSocketUserSessionManagerTest.java b/message-push-ws-gateway/src/test/java/org/kangspace/messagepush/ws/gateway/WebSocketUserSessionManagerTest.java new file mode 100644 index 0000000..bf087cf --- /dev/null +++ b/message-push-ws-gateway/src/test/java/org/kangspace/messagepush/ws/gateway/WebSocketUserSessionManagerTest.java @@ -0,0 +1,41 @@ +package org.kangspace.messagepush.ws.gateway; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.kangspace.messagepush.ws.gateway.filter.session.SessionProxyHolder; +import org.kangspace.messagepush.ws.gateway.filter.session.WebSocketUserSessionManager; + +/** + * WebSocketUserSessionManager 大数据量测试 + * + * @author kango2gler@gmail.com + * @since 2021/10/28 + */ +@RunWith(JUnit4.class) +public class WebSocketUserSessionManagerTest { + private WebSocketUserSessionManager webSocketUserSessionManager = new WebSocketUserSessionManager(); + + /** + * memory size: 6M + */ + @Test + public void add20WTest() { + int dataSize = 20_0000; + for (int i = 0; i < dataSize; i++) { + webSocketUserSessionManager.addSession("uid:" + i, new SessionProxyHolder()); + } + System.out.println(webSocketUserSessionManager.getUserSessionsMap().size()); + while (true) { + try { + Thread.sleep(3_000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + @Test + public void initTest() { + } +} diff --git a/message-push-ws/.gitignore b/message-push-ws/.gitignore new file mode 100644 index 0000000..d4a9d54 --- /dev/null +++ b/message-push-ws/.gitignore @@ -0,0 +1,13 @@ +.idea +/message-push-ws*/target +/message-push-ws-*/target +/message-push-ws*/target/* +/message-push-ws-*/target/* + +.DS_Store +*.iml + +/.idea/* +/target/* + +!.mvn/wrapper/maven-wrapper.jar \ No newline at end of file diff --git a/message-push-ws/README.md b/message-push-ws/README.md new file mode 100644 index 0000000..c54f1ff --- /dev/null +++ b/message-push-ws/README.md @@ -0,0 +1,20 @@ +# message-push-ws + +消息推送Websocket服务 + +## 项目提供服务 + +1. 提供websocket连接服务,并在服务中维护用户和信息 + - 创建Websocket服务,定义WebsocketHandler处理,处理PING,PONG请求 + > 调用websocket的请求,请求头中必须包含当前用户uid(由网关转发时添加该请求头) + - 处理连接信息,注册用户和session + - 提供Http接口,添加持续输出数据到Websocket Session +2. 提供Http接口用于websocket推送 + +Websocket协议消息格式: +PING PONG TEXT BINARY + +## 项目涉及中间件 + + nacos(namespace): + dev: kangspace_dev \ No newline at end of file diff --git a/message-push-ws/message-push-ws-api/pom.xml b/message-push-ws/message-push-ws-api/pom.xml new file mode 100644 index 0000000..7ae922f --- /dev/null +++ b/message-push-ws/message-push-ws-api/pom.xml @@ -0,0 +1,25 @@ + + + + org.kangspace.messagepush + message-push-ws + ${revision} + + 4.0.0 + + message-push-ws-api + ${revision} + + + 8 + 8 + 1.8 + + + + + + + \ No newline at end of file diff --git a/message-push-ws/message-push-ws-core/pom.xml b/message-push-ws/message-push-ws-core/pom.xml new file mode 100644 index 0000000..bdc632c --- /dev/null +++ b/message-push-ws/message-push-ws-core/pom.xml @@ -0,0 +1,72 @@ + + + + org.kangspace.messagepush + message-push-ws + ${revision} + + 4.0.0 + + message-push-ws-core + ${revision} + + + 8 + 8 + + + + + org.kangspace.messagepush + message-push-ws-api + compile + + + + org.kangspace.messagepush + message-push-rest-api + + + + org.kangspace.messagepush + message-push-common + + + + org.springframework.boot + spring-boot-starter-websocket + + + + org.springframework.boot + spring-boot-starter-webflux + + + + org.springframework.cloud + spring-cloud-starter-openfeign + + + + javax.servlet + javax.servlet-api + provided + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.2 + + true + UTF-8 + + + + + + \ No newline at end of file diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/config/ControllerExceptionConfig.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/config/ControllerExceptionConfig.java new file mode 100644 index 0000000..fccd9f7 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/config/ControllerExceptionConfig.java @@ -0,0 +1,95 @@ +package org.kangspace.messagepush.ws.core.config; + + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.dto.response.ApiResponse; +import org.kangspace.messagepush.core.enums.ResponseEnum; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.HttpStatus; +import org.springframework.http.converter.HttpMessageConversionException; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.http.server.reactive.ServerHttpRequest; +import org.springframework.http.server.reactive.ServerHttpResponse; +import org.springframework.validation.BindingResult; +import org.springframework.validation.FieldError; +import org.springframework.validation.ObjectError; +import org.springframework.web.HttpRequestMethodNotSupportedException; +import org.springframework.web.bind.MethodArgumentNotValidException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.bind.support.WebExchangeBindException; +import org.springframework.web.server.ServerWebExchange; +import reactor.core.publisher.Mono; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * @author kango2gler@gmail.com + * @since 2021/8/7 + */ +@Slf4j +@Configuration +public class ControllerExceptionConfig { + + /** + * 接口异常处理 + */ + @RestControllerAdvice + public static class ControllerExceptionHandleAdvice { + @ExceptionHandler + public Mono handler(ServerWebExchange exchange, Exception e) { + ServerHttpRequest request = exchange.getRequest(); + ServerHttpResponse response = exchange.getResponse(); + log.error("Controller 处理异常,url:{},错误信息:{}", request.getURI(), e.getMessage(), e); + ApiResponse responseDTO; + int responseStatus = HttpStatus.INTERNAL_SERVER_ERROR.value(); + if (e instanceof WebExchangeBindException) { +// responseDTO = new ApiResponse(ResponseEnum.BAD_REQUEST); +// responseDTO.setMsg(responseDTO.getMsg() + ":输入参数内容错误,请确认后重试."); + return exceptionHandler((WebExchangeBindException) e); + } else if (e instanceof HttpRequestMethodNotSupportedException) { + responseDTO = new ApiResponse(ResponseEnum.METHOD_NOT_ALLOWED); + responseStatus = HttpStatus.BAD_REQUEST.value(); + } else if (e instanceof HttpMessageNotReadableException) { + responseDTO = new ApiResponse(ResponseEnum.BAD_REQUEST); + responseDTO.setMsg(responseDTO.getMsg() + ":输入参数(JSON)格式错误,请确认后重试."); + responseStatus = HttpStatus.BAD_REQUEST.value(); + } else if (e instanceof HttpMessageConversionException) { + responseDTO = new ApiResponse(ResponseEnum.BAD_REQUEST); + responseStatus = HttpStatus.BAD_REQUEST.value(); + } else { + responseDTO = new ApiResponse(ResponseEnum.INTERNAL_SERVER_ERROR.getValue(), ResponseEnum.INTERNAL_SERVER_ERROR.getReasonPhrase()); + } + response.setRawStatusCode(responseStatus); + return Mono.just(responseDTO); + } + + /** + * 参数错误处理 + * + * @param exception WebExchangeBindException + * @return ApiResponse + */ + @ResponseBody + @ResponseStatus(HttpStatus.BAD_REQUEST) + @ExceptionHandler(MethodArgumentNotValidException.class) + public Mono exceptionHandler(WebExchangeBindException exception) { + BindingResult result = exception.getBindingResult(); + StringBuilder sb = new StringBuilder("参数错误:"); + if (result.hasErrors()) { + List errors = result.getAllErrors(); + if (errors != null) { + sb.append(errors.stream().map(p -> { + FieldError fieldError = (FieldError) p; + log.warn("Bad Request Parameters: dto entity [{}],field [{}],message [{}]", fieldError.getObjectName(), fieldError.getField(), fieldError.getDefaultMessage()); + return fieldError.getDefaultMessage(); + }).collect(Collectors.joining(","))); + } + } + return Mono.just(new ApiResponse(ResponseEnum.BAD_REQUEST.getValue(), sb.toString())); + } + } +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/config/WebsocketConfiguration.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/config/WebsocketConfiguration.java new file mode 100644 index 0000000..cae244a --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/config/WebsocketConfiguration.java @@ -0,0 +1,131 @@ +package org.kangspace.messagepush.ws.core.config; + +import org.kangspace.messagepush.ws.core.websocket.HttpPushMessageConsumer; +import org.kangspace.messagepush.ws.core.websocket.HttpPushMessagePublisher; +import org.kangspace.messagepush.ws.core.websocket.MessagePushHandshakeWebSocketService; +import org.kangspace.messagepush.ws.core.websocket.MessagePushWebSocketHandler; +import org.kangspace.messagepush.ws.core.websocket.session.WebSocketSessionManager; +import org.kangspace.messagepush.ws.core.websocket.session.WebSocketUserSessionManager; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.Ordered; +import org.springframework.web.cors.CorsConfiguration; +import org.springframework.web.cors.reactive.CorsConfigurationSource; +import org.springframework.web.reactive.HandlerMapping; +import org.springframework.web.reactive.handler.SimpleUrlHandlerMapping; +import org.springframework.web.reactive.socket.server.WebSocketService; +import org.springframework.web.reactive.socket.server.support.HandshakeWebSocketService; +import org.springframework.web.reactive.socket.server.support.WebSocketHandlerAdapter; +import org.springframework.web.reactive.socket.server.upgrade.ReactorNettyRequestUpgradeStrategy; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Websocket配置 + * + * @author kango2gler@gmail.com + * @since 2021/10/28 + */ +@Configuration +public class WebsocketConfiguration { + private final static String ALLOWED_ORIGIN_ALL = "*"; + + /** + * 用户Session管理器 + * + * @return WebSocketSessionManager + */ + @Bean + public WebSocketSessionManager webSocketUserSessionManager() { + return new WebSocketUserSessionManager(); + } + + /** + * Http推送消息消费者 + * + * @param webSocketSessionManager WebSocketSessionManager + * @return HttpPushMessageConsumer + */ + @Bean + public HttpPushMessageConsumer httpPushMessagePublisher(WebSocketSessionManager webSocketSessionManager) { + return new HttpPushMessageConsumer(webSocketSessionManager); + } + + /** + * Http推送消息发布者 + * + * @return HttpPushMessagePublisher + */ + @Bean + public HttpPushMessagePublisher httpPushMessagePublisher(HttpPushMessageConsumer httpPushMessagePublisher) { + return new HttpPushMessagePublisher(httpPushMessagePublisher); + } + + /** + * WebsocketHandler + * + * @param webSocketSessionManager WebSocketSessionManager + * @return MessagePushWebSocketHandler + */ + @Bean + public MessagePushWebSocketHandler messagePushWebSocketHandler(WebSocketSessionManager webSocketSessionManager) { + return new MessagePushWebSocketHandler(webSocketSessionManager); + } + + + /** + * 注册Websocket处理器 + * + * @param webSocketHandlers MessagePushWebSocketHandler处理器 + * @see MessagePushWebSocketHandler + */ + @Bean + public HandlerMapping websocketHandlerMapping(List webSocketHandlers, + CorsConfigurationSource corsConfigurationSource) { + Map handlerMap = webSocketHandlers.stream() + .collect(Collectors.toMap(MessagePushWebSocketHandler::getEndpointPath, v -> v)); + SimpleUrlHandlerMapping handlerMapping = new SimpleUrlHandlerMapping(); + handlerMapping.setUrlMap(handlerMap); + handlerMapping.setCorsConfigurationSource(corsConfigurationSource); + handlerMapping.setOrder(Ordered.HIGHEST_PRECEDENCE); + return handlerMapping; + } + + /** + * 注册WebSocketHandlerAdapter + * + * @return {@link WebSocketHandlerAdapter} + */ + @Bean + public WebSocketHandlerAdapter handlerAdapter() { + return new WebSocketHandlerAdapter(webSocketService()); + } + + /** + * 注册WebSocketService + * + * @return {@WebSocketService} + * @see HandshakeWebSocketService + */ + @Bean + public WebSocketService webSocketService() { + return new MessagePushHandshakeWebSocketService(new ReactorNettyRequestUpgradeStrategy()); + } + + /** + * 注册CorsConfigurationSource + * + * @return {@link CorsConfigurationSource} + */ + @Bean + public CorsConfigurationSource corsConfigurationSource() { + CorsConfigurationSource corsConfigurationSource = exchange -> { + CorsConfiguration configuration = new CorsConfiguration(); + configuration.addAllowedOrigin(ALLOWED_ORIGIN_ALL); + return configuration; + }; + return corsConfigurationSource; + } +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/constant/MessagePushCmdEnum.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/constant/MessagePushCmdEnum.java new file mode 100644 index 0000000..a26fc77 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/constant/MessagePushCmdEnum.java @@ -0,0 +1,28 @@ +package org.kangspace.messagepush.ws.core.constant; + +/** + * 命令枚举 + * + * @author kango2gler@gmail.com + * @since 2021/10/29 + */ +public enum MessagePushCmdEnum { + /** + * 登录 + */ + LOGIN, + /** + * 心跳 + */ + HEARTBEAT, + /** + * 响应 + */ + RESPONSE, + + ; + + public String getVal() { + return toString(); + } +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/constant/MessagePushResponseTypeEnum.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/constant/MessagePushResponseTypeEnum.java new file mode 100644 index 0000000..03bb591 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/constant/MessagePushResponseTypeEnum.java @@ -0,0 +1,31 @@ +package org.kangspace.messagepush.ws.core.constant; + +/** + * 命令枚举 + * + * @author kango2gler@gmail.com + * @since 2021/10/29 + */ +public enum MessagePushResponseTypeEnum { + /** + * 登录 + */ + LOGIN, + /** + * 心跳 + */ + HEARTBEAT, + /** + * 消息响应 + */ + MESSAGE, + /** + * 错误 + */ + ERROR, + ; + + public String getVal() { + return toString(); + } +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/constant/MessagePushWsConstants.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/constant/MessagePushWsConstants.java new file mode 100644 index 0000000..3f47c99 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/constant/MessagePushWsConstants.java @@ -0,0 +1,23 @@ +package org.kangspace.messagepush.ws.core.constant; + +/** + * 常量类 + * + * @author kango2gler@gmail.com + * @since 2021/8/7 + */ +public interface MessagePushWsConstants { + /** + * 消息订阅V1路径 + */ + String MESSAGE_V1_ENDPOINT_PATH = "/v1/message"; + + /** + * 任务执行线程数 + */ + int TASK_EXECUTOR_THREAD_NUM = 10; + /** + * Session登录检查超时时间 + */ + long SESSION_CHECK_DELAY_SECONDS = 10; +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/debug/DebugWebsocketSessionCountScheduler.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/debug/DebugWebsocketSessionCountScheduler.java new file mode 100644 index 0000000..8d6b55d --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/debug/DebugWebsocketSessionCountScheduler.java @@ -0,0 +1,58 @@ +package org.kangspace.messagepush.ws.core.debug; + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.ws.core.websocket.session.WebSocketUserSessionManager; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; +import org.springframework.stereotype.Service; + +/** + * websocket session连接数打印 + * + * @author kango2gler@gmail.com + * @since 2021/11/18 + */ +@Slf4j +@Service +public class DebugWebsocketSessionCountScheduler { + /** + * WebSocketUserSessionManager + */ + private final WebSocketUserSessionManager webSocketUserSessionManager; + + private ThreadPoolTaskScheduler scheduler; + + public DebugWebsocketSessionCountScheduler(WebSocketUserSessionManager webSocketUserSessionManager) { + this.webSocketUserSessionManager = webSocketUserSessionManager; + if (!log.isDebugEnabled()) { + return; + } + this.scheduler = new ThreadPoolTaskScheduler(); + this.scheduler.setPoolSize(1); + this.scheduler.initialize(); + startPrint(); + } + + /** + * 开始日志打印任务 + */ + private void startPrint() { + log.debug("WebsocketSession数打印: 定时任务开始!"); + this.scheduler.scheduleAtFixedRate(() -> print(), 1000L); + + } + + /** + * 打印日志 + */ + private void print() { + // 应用数 + int appConnectCnt = webSocketUserSessionManager.getAppUserSessionsMap().size(); + // 用户数 + int userConnectCnt = webSocketUserSessionManager.getAppUserSessionsMap().values() + .stream().mapToInt(t -> t.size()).sum(); + // 总连接数 + long sessionConnectCnt = webSocketUserSessionManager.getAppUserSessionsMap().values() + .stream().flatMap(t -> t.values().stream()).mapToLong(t -> t.size()).sum(); + log.debug("当前Session连接数汇总: 应用数:{}, 用户数:{} ,总连接数:{}", appConnectCnt, userConnectCnt, sessionConnectCnt); + } +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/MessageCmdDTO.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/MessageCmdDTO.java new file mode 100644 index 0000000..388896b --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/MessageCmdDTO.java @@ -0,0 +1,22 @@ +package org.kangspace.messagepush.ws.core.domain.dto; + +import lombok.Data; + +/** + * 
+ * 消息推送命令DTO基础类
+ * (客户端发送命令和服务端影响命令都依赖该类)
+ *
+ * + * @author kango2gler@gmail.com + * @since 2021/10/29 + */ +@Data +public class MessageCmdDTO { + /** + * 命令 + */ + private String cmd; + private Object data; + +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/MessageCmdResponseDTO.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/MessageCmdResponseDTO.java new file mode 100644 index 0000000..19f02b7 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/MessageCmdResponseDTO.java @@ -0,0 +1,34 @@ +package org.kangspace.messagepush.ws.core.domain.dto; + +import lombok.Data; +import lombok.ToString; +import org.kangspace.messagepush.ws.core.constant.MessagePushCmdEnum; +import org.kangspace.messagepush.ws.core.constant.MessagePushResponseTypeEnum; + +/** + * 消息推送命令DTO + * + * @author kango2gler@gmail.com + * @since 2021/10/29 + */ +@Data +@ToString(callSuper = true) +public class MessageCmdResponseDTO extends MessageCmdDTO { + /** + * 消息内容类型,取自 + * + * @see MessagePushResponseTypeEnum + */ + private String type; + + public MessageCmdResponseDTO() { + this(null, null); + } + + public MessageCmdResponseDTO(String type, Object data) { + super(); + setCmd(MessagePushCmdEnum.RESPONSE.toString()); + setType(type); + setData(data); + } +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/request/LoginReqDataDTO.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/request/LoginReqDataDTO.java new file mode 100644 index 0000000..78678ae --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/request/LoginReqDataDTO.java @@ -0,0 +1,21 @@ +package org.kangspace.messagepush.ws.core.domain.dto.request; + +import lombok.Data; + +/** + * 登录命令Data内容DTO + * + * @author kango2gler@gmail.com + * @since 2021/10/29 + */ +@Data +public class LoginReqDataDTO { + /** + * 为当前业务分配的应用ID(与REST PUSH API使用的应用ID相同) + */ + private String appKey; + /** + * 当前平台,值为[H5,Android,iOS],按客户端类型设置 + */ + private String platform; +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/response/HeartBeatRespDataDTO.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/response/HeartBeatRespDataDTO.java new file mode 100644 index 0000000..2e70860 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/response/HeartBeatRespDataDTO.java @@ -0,0 +1,24 @@ +package org.kangspace.messagepush.ws.core.domain.dto.response; + +import lombok.Data; + +/** + * 响应消息 HEARTBEATl类型 Data内容DTO + * + * @author kango2gler@gmail.com + * @since 2021/10/29 + */ +@Data +public class HeartBeatRespDataDTO { + /** + * 服务端当前时间毫秒数 + */ + private Long t; + + public HeartBeatRespDataDTO() { + } + + public HeartBeatRespDataDTO(Long t) { + this.t = t; + } +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/response/MessageRespDataDTO.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/response/MessageRespDataDTO.java new file mode 100644 index 0000000..da35705 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/response/MessageRespDataDTO.java @@ -0,0 +1,35 @@ +package org.kangspace.messagepush.ws.core.domain.dto.response; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +/** + * 消息命令Data内容DTO + * + * @author kango2gler@gmail.com + * @since 2021/10/29 + */ +@Data +public class MessageRespDataDTO { + /** + * 消息标题 + */ + private String title; + + /** + * 消息内容 + */ + private String content; + + /** + * 消息类型，由消息发送方自定义内容类型 + */ + @JsonProperty(value = "content_type") + private String contentType; + + /** + * 扩展内容，由消息发送方自定义扩展 + */ + private String extras; + +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/response/NormalRespDataDTO.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/response/NormalRespDataDTO.java new file mode 100644 index 0000000..9b79505 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/dto/response/NormalRespDataDTO.java @@ -0,0 +1,56 @@ +package org.kangspace.messagepush.ws.core.domain.dto.response; + +import lombok.Data; + +/** + * 响应消息 HEARTBEATl类型 Data内容DTO + * + * @author kango2gler@gmail.com + * @since 2021/10/29 + */ +@Data +public class NormalRespDataDTO { + /** + * 登录结果: 值为: 1:成功, 0:失败 + */ + private Integer code; + /** + * 登录结果描述 + */ + private String msg; + + public NormalRespDataDTO() { + } + + public NormalRespDataDTO(Integer code, String msg) { + this.code = code; + this.msg = msg; + } + + /** + * 默认成功对象 + * + * @return NormalRespDataDTO + */ + public static NormalRespDataDTO success() { + return new NormalRespDataDTO(1, "SUCCESS"); + } + + /** + * 默认失败对象 + * + * @return NormalRespDataDTO + */ + public static NormalRespDataDTO fail() { + return new NormalRespDataDTO(0, "fail"); + } + + /** + * 默认失败对象 + * + * @return NormalRespDataDTO + */ + public static NormalRespDataDTO fail(String msg) { + return new NormalRespDataDTO(0, msg); + } +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/model/HttpPushMessageDTO.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/model/HttpPushMessageDTO.java new file mode 100644 index 0000000..e6866b8 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/model/HttpPushMessageDTO.java @@ -0,0 +1,29 @@ +package org.kangspace.messagepush.ws.core.domain.model; + +import lombok.Data; +import lombok.ToString; +import org.kangspace.messagepush.rest.api.dto.request.MessagePushRequestTimeDTO; +import org.springframework.beans.BeanUtils; + +/** + * Http推送消息DTO类 + * + * @author kango2gler@gmail.com + * @since 2021/11/1 + */ +@Data +@ToString(callSuper = true) +public class HttpPushMessageDTO extends MessagePushRequestTimeDTO { + + /** + * 将messagePushRequestTimeDTO转换为HttpPushMessageDTO + * + * @param messagePushRequestTimeDTO {@link MessagePushRequestTimeDTO} + * @return HttpPushMessageDTO + */ + public static HttpPushMessageDTO from(MessagePushRequestTimeDTO messagePushRequestTimeDTO) { + HttpPushMessageDTO result = new HttpPushMessageDTO(); + BeanUtils.copyProperties(messagePushRequestTimeDTO, result); + return result; + } +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/model/MessageRequestParam.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/model/MessageRequestParam.java new file mode 100644 index 0000000..299f5e7 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/domain/model/MessageRequestParam.java @@ -0,0 +1,36 @@ +package org.kangspace.messagepush.ws.core.domain.model; + +import lombok.Data; + +/** + * 消息推送请求参数 + * + * @author kango2gler@gmail.com + * @since 2021/10/27 + */ +@Data +public class MessageRequestParam { + /** + * 用户Token + * 从请求头 Authorization Bearer Token中获取 + */ + private String token; + /** + * 用户uid + * 从请求头或 用户token中获取 + */ + private String uid; + + public MessageRequestParam() { + } + + public MessageRequestParam(String token, String uid) { + this.token = token; + this.uid = uid; + } + + @Override + public String toString() { + return "token='" + token + '\'' + ", uid='" + uid + '\''; + } +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/feign/DefaultFeignFallBack.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/feign/DefaultFeignFallBack.java new file mode 100644 index 0000000..ff4dab1 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/feign/DefaultFeignFallBack.java @@ -0,0 +1,14 @@ +package org.kangspace.messagepush.ws.core.feign; + +import org.springframework.stereotype.Service; + +/** + * 默认feign降级处理 + * + * @author kango2gler@gmail.com + * @since 2021/8/9 + */ +@Service +public class DefaultFeignFallBack implements TempFeignAPi { + +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/feign/TempFeignAPi.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/feign/TempFeignAPi.java new file mode 100644 index 0000000..52c7c3d --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/feign/TempFeignAPi.java @@ -0,0 +1,12 @@ +package org.kangspace.messagepush.ws.core.feign; + +import org.springframework.cloud.openfeign.FeignClient; + +/** + * @author kango2gler@gmail.com + * @since 2021/8/9 + */ +@FeignClient(value = "temp", path = "/", fallback = DefaultFeignFallBack.class) +public interface TempFeignAPi { + +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/service/BaseService.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/service/BaseService.java new file mode 100644 index 0000000..479e50e --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/service/BaseService.java @@ -0,0 +1,10 @@ +package org.kangspace.messagepush.ws.core.service; + +/** + * 基础Service + * + * @author kango2gler@gmail.com + * @since 2021/10/25 + */ +public class BaseService { +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/service/MessagePushWsService.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/service/MessagePushWsService.java new file mode 100644 index 0000000..c17da5f --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/service/MessagePushWsService.java @@ -0,0 +1,22 @@ +package org.kangspace.messagepush.ws.core.service; + + +import org.kangspace.messagepush.core.dto.response.ApiResponse; +import org.kangspace.messagepush.rest.api.dto.request.MessagePushRequestTimeDTO; + + +/** + * 消息推送Service + * + * @author kango2gler@gmail.com + * @since 2021/10/25 + */ +public interface MessagePushWsService { + /** + * 消息推送处理 + * + * @param messagePushRequestDto messagePushRequestDto + * @return ApiResponse + */ + ApiResponse messagePush(MessagePushRequestTimeDTO messagePushRequestDto); +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/service/impl/MessagePushWsServiceImpl.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/service/impl/MessagePushWsServiceImpl.java new file mode 100644 index 0000000..306b406 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/service/impl/MessagePushWsServiceImpl.java @@ -0,0 +1,39 @@ +package org.kangspace.messagepush.ws.core.service.impl; + + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.dto.response.ApiResponse; +import org.kangspace.messagepush.core.enums.ResponseEnum; +import org.kangspace.messagepush.rest.api.dto.request.MessagePushRequestTimeDTO; +import org.kangspace.messagepush.ws.core.domain.model.HttpPushMessageDTO; +import org.kangspace.messagepush.ws.core.service.BaseService; +import org.kangspace.messagepush.ws.core.service.MessagePushWsService; +import org.kangspace.messagepush.ws.core.websocket.HttpPushMessagePublisher; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + + +/** + * 消息推送Service + * + * @author kango2gler@gmail.com + * @since 2021/10/25 + */ +@Slf4j +@Service +public class MessagePushWsServiceImpl extends BaseService implements MessagePushWsService { + + @Resource + private HttpPushMessagePublisher httpPushMessagePublisher; + + @Override + public ApiResponse messagePush(MessagePushRequestTimeDTO messagePushRequestDto) { + log.info("Http消息推送信息处理: 收到Http推送请求, message:[{}]", messagePushRequestDto); + // 处理消息 + boolean state = httpPushMessagePublisher.publish(HttpPushMessageDTO.from(messagePushRequestDto)); + ApiResponse result = state ? new ApiResponse(ResponseEnum.OK) : new ApiResponse(ResponseEnum.INTERNAL_SERVER_ERROR.getValue(), "推送数据失败!"); + log.info("Http消息推送信息处理: Http推送请求处理结束, result:[{}] message:[{}]", state, messagePushRequestDto); + return result; + } +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/utils/MessagePushHandlerUtils.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/utils/MessagePushHandlerUtils.java new file mode 100644 index 0000000..5e7fd38 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/utils/MessagePushHandlerUtils.java @@ -0,0 +1,28 @@ +package org.kangspace.messagepush.ws.core.utils; + +import org.kangspace.messagepush.core.constant.MessagePushConstants; +import org.kangspace.messagepush.core.util.HttpUtils; +import org.kangspace.messagepush.ws.core.domain.model.MessageRequestParam; +import org.springframework.http.HttpHeaders; +import org.springframework.web.reactive.socket.WebSocketSession; + +/** + * @author kango2gler@gmail.com + * @since 2021/10/29 + */ +public class MessagePushHandlerUtils { + + /** + * 获取消息请求参数 + * + * @param session {@link WebSocketSession} + * @return {@link MessageRequestParam} + */ + public static MessageRequestParam getMessageRequestParam(WebSocketSession session) { + HttpHeaders headers = session.getHandshakeInfo().getHeaders(); + String token = HttpUtils.getHttpBearerToken(headers); + String uid = headers.getFirst(MessagePushConstants.HTTP_HEADER_UID_KEY); + return new MessageRequestParam(token, uid); + } + +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/BaseWebcosketHandler.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/BaseWebcosketHandler.java new file mode 100644 index 0000000..b59fd94 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/BaseWebcosketHandler.java @@ -0,0 +1,45 @@ +package org.kangspace.messagepush.ws.core.websocket; + + +import org.kangspace.messagepush.core.constant.MessagePushConstants; +import org.kangspace.messagepush.core.util.JsonUtil; +import org.kangspace.messagepush.ws.core.constant.MessagePushResponseTypeEnum; +import org.kangspace.messagepush.ws.core.domain.dto.MessageCmdResponseDTO; +import org.kangspace.messagepush.ws.core.domain.dto.response.NormalRespDataDTO; +import org.springframework.web.reactive.socket.WebSocketMessage; +import org.springframework.web.reactive.socket.WebSocketSession; +import reactor.core.publisher.Flux; + +/** + * WebcosketHandler 基类 + * + * @author kango2gler@gmail.com + * @since 2021/10/29 + */ +public class BaseWebcosketHandler { + + /** + * 错误处理 + * + * @param session session + * @param code 错误码 + * @param msg 错误信息 + * @return Flux + */ + public Flux error(WebSocketSession session, int code, String msg) { + return WebsocketUtils.textMessage(session, JsonUtil.toFormatJson(new MessageCmdResponseDTO(MessagePushResponseTypeEnum.ERROR.getVal(), + new NormalRespDataDTO(code, msg)))); + } + + /** + * 错误处理 + * + * @param session session + * @param msg 错误信息 + * @return Flux + */ + public Flux error(WebSocketSession session, String msg) { + return error(session, MessagePushConstants.FAIL_CODE, msg); + } + +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/HttpPushMessageConsumer.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/HttpPushMessageConsumer.java new file mode 100644 index 0000000..a3fa890 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/HttpPushMessageConsumer.java @@ -0,0 +1,102 @@ +package org.kangspace.messagepush.ws.core.websocket; + + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.constant.MessagePushConstants; +import org.kangspace.messagepush.core.util.JsonUtil; +import org.kangspace.messagepush.ws.core.constant.MessagePushResponseTypeEnum; +import org.kangspace.messagepush.ws.core.domain.dto.MessageCmdResponseDTO; +import org.kangspace.messagepush.ws.core.domain.dto.response.MessageRespDataDTO; +import org.kangspace.messagepush.ws.core.domain.model.HttpPushMessageDTO; +import org.kangspace.messagepush.ws.core.websocket.session.SessionHolder; +import org.kangspace.messagepush.ws.core.websocket.session.WebSocketSessionManager; +import org.springframework.util.CollectionUtils; +import org.springframework.web.reactive.socket.WebSocketSession; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +/** + * Http消息消费者 + * + * @author kango2gler@gmail.com + * @since 2021/10/30 + */ +@Slf4j +public class HttpPushMessageConsumer implements Consumer { + private final WebSocketSessionManager webSocketSessionManager; + + public HttpPushMessageConsumer(WebSocketSessionManager webSocketSessionManager) { + this.webSocketSessionManager = webSocketSessionManager; + } + + + /** + * 消息消费 + * + * @param messageDto 消息内容 + */ + @Override + public void accept(HttpPushMessageDTO messageDto) { + log.info("Http推送消息消费: 消费开始, message:[{}]", messageDto); + try { + consume(messageDto); + } catch (Exception e) { + log.error("Http推送消息消费: 异常, error:{}", e.getMessage(), e); + } + + + } + + public void consume(HttpPushMessageDTO messageDto) { + String appKey = messageDto.getAppKey(); + String platform = messageDto.getPlatform(); + String message = JsonUtil.toFormatJson(messageDto.getMessage()); + List audiences = messageDto.getAudience().getUids(); + if (CollectionUtils.isEmpty(audiences)) { + log.info("Http推送消息消费: 消费结束, 无目标用户IDs,message:[{}]", messageDto); + return; + } + Map> sessionHolders = webSocketSessionManager.getKeySessions(appKey); + Set keys = sessionHolders.keySet(); + List existsAlias = audiences.stream().filter(keys::contains).collect(Collectors.toList()); + AtomicInteger pushCount = new AtomicInteger(0); + // 数据发送 + existsAlias.forEach(alias -> { + sessionHolders.get(alias).stream() + .filter(sessionHolder -> sessionHolder.getPlatform() == null || matchPlatform(sessionHolder.getPlatform(), platform)) + .forEach(sessionHolder -> { + WebSocketSession session = sessionHolder.getSession(); + // 推送消息到客户端 + MessageCmdResponseDTO responseMessage = new MessageCmdResponseDTO(MessagePushResponseTypeEnum.MESSAGE.toString(), + JsonUtil.toObject(message, MessageRespDataDTO.class)); + try { + WebsocketUtils.sendTextMessage(session, responseMessage); + pushCount.addAndGet(1); + } catch (Exception e) { + log.error("Http推送消息消费: 消费错误, uid:[{}], platform:[{}], message:[{}]", sessionHolder.getUid(), + sessionHolder.getPlatform(), responseMessage); + } + }); + }); + log.info("Http推送消息消费: 消费结束,消息中的别名数量:{},消息中的目标平台:{},当前服务维持的匹配用户数:[{}] 推送数量:[{}] ", + audiences.size(), platform, existsAlias.size(), pushCount.get()); + } + + /** + * 校验目标平台是否匹配 + * + * @param sessionPlatform sessionPlatform + * @param messageTargetPlatform messageTargetPlatform + * @return boolean + */ + private boolean matchPlatform(String sessionPlatform, String messageTargetPlatform) { + String target = messageTargetPlatform.toLowerCase(); + return "".equals(target) || MessagePushConstants.PUSH_PLATFORM.ALL.toString().equalsIgnoreCase(target) || + target.equals(sessionPlatform); + } +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/HttpPushMessagePublisher.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/HttpPushMessagePublisher.java new file mode 100644 index 0000000..c0590b3 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/HttpPushMessagePublisher.java @@ -0,0 +1,81 @@ +package org.kangspace.messagepush.ws.core.websocket; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.ws.core.domain.model.HttpPushMessageDTO; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxSink; + +import java.util.Objects; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.function.Consumer; + +/** + * Http消息发布者 + * + * @author kango2gler@gmail.com + * @see HttpPushMessageConsumer + * @since 2021/10/30 + */ +@Slf4j +public class HttpPushMessagePublisher { + /** + * 消息Queue + */ + private final BlockingQueue messageQueue = new LinkedBlockingQueue<>(); + private Flux publisher; + + public HttpPushMessagePublisher(HttpPushMessageConsumer messageConsumer) { + Objects.requireNonNull(messageConsumer, "消息消费者不能为null!"); + publisher = Flux.create(new MessageSinkConsumer()).share(); + publisher.subscribe(messageConsumer); + } + + /** + * 发布消息 + * + * @param message message + * @return boolean + */ + public boolean publish(T message) { + try { + messageQueue.put(message); + return true; + } catch (InterruptedException e) { + log.error("Http消息发布者:消息发布失败,错误信息:{}", e.getMessage(), e); + } + return false; + } + + /** + * 
+     * 内部消息消费者
+     * 为Flux publisher提供数据
+     *
+ */ + @NoArgsConstructor + @Data + public class MessageSinkConsumer implements Consumer> { + private final Executor executor = Executors.newSingleThreadExecutor(); + + @Override + public void accept(FluxSink sink) { + executor.execute(() -> { + log.info("Http消息发布者: 监听消息开始！"); + while (true) { + try { + T message = messageQueue.take(); + sink.next(message); + } catch (Exception e) { + log.error("Http消息发布者: 消息消费,消费错误:{}", e.getMessage(), e); + } + } + }); + } + } + +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/MessagePushHandshakeWebSocketService.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/MessagePushHandshakeWebSocketService.java new file mode 100644 index 0000000..fb6b70a --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/MessagePushHandshakeWebSocketService.java @@ -0,0 +1,30 @@ +package org.kangspace.messagepush.ws.core.websocket; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.reactive.socket.WebSocketHandler; +import org.springframework.web.reactive.socket.server.RequestUpgradeStrategy; +import org.springframework.web.reactive.socket.server.support.HandshakeWebSocketService; +import org.springframework.web.server.ServerWebExchange; +import reactor.core.publisher.Mono; + +/** + * 消息推送Websocket握手处理服务 + * + * @author kango2gler@gmail.com + * @since 2021/10/29 + */ +@Slf4j +public class MessagePushHandshakeWebSocketService extends HandshakeWebSocketService { + public MessagePushHandshakeWebSocketService() { + super(); + } + + public MessagePushHandshakeWebSocketService(RequestUpgradeStrategy upgradeStrategy) { + super(upgradeStrategy); + } + + @Override + public Mono handleRequest(ServerWebExchange exchange, WebSocketHandler handler) { + return super.handleRequest(exchange, handler); + } +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/MessagePushWebSocketHandler.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/MessagePushWebSocketHandler.java new file mode 100644 index 0000000..ec883b5 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/MessagePushWebSocketHandler.java @@ -0,0 +1,268 @@ +package org.kangspace.messagepush.ws.core.websocket; + + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.constant.MessagePushConstants; +import org.kangspace.messagepush.core.util.JsonUtil; +import org.kangspace.messagepush.ws.core.constant.MessagePushCmdEnum; +import org.kangspace.messagepush.ws.core.constant.MessagePushResponseTypeEnum; +import org.kangspace.messagepush.ws.core.constant.MessagePushWsConstants; +import org.kangspace.messagepush.ws.core.domain.dto.MessageCmdDTO; +import org.kangspace.messagepush.ws.core.domain.dto.MessageCmdResponseDTO; +import org.kangspace.messagepush.ws.core.domain.dto.request.LoginReqDataDTO; +import org.kangspace.messagepush.ws.core.domain.dto.response.HeartBeatRespDataDTO; +import org.kangspace.messagepush.ws.core.domain.dto.response.NormalRespDataDTO; +import org.kangspace.messagepush.ws.core.domain.model.MessageRequestParam; +import org.kangspace.messagepush.ws.core.utils.MessagePushHandlerUtils; +import org.kangspace.messagepush.ws.core.websocket.session.SessionHolder; +import org.kangspace.messagepush.ws.core.websocket.session.WebSocketSessionManager; +import org.springframework.util.StringUtils; +import org.springframework.web.reactive.socket.*; +import org.springframework.web.server.ServerWebInputException; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * 消息推送WebsocketHandler + * + * @author kango2gler@gmail.com + * @see WebSocketHandler + * @since 2021/10/28 + */ +@Slf4j +@Data +public class MessagePushWebSocketHandler extends BaseWebcosketHandler implements WebSocketHandler { + + /** + * 支持的消息类型 + */ + private final List SUPPORT_MESSAGE_TYPES = Arrays.asList( + WebSocketMessage.Type.PING, + WebSocketMessage.Type.TEXT); + /** + * session 管理器 + */ + private final WebSocketSessionManager webSocketSessionManager; + /** + * 监听路径 + */ + private String endpointPath = MessagePushWsConstants.MESSAGE_V1_ENDPOINT_PATH; + /** + * session检查executor + */ + private ScheduledThreadPoolExecutor sessionCheckExecutor; + + + public MessagePushWebSocketHandler(WebSocketSessionManager webSocketSessionManager) { + this.webSocketSessionManager = webSocketSessionManager; + this.sessionCheckExecutor = new ScheduledThreadPoolExecutor(MessagePushWsConstants.TASK_EXECUTOR_THREAD_NUM); + } + + @Override + public Mono handle(WebSocketSession session) { + // 1. 获取握手信息 + HandshakeInfo handshakeInfo = session.getHandshakeInfo(); + log.info("Websocket处理器: 建立连接, session:[{}], handshakeInfo:[{}]", session.getId(), handshakeInfo); + MessageRequestParam requestParam = MessagePushHandlerUtils.getMessageRequestParam(session); + // 2. 校验请求参数 + CloseStatus isValid = valid(requestParam, session); + if (isValid != null) { + return session.close(isValid); + } + String uid = requestParam.getUid(); + // 3. 消息处理 + return session.receive().doOnSubscribe(s -> sessionCheckSchedule(session)) + .doOnTerminate(() -> unRegisterUserSession(uid, session, "Terminate", "连接已断开")) + .doOnCancel(() -> unRegisterUserSession(uid, session, "Cancel", "连接已取消")) + .doOnComplete(() -> { + }) + .doOnNext(message -> { + String plainMessage = message.getPayloadAsText(); + log.info("Websocket处理器: 收到消息, url:[{}], uid:[{}], session:[{}],message:[{}]", endpointPath, uid, session, plainMessage); + MessageCmdDTO messageCmdDTO = null; + try { + messageCmdDTO = JsonUtil.toObject(plainMessage, MessageCmdDTO.class); + } catch (Exception e) { + log.error("Websocket处理器: 收到消息, 消息反序列化失败,url:[{}], uid:[{}], session:[{}],message:[{}], error:[{}]", endpointPath, uid, session, message, e.getMessage(), e); + session.send(error(session, "消息格式错误,请确认JSON格式是否正确,并检查参数内容和参数值。")).toProcessor(); + return; + } + if (!StringUtils.hasText(messageCmdDTO.getCmd())) { + log.error("Websocket处理器: 收到消息, 消息格式错误,cmd字段值不存在,url:[{}], uid:[{}], session:[{}],message:[{}]", endpointPath, uid, session, message); + session.send(error(session, "消息格式错误,cmd字段不能为空")).toProcessor(); + } + messageHandle(message, messageCmdDTO, session, uid); + }).doOnError((e) -> unRegisterUserSession(uid, session, "Error", "连接发生错误")).then(); + } + + /** + * 验证请求参数 + * + * @param requestParam + * @return boolean + */ + private CloseStatus valid(MessageRequestParam requestParam, WebSocketSession session) { + log.info("Websocket处理器: 参数验证, session:[{}], requestParam:[{}]", session, requestParam); + if (!StringUtils.hasText(requestParam.getUid())) { + Exception e = new ServerWebInputException("Invalid " + MessagePushConstants.HTTP_HEADER_UID_KEY + " header: value must be not null!"); + log.error("Websocket处理器: 参数验证失败,session:[{}], 错误信息:[{}]", session.getId(), e.getMessage(), e); + return new CloseStatus(CloseStatus.POLICY_VIOLATION.getCode(), e.getMessage()); + } + return null; + } + + /** + * 注册用户session信息 + * + * @param uid 用户uid + * @param session {@link WebSocketSession} + * @return uid + */ + private String registerUserSession(WebSocketSession session, String uid, LoginReqDataDTO loginReqData) { + log.info("Websocket处理器: 注册用户Session信息, url:[{}], uid:[{}], session:[{}], loginReqData:[{}] ", endpointPath, uid, session.getId(), loginReqData); + webSocketSessionManager.addSession(loginReqData.getAppKey(), uid, new SessionHolder(session, uid, loginReqData.getPlatform())); + return uid; + } + + /** + * 删除用户Session注册信息 + * + * @param key uid + * @param session {@link WebSocketSession} + * @param reason 删除原因(连接断开,连接取消等) + */ + private void unRegisterUserSession(String key, WebSocketSession session, String code, String reason) { + log.info("Websocket处理器, 删除用户Session信息,url[{}], uid:[{}], session:[{}], {} {}", endpointPath, key, session.getId(), code, reason); + webSocketSessionManager.removeSession(key, session); + } + + /** + * 消息处理 + * + * @param message 消息内容 + * @param session session + */ + public void messageHandle(WebSocketMessage message, MessageCmdDTO messageCmd, WebSocketSession session, String uid) { + log.info("Websocket处理器: 消息处理开始, url[{}], uid:[{}], session:[{}], message:[{}], ", endpointPath, uid, session.getId(), message.getPayloadAsText()); + WebSocketMessage.Type messageType = message.getType(); + boolean isSupportedMessageType = supportMessageTypes(message); + if (!isSupportedMessageType) { + log.warn("Websocket处理器: 消息处理结束, 不支持的消息类型,忽略处理。 url[{}], uid:[{}], session:[{}], message:[{}], ", endpointPath, uid, session.getId(), message.getPayloadAsText()); + return; + } + switch (messageType) { + case PING: + // 心跳 + heartbeat(session, true); + break; + case TEXT: + // 登录 + String cmd = messageCmd.getCmd().toUpperCase(); + if (MessagePushCmdEnum.LOGIN.toString().equals(cmd)) { + login(session, uid, cmd, messageCmd); + } else if (MessagePushCmdEnum.HEARTBEAT.toString().equals(cmd)) { + heartbeat(session, false); + } + break; + default: + break; + } + log.info("Websocket处理器: 消息处理结束, url[{}], uid:[{}], session:[{}]", endpointPath, uid, session.getId()); + } + + /** + * 心跳响应 + * + * @param session WebSocketSession + */ + private void heartbeat(WebSocketSession session, boolean isPong) { + String pongMessage = JsonUtil.toFormatJson(new MessageCmdResponseDTO(MessagePushResponseTypeEnum.HEARTBEAT.toString(), + new HeartBeatRespDataDTO(System.currentTimeMillis()))); + Flux pong; + if (isPong) { + pong = Flux.just(session.pongMessage((dbf) -> dbf.wrap(pongMessage.getBytes()))); + } else { + pong = Flux.just(session.textMessage(pongMessage)); + } + session.send(pong).toProcessor(); + } + + /** + * 登录处理 + * + * @param session session + * @param uid 用户ID + * @param cmd 命令 + * @param messageCmd 命令对象 + */ + private void login(WebSocketSession session, String uid, String cmd, MessageCmdDTO messageCmd) { + // 验证登录信息字段 + String loginInfoStr = JsonUtil.toFormatJson(messageCmd.getData()); + LoginReqDataDTO loginReqData = JsonUtil.toObject(loginInfoStr, LoginReqDataDTO.class); + if (!StringUtils.hasText(loginReqData.getAppKey()) || + !StringUtils.hasText(loginReqData.getPlatform())) { + MessageCmdResponseDTO cmdDTO = new MessageCmdResponseDTO(cmd, + new NormalRespDataDTO(MessagePushConstants.FAIL_CODE, + "登录接口参数错误,请检查输入参数,必填参数项不能为空")); + session.send(WebsocketUtils.textMessage(session, JsonUtil.toFormatJson(cmdDTO))).toProcessor(); + return; + } + if (sessionCheck(session)) { + log.warn("Websocket处理器: 当前Session已登录,不可重复登录, url[{}], uid:[{}], session:[{}]", endpointPath, uid, session.getId()); + session.send(WebsocketUtils.textMessage(session, new MessageCmdResponseDTO(cmd, NormalRespDataDTO.fail("当前连接已登录,请勿多次登录")))).toProcessor(); + return; + } + // 注册用户session信息 + registerUserSession(session, uid, loginReqData); + session.send(WebsocketUtils.textMessage(session, new MessageCmdResponseDTO(cmd, NormalRespDataDTO.success()))).toProcessor(); + log.info("Websocket处理器: 登录成功, url[{}], uid:[{}], session:[{}]", endpointPath, uid, session.getId()); + } + + /** + * 
+     * Session 检查
+     * 检查规定时间内的连接是否已登录,若未登录则断开连接
+     *
+ * + * @param session session + */ + private void sessionCheckSchedule(WebSocketSession session) { + // 登录超时处理 + sessionCheckExecutor.schedule(() -> { + boolean isLogin = sessionCheck(session); + if (!isLogin) { + session.send(WebsocketUtils.textMessage(session, new MessageCmdResponseDTO(MessagePushResponseTypeEnum.ERROR.getVal(), + NormalRespDataDTO.fail("login timeout!")))).toProcessor(); + session.close(CloseStatus.POLICY_VIOLATION.withReason("login timeout!")).toProcessor(); + log.info("Websocket处理器: Session长时间未登录自动断开,session:[{}]", session.getId()); + } + }, MessagePushWsConstants.SESSION_CHECK_DELAY_SECONDS, TimeUnit.SECONDS); + } + + /** + * 检查当前session是否已登录 + * + * @param session WebSocketSession + * @return true:已登录,false: 未登录 + */ + private boolean sessionCheck(WebSocketSession session) { + return webSocketSessionManager.sessionCheck(session); + } + + /** + * 是否支持当前消息类型 + * + * @param message WebSocketMessage + * @return boolean + */ + public boolean supportMessageTypes(WebSocketMessage message) { + return SUPPORT_MESSAGE_TYPES.contains(message.getType()); + } + +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/WebsocketUtils.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/WebsocketUtils.java new file mode 100644 index 0000000..3783c19 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/WebsocketUtils.java @@ -0,0 +1,60 @@ +package org.kangspace.messagepush.ws.core.websocket; + + +import org.kangspace.messagepush.core.util.JsonUtil; +import org.springframework.web.reactive.socket.WebSocketMessage; +import org.springframework.web.reactive.socket.WebSocketSession; +import reactor.core.publisher.Flux; + +/** + * Websocket相关Util + * + * @author kango2gler@gmail.com + * @since 2021/11/1 + */ +public class WebsocketUtils { + + + /** + * 输出文本消息 + * + * @param session session + * @param msg 消息内容 + * @return Flux + */ + public static Flux textMessage(WebSocketSession session, Object msg) { + return textMessage(session, JsonUtil.toFormatJson(msg)); + } + + /** + * 输出文本消息 + * + * @param session session + * @param msg 消息内容 + * @return Flux + */ + public static Flux textMessage(WebSocketSession session, String msg) { + return Flux.just(session.textMessage(msg)); + } + + /** + * 输出文本消息 + * + * @param session session + * @param msg 消息内容 + */ + public static void sendTextMessage(WebSocketSession session, String msg) { + session.send(textMessage(session, msg)).toProcessor(); + } + + /** + * 输出文本消息 + * + * @param session session + * @param msg 消息内容 + */ + public static void sendTextMessage(WebSocketSession session, Object msg) { + session.send(textMessage(session, msg)).toProcessor(); + } + +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/session/SessionHolder.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/session/SessionHolder.java new file mode 100644 index 0000000..55224d6 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/session/SessionHolder.java @@ -0,0 +1,31 @@ +package org.kangspace.messagepush.ws.core.websocket.session; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.web.reactive.socket.WebSocketSession; + +/** + * Session管理器中保存实体 + * + * @author kango2gler@gmail.com + * @since 2021/10/29 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class SessionHolder { + /** + * session + */ + private WebSocketSession session; + /** + * 用户ID + */ + private String uid; + /** + * 目标平台 + */ + private String platform; + +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/session/WebSocketSessionManager.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/session/WebSocketSessionManager.java new file mode 100644 index 0000000..432fa08 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/session/WebSocketSessionManager.java @@ -0,0 +1,58 @@ +package org.kangspace.messagepush.ws.core.websocket.session; + +import org.springframework.web.reactive.socket.WebSocketSession; + +import java.util.List; +import java.util.Map; + +/** + * WebSocket用户Session管理器 + * + * @author kango2gler@gmail.com + * @since 2021/10/28 + */ +public interface WebSocketSessionManager { + /** + * 添加Session + * + * @param key session关联的推送对象 + * @param sessionHolder sessionHolder + * @return SessionHolder + */ + SessionHolder addSession(String key, SessionHolder sessionHolder); + + /** + * 删除Session + * + * @param key session关联的推送对象 + * @param socketSession WebSocketSession + * @return boolean + */ + boolean removeSession(String key, WebSocketSession socketSession); + + /** + * 为应用添加用户Session关系 + * + * @param appKey 应用ID + * @param key session关联的推送对象 + * @param sessionHolder sessionHolder + * @return SessionHolder + */ + SessionHolder addSession(String appKey, String key, SessionHolder sessionHolder); + + /** + * 检查Session是否已添加 + * + * @param session WebSocketSession + * @return boolean + */ + boolean sessionCheck(WebSocketSession session); + + /** + * 通过appKey获取所有用户Session信息 + * + * @param appKey appKey + * @return 用户SessionMap + */ + Map> getKeySessions(String appKey); +} diff --git a/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/session/WebSocketUserSessionManager.java b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/session/WebSocketUserSessionManager.java new file mode 100644 index 0000000..694ed16 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/main/java/org/kangspace/messagepush/ws/core/websocket/session/WebSocketUserSessionManager.java @@ -0,0 +1,161 @@ +package org.kangspace.messagepush.ws.core.websocket.session; + +import cn.hutool.core.collection.ConcurrentHashSet; +import lombok.Getter; +import org.springframework.util.CollectionUtils; +import org.springframework.web.reactive.socket.WebSocketSession; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.locks.ReentrantLock; + +/** + * WebSocket用户Session管理器 + * + * @author kango2gler@gmail.com + * @since 2021/10/28 + */ +@Getter +public class WebSocketUserSessionManager implements WebSocketSessionManager { + /** + * 同步操作锁 + */ + private ReentrantLock lock = new ReentrantLock(); + + /** + * 用户和Session关系 + * + * @see CopyOnWriteArrayList + */ + private ConcurrentHashMap> userSessionsMap; + /** + * AppKey下用户和Session关系 + * + * @see CopyOnWriteArrayList + */ + private ConcurrentHashMap>> appUserSessionsMap; + + private ConcurrentHashSet managedSessions; + + + public WebSocketUserSessionManager() { + this.userSessionsMap = new ConcurrentHashMap<>(16); + this.appUserSessionsMap = new ConcurrentHashMap<>(16); + this.managedSessions = new ConcurrentHashSet<>(16); + } + + + @Override + public SessionHolder addSession(String key, SessionHolder sessionHolder) { + lock.lock(); + try { + List sessionHolders = this.userSessionsMap.getOrDefault(key, new CopyOnWriteArrayList<>()); + boolean isNewKey = CollectionUtils.isEmpty(sessionHolders); + sessionHolders.add(sessionHolder); + if (isNewKey) { + this.userSessionsMap.put(key, sessionHolders); + } + managedSessions.add(sessionHolder.getSession()); + } finally { + lock.unlock(); + } + return sessionHolder; + } + + @Override + public boolean removeSession(String key, WebSocketSession session) { + lock.lock(); + try { + List sessionHolders = this.userSessionsMap.getOrDefault(key, new CopyOnWriteArrayList<>()); + boolean del = removeUserSessionMapSession(sessionHolders, session); + if (!CollectionUtils.isEmpty(this.userSessionsMap)) { + this.userSessionsMap.forEach((k, v) -> { + if (CollectionUtils.isEmpty(v)) { + this.userSessionsMap.remove(k); + } + }); + } + if (!CollectionUtils.isEmpty(this.appUserSessionsMap)) { + this.appUserSessionsMap.values().stream().filter(k -> !CollectionUtils.isEmpty(k.values())) + .forEach(userSessionMaps -> { + userSessionMaps.values().forEach(userSessionMap -> removeUserSessionMapSession(userSessionMap, session)); + userSessionMaps.forEach((k, v) -> { + if (CollectionUtils.isEmpty(v)) { + userSessionMaps.remove(k); + } + }); + }); + this.appUserSessionsMap.forEach((k, v) -> { + if (CollectionUtils.isEmpty(v)) { + this.appUserSessionsMap.remove(k); + } + }); + } + managedSessions.remove(session); + return del; + } finally { + lock.unlock(); + } + } + + /** + * 删除用户sessionMap中的Session + * + * @param sessionHolders + * @param session + */ + private boolean removeUserSessionMapSession(Collection sessionHolders, WebSocketSession session) { + lock.lock(); + try { + boolean del = false; + if (!CollectionUtils.isEmpty(sessionHolders)) { + for (Iterator iterator = sessionHolders.iterator(); iterator.hasNext(); ) { + SessionHolder sessionHolder = iterator.next(); + if (sessionHolder.getSession().equals(session)) { + iterator.remove(); + del = true; + } + } + } + return del; + } finally { + lock.unlock(); + } + } + + @Override + public SessionHolder addSession(String appKey, String key, SessionHolder sessionHolder) { + lock.lock(); + try { + ConcurrentHashMap> userSessionsMap = this.appUserSessionsMap.getOrDefault(appKey, new ConcurrentHashMap<>(16)); + // 是否新APP加入 + boolean isNewApp = CollectionUtils.isEmpty(userSessionsMap); + List sessionHolders = userSessionsMap.getOrDefault(key, new ArrayList<>()); + // 是否新用户Session加入 + boolean isNewUserSessions = CollectionUtils.isEmpty(sessionHolders); + sessionHolders.add(sessionHolder); + if (isNewUserSessions) { + userSessionsMap.put(key, sessionHolders); + } + if (isNewApp) { + this.appUserSessionsMap.put(appKey, userSessionsMap); + } + managedSessions.add(sessionHolder.getSession()); + } finally { + lock.unlock(); + } + return sessionHolder; + } + + @Override + public boolean sessionCheck(WebSocketSession session) { + return managedSessions.contains(session); + } + + @Override + public Map> getKeySessions(String appKey) { + ConcurrentHashMap> concurrentHashMap = appUserSessionsMap.get(appKey); + return CollectionUtils.isEmpty(concurrentHashMap) ? Collections.emptyMap() : concurrentHashMap; + } +} diff --git a/message-push-ws/message-push-ws-core/src/test/java/org/kangspace/messagepush/ws/core/FluxTest.java b/message-push-ws/message-push-ws-core/src/test/java/org/kangspace/messagepush/ws/core/FluxTest.java new file mode 100644 index 0000000..01e6c7c --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/test/java/org/kangspace/messagepush/ws/core/FluxTest.java @@ -0,0 +1,59 @@ +package org.kangspace.messagepush.ws.core; + +import lombok.SneakyThrows; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import reactor.core.publisher.Flux; + +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + + +/** + * 公共测试类型 + * + * @author kango2gler@gmail.com + * @since 2021/8/7 + */ +@RunWith(JUnit4.class) +public class FluxTest { + private final BlockingQueue queue = new LinkedBlockingQueue<>(); + + /** + * Flux测试 + * https://stackoverflow.com/questions/54248336/springboot2-webflux-websocket + */ + @Test + public void fluxTest() throws InterruptedException { + Flux publisher = Flux.create(sink -> { + try { + while (true) { + sink.next(queue.take()); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + }); + + new Timer().schedule(new TimerTask() { + @SneakyThrows + @Override + public void run() { + String data = "beat:" + System.currentTimeMillis(); + queue.put(data); + } + }, 0, 1000L); + + publisher.subscribe(this::subscribe); + Thread.sleep(1 * 60 * 1000L); + } + + public void subscribe(String temp) { + System.out.println("subscribe: thread:" + Thread.currentThread().getName() + ", data:" + temp); + } + + +} diff --git a/message-push-ws/message-push-ws-core/src/test/java/org/kangspace/messagepush/ws/core/HttpPushMessagePublisherTest.java b/message-push-ws/message-push-ws-core/src/test/java/org/kangspace/messagepush/ws/core/HttpPushMessagePublisherTest.java new file mode 100644 index 0000000..c6d7949 --- /dev/null +++ b/message-push-ws/message-push-ws-core/src/test/java/org/kangspace/messagepush/ws/core/HttpPushMessagePublisherTest.java @@ -0,0 +1,51 @@ +package org.kangspace.messagepush.ws.core; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.kangspace.messagepush.rest.api.dto.request.MessagePushRequestDTO; +import org.kangspace.messagepush.ws.core.domain.model.HttpPushMessageDTO; +import org.kangspace.messagepush.ws.core.websocket.HttpPushMessageConsumer; +import org.kangspace.messagepush.ws.core.websocket.HttpPushMessagePublisher; + +import java.util.Timer; +import java.util.TimerTask; + +/** + * Http消息发布者测试 + * + * @author kango2gler@gmail.com + * @since 2021/10/30 + */ +@RunWith(JUnit4.class) +@Slf4j +public class HttpPushMessagePublisherTest { + + private HttpPushMessagePublisher publisher; + + @Before + public void init() { + this.publisher = new HttpPushMessagePublisher(new HttpPushMessageConsumer(null)); + } + + @Test + public void test() throws InterruptedException { + new Timer().schedule(new TimerTask() { + @SneakyThrows + @Override + public void run() { + String data = "beat:" + System.currentTimeMillis(); + HttpPushMessagePublisher publisher = HttpPushMessagePublisherTest.this.publisher; + HttpPushMessageDTO message = new HttpPushMessageDTO(); + MessagePushRequestDTO.Message coreMsg = new MessagePushRequestDTO.Message(); + coreMsg.setContent(data); + message.setMessage(coreMsg); + publisher.publish(message); + } + }, 0, 1000L); + Thread.sleep(1 * 60 * 1000L); + } +} diff --git a/message-push-ws/message-push-ws-microservice/pom.xml b/message-push-ws/message-push-ws-microservice/pom.xml new file mode 100644 index 0000000..ddb0f37 --- /dev/null +++ b/message-push-ws/message-push-ws-microservice/pom.xml @@ -0,0 +1,101 @@ + + + + org.kangspace.messagepush + message-push-ws + ${revision} + + 4.0.0 + + message-push-ws-microservice + ${revision} + + + 8 + 8 + + + + + org.kangspace.messagepush + message-push-ws-core + + + + org.kangspace.messagepush + message-push-ws-api + + + + org.springframework.boot + spring-boot-starter-test + test + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-tomcat + + + + + + org.springframework.boot + spring-boot-starter-undertow + + + + org.springframework.cloud + spring-cloud-starter-openfeign + + + + + + + message-push-ws-microservice + + + org.springframework.boot + spring-boot-maven-plugin + + + + repackage + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.2 + + true + UTF-8 + + + + + + + src/main/resources + true + + + + + \ No newline at end of file diff --git a/message-push-ws/message-push-ws-microservice/src/main/java/org/kangspace/messagepush/ws/MessagePushWsApplication.java b/message-push-ws/message-push-ws-microservice/src/main/java/org/kangspace/messagepush/ws/MessagePushWsApplication.java new file mode 100644 index 0000000..eb8eb75 --- /dev/null +++ b/message-push-ws/message-push-ws-microservice/src/main/java/org/kangspace/messagepush/ws/MessagePushWsApplication.java @@ -0,0 +1,27 @@ +package org.kangspace.messagepush.ws; + + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.cloud.openfeign.EnableFeignClients; +import org.springframework.context.annotation.EnableAspectJAutoProxy; + +/** + * 服务主入口 + * + * @author kango2gler@gmail.com + * @since 2021/10/25 + */ +@Slf4j +@EnableAspectJAutoProxy +@EnableFeignClients +@SpringBootApplication(exclude = DataSourceAutoConfiguration.class) +public class MessagePushWsApplication { + + public static void main(String[] args) { + SpringApplication.run(MessagePushWsApplication.class, args); + } + +} diff --git a/message-push-ws/message-push-ws-microservice/src/main/java/org/kangspace/messagepush/ws/controller/MessagePushRestController.java b/message-push-ws/message-push-ws-microservice/src/main/java/org/kangspace/messagepush/ws/controller/MessagePushRestController.java new file mode 100644 index 0000000..b7359af --- /dev/null +++ b/message-push-ws/message-push-ws-microservice/src/main/java/org/kangspace/messagepush/ws/controller/MessagePushRestController.java @@ -0,0 +1,40 @@ +package org.kangspace.messagepush.ws.controller; + + +import lombok.extern.slf4j.Slf4j; +import org.kangspace.messagepush.core.dto.response.ApiResponse; +import org.kangspace.messagepush.rest.api.dto.request.MessagePushRequestTimeDTO; +import org.kangspace.messagepush.ws.core.service.MessagePushWsService; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import javax.annotation.Resource; + +/** + * 消息推送REST API接口 + * + * @author kango2gler@gmail.com + */ +@Slf4j +@Validated +@RestController +@RequestMapping(value = "v1/inner/push") +public class MessagePushRestController { + + @Resource + private MessagePushWsService messagePushWsService; + + /** + * 消息推送PUSH接口 + * + * @param messagePushRequestDto MessagePushRequestTimeDTO + * @return ApiResponse + */ + @PostMapping("") + public ApiResponse messagePush(@Validated @RequestBody MessagePushRequestTimeDTO messagePushRequestDto) { + return messagePushWsService.messagePush(messagePushRequestDto); + } +} diff --git a/message-push-ws/message-push-ws-microservice/src/main/resources/bootstrap.yml b/message-push-ws/message-push-ws-microservice/src/main/resources/bootstrap.yml new file mode 100644 index 0000000..e840c62 --- /dev/null +++ b/message-push-ws/message-push-ws-microservice/src/main/resources/bootstrap.yml @@ -0,0 +1,17 @@ +spring: + application: + name: @artifactId@ + cloud: + nacos: + discovery: + server-addr: ${SERVICE_DISCOVERY_ADDR:discory.kangspace.org:8443} + namespace: ${SERVICE_DISCOVERY_NAMESPACE:kangspace_dev} + metadata: + version: @project.version@ + config: + server-addr: ${spring.cloud.nacos.discovery.server-addr} + namespace: ${spring.cloud.nacos.discovery.namespace} + file-extension: yaml + shared-configs: + - application.${spring.cloud.nacos.config.file-extension} + - message-push-ws-microservice.${spring.cloud.nacos.config.file-extension} \ No newline at end of file diff --git a/message-push-ws/message-push-ws-microservice/src/test/java/org/kangspace/messagepush/ws/Test.java b/message-push-ws/message-push-ws-microservice/src/test/java/org/kangspace/messagepush/ws/Test.java new file mode 100644 index 0000000..49a46f8 --- /dev/null +++ b/message-push-ws/message-push-ws-microservice/src/test/java/org/kangspace/messagepush/ws/Test.java @@ -0,0 +1,13 @@ +package org.kangspace.messagepush.ws; + +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * @author kango2gler@gmail.com + * @since 2021/8/9 + */ +@RunWith(JUnit4.class) +public class Test { + +} diff --git a/message-push-ws/pom.xml b/message-push-ws/pom.xml new file mode 100644 index 0000000..28fcde8 --- /dev/null +++ b/message-push-ws/pom.xml @@ -0,0 +1,64 @@ + + + 4.0.0 + + + org.kangspace.messagepush + message-push + ${revision} + + + message-push-ws + ${revision} + pom + + + message-push-ws-api + message-push-ws-core + message-push-ws-microservice + + + + 8 + 8 + + + + + + org.kangspace.messagepush + message-push-common + ${revision} + + + + org.kangspace.messagepush + message-push-ws-api + ${revision} + + + + org.kangspace.messagepush + message-push-ws-core + ${revision} + + + + org.kangspace.messagepush + message-push-rest-api + ${revision} + + + + javax.servlet + javax.servlet-api + 4.0.1 + + + + + + + \ No newline at end of file