[merge]: STT 구현 (#69)

build.gradle

@@ -2,6 +2,8 @@ plugins {
 	id 'java'
 	id 'org.springframework.boot' version '3.3.3'
 	id 'io.spring.dependency-management' version '1.1.6'
+	id 'com.google.protobuf' version '0.9.4'
+
 }
 
 group = 'com.splanet'
@@ -38,7 +40,19 @@ dependencies {
     implementation 'io.jsonwebtoken:jjwt:0.9.1'
 	implementation 'org.hibernate.validator:hibernate-validator:8.0.0.Final'
 	implementation 'jakarta.validation:jakarta.validation-api:3.0.2'
-	implementation 'org.springframework.boot:spring-boot-starter-data-redis'
+	implementation 'org.apache.httpcomponents.client5:httpclient5:5.2.1'
+	implementation 'org.springframework.boot:spring-boot-starter-websocket'
+	// gRPC 및 Protocol Buffers 의존성
+	implementation 'io.grpc:grpc-netty-shaded:1.56.1'
+	implementation 'io.grpc:grpc-protobuf:1.56.1'
+	implementation 'io.grpc:grpc-stub:1.56.1'
+	implementation 'com.google.protobuf:protobuf-java:3.23.4'
+
+	// gRPC 관련 필요한 의존성
+	implementation 'javax.annotation:javax.annotation-api:1.3.2'
+	implementation 'com.google.code.gson:gson:2.8.9'
+
+
 	compileOnly 'org.projectlombok:lombok'
 	runtimeOnly 'com.mysql:mysql-connector-j'
 	annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
@@ -52,3 +66,29 @@ dependencies {
 tasks.named('test') {
 	useJUnitPlatform()
 }
+
+protobuf {
+	protoc {
+		artifact = 'com.google.protobuf:protoc:3.23.4'
+	}
+	plugins {
+		grpc {
+			artifact = 'io.grpc:protoc-gen-grpc-java:1.66.0'
+		}
+	}
+	generateProtoTasks {
+		all().forEach { task ->
+			task.plugins {
+				grpc {}
+			}
+		}
+	}
+}
+
+sourceSets {
+	main {
+		java {
+			srcDirs 'build/generated/source/proto/main/java', 'build/generated/source/proto/main/grpc'
+		}
+	}
+}

src/main/java/com/splanet/splanet/config/WebConfig.java

@@ -0,0 +1,16 @@
+package com.splanet.splanet.config;
+
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.servlet.config.annotation.CorsRegistry;
+import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
+
+@Configuration
+public class WebConfig implements WebMvcConfigurer {
+
+    @Override
+    public void addCorsMappings(CorsRegistry registry) { //인터페이스 WebMvcConfigurer 상속
+        registry.addMapping("/**") //모든 경로를 허용해줄것이므로
+                .allowedOrigins("*") //리소스 공유 허락할 origin 지정
+                .allowedMethods("*"); //모든 메소드를 허용
+    }
+}

src/main/java/com/splanet/splanet/config/WebSocketConfig.java

@@ -0,0 +1,24 @@
+package com.splanet.splanet.config;
+
+import com.splanet.splanet.core.handler.SpeechWebSocketHandler;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.socket.config.annotation.EnableWebSocket;
+import org.springframework.web.socket.config.annotation.WebSocketConfigurer;
+import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry;
+
+@Configuration
+@EnableWebSocket
+public class WebSocketConfig implements WebSocketConfigurer {
+
+    private final SpeechWebSocketHandler speechWebSocketHandler;
+
+    public WebSocketConfig(SpeechWebSocketHandler speechWebSocketHandler) {
+        this.speechWebSocketHandler = speechWebSocketHandler;
+    }
+
+    @Override
+    public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) {
+        registry.addHandler(speechWebSocketHandler, "/ws/stt")
+                .setAllowedOrigins("*");
+    }
+}

src/main/java/com/splanet/splanet/core/handler/SpeechWebSocketHandler.java

@@ -0,0 +1,100 @@
+package com.splanet.splanet.core.handler;
+
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.protobuf.ByteString;
+import com.nbp.cdncp.nest.grpc.proto.v1.NestResponse;
+import com.splanet.splanet.stt.service.ClovaSpeechGrpcService;
+import io.grpc.stub.StreamObserver;
+import org.springframework.stereotype.Component;
+import org.springframework.web.socket.BinaryMessage;
+import org.springframework.web.socket.CloseStatus;
+import org.springframework.web.socket.TextMessage;
+import org.springframework.web.socket.WebSocketSession;
+import org.springframework.web.socket.handler.BinaryWebSocketHandler;
+
+import java.io.IOException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.Map;
+
+@Component
+public class SpeechWebSocketHandler extends BinaryWebSocketHandler {
+
+    private final ClovaSpeechGrpcService clovaSpeechGrpcService;
+    private final Map<String, StreamObserver<ByteString>> clientObservers = new ConcurrentHashMap<>();
+
+    public SpeechWebSocketHandler(ClovaSpeechGrpcService clovaSpeechGrpcService) {
+        this.clovaSpeechGrpcService = clovaSpeechGrpcService;
+    }
+
+    @Override
+    public void afterConnectionEstablished(WebSocketSession session) throws Exception {
+        // 세션이 열릴 때마다 새로운 gRPC 스트림을 생성
+        StreamObserver<NestResponse> responseObserver = new StreamObserver<NestResponse>() {
+            @Override
+            public void onNext(NestResponse value) {
+                // 서버로부터 받은 응답 처리
+                try {
+                    String contents = value.getContents(); // JSON 문자열
+
+                    // JSON 파싱
+                    JsonParser parser = new JsonParser();
+                    JsonObject jsonObject = parser.parse(contents).getAsJsonObject();
+
+                    if (jsonObject.has("transcription")) {
+                        JsonObject transcription = jsonObject.getAsJsonObject("transcription");
+                        String text = transcription.get("text").getAsString();
+                        // 클라이언트로 text 필드만 전송
+                        session.sendMessage(new TextMessage(text));
+                    }
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+
+            @Override
+            public void onError(Throwable t) {
+                t.printStackTrace();
+                try {
+                    session.sendMessage(new TextMessage("오류 발생: " + t.getMessage()));
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+
+            @Override
+            public void onCompleted() {
+                // 스트림 완료 처리
+                try {
+                    session.close();
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+        };
+
+        // 오디오 데이터를 전송할 StreamObserver 생성
+        StreamObserver<ByteString> requestObserver = clovaSpeechGrpcService.recognize(responseObserver);
+        clientObservers.put(session.getId(), requestObserver);
+    }
+
+    @Override
+    protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) throws Exception {
+        // 클라이언트로부터 받은 오디오 데이터를 gRPC 서비스로 전달
+        StreamObserver<ByteString> requestObserver = clientObservers.get(session.getId());
+        if (requestObserver != null) {
+            byte[] audioData = message.getPayload().array();
+            ByteString audioChunk = ByteString.copyFrom(audioData);
+            requestObserver.onNext(audioChunk);
+        }
+    }
+
+    @Override
+    public void afterConnectionClosed(WebSocketSession session, CloseStatus status) throws Exception {
+        // 세션이 종료되면 gRPC 스트림도 종료
+        StreamObserver<ByteString> requestObserver = clientObservers.remove(session.getId());
+        if (requestObserver != null) {
+            requestObserver.onCompleted();
+        }
+    }
+}

src/main/java/com/splanet/splanet/core/properties/ClovaProperties.java

@@ -0,0 +1,15 @@
+package com.splanet.splanet.core.properties;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+@Configuration
+@ConfigurationProperties(prefix = "clova.speech")
+public class ClovaProperties {
+    private String clientSecret;
+    private String language;
+}

src/main/java/com/splanet/splanet/jwt/JwtAuthenticationFilter.java

@@ -76,7 +76,7 @@ private boolean isApiPath(String requestURI) {
   }
 
   private boolean isExemptedPath(String requestURI) {
-    return requestURI.equals("/api/users/create") || requestURI.startsWith("/api/token");
+    return requestURI.equals("/api/users/create") || requestURI.startsWith("/api/token") || requestURI.startsWith("/api/stt");
   }
 
   private void sendErrorResponse(HttpServletResponse response, int status, String message) throws IOException {

src/main/java/com/splanet/splanet/stt/service/ClovaSpeechGrpcService.java

@@ -0,0 +1,89 @@
+package com.splanet.splanet.stt.service;
+
+import com.google.protobuf.ByteString;
+import com.nbp.cdncp.nest.grpc.proto.v1.*;
+import com.splanet.splanet.core.properties.ClovaProperties;
+import io.grpc.ManagedChannel;
+import io.grpc.Metadata;
+import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder;
+import io.grpc.stub.MetadataUtils;
+import io.grpc.stub.StreamObserver;
+import org.springframework.stereotype.Service;
+
+@Service
+public class ClovaSpeechGrpcService implements ClovaSpeechService {
+
+    private final NestServiceGrpc.NestServiceStub nestServiceStub;
+    private final ClovaProperties clovaProperties;
+
+    public ClovaSpeechGrpcService(ClovaProperties clovaProperties) {
+        this.clovaProperties = clovaProperties;
+
+        // gRPC 채널 생성
+        ManagedChannel channel = NettyChannelBuilder
+                .forAddress("clovaspeech-gw.ncloud.com", 50051)
+                .useTransportSecurity()
+                .build();
+
+        // Stub 생성 및 인증 정보 설정
+        NestServiceGrpc.NestServiceStub stub = NestServiceGrpc.newStub(channel);
+        Metadata metadata = new Metadata();
+        metadata.put(Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER), "Bearer " + clovaProperties.getClientSecret());
+        this.nestServiceStub = MetadataUtils.attachHeaders(stub, metadata);
+    }
+
+    @Override
+    public StreamObserver<ByteString> recognize(StreamObserver<NestResponse> responseObserver) {
+        StreamObserver<NestRequest> requestObserver = nestServiceStub.recognize(responseObserver);
+
+        // Config 메시지 전송
+        requestObserver.onNext(createConfigRequest(clovaProperties.getLanguage()));
+
+        return new StreamObserver<ByteString>() {
+            private int sequenceId = 0;
+
+            @Override
+            public void onNext(ByteString audioChunk) {
+                NestRequest dataRequest = createDataRequest(audioChunk, sequenceId, false);
+                requestObserver.onNext(dataRequest);
+                sequenceId++;
+            }
+
+            @Override
+            public void onError(Throwable t) {
+                t.printStackTrace();
+                requestObserver.onError(t);
+            }
+
+            @Override
+            public void onCompleted() {
+                requestObserver.onCompleted();
+            }
+        };
+    }
+
+    // Config 설정
+    private NestRequest createConfigRequest(String language) {
+        NestConfig config = NestConfig.newBuilder()
+                .setConfig("{\"transcription\":{\"language\":\"" + language + "\"}}")
+                .build();
+
+        return NestRequest.newBuilder()
+                .setType(RequestType.CONFIG)
+                .setConfig(config)
+                .build();
+    }
+
+    // 데이터 구성
+    private NestRequest createDataRequest(ByteString audioChunk, int sequenceId, boolean epFlag) {
+        NestData data = NestData.newBuilder()
+                .setChunk(audioChunk)
+                .setExtraContents("{\"seqId\":" + sequenceId + ",\"epFlag\":" + epFlag + "}")
+                .build();
+
+        return NestRequest.newBuilder()
+                .setType(RequestType.DATA)
+                .setData(data)
+                .build();
+    }
+}

src/main/java/com/splanet/splanet/stt/service/ClovaSpeechService.java

@@ -0,0 +1,9 @@
+package com.splanet.splanet.stt.service;
+
+import com.google.protobuf.ByteString;
+import com.nbp.cdncp.nest.grpc.proto.v1.NestResponse;
+import io.grpc.stub.StreamObserver;
+
+public interface ClovaSpeechService {
+    StreamObserver<ByteString> recognize(StreamObserver<NestResponse> responseObserver);
+}

src/main/proto/nest.proto

@@ -0,0 +1,34 @@
+syntax = "proto3";
+option java_multiple_files = true;
+
+package com.nbp.cdncp.nest.grpc.proto.v1;
+
+enum RequestType {
+  CONFIG = 0;
+  DATA = 1;
+}
+
+message NestConfig {
+  string config = 1;
+}
+
+message NestData {
+  bytes chunk = 1;
+  string extra_contents = 2;
+}
+
+message NestRequest {
+  RequestType type = 1;
+  oneof part {
+    NestConfig config = 2;
+    NestData data = 3;
+  }
+}
+
+message NestResponse {
+  string contents = 1;
+}
+
+service NestService {
+  rpc recognize(stream NestRequest) returns (stream NestResponse) {}
+}