Merge pull request #5 from hubo-gillajabi/HUBOBE-3-be-두루누비-정보-서비스-가져오기

Hubo be 3 두루누비 정보 서비스 가져오기

.github/workflows/ci-cd-develop.yml

@@ -28,7 +28,7 @@ jobs:
       - name: copy application.yml files from submodule
         run: |
           mkdir -p src/main/resources
-          cp BE-hubo-gillajabi-resources/resources/application*.yml src/main/resources/
+          cp -r BE-hubo-gillajabi-resources/resources/application*.yml src/main/resources/
 
       - name: Grant execute permission for gradlew
         run: chmod +x ./gradlew

.github/workflows/opened-pr-notification.yml

@@ -1,7 +1,7 @@
 name: Opened pr notification
 on:
   pull_request:
-    branches: [ dev, hotfix ]
+    branches: [ dev, hotfix, "HUBOBE-*" ]
     types: [ reopened, opened ]
 
 jobs:

.github/workflows/opened-pr-test.yaml

@@ -2,7 +2,7 @@ name: Test workflows
 
 on:
   pull_request:
-    branches: [ "develop", "main", "hotfix" ]
+    branches: [ "develop", "main", "hotfix" , "HUBOBE-*" ]
     types: [ reopened, opened, synchronize ]
     paths:
       - 'src/main/java/**'
@@ -12,14 +12,17 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
-
       - name: Checkout secret code
         uses: actions/checkout@v3
         with:
           submodules: true
           token: ${{ secrets.ACTION_SECRETS }}
 
+      - name: Initialize and update submodules
+        run: |
+          git submodule sync
+          git submodule update --init --recursive
+
       - name: set up JDK 17
         uses: actions/setup-java@v3
         with:
@@ -29,7 +32,20 @@ jobs:
       - name: Copy application.yml files from submodule
         run: |
           mkdir -p src/main/resources
-          cp BE-hubo-gillajabi-resources/resources/application*.yml src/main/resources/
+          cp -r BE-hubo-gillajabi-resources/resources/application*.yml src/main/resources/
+          echo "Copied $(ls -1 src/main/resources/application*.yml | wc -l) application.yml files"
+
+      - name: Copy resources db files from submodule
+        run: |
+          mkdir -p src/main/resources/db
+          cp -r BE-hubo-gillajabi-resources/resources/db/* src/main/resources/db/
+          echo "Copied $(ls -1 src/main/resources/db/* | wc -l) db files"
+
+      - name: Copy test mocked files from submodule
+        run: |
+          mkdir -p src/test/resources
+          cp -r BE-hubo-gillajabi-resources/test/mockedResponses/* src/test/resources/
+          echo "Copied $(ls -1 src/test/resources/* | wc -l) test mocked files"
 
       - name: Gradle Caching
         uses: actions/cache@v3
@@ -48,10 +64,13 @@ jobs:
         run: ./gradlew test -Dspring.profiles.active=test
 
       - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v4.0.1
+        uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           file: ./build/reports/jacoco/test/jacocoTestReport.xml
 
 
 
+
+
+

.gitignore

@@ -39,3 +39,7 @@ out/
 
 ### yml ###
 **/src/main/resources/*.yml
+**/src/test/resources/**/*.json
+
+### sql ##
+**/src/main/resources/db/migration/*.sql

build.gradle

@@ -16,7 +16,7 @@ repositories {
     mavenCentral()
 }
 
-jacoco{
+jacoco {
     toolVersion = '0.8.12'
 }
 
@@ -26,6 +26,9 @@ dependencies {
     implementation 'org.springframework.boot:spring-boot-starter-web'
     testImplementation 'org.springframework.boot:spring-boot-starter-test'
 
+    // https://mvnrepository.com/artifact/junit/junit
+    testImplementation 'junit:junit:4.13.1'
+
     compileOnly 'org.projectlombok:lombok'
     annotationProcessor 'org.projectlombok:lombok'
 
@@ -35,21 +38,48 @@ dependencies {
     testImplementation 'com.h2database:h2'
 
     implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.2.0'
+
+    // https://mvnrepository.com/artifact/org.springframework.retry/spring-retry
+    implementation group: 'org.springframework.retry', name: 'spring-retry', version: '2.0.6'
+
+
+    implementation 'org.flywaydb:flyway-core'
+    implementation 'org.flywaydb:flyway-mysql'
+
+    // https://mvnrepository.com/artifact/org.jsoup/jsoup
+    implementation 'org.jsoup:jsoup:1.17.2'
+
+//    implementation 'com.graphql-java-kickstart:graphql-spring-boot-starter:14.0.0'
+//
+//    // testing facilities
+//    testImplementation 'com.graphql-java-kickstart:graphql-spring-boot-starter-test:14.0.0'
+
+    testImplementation("com.navercorp.fixturemonkey:fixture-monkey-starter:1.0.17")
+
+    // https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml
+    implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.15.4'
 }
 
 tasks.register('copyResources', Copy) {
     from './BE-hubo-gillajabi-resources/resources'
-    include 'application.yml', 'application-*.yml'
+    include 'application.yml', 'application-*.yml', "db/*", "db/*/*"
     into 'src/main/resources'
 }
 
+tasks.register('copyTestResources', Copy) {
+    from './BE-hubo-gillajabi-resources/test'
+    include 'mockedResponses/*'
+    into 'src/test/resources'
+}
+
 processResources.dependsOn('copyResources')
+processTestResources.dependsOn('copyTestResources')
 
 
 jacocoTestReport {
 
     reports {
-        html.required = false
+        html.required = true
         xml.required = true
         csv.required = false
     }

codecov.yml

@@ -5,9 +5,7 @@ codecov:
 comment:
   layout: "reach,diff,flags,files,footer"
   behavior: default
-  require_changes: false
-  require_base: false
-  hide_project_coverage : true
+  require_changes: true
 
 github_checks:
-  annotations: true
+  annotations: true

lombok.config

@@ -0,0 +1 @@
+lombok.anyConstructor.addConstructorProperties=true

src/main/java/com/hubo/gillajabi/HuboApplication.java

@@ -2,7 +2,9 @@
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.data.jpa.repository.config.EnableJpaAuditing;
 
+@EnableJpaAuditing
 @SpringBootApplication
 public class HuboApplication {
 

src/main/java/com/hubo/gillajabi/crawl/application/dto/response/CrawlResponse.java

@@ -0,0 +1,52 @@
+package com.hubo.gillajabi.crawl.application.dto.response;
+
+import com.hubo.gillajabi.crawl.domain.entity.*;
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+import java.util.List;
+
+@Getter
+@Setter
+public class CrawlResponse {
+
+    @Getter
+    @Setter
+    @NoArgsConstructor(access = AccessLevel.PROTECTED)
+    @Schema(name = "CrawlResponse.Course", description = "코스 크롤링 갯수 DTO")
+    public static class CourseResult {
+        private Integer cityCount;
+        private Integer courseCount;
+        private Integer courseDetailCount;
+        private Integer gpxInfoCount;
+
+        public static CourseResult of(final List<City> cities, final List<Course> courses,
+                                      final List<CourseDetail> courseDetails, final List<GpxInfo> gpxInfos) {
+            CourseResult courseResult = new CourseResult();
+            courseResult.setCityCount(cities.size());
+            courseResult.setCourseCount(courses.size());
+            courseResult.setCourseDetailCount(courseDetails.size());
+            courseResult.setGpxInfoCount(gpxInfos.size());
+            return courseResult;
+        }
+    }
+
+    @Getter
+    @Setter
+    @NoArgsConstructor(access = AccessLevel.PROTECTED)
+    @Schema(name = "CrawlResponse.Theme", description = "코스 테마 갯수 DTO")
+    public static class ThemeResult{
+        private Integer themeCount;
+
+        public static ThemeResult from(final List<CourseTheme> themes) {
+            ThemeResult theme = new ThemeResult();
+            theme.setThemeCount(themes.size());
+            return theme;
+        }
+    }
+
+
+}

src/main/java/com/hubo/gillajabi/crawl/application/presenation/CrawlController.java

@@ -0,0 +1,41 @@
+package com.hubo.gillajabi.crawl.application.presenation;
+
+
+import com.hubo.gillajabi.crawl.application.dto.response.CrawlResponse;
+import com.hubo.gillajabi.crawl.application.service.CrawlFacadeService;
+import com.hubo.gillajabi.crawl.domain.constant.CityName;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import jakarta.validation.Valid;
+import lombok.RequiredArgsConstructor;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+
+@RestController
+@RequestMapping("/api/crawl")
+@RequiredArgsConstructor
+@Tag(name = "crawl 컨트롤러", description = "관리자 공공데이터 포털 호출 api")
+public class CrawlController {
+
+    private final CrawlFacadeService crawlFacadeService;
+
+    @Operation(summary = "전국 길 크롤링 ", description = "만약 해당 정보가 존재한다면 생략합니다.")
+    @GetMapping("/courses")
+    public ResponseEntity<CrawlResponse.CourseResult> startCrawlingCurse(@Valid @RequestParam CityName cityName) {
+
+        CrawlResponse.CourseResult response = crawlFacadeService.getCourse(cityName);
+
+        return ResponseEntity.ok().body(response);
+    }
+
+    @Operation(summary = "전국 길 테마 크롤링", description = "전국 길 테마 크롤링 (ex: 남파랑길)")
+    @GetMapping("/themes")
+    public ResponseEntity<CrawlResponse.ThemeResult> startCrawlingTheme(@Valid @RequestParam CityName cityName) {
+
+        CrawlResponse.ThemeResult response = crawlFacadeService.getTheme(cityName);
+
+        return ResponseEntity.ok().body(response);
+    }
+}
+
+

src/main/java/com/hubo/gillajabi/crawl/application/service/BusanCourseHandler.java

@@ -0,0 +1,12 @@
+package com.hubo.gillajabi.crawl.application.service;
+
+import com.hubo.gillajabi.crawl.application.dto.response.CrawlResponse;
+import org.springframework.stereotype.Service;
+
+@Service
+public class BusanCourseHandler {
+
+    public CrawlResponse.CourseResult handle() {
+        return null;
+    }
+}

src/main/java/com/hubo/gillajabi/crawl/application/service/BusanThemeHandler.java

@@ -0,0 +1,14 @@
+package com.hubo.gillajabi.crawl.application.service;
+
+import com.hubo.gillajabi.crawl.application.dto.response.CrawlResponse;
+import lombok.RequiredArgsConstructor;
+import org.springframework.stereotype.Service;
+
+@Service
+@RequiredArgsConstructor
+public class BusanThemeHandler {
+
+    public CrawlResponse.ThemeResult handle() {
+        return null;
+    }
+}

src/main/java/com/hubo/gillajabi/crawl/application/service/CrawlFacadeService.java

@@ -0,0 +1,36 @@
+package com.hubo.gillajabi.crawl.application.service;
+
+
+import com.hubo.gillajabi.crawl.application.dto.response.CrawlResponse;
+import com.hubo.gillajabi.crawl.domain.constant.CityName;
+import lombok.RequiredArgsConstructor;
+import org.springframework.stereotype.Service;
+
+
+@Service
+@RequiredArgsConstructor
+public class CrawlFacadeService {
+
+    private final DuruCourseHandler duruCourseHandler;
+    private final BusanCourseHandler busanCourseHandler;
+    private final DuruThemeHandler duruThemeHandler;
+    private final BusanThemeHandler busanThemeHandler;
+
+    public CrawlResponse.CourseResult getCourse(final CityName cityName) {
+        return switch (cityName) {
+            case DURU -> duruCourseHandler.handle();
+            case BUSAN -> busanCourseHandler.handle();
+        };
+    }
+
+    public CrawlResponse.ThemeResult getTheme(final CityName cityName) {
+        return switch (cityName) {
+            case DURU -> duruThemeHandler.handle();
+            case BUSAN -> busanThemeHandler.handle();
+        };
+    }
+}
+
+
+
+

src/main/java/com/hubo/gillajabi/crawl/application/service/DuruCourseHandler.java

@@ -0,0 +1,35 @@
+package com.hubo.gillajabi.crawl.application.service;
+
+import com.hubo.gillajabi.crawl.application.dto.response.CrawlResponse;
+import com.hubo.gillajabi.crawl.domain.entity.City;
+import com.hubo.gillajabi.crawl.domain.entity.Course;
+import com.hubo.gillajabi.crawl.domain.entity.CourseDetail;
+import com.hubo.gillajabi.crawl.domain.entity.GpxInfo;
+import com.hubo.gillajabi.crawl.domain.service.duru.*;
+import com.hubo.gillajabi.crawl.infrastructure.dto.response.DuruCourseResponse;
+import lombok.RequiredArgsConstructor;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+@Service
+@RequiredArgsConstructor
+public class DuruCourseHandler {
+
+    private final CrawlDuruServiceImpl duruCrawlService;
+    private final CityDuruService cityService;
+    private final CourseDuruService courseDuruService;
+    private final CourseDetailDuruService courseDetailDuruService;
+    private final GpxInfoDuruService gpxInfoDuruService;
+
+    public CrawlResponse.CourseResult handle() {
+        List<DuruCourseResponse.Course> rawCourses = duruCrawlService.crawlCourse();
+        List<City> cities = cityService.saveCity(rawCourses);
+        List<Course> courses = courseDuruService.saveDuruCourse(rawCourses, cities);
+        List<CourseDetail> courseDetails = courseDetailDuruService.saveDuruCourseDetail(rawCourses, courses);
+        List<GpxInfo> gpxInfos = gpxInfoDuruService.saveGpxInfo(courseDetails);
+
+        return CrawlResponse.CourseResult.of(cities, courses, courseDetails, gpxInfos);
+
+    }
+}