From 444b1a3543fc9d8200704e242332a128007b6e42 Mon Sep 17 00:00:00 2001 From: bongsh0112 Date: Tue, 10 Oct 2023 12:18:35 +0900 Subject: [PATCH] =?UTF-8?q?feat:=20=EB=AC=B4=EC=8B=A0=EC=82=AC,=2029cm=20?= =?UTF-8?q?=ED=81=AC=EB=A1=A4=EB=A7=81=20=EB=A1=9C=EC=A7=81=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80=20&=20=EA=B3=B5=ED=86=B5=20=ED=81=AC=EB=A1=A4?= =?UTF-8?q?=EB=A7=81=20=EB=A1=9C=EC=A7=81=20=EB=A6=AC=ED=8C=A9=ED=86=A0?= =?UTF-8?q?=EB=A7=81=20#59?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../product/controller/ProductController.java | 12 ++++ .../api/product/service/CrawlingUseCase.java | 28 ++++++++- .../product/adaptor/ProductAdaptor.java | 4 ++ .../domain/domains/product/domain/Site.java | 4 +- .../repository/ProductCustomRepository.java | 3 + .../ProductCustomRepositoryImpl.java | 12 ++++ .../outer/crawling/MusinsaCrawl.java | 53 +++++++++++++++++ .../outer/crawling/OliveYoungCrawl.java | 5 ++ .../outer/crawling/TwentyNineCrawl.java | 57 +++++++++++++++++++ 9 files changed, 175 insertions(+), 3 deletions(-) create mode 100644 Infrastructure/src/main/java/tify/server/infrastructure/outer/crawling/MusinsaCrawl.java create mode 100644 Infrastructure/src/main/java/tify/server/infrastructure/outer/crawling/TwentyNineCrawl.java diff --git a/Api/src/main/java/tify/server/api/product/controller/ProductController.java b/Api/src/main/java/tify/server/api/product/controller/ProductController.java index d2b269a2..9e816d7d 100644 --- a/Api/src/main/java/tify/server/api/product/controller/ProductController.java +++ b/Api/src/main/java/tify/server/api/product/controller/ProductController.java @@ -36,6 +36,18 @@ public void oliveYoungCrawl() { crawlingUseCase.executeForOliveYoung(); } + @Operation(summary = "무신사 크롤링을 통해 상품명과 상품 이미지를 가져옵니다.") + @PatchMapping("/musinsa") + public void musinsaCrawl() { + crawlingUseCase.executeForMusinsa(); + } + + @Operation(summary = "29cm 크롤링을 통해 상품명과 상품 이미지를 가져옵니다.") + @PatchMapping("/twentynine") + public void twentynineCrawl() { + crawlingUseCase.executeForTwentyNine(); + } + @Operation(summary = "키워드를 이용하여 관련된 상품을 검색합니다.") @GetMapping() public SliceResponse productSearch( diff --git a/Api/src/main/java/tify/server/api/product/service/CrawlingUseCase.java b/Api/src/main/java/tify/server/api/product/service/CrawlingUseCase.java index 874414cd..8f8baf4b 100644 --- a/Api/src/main/java/tify/server/api/product/service/CrawlingUseCase.java +++ b/Api/src/main/java/tify/server/api/product/service/CrawlingUseCase.java @@ -6,19 +6,24 @@ import lombok.RequiredArgsConstructor; import tify.server.core.annotation.UseCase; import tify.server.domain.domains.product.adaptor.ProductAdaptor; +import tify.server.domain.domains.product.domain.Site; import tify.server.domain.domains.product.dto.ProductCrawlingDto; +import tify.server.infrastructure.outer.crawling.MusinsaCrawl; import tify.server.infrastructure.outer.crawling.OliveYoungCrawl; +import tify.server.infrastructure.outer.crawling.TwentyNineCrawl; @UseCase @RequiredArgsConstructor public class CrawlingUseCase { private final OliveYoungCrawl oliveYoungCrawl; + private final MusinsaCrawl musinsaCrawl; + private final TwentyNineCrawl twentyNineCrawl; private final ProductAdaptor productAdaptor; @Transactional public void executeForOliveYoung() { - List productCrawlingDtos = productAdaptor.searchByName(); + List productCrawlingDtos = productAdaptor.searchByCompany(Site.OLIVE_YOUNG); productCrawlingDtos.forEach( dto -> { String imgSrc = oliveYoungCrawl.process(dto.getCrawlUrl()); @@ -26,6 +31,27 @@ public void executeForOliveYoung() { }); } + @Transactional + public void executeForMusinsa() { + List productCrawlingDtos = productAdaptor.searchByCompany(Site.MUSINSA); + productCrawlingDtos.forEach( + dto -> { + System.out.println(dto.getName() + dto.getCrawlUrl()); + String imgSrc = musinsaCrawl.process(dto.getCrawlUrl()); + updateImageUrl(dto.getName(), imgSrc); + }); + } + + @Transactional + public void executeForTwentyNine() { + List productCrawlingDtos = productAdaptor.searchByCompany(Site.CM); + productCrawlingDtos.forEach( + dto -> { + String imgSrc = twentyNineCrawl.process(dto.getCrawlUrl()); + updateImageUrl(dto.getName(), imgSrc); + }); + } + @Transactional public void updateImageUrl(String name, String imgSrc) { productAdaptor diff --git a/Domain/src/main/java/tify/server/domain/domains/product/adaptor/ProductAdaptor.java b/Domain/src/main/java/tify/server/domain/domains/product/adaptor/ProductAdaptor.java index 4328f0f2..0b4ab010 100644 --- a/Domain/src/main/java/tify/server/domain/domains/product/adaptor/ProductAdaptor.java +++ b/Domain/src/main/java/tify/server/domain/domains/product/adaptor/ProductAdaptor.java @@ -26,6 +26,10 @@ public List searchByName() { return productRepository.search(); } + public List searchByCompany(Site site) { + return productRepository.searchByCompany(site); + } + public List queryAllBySite(Site site) { return productRepository.findAllBySite(site); } diff --git a/Domain/src/main/java/tify/server/domain/domains/product/domain/Site.java b/Domain/src/main/java/tify/server/domain/domains/product/domain/Site.java index c882a030..baf56c3a 100644 --- a/Domain/src/main/java/tify/server/domain/domains/product/domain/Site.java +++ b/Domain/src/main/java/tify/server/domain/domains/product/domain/Site.java @@ -8,8 +8,8 @@ @Getter @AllArgsConstructor public enum Site { - OLIVE_YOUNG("올리브영"), - MUSINSA("무신사"), + OLIVE_YOUNG("oliveyoung"), + MUSINSA("musinsa"), CM("29cm"), ; diff --git a/Domain/src/main/java/tify/server/domain/domains/product/repository/ProductCustomRepository.java b/Domain/src/main/java/tify/server/domain/domains/product/repository/ProductCustomRepository.java index bb015662..17e5e999 100644 --- a/Domain/src/main/java/tify/server/domain/domains/product/repository/ProductCustomRepository.java +++ b/Domain/src/main/java/tify/server/domain/domains/product/repository/ProductCustomRepository.java @@ -4,6 +4,7 @@ import java.util.List; import org.springframework.data.domain.Slice; import tify.server.domain.domains.product.domain.Product; +import tify.server.domain.domains.product.domain.Site; import tify.server.domain.domains.product.dto.ProductCondition; import tify.server.domain.domains.product.dto.ProductCrawlingDto; import tify.server.domain.domains.product.dto.ProductRetrieveDTO; @@ -11,6 +12,8 @@ public interface ProductCustomRepository { List search(); + List searchByCompany(Site site); + List searchAllToRecommendation(String categoryName, String answer); Slice searchByKeyword(ProductCondition productCondition); diff --git a/Domain/src/main/java/tify/server/domain/domains/product/repository/ProductCustomRepositoryImpl.java b/Domain/src/main/java/tify/server/domain/domains/product/repository/ProductCustomRepositoryImpl.java index 6ad7cf27..9c290b91 100644 --- a/Domain/src/main/java/tify/server/domain/domains/product/repository/ProductCustomRepositoryImpl.java +++ b/Domain/src/main/java/tify/server/domain/domains/product/repository/ProductCustomRepositoryImpl.java @@ -10,6 +10,7 @@ import org.springframework.data.domain.Slice; import tify.server.domain.common.util.SliceUtil; import tify.server.domain.domains.product.domain.Product; +import tify.server.domain.domains.product.domain.Site; import tify.server.domain.domains.product.dto.ProductCondition; import tify.server.domain.domains.product.dto.ProductCrawlingDto; import tify.server.domain.domains.product.dto.ProductRetrieveDTO; @@ -29,6 +30,17 @@ public List search() { .fetch(); } + @Override + public List searchByCompany(Site site) { + return queryFactory + .select(new QProductCrawlingDto(product.name, product.crawlUrl)) + .from(product) + .where(product.crawlUrl.contains(site.getValue())) + .groupBy(product.name) + .fetch(); + } + + @Override public List searchAllToRecommendation(String categoryName, String answer) { return queryFactory diff --git a/Infrastructure/src/main/java/tify/server/infrastructure/outer/crawling/MusinsaCrawl.java b/Infrastructure/src/main/java/tify/server/infrastructure/outer/crawling/MusinsaCrawl.java new file mode 100644 index 00000000..2decd9f7 --- /dev/null +++ b/Infrastructure/src/main/java/tify/server/infrastructure/outer/crawling/MusinsaCrawl.java @@ -0,0 +1,53 @@ +package tify.server.infrastructure.outer.crawling; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.openqa.selenium.By; +import org.openqa.selenium.UnhandledAlertException; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.WebElement; +import org.openqa.selenium.chrome.ChromeDriver; +import org.openqa.selenium.chrome.ChromeOptions; +import org.springframework.stereotype.Component; +import tify.server.infrastructure.exception.FeignException; + +@Component +@RequiredArgsConstructor +@Slf4j +public class MusinsaCrawl { + private WebDriver driver; + + public String process(String url) { + System.setProperty( + "webdriver.chrome.driver", + "/Users/sehwan/Downloads/chromedriver-mac-arm64/chromedriver"); + + ChromeOptions options = new ChromeOptions(); + options.addArguments("--remote-allow-origins=*"); + + driver = new ChromeDriver(options); + + String imgSrc = ""; + + try { + imgSrc = getDataList(url); + } catch (InterruptedException e) { + throw FeignException.EXCEPTION; + } catch (UnhandledAlertException e) { + log.info("유효하지 않은 url : {}", url); + } + + driver.close(); + driver.quit(); + + return imgSrc; + } + + private String getDataList(String url) throws InterruptedException { + driver.get(url); + Thread.sleep(1000); + WebElement element = driver.findElement(By.id("detail_bigimg")).findElement(By.className("product-img")).findElement(By.id("bigimg")); + System.out.println(element.toString()); + return element.getAttribute("src"); + } +} diff --git a/Infrastructure/src/main/java/tify/server/infrastructure/outer/crawling/OliveYoungCrawl.java b/Infrastructure/src/main/java/tify/server/infrastructure/outer/crawling/OliveYoungCrawl.java index 590aa3a1..449e885d 100644 --- a/Infrastructure/src/main/java/tify/server/infrastructure/outer/crawling/OliveYoungCrawl.java +++ b/Infrastructure/src/main/java/tify/server/infrastructure/outer/crawling/OliveYoungCrawl.java @@ -2,7 +2,9 @@ import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.openqa.selenium.By; +import org.openqa.selenium.UnhandledAlertException; import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebElement; import org.openqa.selenium.chrome.ChromeDriver; @@ -12,6 +14,7 @@ @Component @RequiredArgsConstructor +@Slf4j public class OliveYoungCrawl { private WebDriver driver; @@ -32,6 +35,8 @@ public String process(String url) { imgSrc = getDataList(url); } catch (InterruptedException e) { throw FeignException.EXCEPTION; + } catch (UnhandledAlertException e) { + log.info("유효하지 않은 url : {}", url); } driver.close(); diff --git a/Infrastructure/src/main/java/tify/server/infrastructure/outer/crawling/TwentyNineCrawl.java b/Infrastructure/src/main/java/tify/server/infrastructure/outer/crawling/TwentyNineCrawl.java new file mode 100644 index 00000000..2fd7f9fb --- /dev/null +++ b/Infrastructure/src/main/java/tify/server/infrastructure/outer/crawling/TwentyNineCrawl.java @@ -0,0 +1,57 @@ +package tify.server.infrastructure.outer.crawling; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.openqa.selenium.By; +import org.openqa.selenium.UnhandledAlertException; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.WebElement; +import org.openqa.selenium.chrome.ChromeDriver; +import org.openqa.selenium.chrome.ChromeOptions; +import org.springframework.stereotype.Component; +import tify.server.infrastructure.exception.FeignException; + +@Component +@RequiredArgsConstructor +@Slf4j +public class TwentyNineCrawl { + private WebDriver driver; + + public String process(String url) { + System.setProperty( + "webdriver.chrome.driver", + "/Users/sehwan/Downloads/chromedriver-mac-arm64/chromedriver"); + + ChromeOptions options = new ChromeOptions(); + options.addArguments("--remote-allow-origins=*"); + + driver = new ChromeDriver(options); + + String imgSrc = ""; + + try { + imgSrc = getDataList(url); + } catch (InterruptedException e) { +// throw FeignException.EXCEPTION; + log.info("에러 발생 ㅠㅠ"); + } catch (UnhandledAlertException e) { + log.info("유효하지 않은 url : {}", url); + } finally { + driver.close(); + driver.quit(); + + return imgSrc; + } + } + + private String getDataList(String url) throws InterruptedException { + driver.get(url); + Thread.sleep(1000); + if (!driver.findElements(By.cssSelector(".css-12qah06.ewptmlp5")).isEmpty()) { + WebElement element = driver.findElements(By.cssSelector(".css-12qah06.ewptmlp5")).get(0); + System.out.println(element.toString()); + return element.getAttribute("src"); + } + return null; + } +}