diff --git a/src/main/java/team7/inplace/crawling/application/AddressUtil.java b/src/main/java/team7/inplace/crawling/application/AddressUtil.java new file mode 100644 index 00000000..5ebd55cb --- /dev/null +++ b/src/main/java/team7/inplace/crawling/application/AddressUtil.java @@ -0,0 +1,25 @@ +package team7.inplace.crawling.application; + +import static lombok.AccessLevel.PRIVATE; + +import com.fasterxml.jackson.databind.JsonNode; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import lombok.NoArgsConstructor; + +@NoArgsConstructor(access = PRIVATE) +public class AddressUtil { + private static final String ADDRESS_REGEX = "[가-힣0-9]+(?:도|시|구|군|읍|면|동|리|로|길)[^#,\\n()]+(?:동|읍|면|리|로|길|호|층|번지)[^#,\\n()]+"; + + public static String extractAddress(JsonNode snippet) { + + String videoDescription = snippet.path("description").asText(); + + Pattern pattern = Pattern.compile(ADDRESS_REGEX); + Matcher matcher = pattern.matcher(videoDescription); + if (matcher.find()) { + return matcher.group(); + } + return null; + } +} diff --git a/src/main/java/team7/inplace/crawling/application/CrawlingFacade.java b/src/main/java/team7/inplace/crawling/application/CrawlingFacade.java new file mode 100644 index 00000000..cf92a42d --- /dev/null +++ b/src/main/java/team7/inplace/crawling/application/CrawlingFacade.java @@ -0,0 +1,22 @@ +package team7.inplace.crawling.application; + +import lombok.RequiredArgsConstructor; +import team7.inplace.global.annotation.Facade; +import team7.inplace.video.application.VideoFacade; + +@Facade +@RequiredArgsConstructor +public class CrawlingFacade { + private final YoutubeCrawlingService youtubeCrawlingService; + private final VideoFacade videoFacade; + + public void updateVideos() { + var crawlingInfos = youtubeCrawlingService.crawlAllVideos(); + for (var crawlingInfo : crawlingInfos) { + var videoCommands = crawlingInfo.toVideoCommands(); + var placesCommands = crawlingInfo.toPlacesCommands(); + + videoFacade.createVideos(videoCommands, placesCommands); + } + } +} diff --git a/src/main/java/team7/inplace/crawling/application/YoutubeCrawlingService.java b/src/main/java/team7/inplace/crawling/application/YoutubeCrawlingService.java index 375866bd..9a3020d1 100644 --- a/src/main/java/team7/inplace/crawling/application/YoutubeCrawlingService.java +++ b/src/main/java/team7/inplace/crawling/application/YoutubeCrawlingService.java @@ -1,9 +1,11 @@ package team7.inplace.crawling.application; +import java.util.List; import java.util.Objects; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import team7.inplace.crawling.application.dto.CrawlingInfo; import team7.inplace.crawling.client.KakaoMapClient; import team7.inplace.crawling.client.YoutubeClient; import team7.inplace.crawling.persistence.YoutubeChannelRepository; @@ -22,17 +24,30 @@ public class YoutubeCrawlingService { 3. 마지막 비디오 UUID를 업데이트 한다. 4. 카카오 API를 호출해 장소 정보를 가져온다 */ - public void crawlAllVideos() { + public List crawlAllVideos() { var youtubeChannels = youtubeChannelRepository.findAll(); - for (var channel : youtubeChannels) { - var rawVideoInfos = youtubeClient.getVideos(channel.getPlayListUUID(), channel.getLastVideoUUID()); - channel.updateLastVideoUUID(rawVideoInfos.get(0).videoId()); - - var videos = rawVideoInfos.stream() - .map(rawVideoInfo -> kakaoMapClient.search(rawVideoInfo, channel.getChannelType().getCode())) - .filter(Objects::nonNull) - .toList(); - } + + var crawlInfos = youtubeChannels.stream() + .map(channel -> { + var videoSnippets = youtubeClient.getVideos(channel.getPlayListUUID(), channel.getLastVideoUUID()); + + var videoAddresses = videoSnippets.stream() + .map(AddressUtil::extractAddress) + .toList(); + + var placeNodes = videoAddresses.stream() + .map(address -> { + if (Objects.isNull(address)) { + return null; + } + return kakaoMapClient.search(address, channel.getChannelType().getCode()); + }) + .toList(); + + return new CrawlingInfo(channel.getInfluencerId(), videoSnippets, placeNodes); + }).toList(); + + return crawlInfos; } } diff --git a/src/main/java/team7/inplace/crawling/application/dto/CrawlingInfo.java b/src/main/java/team7/inplace/crawling/application/dto/CrawlingInfo.java new file mode 100644 index 00000000..7ed84962 --- /dev/null +++ b/src/main/java/team7/inplace/crawling/application/dto/CrawlingInfo.java @@ -0,0 +1,25 @@ +package team7.inplace.crawling.application.dto; + +import com.fasterxml.jackson.databind.JsonNode; +import java.util.List; +import team7.inplace.crawling.client.dto.PlaceNode; +import team7.inplace.place.application.command.PlacesCommand; +import team7.inplace.video.application.command.VideoCommand; + +public record CrawlingInfo( + Long influencerId, + List videoSnippets, + List placeNodes +) { + public List toVideoCommands() { + return videoSnippets.stream() + .map(snippet -> VideoCommand.Create.from(snippet, influencerId)) + .toList(); + } + + public List toPlacesCommands() { + return placeNodes.stream() + .map(placeNode -> PlacesCommand.Create.from(placeNode.locationNode(), placeNode.placeNode())) + .toList(); + } +} diff --git a/src/main/java/team7/inplace/crawling/client/KakaoMapClient.java b/src/main/java/team7/inplace/crawling/client/KakaoMapClient.java index 992f5cdb..b56fc71f 100644 --- a/src/main/java/team7/inplace/crawling/client/KakaoMapClient.java +++ b/src/main/java/team7/inplace/crawling/client/KakaoMapClient.java @@ -10,8 +10,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; import org.springframework.web.client.RestTemplate; -import team7.inplace.crawling.client.dto.RawPlace; -import team7.inplace.crawling.client.dto.RawVideoInfo; +import team7.inplace.crawling.client.dto.PlaceNode; import team7.inplace.global.kakao.config.KakaoApiProperties; @Slf4j @@ -24,8 +23,7 @@ public class KakaoMapClient { private final KakaoApiProperties kakaoApiProperties; private final RestTemplate restTemplate; - public RawPlace.Info search(RawVideoInfo videoInfo, String category) { - var address = videoInfo.address(); + public PlaceNode search(String address, String category) { var locationInfo = getLocateInfo(address, category); var placeId = locationInfo.has("documents") ? locationInfo.get("documents").get(0).get("id").asText() : null; @@ -34,7 +32,7 @@ public RawPlace.Info search(RawVideoInfo videoInfo, String category) { } var placeInfo = getPlaceInfo(placeId); - return RawPlace.Info.from(locationInfo, placeInfo); + return PlaceNode.of(locationInfo, placeInfo); } private JsonNode getLocateInfo(String address, String category) { diff --git a/src/main/java/team7/inplace/crawling/client/YoutubeClient.java b/src/main/java/team7/inplace/crawling/client/YoutubeClient.java index 94e86359..b9187104 100644 --- a/src/main/java/team7/inplace/crawling/client/YoutubeClient.java +++ b/src/main/java/team7/inplace/crawling/client/YoutubeClient.java @@ -4,31 +4,26 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.web.client.RestTemplate; -import team7.inplace.crawling.client.dto.RawVideoInfo; @Slf4j @Component public class YoutubeClient { private static final String PLAY_LIST_ITEMS_BASE_URL = "https://www.googleapis.com/youtube/v3/playlistItems"; private static final String PLAY_LIST_PARAMS = "?part=snippet&playlistId=%s&key=%s&maxResults=50"; - private static final String ADDRESS_REGEX = "[가-힣0-9]+(?:도|시|구|군|읍|면|동|리|로|길)[^#,\\n()]+(?:동|읍|면|리|로|길|호|층|번지)[^#,\\n()]+"; private final RestTemplate restTemplate; private final String apiKey; public YoutubeClient(@Value("${youtube.api.key}") String apiKey, RestTemplate restTemplate) { - log.info("Youtube API Key: {}", apiKey); this.restTemplate = restTemplate; this.apiKey = apiKey; } - public List getVideos(String playListId, String finalVideoUUID) { - List videoInfos = new ArrayList<>(); + public List getVideos(String playListId, String finalVideoUUID) { + List snippets = new ArrayList<>(); String nextPageToken = null; while (true) { String url = PLAY_LIST_ITEMS_BASE_URL + String.format(PLAY_LIST_PARAMS, playListId, apiKey); @@ -41,7 +36,6 @@ public List getVideos(String playListId, String finalVideoUUID) { response = restTemplate.getForObject(url, JsonNode.class); } catch (Exception e) { log.error("Youtube API 호출이 실패했습니다. Youtuber Id {}", playListId); - log.info(e.getMessage()); break; } if (Objects.isNull(response)) { @@ -49,7 +43,7 @@ public List getVideos(String playListId, String finalVideoUUID) { break; } - var containsLastVideo = extractRawVideoInfo(videoInfos, response.path("items"), finalVideoUUID); + var containsLastVideo = extractSnippets(snippets, response.path("items"), finalVideoUUID); if (containsLastVideo) { break; } @@ -58,39 +52,22 @@ public List getVideos(String playListId, String finalVideoUUID) { break; } } - return videoInfos; + return snippets; } private boolean isLastPage(String nextPageToken) { return Objects.isNull(nextPageToken) || nextPageToken.isEmpty(); } - private boolean extractRawVideoInfo(List videoInfos, JsonNode items, String finalVideoUUID) { + private boolean extractSnippets(List snippets, JsonNode items, String finalVideoUUID) { for (JsonNode item : items) { var snippet = item.path("snippet"); var videoId = snippet.path("resourceId").path("videoId").asText(); - var videoTitle = snippet.path("title").asText(); - var videoDescription = snippet.path("description").asText(); if (videoId.equals(finalVideoUUID)) { return true; } - - var address = extractAddress(videoDescription); - if (Objects.nonNull(address)) { - videoInfos.add(new RawVideoInfo(videoId, videoTitle, address)); - continue; - } - log.info("주소를 찾을 수 없습니다. {}", videoDescription); + snippets.add(snippet); } return false; } - - private String extractAddress(String description) { - Pattern pattern = Pattern.compile(ADDRESS_REGEX); - Matcher matcher = pattern.matcher(description); - if (matcher.find()) { - return matcher.group(); - } - return null; - } } diff --git a/src/main/java/team7/inplace/crawling/client/dto/PlaceNode.java b/src/main/java/team7/inplace/crawling/client/dto/PlaceNode.java new file mode 100644 index 00000000..4f919cc3 --- /dev/null +++ b/src/main/java/team7/inplace/crawling/client/dto/PlaceNode.java @@ -0,0 +1,12 @@ +package team7.inplace.crawling.client.dto; + +import com.fasterxml.jackson.databind.JsonNode; + +public record PlaceNode( + JsonNode locationNode, + JsonNode placeNode +) { + public static PlaceNode of(JsonNode locationNode, JsonNode placeNode) { + return new PlaceNode(locationNode, placeNode); + } +} diff --git a/src/main/java/team7/inplace/crawling/client/dto/RawPlace.java b/src/main/java/team7/inplace/crawling/client/dto/RawPlace.java deleted file mode 100644 index feb9f7ad..00000000 --- a/src/main/java/team7/inplace/crawling/client/dto/RawPlace.java +++ /dev/null @@ -1,138 +0,0 @@ -package team7.inplace.crawling.client.dto; - -import com.fasterxml.jackson.databind.JsonNode; -import java.util.ArrayList; -import java.util.List; - -public class RawPlace { - public record Info( - String placeName, - String facility, - String menuImgUrl, - String category, - String address, - String x, - String y, - List offDays, - List openPeriods, - List menus - ) { - public static Info from(JsonNode locationNode, JsonNode placeNode) { - var basicInfo = placeNode.get("basicInfo"); - - String placeName = - basicInfo.has("placenamefull") ? basicInfo.get("placenamefull").asText() : "Unknown Place"; - String facility = basicInfo.has("facilityInfo") - ? basicInfo.get("facilityInfo").toString() : "N/A"; - - String menuImgUrl = basicInfo.has("mainphotourl") ? basicInfo.get("mainphotourl").asText() : ""; - String category = basicInfo.has("category") && basicInfo.get("category").has("catename") - ? basicInfo.get("category").get("catename").asText() : "Unknown Category"; - String address = - basicInfo.has("address") && basicInfo.get("address").has("region") && basicInfo.get("address") - .get("region").has("newaddrfullname") - ? basicInfo.get("address").get("region").get("newaddrfullname").asText() - : "Unknown Address"; - String addressDetail = - basicInfo.has("address") && basicInfo.get("address").has("newaddr") && basicInfo.get("address") - .get("newaddr").has("newaddrfull") - ? basicInfo.get("address").get("newaddr").get("newaddrfull").asText() : ""; - - String x = locationNode.has("documents") && locationNode.get("documents").get(0).has("x") - ? locationNode.get("documents").get(0).get("x").asText() : "0.0"; - String y = locationNode.has("documents") && locationNode.get("documents").get(0).has("y") - ? locationNode.get("documents").get(0).get("y").asText() : "0.0"; - - var timeList = basicInfo.has("openHour") ? basicInfo.get("openHour") : null; - var openPeriods = extractOpenPeriods(timeList.has("periodList") ? timeList.get("periodList") : null); - var offDays = extractOffDays(timeList.has("offdayList") ? timeList.get("offdayList") : null); - var menus = extractMenus(placeNode.has("menuInfo") ? placeNode.get("menuInfo") : null); - - return new Info(placeName, facility, menuImgUrl, category, address + " " + addressDetail, x, y, offDays, - openPeriods, menus); - } - - private static List extractOpenPeriods(JsonNode openTimeList) { - if (openTimeList == null) { - return new ArrayList<>(); - } - List openTimes = new ArrayList<>(); - for (JsonNode openTimeNode : openTimeList) { - for (JsonNode timeNode : openTimeNode.get("timeList")) { - openTimes.add(OpenTime.from(timeNode)); - } - } - return openTimes; - } - - private static List extractOffDays(JsonNode offdayList) { - if (offdayList == null) { - return new ArrayList<>(); // 빈 리스트를 반환하여 null 회피 - } - List offDays = new ArrayList<>(); - for (JsonNode offDayNode : offdayList) { - offDays.add(OffDay.from(offDayNode)); - } - return offDays; - } - - private static List extractMenus(JsonNode menuList) { - if (menuList == null) { - return new ArrayList<>(); - } - List menus = new ArrayList<>(); - for (JsonNode menuNode : menuList.get("menuList")) { - menus.add(Menu.from(menuNode)); - } - return menus; - } - } - - public record OffDay( - String holidayName, - String weekAndDay, - String temporaryHolidays - ) { - public static OffDay from(JsonNode offDayNode) { - String holidayName = offDayNode != null && offDayNode.has("holidayName") - ? offDayNode.get("holidayName").asText() : "Unknown Holiday"; - String weekAndDay = offDayNode != null && offDayNode.has("weekAndDay") - ? offDayNode.get("weekAndDay").asText() : "Unknown Week And Day"; - String temporaryHolidays = offDayNode != null && offDayNode.has("temporaryHolidays") - ? offDayNode.get("temporaryHolidays").asText() : "No Temporary Holidays"; - return new OffDay(holidayName, weekAndDay, temporaryHolidays); - } - } - - public record OpenTime( - String timeName, - String timeSE, - String dayOfWeek - ) { - public static OpenTime from(JsonNode openTimeNode) { - String timeName = openTimeNode != null && openTimeNode.has("timeName") - ? openTimeNode.get("timeName").asText() : "Unknown Time Name"; - String timeSE = openTimeNode != null && openTimeNode.has("timeSE") - ? openTimeNode.get("timeSE").asText() : "Unknown Time Range"; - String dayOfWeek = openTimeNode != null && openTimeNode.has("dayOfWeek") - ? openTimeNode.get("dayOfWeek").asText() : "Unknown Day Of Week"; - return new OpenTime(timeName, timeSE, dayOfWeek); - } - } - - public record Menu( - String menuName, - String menuPrice, - boolean recommend - ) { - public static Menu from(JsonNode menuNode) { - String menuName = menuNode != null && menuNode.has("menu") - ? menuNode.get("menu").asText() : "Unknown Menu"; - String menuPrice = menuNode != null && menuNode.has("price") - ? menuNode.get("price").asText() : "0"; - boolean recommend = menuNode != null && menuNode.has("recommend") - && menuNode.get("recommend").asBoolean(); - return new Menu(menuName, menuPrice, recommend); - } - } -} \ No newline at end of file diff --git a/src/main/java/team7/inplace/crawling/client/dto/RawVideoInfo.java b/src/main/java/team7/inplace/crawling/client/dto/RawVideoInfo.java deleted file mode 100644 index 0faedd74..00000000 --- a/src/main/java/team7/inplace/crawling/client/dto/RawVideoInfo.java +++ /dev/null @@ -1,8 +0,0 @@ -package team7.inplace.crawling.client.dto; - -public record RawVideoInfo( - String videoId, - String videoTitle, - String address -) { -} diff --git a/src/main/java/team7/inplace/global/annotation/Facade.java b/src/main/java/team7/inplace/global/annotation/Facade.java new file mode 100644 index 00000000..1b0f2071 --- /dev/null +++ b/src/main/java/team7/inplace/global/annotation/Facade.java @@ -0,0 +1,20 @@ +package team7.inplace.global.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.springframework.core.annotation.AliasFor; +import org.springframework.stereotype.Component; + +@Target({ElementType.TYPE}) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@Component +public @interface Facade { + @AliasFor( + annotation = Component.class + ) + String value() default ""; +} diff --git a/src/main/java/team7/inplace/place/application/PlaceService.java b/src/main/java/team7/inplace/place/application/PlaceService.java index 1fc5f560..5b118124 100644 --- a/src/main/java/team7/inplace/place/application/PlaceService.java +++ b/src/main/java/team7/inplace/place/application/PlaceService.java @@ -3,11 +3,13 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.stereotype.Service; +import team7.inplace.place.application.command.PlacesCommand.Create; import team7.inplace.place.application.command.PlacesCommand.PlacesCoordinateCommand; import team7.inplace.place.application.command.PlacesCommand.PlacesFilterParamsCommand; import team7.inplace.place.application.dto.PlaceInfo; @@ -25,8 +27,8 @@ public class PlaceService { private final VideoRepository videoRepository; public Page getPlacesWithinRadius( - PlacesCoordinateCommand placesCoordinateCommand, - PlacesFilterParamsCommand placesFilterParamsCommand) { + PlacesCoordinateCommand placesCoordinateCommand, + PlacesFilterParamsCommand placesFilterParamsCommand) { // categories와 influencers 필터 처리 List categoryFilters = null; @@ -35,55 +37,80 @@ public Page getPlacesWithinRadius( // 필터 값이 있을 경우에만 split 처리 if (placesFilterParamsCommand.isCategoryFilterExists()) { categoryFilters = Arrays.stream(placesFilterParamsCommand.categories().split(",")) - .toList(); + .toList(); } if (placesFilterParamsCommand.isInfluencerFilterExists()) { influencerFilters = Arrays.stream(placesFilterParamsCommand.influencers().split(",")) - .toList(); + .toList(); } // 주어진 좌표로 장소를 찾고, 해당 페이지의 결과를 가져옵니다. Page placesPage = getPlacesByDistance(placesCoordinateCommand, categoryFilters, - influencerFilters); + influencerFilters); // Place ID 목록 추출 List placeIds = placesPage.getContent().stream() - .map(Place::getId) - .toList(); + .map(Place::getId) + .toList(); // influencer 조회와 PlaceInfo 변환 List