스마트스토어 크롤링 기능 수정

backend/src/cron/cron.service.ts

@@ -15,6 +15,7 @@ import Redis from 'ioredis';
 import { InjectRedis } from '@songkeys/nestjs-redis';
 import { getProductInfo } from 'src/utils/product.info';
 import { CacheService } from 'src/cache/cache.service';
+import { RecentProductInfoMap } from 'src/types/product';
 
 @Injectable()
 export class CronService {
@@ -70,10 +71,18 @@ export class CronService {
             }
         }
 
+        const recentProductInfoMap: RecentProductInfoMap = recentProductInfo.reduce((map, product) => {
+            map[product.productId] = { productName: product.productName, imageUrl: product.imageUrl };
+            return map;
+        }, {} as RecentProductInfoMap);
+
         totalProducts.sort((a, b) => a.id.localeCompare(b.id));
-        recentProductInfo.sort((a, b) => a.productId.localeCompare(b.productId));
-        const infoUpdatedProducts = totalProducts.filter((product, idx) => {
-            const recentInfo = recentProductInfo[idx];
+        const infoUpdatedProducts = totalProducts.filter((product) => {
+            const recentInfo = recentProductInfoMap[product.id];
+            if (recentInfo === undefined) {
+                console.log('fail', product.productCode);
+                return false;
+            }
             if (product.productName !== recentInfo.productName || product.imageUrl !== recentInfo.imageUrl) {
                 product.productName = recentInfo.productName;
                 product.imageUrl = recentInfo.imageUrl;

backend/src/types/product.ts

@@ -0,0 +1,6 @@
+export type RecentProductInfoMap = {
+    [key: string]: {
+        productName: string;
+        imageUrl: string;
+    };
+};

backend/src/utils/product.info.ts

@@ -1,19 +1,12 @@
 import { HttpException, HttpStatus } from '@nestjs/common';
 import { ProductInfoDto } from 'src/dto/product.info.dto';
-import {
-    API_VERSION_1,
-    API_VERSION_NEUTRAL,
-    BASE_URL_11ST,
-    BROWSER_VERSION_20,
-    OPEN_API_KEY_11ST,
-    REGEX_SHOP,
-} from 'src/constants';
+import { API_VERSION_1, API_VERSION_NEUTRAL, BASE_URL_11ST, OPEN_API_KEY_11ST, REGEX_SHOP } from 'src/constants';
 import { JSDOM } from 'jsdom';
 import * as convert from 'xml-js';
 import * as iconv from 'iconv-lite';
 import axios from 'axios';
-import * as randomUseragent from 'random-useragent';
 import { ProductIdentifierDto } from 'src/dto/product.identifier';
+import { getRandomUserAgent } from './util';
 
 export async function getProductInfo(shop: string, productCode: string): Promise<ProductInfoDto> {
     if (shop === '11번가') {
@@ -66,9 +59,7 @@ async function getProductInfo11st(productCode: string): Promise<ProductInfoDto>
 }
 
 async function getProductInfoByBrandSmartStore(productCode: string): Promise<ProductInfoDto> {
-    const userAgent = randomUseragent.getRandom(function (ua) {
-        return parseFloat(ua.browserVersion) >= BROWSER_VERSION_20;
-    });
+    const userAgent = getRandomUserAgent();
     const instance = axios.create();
     const smartstoreURL = `https://smartstore.naver.com/main/products/${productCode}`;
     instance.interceptors.request.use((req: any) => {

backend/src/utils/util.ts

@@ -1,3 +1,5 @@
+import * as randomUseragent from 'random-useragent';
+
 export function createRandomNumber(min: number, max: number) {
     return Math.floor(Math.random() * (max - min + 1)) + min;
 }
@@ -9,3 +11,24 @@ export function getSecondsUntilMidnight() {
     const secondsUntilMidnight = Math.floor((midnight.getTime() - now.getTime()) / 1000);
     return secondsUntilMidnight;
 }
+
+function isVersionInRange(version: string) {
+    const numericVersion = parseFloat(version);
+    return (numericVersion >= 20 && numericVersion < 29) || (numericVersion >= 535 && numericVersion < 538);
+}
+
+function isOsNameValid(osName: string) {
+    return osName === 'Windows' || osName === 'Mac OS';
+}
+
+export function getRandomUserAgent() {
+    const userAgent = randomUseragent.getRandom(function (ua) {
+        return (
+            ua.userAgent.includes('AppleWebKit') &&
+            ua.browserName === 'Chrome' &&
+            isOsNameValid(ua.osName) &&
+            isVersionInRange(ua.engineVersion)
+        );
+    });
+    return userAgent;
+}