Feature/#312 - 기간 별 데이터와 지수 데이터 큐 적용 (#316)

boostcampwm-2024 · Dec 1, 2024 · aa1f76d · aa1f76d
2 parents a2c4be9 + c5132b5
commit aa1f76d
Show file tree

Hide file tree

Showing 3 changed files with 174 additions and 81 deletions.
diff --git a/packages/backend/src/scraper/openapi/api/openapiIndex.api.ts b/packages/backend/src/scraper/openapi/api/openapiIndex.api.ts
@@ -20,6 +20,7 @@ import {
 import { TR_ID } from '../type/openapiUtil.type';
 import { getOpenApi, getTodayDate } from '../util/openapiUtil.api';
 import { OpenapiLiveData } from './openapiLiveData.api';
+import { Json, OpenapiQueue } from '@/scraper/openapi/queue/openapi.queue';
 import { Stock } from '@/stock/domain/stock.entity';
 import { StockLiveData } from '@/stock/domain/stockLiveData.entity';
 
@@ -43,6 +44,7 @@ export class OpenapiIndex extends Openapi {
     protected readonly datasource: DataSource,
     protected readonly config: OpenapiTokenApi,
     private readonly openapiLiveData: OpenapiLiveData,
+    private readonly openapiQueue: OpenapiQueue,
   ) {
     const interval = 1000;
     super(datasource, config, interval);
@@ -53,52 +55,8 @@ export class OpenapiIndex extends Openapi {
   @Cron('* 9-14 * * 1-5')
   @Cron('0-30 15 * * 1-5')
   async start() {
-    await this.step((await this.config.configs()).length - 1);
-  }
-
-  private initKospiData() {
-    const name = '코스피';
-    const initStockData = new Stock();
-    initStockData.id = IndexRateStockId.kospi;
-    initStockData.groupCode = IndexRateGroupCodeStock.kospi;
-    initStockData.name = name;
-    return initStockData;
-  }
-
-  private initKosdaqData() {
-    const name = '코스닥';
-    const initStockData = new Stock();
-    initStockData.id = IndexRateStockId.kosdaq;
-    initStockData.groupCode = IndexRateGroupCodeStock.kosdaq;
-    initStockData.name = name;
-    return initStockData;
-  }
-
-  private initUsdKrwData() {
-    const name = '원 달러 환율';
-    const initStockData = new Stock();
-    initStockData.id = IndexRateStockId.usd_krw;
-    initStockData.groupCode = IndexRateGroupCodeStock.usd_krw;
-    initStockData.name = name;
-    return initStockData;
-  }
-
-  private async initData() {
-    await this.saveStock(this.initKosdaqData());
-    await this.saveStock(this.initKospiData());
-    await this.saveStock(this.initUsdKrwData());
-  }
-
-  private async saveStock(data: Stock) {
-    const target = Stock;
-
-    await this.datasource.manager
-      .getRepository(target)
-      .createQueryBuilder()
-      .insert()
-      .values(data)
-      .orUpdate(['is_trading'], ['id'])
-      .execute();
+    // await this.step((await this.config.configs()).length - 1);
+    await this.getIndexData();
   }
 
   protected async step(idx: number) {
@@ -167,32 +125,6 @@ export class OpenapiIndex extends Openapi {
     }
   }
 
-  private convertResToStockIndex(res: StockIndex, stockId: string) {
-    const result = new StockLiveData();
-    result.currentPrice = parseFloat(res.bstp_nmix_prpr);
-    result.changeRate = parseFloat(res.bstp_nmix_prdy_ctrt);
-    result.high = parseFloat(res.bstp_nmix_hgpr);
-    result.low = parseFloat(res.bstp_nmix_lwpr);
-    result.open = parseFloat(res.bstp_nmix_oprc);
-    result.volume = parseInt(res.acml_vol);
-    result.updatedAt = new Date();
-    result.stock = { id: stockId } as Stock;
-    return result;
-  }
-
-  private convertResToExchangeRate(res: ExchangeRate, stockId: string) {
-    const result = new StockLiveData();
-    result.currentPrice = parseFloat(res.ovrs_nmix_prpr);
-    result.changeRate = parseFloat(res.prdy_ctrt);
-    result.high = parseFloat(res.ovrs_prod_hgpr);
-    result.low = parseFloat(res.ovrs_prod_lwpr);
-    result.open = parseFloat(res.ovrs_prod_oprc);
-    result.volume = parseInt(res.acml_vol);
-    result.updatedAt = new Date();
-    result.stock = { id: stockId } as Stock;
-    return result;
-  }
-
   protected convertResToEntity(
     res: StockIndex | ExchangeRate,
     stockId: string,
@@ -230,4 +162,109 @@ export class OpenapiIndex extends Openapi {
       fid_period_div_code: period,
     };
   }
+
+  private async getIndexData() {
+    this.openapiQueue.enqueue({
+      url: this.INDEX_URL,
+      query: this.indexQuery(this.KOSPI_ID),
+      trId: this.TR_ID_INDEX,
+      callback: this.getIndexDataCallback(this.KOSPI_ID, true),
+    });
+    this.openapiQueue.enqueue({
+      url: this.INDEX_URL,
+      query: this.indexQuery(this.KOSDAQ_ID),
+      trId: this.TR_ID_INDEX,
+      callback: this.getIndexDataCallback(this.KOSDAQ_ID, true),
+    });
+    this.openapiQueue.enqueue({
+      url: this.INDEX_URL,
+      query: this.indexQuery(this.USD_KRW_RATE),
+      trId: this.TR_ID_INDEX,
+      callback: this.getIndexDataCallback(this.USD_KRW_RATE, true),
+    });
+  }
+
+  private getIndexDataCallback(stockId: string, isStock: boolean) {
+    return async (data: Json) => {
+      if (!data.output) return;
+      if (isStock && isStockIndex(data.output)) {
+        const indexData = this.convertResToEntity(data.output, stockId);
+        await this.save(indexData);
+      } else if (isExchangeRate(data.output)) {
+        const rateData = this.convertResToEntity(data.output, stockId);
+        await this.save(rateData);
+      }
+    };
+  }
+
+  private initKospiData() {
+    const name = '코스피';
+    const initStockData = new Stock();
+    initStockData.id = IndexRateStockId.kospi;
+    initStockData.groupCode = IndexRateGroupCodeStock.kospi;
+    initStockData.name = name;
+    return initStockData;
+  }
+
+  private initKosdaqData() {
+    const name = '코스닥';
+    const initStockData = new Stock();
+    initStockData.id = IndexRateStockId.kosdaq;
+    initStockData.groupCode = IndexRateGroupCodeStock.kosdaq;
+    initStockData.name = name;
+    return initStockData;
+  }
+
+  private initUsdKrwData() {
+    const name = '원 달러 환율';
+    const initStockData = new Stock();
+    initStockData.id = IndexRateStockId.usd_krw;
+    initStockData.groupCode = IndexRateGroupCodeStock.usd_krw;
+    initStockData.name = name;
+    return initStockData;
+  }
+
+  private async initData() {
+    await this.saveStock(this.initKosdaqData());
+    await this.saveStock(this.initKospiData());
+    await this.saveStock(this.initUsdKrwData());
+  }
+
+  private async saveStock(data: Stock) {
+    const target = Stock;
+
+    await this.datasource.manager
+      .getRepository(target)
+      .createQueryBuilder()
+      .insert()
+      .values(data)
+      .orUpdate(['is_trading'], ['id'])
+      .execute();
+  }
+
+  private convertResToStockIndex(res: StockIndex, stockId: string) {
+    const result = new StockLiveData();
+    result.currentPrice = parseFloat(res.bstp_nmix_prpr);
+    result.changeRate = parseFloat(res.bstp_nmix_prdy_ctrt);
+    result.high = parseFloat(res.bstp_nmix_hgpr);
+    result.low = parseFloat(res.bstp_nmix_lwpr);
+    result.open = parseFloat(res.bstp_nmix_oprc);
+    result.volume = parseInt(res.acml_vol);
+    result.updatedAt = new Date();
+    result.stock = { id: stockId } as Stock;
+    return result;
+  }
+
+  private convertResToExchangeRate(res: ExchangeRate, stockId: string) {
+    const result = new StockLiveData();
+    result.currentPrice = parseFloat(res.ovrs_nmix_prpr);
+    result.changeRate = parseFloat(res.prdy_ctrt);
+    result.high = parseFloat(res.ovrs_prod_hgpr);
+    result.low = parseFloat(res.ovrs_prod_lwpr);
+    result.open = parseFloat(res.ovrs_prod_oprc);
+    result.volume = parseInt(res.acml_vol);
+    result.updatedAt = new Date();
+    result.stock = { id: stockId } as Stock;
+    return result;
+  }
 }
diff --git a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts
@@ -15,12 +15,13 @@ import {
   getTodayDate,
 } from '../util/openapiUtil.api';
 import { OpenapiTokenApi } from './openapiToken.api';
+import { Json, OpenapiQueue } from '@/scraper/openapi/queue/openapi.queue';
 import { Stock } from '@/stock/domain/stock.entity';
 import {
-  StockData,
   StockDaily,
-  StockWeekly,
+  StockData,
   StockMonthly,
+  StockWeekly,
   StockYearly,
 } from '@/stock/domain/stockData.entity';
 
@@ -47,8 +48,11 @@ export class OpenapiPeriodData {
   constructor(
     private readonly datasource: DataSource,
     private readonly openApiToken: OpenapiTokenApi,
+    private readonly openApiQueue: OpenapiQueue,
     @Inject('winston') private readonly logger: Logger,
-  ) {}
+  ) {
+    // this.getItemChartPriceCheck();
+  }
 
   @Cron('0 1 * * 1-5')
   async getItemChartPriceCheck() {
@@ -58,24 +62,75 @@ export class OpenapiPeriodData {
         isTrading: true,
       },
     });
-
     await this.getChartData(stocks, 'Y');
     await this.getChartData(stocks, 'M');
     await this.getChartData(stocks, 'W');
     await this.getChartData(stocks, 'D');
   }
 
+  /**
+   * 월, 년의 경우 마지막 데이터를 업데이트 하는 형식으로 변경해야됨
+   */
+  private getLiveDataSaveCallback(
+    stockId: string,
+    entity: typeof StockData,
+    period: Period,
+    end: string,
+  ) {
+    return async (data: Json) => {
+      if (!data.output2 || !Array.isArray(data.output2)) return;
+      // 이거 빈값들어오는 케이스 있음(빈값 필터링 안하면 요청이 매우 많아짐)
+      data.output2 = data.output2.filter(
+        (data) => Object.keys(data).length !== 0,
+      );
+      if (data.output2.length === 0) return;
+      await this.saveChartData(entity, stockId, data.output2 as ChartData[]);
+      const { endDate, startDate } = this.updateDates(end, period);
+      const query = this.getItemChartPriceQuery(
+        stockId,
+        startDate,
+        endDate,
+        period,
+      );
+      this.openApiQueue.enqueue({
+        url: this.url,
+        query,
+        trId: TR_IDS.ITEM_CHART_PRICE,
+        callback: this.getLiveDataSaveCallback(
+          stockId,
+          entity,
+          period,
+          endDate,
+        ),
+      });
+    };
+  }
+
   private async getChartData(chunk: Stock[], period: Period) {
-    const baseTime = INTERVALS;
     const entity = DATE_TO_ENTITY[period];
-
-    let time = 0;
     for (const stock of chunk) {
-      time += baseTime;
-      setTimeout(() => this.processStockData(stock, period, entity), time);
+      this.processStockData2(stock, period, entity);
     }
   }
 
+  private async processStockData2(
+    stock: Stock,
+    period: Period,
+    entity: typeof StockData,
+  ) {
+    const end = getTodayDate();
+    const start = getPreviousDate(end, DATE_TO_MONTH[period]);
+
+    const query = this.getItemChartPriceQuery(stock.id!, start, end, period);
+
+    this.openApiQueue.enqueue({
+      url: this.url,
+      query,
+      trId: TR_IDS.ITEM_CHART_PRICE,
+      callback: this.getLiveDataSaveCallback(stock.id!, entity, period, end),
+    });
+  }
+
   private async processStockData(
     stock: Stock,
     period: Period,

diff --git a/packages/backend/src/scraper/openapi/queue/openapi.queue.ts b/packages/backend/src/scraper/openapi/queue/openapi.queue.ts
@@ -7,6 +7,7 @@ import { PriorityQueue } from '@/scraper/openapi/util/priorityQueue';
 
 export interface Json {
   output: Record<string, string> | Record<string, string>[];
+  output2: Record<string, string>[];
 }
 
 export interface OpenapiQueueNodeValue {