diff --git a/packages/backend/src/scraper/openapi/api/openapiIndex.api.ts b/packages/backend/src/scraper/openapi/api/openapiIndex.api.ts index d303f076..80748faa 100644 --- a/packages/backend/src/scraper/openapi/api/openapiIndex.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiIndex.api.ts @@ -20,6 +20,7 @@ import { import { TR_ID } from '../type/openapiUtil.type'; import { getOpenApi, getTodayDate } from '../util/openapiUtil.api'; import { OpenapiLiveData } from './openapiLiveData.api'; +import { Json, OpenapiQueue } from '@/scraper/openapi/queue/openapi.queue'; import { Stock } from '@/stock/domain/stock.entity'; import { StockLiveData } from '@/stock/domain/stockLiveData.entity'; @@ -43,6 +44,7 @@ export class OpenapiIndex extends Openapi { protected readonly datasource: DataSource, protected readonly config: OpenapiTokenApi, private readonly openapiLiveData: OpenapiLiveData, + private readonly openapiQueue: OpenapiQueue, ) { const interval = 1000; super(datasource, config, interval); @@ -53,52 +55,8 @@ export class OpenapiIndex extends Openapi { @Cron('* 9-14 * * 1-5') @Cron('0-30 15 * * 1-5') async start() { - await this.step((await this.config.configs()).length - 1); - } - - private initKospiData() { - const name = '코스피'; - const initStockData = new Stock(); - initStockData.id = IndexRateStockId.kospi; - initStockData.groupCode = IndexRateGroupCodeStock.kospi; - initStockData.name = name; - return initStockData; - } - - private initKosdaqData() { - const name = '코스닥'; - const initStockData = new Stock(); - initStockData.id = IndexRateStockId.kosdaq; - initStockData.groupCode = IndexRateGroupCodeStock.kosdaq; - initStockData.name = name; - return initStockData; - } - - private initUsdKrwData() { - const name = '원 달러 환율'; - const initStockData = new Stock(); - initStockData.id = IndexRateStockId.usd_krw; - initStockData.groupCode = IndexRateGroupCodeStock.usd_krw; - initStockData.name = name; - return initStockData; - } - - private async initData() { - await this.saveStock(this.initKosdaqData()); - await this.saveStock(this.initKospiData()); - await this.saveStock(this.initUsdKrwData()); - } - - private async saveStock(data: Stock) { - const target = Stock; - - await this.datasource.manager - .getRepository(target) - .createQueryBuilder() - .insert() - .values(data) - .orUpdate(['is_trading'], ['id']) - .execute(); + // await this.step((await this.config.configs()).length - 1); + await this.getIndexData(); } protected async step(idx: number) { @@ -167,32 +125,6 @@ export class OpenapiIndex extends Openapi { } } - private convertResToStockIndex(res: StockIndex, stockId: string) { - const result = new StockLiveData(); - result.currentPrice = parseFloat(res.bstp_nmix_prpr); - result.changeRate = parseFloat(res.bstp_nmix_prdy_ctrt); - result.high = parseFloat(res.bstp_nmix_hgpr); - result.low = parseFloat(res.bstp_nmix_lwpr); - result.open = parseFloat(res.bstp_nmix_oprc); - result.volume = parseInt(res.acml_vol); - result.updatedAt = new Date(); - result.stock = { id: stockId } as Stock; - return result; - } - - private convertResToExchangeRate(res: ExchangeRate, stockId: string) { - const result = new StockLiveData(); - result.currentPrice = parseFloat(res.ovrs_nmix_prpr); - result.changeRate = parseFloat(res.prdy_ctrt); - result.high = parseFloat(res.ovrs_prod_hgpr); - result.low = parseFloat(res.ovrs_prod_lwpr); - result.open = parseFloat(res.ovrs_prod_oprc); - result.volume = parseInt(res.acml_vol); - result.updatedAt = new Date(); - result.stock = { id: stockId } as Stock; - return result; - } - protected convertResToEntity( res: StockIndex | ExchangeRate, stockId: string, @@ -230,4 +162,109 @@ export class OpenapiIndex extends Openapi { fid_period_div_code: period, }; } + + private async getIndexData() { + this.openapiQueue.enqueue({ + url: this.INDEX_URL, + query: this.indexQuery(this.KOSPI_ID), + trId: this.TR_ID_INDEX, + callback: this.getIndexDataCallback(this.KOSPI_ID, true), + }); + this.openapiQueue.enqueue({ + url: this.INDEX_URL, + query: this.indexQuery(this.KOSDAQ_ID), + trId: this.TR_ID_INDEX, + callback: this.getIndexDataCallback(this.KOSDAQ_ID, true), + }); + this.openapiQueue.enqueue({ + url: this.INDEX_URL, + query: this.indexQuery(this.USD_KRW_RATE), + trId: this.TR_ID_INDEX, + callback: this.getIndexDataCallback(this.USD_KRW_RATE, true), + }); + } + + private getIndexDataCallback(stockId: string, isStock: boolean) { + return async (data: Json) => { + if (!data.output) return; + if (isStock && isStockIndex(data.output)) { + const indexData = this.convertResToEntity(data.output, stockId); + await this.save(indexData); + } else if (isExchangeRate(data.output)) { + const rateData = this.convertResToEntity(data.output, stockId); + await this.save(rateData); + } + }; + } + + private initKospiData() { + const name = '코스피'; + const initStockData = new Stock(); + initStockData.id = IndexRateStockId.kospi; + initStockData.groupCode = IndexRateGroupCodeStock.kospi; + initStockData.name = name; + return initStockData; + } + + private initKosdaqData() { + const name = '코스닥'; + const initStockData = new Stock(); + initStockData.id = IndexRateStockId.kosdaq; + initStockData.groupCode = IndexRateGroupCodeStock.kosdaq; + initStockData.name = name; + return initStockData; + } + + private initUsdKrwData() { + const name = '원 달러 환율'; + const initStockData = new Stock(); + initStockData.id = IndexRateStockId.usd_krw; + initStockData.groupCode = IndexRateGroupCodeStock.usd_krw; + initStockData.name = name; + return initStockData; + } + + private async initData() { + await this.saveStock(this.initKosdaqData()); + await this.saveStock(this.initKospiData()); + await this.saveStock(this.initUsdKrwData()); + } + + private async saveStock(data: Stock) { + const target = Stock; + + await this.datasource.manager + .getRepository(target) + .createQueryBuilder() + .insert() + .values(data) + .orUpdate(['is_trading'], ['id']) + .execute(); + } + + private convertResToStockIndex(res: StockIndex, stockId: string) { + const result = new StockLiveData(); + result.currentPrice = parseFloat(res.bstp_nmix_prpr); + result.changeRate = parseFloat(res.bstp_nmix_prdy_ctrt); + result.high = parseFloat(res.bstp_nmix_hgpr); + result.low = parseFloat(res.bstp_nmix_lwpr); + result.open = parseFloat(res.bstp_nmix_oprc); + result.volume = parseInt(res.acml_vol); + result.updatedAt = new Date(); + result.stock = { id: stockId } as Stock; + return result; + } + + private convertResToExchangeRate(res: ExchangeRate, stockId: string) { + const result = new StockLiveData(); + result.currentPrice = parseFloat(res.ovrs_nmix_prpr); + result.changeRate = parseFloat(res.prdy_ctrt); + result.high = parseFloat(res.ovrs_prod_hgpr); + result.low = parseFloat(res.ovrs_prod_lwpr); + result.open = parseFloat(res.ovrs_prod_oprc); + result.volume = parseInt(res.acml_vol); + result.updatedAt = new Date(); + result.stock = { id: stockId } as Stock; + return result; + } } diff --git a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts index 778f82ee..cc290472 100644 --- a/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts +++ b/packages/backend/src/scraper/openapi/api/openapiPeriodData.api.ts @@ -15,12 +15,13 @@ import { getTodayDate, } from '../util/openapiUtil.api'; import { OpenapiTokenApi } from './openapiToken.api'; +import { Json, OpenapiQueue } from '@/scraper/openapi/queue/openapi.queue'; import { Stock } from '@/stock/domain/stock.entity'; import { - StockData, StockDaily, - StockWeekly, + StockData, StockMonthly, + StockWeekly, StockYearly, } from '@/stock/domain/stockData.entity'; @@ -47,8 +48,11 @@ export class OpenapiPeriodData { constructor( private readonly datasource: DataSource, private readonly openApiToken: OpenapiTokenApi, + private readonly openApiQueue: OpenapiQueue, @Inject('winston') private readonly logger: Logger, - ) {} + ) { + // this.getItemChartPriceCheck(); + } @Cron('0 1 * * 1-5') async getItemChartPriceCheck() { @@ -58,24 +62,75 @@ export class OpenapiPeriodData { isTrading: true, }, }); - await this.getChartData(stocks, 'Y'); await this.getChartData(stocks, 'M'); await this.getChartData(stocks, 'W'); await this.getChartData(stocks, 'D'); } + /** + * 월, 년의 경우 마지막 데이터를 업데이트 하는 형식으로 변경해야됨 + */ + private getLiveDataSaveCallback( + stockId: string, + entity: typeof StockData, + period: Period, + end: string, + ) { + return async (data: Json) => { + if (!data.output2 || !Array.isArray(data.output2)) return; + // 이거 빈값들어오는 케이스 있음(빈값 필터링 안하면 요청이 매우 많아짐) + data.output2 = data.output2.filter( + (data) => Object.keys(data).length !== 0, + ); + if (data.output2.length === 0) return; + await this.saveChartData(entity, stockId, data.output2 as ChartData[]); + const { endDate, startDate } = this.updateDates(end, period); + const query = this.getItemChartPriceQuery( + stockId, + startDate, + endDate, + period, + ); + this.openApiQueue.enqueue({ + url: this.url, + query, + trId: TR_IDS.ITEM_CHART_PRICE, + callback: this.getLiveDataSaveCallback( + stockId, + entity, + period, + endDate, + ), + }); + }; + } + private async getChartData(chunk: Stock[], period: Period) { - const baseTime = INTERVALS; const entity = DATE_TO_ENTITY[period]; - - let time = 0; for (const stock of chunk) { - time += baseTime; - setTimeout(() => this.processStockData(stock, period, entity), time); + this.processStockData2(stock, period, entity); } } + private async processStockData2( + stock: Stock, + period: Period, + entity: typeof StockData, + ) { + const end = getTodayDate(); + const start = getPreviousDate(end, DATE_TO_MONTH[period]); + + const query = this.getItemChartPriceQuery(stock.id!, start, end, period); + + this.openApiQueue.enqueue({ + url: this.url, + query, + trId: TR_IDS.ITEM_CHART_PRICE, + callback: this.getLiveDataSaveCallback(stock.id!, entity, period, end), + }); + } + private async processStockData( stock: Stock, period: Period, diff --git a/packages/backend/src/scraper/openapi/queue/openapi.queue.ts b/packages/backend/src/scraper/openapi/queue/openapi.queue.ts index 051995a4..e46f46db 100644 --- a/packages/backend/src/scraper/openapi/queue/openapi.queue.ts +++ b/packages/backend/src/scraper/openapi/queue/openapi.queue.ts @@ -7,6 +7,7 @@ import { PriorityQueue } from '@/scraper/openapi/util/priorityQueue'; export interface Json { output: Record | Record[]; + output2: Record[]; } export interface OpenapiQueueNodeValue {