Skip to content

Commit

Permalink
Bug/#235 websocket 버그 해결, token db에 저장 (#240)
Browse files Browse the repository at this point in the history
* ✨ feat: token entity 추가

* ✨ feat: entity 에 저장, expire 검사 로직 추가

* 🐛 fix: token 주입으로 로직 변경

* ♻️ refactor: token 주입으로 변경, 그로 인한 오류 수정 및 console.log 삭제

* 🐛 fix: live 데이터 수집 오류 해결, 데이터 없을 때 insert 오류 해결

* 🐛 fix: stock, livedata entity 수정

* ♻️ refactor: develop 환경시 logging 활성화

* 📦️ ci: production 환경일 때 작동되게 변경
  • Loading branch information
swkim12345 authored Nov 25, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent cb22ad5 commit e82a6dd
Showing 11 changed files with 210 additions and 64 deletions.
2 changes: 1 addition & 1 deletion packages/backend/src/configs/typeormConfig.ts
Original file line number Diff line number Diff line change
@@ -21,6 +21,6 @@ export const typeormDevelopConfig: TypeOrmModuleOptions = {
password: process.env.DB_PASS,
database: process.env.DB_NAME,
entities: [__dirname + '/../**/*.entity.{js,ts}'],
//logging: true,
logging: true,
synchronize: true,
};
28 changes: 28 additions & 0 deletions packages/backend/src/scraper/domain/openapiToken.entity.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
import { Column, Entity, PrimaryColumn } from 'typeorm';

@Entity({ name: 'openapi_token' })
export class OpenapiToken {
@PrimaryColumn({ name: 'account' })
account: string;

@Column({ name: 'apiUrl' })
api_url: string;

@Column({ name: 'key' })
api_key: string;

@Column({ name: 'password' })
api_password: string;

@Column({ name: 'token', length: 512 })
api_token?: string;

@Column({ name: 'tokenExpire' })
api_token_expire?: Date;

@Column({ name: 'websocketKey' })
websocket_key?: string;

@Column({ name: 'websocketKeyExpire' })
websocket_key_expire?: Date;
}
Original file line number Diff line number Diff line change
@@ -13,7 +13,7 @@ import {
} from '../type/openapiDetailData.type';
import { TR_IDS } from '../type/openapiUtil.type';
import { getOpenApi } from '../util/openapiUtil.api';
import { openApiToken } from './openapiToken.api';
import { OpenapiTokenApi } from './openapiToken.api';
import { KospiStock } from '@/stock/domain/kospiStock.entity';
import { Stock } from '@/stock/domain/stock.entity';
import { StockDaily } from '@/stock/domain/stockData.entity';
@@ -27,6 +27,7 @@ export class OpenapiDetailData {
'/uapi/domestic-stock/v1/quotations/search-stock-info';
private readonly intervals = 1000;
constructor(
private readonly openApiToken: OpenapiTokenApi,
private readonly datasource: DataSource,
@Inject('winston') private readonly logger: Logger,
) {
@@ -38,13 +39,13 @@ export class OpenapiDetailData {
if (process.env.NODE_ENV !== 'production') return;
const entityManager = this.datasource.manager;
const stocks = await entityManager.find(Stock);
const configCount = openApiToken.configs.length;
const configCount = this.openApiToken.configs.length;
const chunkSize = Math.ceil(stocks.length / configCount);

for (let i = 0; i < configCount; i++) {
this.logger.info(openApiToken.configs[i]);
this.logger.info(this.openApiToken.configs[i]);
const chunk = stocks.slice(i * chunkSize, (i + 1) * chunkSize);
this.getDetailDataChunk(chunk, openApiToken.configs[i]);
this.getDetailDataChunk(chunk, this.openApiToken.configs[i]);
}
}

39 changes: 30 additions & 9 deletions packages/backend/src/scraper/openapi/api/openapiLiveData.api.ts
Original file line number Diff line number Diff line change
@@ -1,26 +1,47 @@
import { Injectable } from '@nestjs/common';
import { Inject, Injectable } from '@nestjs/common';
import { DataSource } from 'typeorm';
import { Logger } from 'winston';
import { Stock } from '@/stock/domain/stock.entity';
import { StockLiveData } from '@/stock/domain/stockLiveData.entity';

@Injectable()
export class OpenapiLiveData {
public readonly TR_ID: string = 'H0STCNT0';
constructor(private readonly datasource: DataSource) {}
constructor(
private readonly datasource: DataSource,
@Inject('winston') private readonly logger: Logger,
) {}

async saveLiveData(data: StockLiveData[]) {
await this.datasource.manager
.getRepository(StockLiveData)
.createQueryBuilder()
.insert()
.into(StockLiveData)
.values(data)
.execute();
const exists = await this.datasource.manager.exists(StockLiveData, {
where: {
stock: { id: data[0].stock.id },
},
});
if (exists) {
await this.datasource.manager
.getRepository(StockLiveData)
.createQueryBuilder()
.update()
.set(data[0])
.where('stock.id = :stockId', { stockId: data[0].stock.id })
.execute();
} else {
await this.datasource.manager
.getRepository(StockLiveData)
.createQueryBuilder()
.insert()
.into(StockLiveData)
.values(data)
.execute();
}
}

convertLiveData(messages: Record<string, string>[]): StockLiveData[] {
const stockData: StockLiveData[] = [];
messages.map((message) => {
const stockLiveData = new StockLiveData();
stockLiveData.stock = { id: message.STOCK_ID } as Stock;
stockLiveData.currentPrice = parseFloat(message.STCK_PRPR);
stockLiveData.changeRate = parseFloat(message.PRDY_CTRT);
stockLiveData.volume = parseInt(message.CNTG_VOL);
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@ import {
} from '../type/openapiMinuteData.type';
import { TR_IDS } from '../type/openapiUtil.type';
import { getCurrentTime, getOpenApi } from '../util/openapiUtil.api';
import { openApiToken } from './openapiToken.api';
import { OpenapiTokenApi } from './openapiToken.api';
import { Stock } from '@/stock/domain/stock.entity';
import { StockData, StockMinutely } from '@/stock/domain/stockData.entity';

@@ -27,9 +27,10 @@ export class OpenapiMinuteData {
private flip: number = 0;
constructor(
private readonly datasource: DataSource,
private readonly openApiToken: OpenapiTokenApi,
@Inject('winston') private readonly logger: Logger,
) {
this.getStockData();
//this.getStockData();
}

@Cron('0 1 * * 1-5')
@@ -126,16 +127,16 @@ export class OpenapiMinuteData {
}
}

@Cron(`*/${STOCK_CUT} 9-15 * * 1-5`)
//@Cron(`*/${STOCK_CUT} 9-15 * * 1-5`)
getMinuteData() {
if (process.env.NODE_ENV !== 'production') return;
const configCount = openApiToken.configs.length;
const configCount = this.openApiToken.configs.length;
const stock = this.stock[this.flip % STOCK_CUT];
this.flip++;
const chunkSize = Math.ceil(stock.length / configCount);
for (let i = 0; i < configCount; i++) {
const chunk = stock.slice(i * chunkSize, (i + 1) * chunkSize);
this.getMinuteDataChunk(chunk, openApiToken.configs[i]);
this.getMinuteDataChunk(chunk, this.openApiToken.configs[i]);
}
}

Original file line number Diff line number Diff line change
@@ -14,7 +14,7 @@ import {
getPreviousDate,
getTodayDate,
} from '../util/openapiUtil.api';
import { openApiToken } from './openapiToken.api';
import { OpenapiTokenApi } from './openapiToken.api';
import { Stock } from '@/stock/domain/stock.entity';
import {
StockData,
@@ -46,6 +46,7 @@ export class OpenapiPeriodData {
'/uapi/domestic-stock/v1/quotations/inquire-daily-itemchartprice';
constructor(
private readonly datasource: DataSource,
private readonly openApiToken: OpenapiTokenApi,
@Inject('winston') private readonly logger: Logger,
) {
//this.getItemChartPriceCheck();
@@ -59,7 +60,7 @@ export class OpenapiPeriodData {
isTrading: true,
},
});
const configCount = openApiToken.configs.length;
const configCount = this.openApiToken.configs.length;
const chunkSize = Math.ceil(stocks.length / configCount);

for (let i = 0; i < configCount; i++) {
@@ -95,7 +96,7 @@ export class OpenapiPeriodData {
let isFail = false;

while (!isFail) {
configIdx = (configIdx + 1) % openApiToken.configs.length;
configIdx = (configIdx + 1) % this.openApiToken.configs.length;
this.setStockPeriod(stockPeriod, stock.id!, end);

// chart 데이터가 있는 지 확인 -> 리턴
@@ -129,7 +130,7 @@ export class OpenapiPeriodData {
try {
const response = await getOpenApi(
this.url,
openApiToken.configs[configIdx],
this.openApiToken.configs[configIdx],
query,
TR_IDS.ITEM_CHART_PRICE,
);
125 changes: 112 additions & 13 deletions packages/backend/src/scraper/openapi/api/openapiToken.api.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,21 @@
import { Inject } from '@nestjs/common';
import { Global, Inject, Injectable } from '@nestjs/common';
import { Cron } from '@nestjs/schedule';
import { DataSource } from 'typeorm';
import { Logger } from 'winston';
import { OpenapiToken } from '../../domain/openapiToken.entity';
import { openApiConfig } from '../config/openapi.config';
import { OpenapiException } from '../util/openapiCustom.error';
import { postOpenApi } from '../util/openapiUtil.api';
import { logger } from '@/configs/logger.config';

class OpenapiTokenApi {
@Global()
@Injectable()
export class OpenapiTokenApi {
private config: (typeof openApiConfig)[] = [];
constructor(@Inject('winston') private readonly logger: Logger) {
constructor(
@Inject('winston') private readonly logger: Logger,
private readonly datasource: DataSource,
) {
if (process.env.NODE_ENV !== 'production') return;
const accounts = openApiConfig.STOCK_ACCOUNT!.split(',');
const api_keys = openApiConfig.STOCK_API_KEY!.split(',');
const api_passwords = openApiConfig.STOCK_API_PASSWORD!.split(',');
@@ -27,14 +34,111 @@ class OpenapiTokenApi {
STOCK_API_PASSWORD: api_passwords[i],
});
}
this.initAuthenValue();
this.init();
}

get configs() {
//TODO : 현재 구조에서 받아올 때마다 확인후 할당으로 변경
this.init();
return this.config;
}

@Cron('30 0 * * 1-5')
async init() {
const tokens = await this.convertConfigToTokenEntity(this.config);
const config = await this.getPropertyFromDB(tokens);
const expired = config.filter(
(val) =>
this.isTokenExpired(val.api_token_expire) &&
this.isTokenExpired(val.websocket_key_expire),
);

if (expired.length || !config.length) {
await this.initAuthenValue();
const newTokens = await this.convertConfigToTokenEntity(this.config);
this.savePropertyToDB(newTokens);
} else {
this.config = await this.convertTokenEntityToConfig(config);
}
}

private isTokenExpired(startDate?: Date) {
if (!startDate) return true;
const now = new Date();
//실제 만료 시간은 24시간이지만, 문제의 소지가 발생하는 것을 방지하기 위해 20시간으로 설정함.
const baseTimeToMilliSec = 20 * 60 * 60 * 1000;
const timeDiff = now.getTime() - startDate.getTime();

return timeDiff >= baseTimeToMilliSec;
}

private async convertTokenEntityToConfig(tokens: OpenapiToken[]) {
const result: (typeof openApiConfig)[] = [];
tokens.forEach((val) => {
const config: typeof openApiConfig = {
STOCK_ACCOUNT: val.account,
STOCK_API_KEY: val.api_key,
STOCK_API_PASSWORD: val.api_password,
STOCK_API_TOKEN: val.api_token,
STOCK_URL: val.api_url,
STOCK_WEBSOCKET_KEY: val.websocket_key,
};
result.push(config);
});
return result;
}

private async convertConfigToTokenEntity(config: (typeof openApiConfig)[]) {
const result: OpenapiToken[] = [];
config.forEach((val) => {
const token = new OpenapiToken();
if (
val.STOCK_URL &&
val.STOCK_ACCOUNT &&
val.STOCK_API_KEY &&
val.STOCK_API_PASSWORD
) {
token.api_url = val.STOCK_URL;
token.account = val.STOCK_ACCOUNT;
token.api_key = val.STOCK_API_KEY;
token.api_password = val.STOCK_API_PASSWORD;
}
token.api_token = val.STOCK_API_TOKEN;
token.websocket_key = val.STOCK_WEBSOCKET_KEY;
token.api_token_expire = new Date();
token.websocket_key_expire = new Date();
result.push(token);
});
return result;
}

private async savePropertyToDB(tokens: OpenapiToken[]) {
tokens.forEach(async (val) => {
this.datasource.manager.save(OpenapiToken, val);
});
}

private async getPropertyFromDB(tokens: OpenapiToken[]) {
const result: OpenapiToken[] = [];
await Promise.all(
tokens.map(async (val) => {
const findByToken = await this.datasource.manager.findOne(
OpenapiToken,
{
where: {
account: val.account,
api_key: val.api_key,
api_password: val.api_password,
},
},
);
if (findByToken) {
result.push(findByToken);
}
}),
);
return result;
}

private async initAuthenValue() {
const delay = 60000;
const delayMinute = delay / 1000 / 60;
@@ -59,8 +163,7 @@ class OpenapiTokenApi {
}
}

@Cron('50 0 * * 1-5')
async initAccessToken() {
private async initAccessToken() {
const updatedConfig = await Promise.all(
this.config.map(async (val) => {
val.STOCK_API_TOKEN = await this.getToken(val)!;
@@ -70,8 +173,7 @@ class OpenapiTokenApi {
this.config = updatedConfig;
}

@Cron('50 0 * * 1-5')
async initWebSocketKey() {
private async initWebSocketKey() {
const updatedConfig = await Promise.all(
this.config.map(async (val) => {
val.STOCK_WEBSOCKET_KEY = await this.getWebSocketKey(val)!;
@@ -107,6 +209,3 @@ class OpenapiTokenApi {
return tmp.approval_key as string;
}
}

const openApiToken = new OpenapiTokenApi(logger);
export { openApiToken };
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@ import { OpenapiDetailData } from './api/openapiDetailData.api';
import { OpenapiLiveData } from './api/openapiLiveData.api';
import { OpenapiMinuteData } from './api/openapiMinuteData.api';
import { OpenapiPeriodData } from './api/openapiPeriodData.api';
import { OpenapiTokenApi } from './api/openapiToken.api';
import { OpenapiScraperService } from './openapi-scraper.service';
import { WebsocketClient } from './websocketClient.service';
import { Stock } from '@/stock/domain/stock.entity';
@@ -32,6 +33,7 @@ import { StockLiveData } from '@/stock/domain/stockLiveData.entity';
],
controllers: [],
providers: [
OpenapiTokenApi,
OpenapiPeriodData,
OpenapiMinuteData,
OpenapiDetailData,
Loading

0 comments on commit e82a6dd

Please sign in to comment.