Skip to content

Commit

Permalink
Update dependencies and fix error handling in interactive.js
Browse files Browse the repository at this point in the history
  • Loading branch information
g1nt0ki committed Feb 3, 2024
1 parent 3ad9922 commit f087417
Show file tree
Hide file tree
Showing 8 changed files with 141 additions and 1,179 deletions.
63 changes: 32 additions & 31 deletions adapters/types.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { util } from '@defillama/sdk';
import { Balances, util } from '@defillama/sdk';

const { blocks: { getChainBlocks } } = util

Expand All @@ -14,7 +14,7 @@ export type FetchResultBase = {
};

export type FetchResultGeneric = FetchResultBase & {
[key: string]: number | string | undefined | IJSON<string>
[key: string]: FetchResponseValue | undefined;
}

export type Fetch = (
Expand All @@ -26,7 +26,7 @@ export type IStartTimestamp = () => Promise<number>

export type BaseAdapter = {
[chain: string]: {
start: IStartTimestamp
start: IStartTimestamp | number
fetch: Fetch;
runAtCurrTime?: boolean;
customBackfill?: Fetch;
Expand Down Expand Up @@ -58,57 +58,58 @@ export type BreakdownAdapter = {
};

export type Adapter = SimpleAdapter | BreakdownAdapter;
export type FetchResponseValue = string | number | Balances;

/**
* Include here new adaptors types
*/

// VOLUME
export type FetchResultVolume = FetchResultBase & {
dailyVolume?: string // | IJSON<string>;
totalVolume?: string // | IJSON<string>;
dailyShortOpenInterest?: string // | IJSON<string>;
dailyLongOpenInterest?: string;
dailyOpenInterest?: string;
dailyVolume?: FetchResponseValue
totalVolume?: FetchResponseValue
dailyShortOpenInterest?: FetchResponseValue
dailyLongOpenInterest?: FetchResponseValue
dailyOpenInterest?: FetchResponseValue
};

// FEES
export type FetchResultFees = FetchResultBase & {
totalFees?: string | IJSON<string>;
dailyFees?: string | IJSON<string>;
dailyUserFees?: string | IJSON<string>;
totalRevenue?: string | IJSON<string>;
dailyRevenue?: string | IJSON<string>;
dailyProtocolRevenue?: string | IJSON<string>;
dailyHoldersRevenue?: string | IJSON<string>;
dailySupplySideRevenue?: string | IJSON<string>;
totalProtocolRevenue?: string | IJSON<string>;
totalSupplySideRevenue?: string | IJSON<string>;
totalUserFees?: string | IJSON<string>;
dailyBribesRevenue?: string | IJSON<string>;
dailyTokenTaxes?: string | IJSON<string>;
totalFees?: FetchResponseValue;
dailyFees?: FetchResponseValue;
dailyUserFees?: FetchResponseValue;
totalRevenue?: FetchResponseValue;
dailyRevenue?: FetchResponseValue;
dailyProtocolRevenue?: FetchResponseValue;
dailyHoldersRevenue?: FetchResponseValue;
dailySupplySideRevenue?: FetchResponseValue;
totalProtocolRevenue?: FetchResponseValue;
totalSupplySideRevenue?: FetchResponseValue;
totalUserFees?: FetchResponseValue;
dailyBribesRevenue?: FetchResponseValue;
dailyTokenTaxes?: FetchResponseValue;
};

// INCENTIVES
export type FetchResultIncentives = FetchResultBase & {
tokenIncentives?: string // | IJSON<string>;
tokenIncentives?: FetchResponseValue
};

// AGGREGATORS
export type FetchResultAggregators = FetchResultBase & {
dailyVolume?: string // | IJSON<string>;
totalVolume?: string // | IJSON<string>;
dailyVolume?: FetchResponseValue
totalVolume?: FetchResponseValue
};

// OPTIONS
export type FetchResultOptions = FetchResultBase & {
totalPremiumVolume?: string // | IJSON<string>
totalNotionalVolume?: string // | IJSON<string>
dailyPremiumVolume?: string // | IJSON<string>
dailyNotionalVolume?: string // | IJSON<string>
dailyShortOpenInterest?: string // | IJSON<string>;
dailyLongOpenInterest?: string;
dailyOpenInterest?: string;
totalPremiumVolume?: FetchResponseValue
totalNotionalVolume?: FetchResponseValue
dailyPremiumVolume?: FetchResponseValue
dailyNotionalVolume?: FetchResponseValue
dailyShortOpenInterest?: FetchResponseValue
dailyLongOpenInterest?: FetchResponseValue
dailyOpenInterest?: FetchResponseValue
};


Expand Down
114 changes: 56 additions & 58 deletions adapters/utils/runAdapter.ts
Original file line number Diff line number Diff line change
@@ -1,65 +1,63 @@
import allSettled, { PromiseRejection, PromiseResolution, PromiseResult } from 'promise.allsettled'
import { BaseAdapter, ChainBlocks, DISABLED_ADAPTER_KEY, FetchResult, FetchResultGeneric, IJSON } from '../types'
import { Balances } from '@defillama/sdk'
import { BaseAdapter, ChainBlocks, DISABLED_ADAPTER_KEY, FetchResultGeneric, } from '../types'

const ONE_DAY_IN_SECONDS = 60 * 60 * 24

export type IRunAdapterResponseFulfilled = FetchResult & {
chain: string
startTimestamp: number
}
export interface IRunAdapterResponseRejected {
chain: string
timestamp: number
error: Error
}

export default async function runAdapter(volumeAdapter: BaseAdapter, cleanCurrentDayTimestamp: number, chainBlocks: ChainBlocks, id?: string, version?: string) {
const cleanPreviousDayTimestamp = cleanCurrentDayTimestamp - ONE_DAY_IN_SECONDS
const chains = Object.keys(volumeAdapter).filter(c => c !== DISABLED_ADAPTER_KEY)
const validStart = ((await Promise.all(chains.map(async (chain) => {
const start = await volumeAdapter[chain]?.start().catch(() => {
console.error(`Failed to get start time for ${id} ${version} ${chain}`)
return Math.trunc(Date.now() / 1000)
})
return [chain, start !== undefined && (start <= cleanPreviousDayTimestamp), start]
}))) as [string, boolean, number][]).reduce((acc, curr) => ({ ...acc, [curr[0]]: [curr[1], curr[2]] }), {} as IJSON<(boolean | number)[]>)
return allSettled(chains
.filter(chain => validStart[chain][0])
.map(async (chain) => {
const fetchFunction = volumeAdapter[chain].customBackfill ?? volumeAdapter[chain].fetch
try {
const startTimestamp = validStart[chain][1]
const result: FetchResultGeneric = await fetchFunction(cleanCurrentDayTimestamp - 1, chainBlocks);
if (id)
console.log("Result before cleaning", id, version, cleanCurrentDayTimestamp, chain, result, JSON.stringify(chainBlocks ?? {}))
cleanResult(result)
return Promise.resolve({
chain,
startTimestamp,
...result
})
} catch (e) {
console.error(`Failed to get value on ${chain}: ${e}`)
return Promise.reject({ chain, error: e, timestamp: cleanPreviousDayTimestamp });
}
})) as Promise<PromiseResult<IRunAdapterResponseFulfilled, IRunAdapterResponseRejected>[]>
}
const cleanPreviousDayTimestamp = cleanCurrentDayTimestamp - ONE_DAY_IN_SECONDS
const chains = Object.keys(volumeAdapter).filter(c => c !== DISABLED_ADAPTER_KEY)
const validStart = {} as {
[chain: string]: {
canRun: boolean,
startTimestamp: number
}
}
await Promise.all(chains.map(setChainValidStart))

const cleanResult = (obj: any) => {
Object.keys(obj).forEach(key => {
if (typeof obj[key] === 'object') cleanResult(obj[key])
else if (!okAttribute(obj[key])) {
console.log("Wrong value", obj[key], "with key", key)
delete obj[key]
}
})
}
return Promise.all(chains.filter(chain => validStart[chain]?.canRun).map(getChainResult))

const okAttribute = (value: any) => {
return !(value && Number.isNaN(+value))
}
async function getChainResult(chain: string) {
const fetchFunction = volumeAdapter[chain].customBackfill ?? volumeAdapter[chain].fetch
try {
const result: FetchResultGeneric = await fetchFunction(cleanCurrentDayTimestamp - 1, chainBlocks);
const ignoreKeys = ['timestamp', 'block']
if (id)
console.log("Result before cleaning", id, version, cleanCurrentDayTimestamp, chain, result, JSON.stringify(chainBlocks ?? {}))
for (const [key, value] of Object.entries(result)) {
if (ignoreKeys.includes(key)) continue;
if (value === undefined || value === null) throw new Error(`Value: ${value} ${key} is undefined or null`)
if (value instanceof Balances) result[key] = await value.getUSDString()
result[key] = +Number(value).toFixed(0)
if (isNaN(result[key] as number)) throw new Error(`Value: ${value} ${key} is NaN`)
}
return {
chain,
startTimestamp: validStart[chain].startTimestamp,
...result
}
} catch (error) {
throw { chain, error }
}
}

const isFulfilled = <T,>(p: PromiseResult<T>): p is PromiseResolution<T> => p.status === 'fulfilled';
const isRejected = <T, E>(p: PromiseResult<T, E>): p is PromiseRejection<E> => p.status === 'rejected';
export const getFulfilledResults = <T,>(results: PromiseResult<T>[]) => results.filter(isFulfilled).map(r => r.value)
export const getRejectedResults = <T, E>(results: PromiseResult<T, E>[]) => results.filter(isRejected).map(r => r.reason)
async function setChainValidStart(chain: string) {
const _start = volumeAdapter[chain]?.start
if (_start === undefined) return;
if (typeof _start === 'number') {
validStart[chain] = {
canRun: _start <= cleanPreviousDayTimestamp,
startTimestamp: _start
}
} else if (_start) {
const defaultStart = Math.trunc(Date.now() / 1000)
const start = await (_start as any)().catch(() => {
console.error(`Failed to get start time for ${id} ${version} ${chain}`)
return defaultStart
})
validStart[chain] = {
canRun: typeof start === 'number' && start <= cleanPreviousDayTimestamp,
startTimestamp: start
}
}
}
}
2 changes: 1 addition & 1 deletion cli/interactive.js
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,10 @@ async function runAdapter(adapterPath, debugMode) {
const startTime = Date.now()
const child = childProcess.spawn('ts-node', ['--transpile-only', 'cli/testAdapter.ts', ...adapterPath.split('/')], {
env,
stdio: ['pipe', 'pipe', 'pipe', 'ipc']
})

child.stdout.pipe(process.stdout);
child.stderr.pipe(process.stderr);

child.on('error', reject)
child.on('close', function (code) {
Expand Down
90 changes: 38 additions & 52 deletions cli/testAdapter.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
require('dotenv').config()
import * as path from 'path'
import { Adapter, AdapterType, ChainBlocks, FetchResultVolume } from '../adapters/types';
import { checkArguments, ERROR_STRING, formatTimestampAsDate, printRejectedVolumes, printVolumes, upperCaseFirst } from './utils';
import { Adapter, AdapterType, ChainBlocks, } from '../adapters/types';
import { checkArguments, ERROR_STRING, formatTimestampAsDate, printVolumes, upperCaseFirst } from './utils';
import { getUniqStartOfTodayTimestamp } from '../helpers/getUniSubgraphVolume';
import runAdapter, { getFulfilledResults, getRejectedResults } from '../adapters/utils/runAdapter'
import runAdapter from '../adapters/utils/runAdapter'
import { canGetBlock, getBlock } from '../helpers/getBlock';
import allSettled from 'promise.allsettled';
import getChainsFromDexAdapter from '../adapters/utils/getChainsFromDexAdapter';

// tmp
Expand All @@ -22,55 +21,42 @@ checkArguments(process.argv)
const adapterType: AdapterType = process.argv[2] as AdapterType
const passedFile = path.resolve(process.cwd(), `./${adapterType}/${process.argv[3]}`);
(async () => {
try {
const cleanDayTimestamp = process.argv[4] ? getUniqStartOfTodayTimestamp(new Date(+process.argv[4] * 1000 + 60 * 60 * 24 * 1000)) : getUniqStartOfTodayTimestamp(new Date())
const endCleanDayTimestamp = cleanDayTimestamp - 1
console.info(`🦙 Running ${process.argv[3].toUpperCase()} adapter 🦙`)
console.info(`_______________________________________`)
// Import module to test
let module: Adapter = (await import(passedFile)).default
console.info(`${upperCaseFirst(adapterType)} for ${formatTimestampAsDate(String(getUniqStartOfTodayTimestamp(new Date(endCleanDayTimestamp * 1000))))}`)
console.info(`_______________________________________\n`)
const cleanDayTimestamp = process.argv[4] ? getUniqStartOfTodayTimestamp(new Date(+process.argv[4] * 1000 + 60 * 60 * 24 * 1000)) : getUniqStartOfTodayTimestamp(new Date())
const endCleanDayTimestamp = cleanDayTimestamp - 1
console.info(`🦙 Running ${process.argv[3].toUpperCase()} adapter 🦙`)
console.info(`_______________________________________`)
// Import module to test
let module: Adapter = (await import(passedFile)).default
console.info(`${upperCaseFirst(adapterType)} for ${formatTimestampAsDate(String(getUniqStartOfTodayTimestamp(new Date(endCleanDayTimestamp * 1000))))}`)
console.info(`_______________________________________\n`)

// Get closest block to clean day. Only for EVM compatible ones.
const allChains = getChainsFromDexAdapter(module).filter(canGetBlock)
// Get closest block to clean day. Only for EVM compatible ones.
const allChains = getChainsFromDexAdapter(module).filter(canGetBlock)

const chainBlocks: ChainBlocks = {};
await allSettled(
allChains.map(async (chain) => {
try {
const latestBlock = await getBlock(endCleanDayTimestamp, chain, chainBlocks).catch((e: any) => console.error(`${e.message}; ${endCleanDayTimestamp}, ${chain}`))
if (latestBlock)
chainBlocks[chain] = latestBlock
} catch (e) { console.log(e) }
})
);
const chainBlocks: ChainBlocks = {};
await Promise.all(allChains.map(async (chain) => {
try {
const latestBlock = await getBlock(endCleanDayTimestamp, chain, chainBlocks).catch((e: any) => console.error(`${e.message}; ${endCleanDayTimestamp}, ${chain}`))
if (latestBlock)
chainBlocks[chain] = latestBlock
} catch (e) { console.log(e) }
}))

if ("adapter" in module) {
const adapter = module.adapter
// Get adapter
const volumes = await runAdapter(adapter, endCleanDayTimestamp, chainBlocks)
const fulfilledResults = getFulfilledResults(volumes)
const rejectedResults = getRejectedResults(volumes)
printVolumes(fulfilledResults, adapter)
printRejectedVolumes(rejectedResults)
console.info("\n")
} else if ("breakdown" in module) {
const breakdownAdapter = module.breakdown
const allVolumes = await Promise.all(Object.entries(breakdownAdapter).map(async ([version, adapter]) =>
await runAdapter(adapter, cleanDayTimestamp, chainBlocks).then(res => ({ version, res }))
))
allVolumes.forEach((promise) => {
console.info("Version ->", promise.version.toUpperCase())
console.info("---------")
const fulfilledResults = getFulfilledResults(promise.res)
const rejectedResults = getRejectedResults(promise.res)
printVolumes(fulfilledResults, breakdownAdapter[promise.version])
printRejectedVolumes(rejectedResults)
})
} else throw new Error("No compatible adapter found")
} catch (error) {
console.error(ERROR_STRING)
console.error(error)
}
if ("adapter" in module) {
const adapter = module.adapter
// Get adapter
const volumes = await runAdapter(adapter, endCleanDayTimestamp, chainBlocks)
printVolumes(volumes, adapter)
console.info("\n")
} else if ("breakdown" in module) {
const breakdownAdapter = module.breakdown
const allVolumes = await Promise.all(Object.entries(breakdownAdapter).map(([version, adapter]) =>
runAdapter(adapter, cleanDayTimestamp, chainBlocks).then(res => ({ version, res }))
))
allVolumes.forEach(({ version, res }) => {
console.info("Version ->", version.toUpperCase())
console.info("---------")
printVolumes(res, breakdownAdapter[version])
})
} else throw new Error("No compatible adapter found")
})()
18 changes: 2 additions & 16 deletions cli/utils.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
import { getLatestBlock } from "@defillama/sdk/build/util";
import { clearTimeout } from "timers";
import { Adapter, BaseAdapter, IJSON } from "../adapters/types";
import { IRunAdapterResponseFulfilled, IRunAdapterResponseRejected } from "../adapters/utils/runAdapter";
import { BaseAdapter, } from "../adapters/types";
import { humanizeNumber } from "@defillama/sdk/build/computeTVL/humanizeNumber";

export const ERROR_STRING = '------ ERROR ------'
Expand All @@ -24,7 +22,7 @@ export async function getLatestBlockRetry(chain: string) {
}
}

export function printVolumes(volumes: IRunAdapterResponseFulfilled[], baseAdapter?: BaseAdapter) {
export function printVolumes(volumes: any[], baseAdapter?: BaseAdapter) {
const exclude2Print = ['startTimestamp', 'chain']
volumes.forEach((element) => {
const methodology = baseAdapter?.[element.chain].meta?.methodology
Expand All @@ -47,18 +45,6 @@ export function printVolumes(volumes: IRunAdapterResponseFulfilled[], baseAdapte
});
}

export function printRejectedVolumes(volumes: IRunAdapterResponseRejected[]) {
volumes.forEach((element) => {
if (typeof element.chain === 'string')
console.info(element.chain.toUpperCase(), "👇")
if (element.timestamp !== undefined)
console.info(`Timestamp attempted: ${formatTimestampAsDate(String(element.timestamp))}`)
else console.info("No timestamp defined")
console.info(element.error)
console.info('\n')
});
}

export function formatTimestampAsDate(timestamp: string) {
const date = new Date(Number(timestamp) * 1000);
return `${date.getUTCDate()}/${date.getUTCMonth() + 1}/${date.getUTCFullYear()}`;
Expand Down
Loading

0 comments on commit f087417

Please sign in to comment.