forked from DefiLlama/dimension-adapters
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request DefiLlama#1 from DefiLlama/master
Taking commits from base branch
- Loading branch information
Showing
915 changed files
with
15,183 additions
and
19,797 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,65 +1,150 @@ | ||
| import allSettled, { PromiseRejection, PromiseResolution, PromiseResult } from 'promise.allsettled' | ||
| import { BaseAdapter, ChainBlocks, DISABLED_ADAPTER_KEY, FetchResult, FetchResultGeneric, IJSON } from '../types' | ||
| import { Balances, ChainApi, getEventLogs, getProvider } from '@defillama/sdk' | ||
| import { BaseAdapter, ChainBlocks, DISABLED_ADAPTER_KEY, Fetch, FetchGetLogsOptions, FetchOptions, FetchResultGeneric, FetchV2, } from '../types' | ||
| import { getBlock } from "../../helpers/getBlock"; | ||
| import { getUniqStartOfTodayTimestamp } from '../../helpers/getUniSubgraphFees'; | ||
|
|
||
| const ONE_DAY_IN_SECONDS = 60 * 60 * 24 | ||
|
|
||
| export type IRunAdapterResponseFulfilled = FetchResult & { | ||
| chain: string | ||
| startTimestamp: number | ||
| } | ||
| export interface IRunAdapterResponseRejected { | ||
| chain: string | ||
| timestamp: number | ||
| error: Error | ||
| } | ||
| export default async function runAdapter(volumeAdapter: BaseAdapter, cleanCurrentDayTimestamp: number, chainBlocks: ChainBlocks, id?: string, version?: string, { | ||
| adapterVersion = 1 | ||
| }: any = {}) { | ||
| const closeToCurrentTime = Math.trunc(Date.now() / 1000) - cleanCurrentDayTimestamp < 24 * 60 * 60 // 12 hours | ||
| const chains = Object.keys(volumeAdapter).filter(c => c !== DISABLED_ADAPTER_KEY) | ||
| const validStart = {} as { | ||
| [chain: string]: { | ||
| canRun: boolean, | ||
| startTimestamp: number | ||
| } | ||
| } | ||
| await Promise.all(chains.map(setChainValidStart)) | ||
|
|
||
| export default async function runAdapter(volumeAdapter: BaseAdapter, cleanCurrentDayTimestamp: number, chainBlocks: ChainBlocks, id?: string, version?: string) { | ||
| const cleanPreviousDayTimestamp = cleanCurrentDayTimestamp - ONE_DAY_IN_SECONDS | ||
| const chains = Object.keys(volumeAdapter).filter(c => c !== DISABLED_ADAPTER_KEY) | ||
| const validStart = ((await Promise.all(chains.map(async (chain) => { | ||
| const start = await volumeAdapter[chain]?.start().catch(() => { | ||
| console.error(`Failed to get start time for ${id} ${version} ${chain}`) | ||
| return Math.trunc(Date.now() / 1000) | ||
| }) | ||
| return [chain, start !== undefined && (start <= cleanPreviousDayTimestamp), start] | ||
| }))) as [string, boolean, number][]).reduce((acc, curr) => ({ ...acc, [curr[0]]: [curr[1], curr[2]] }), {} as IJSON<(boolean | number)[]>) | ||
| return allSettled(chains | ||
| .filter(chain => validStart[chain][0]) | ||
| .map(async (chain) => { | ||
| const fetchFunction = volumeAdapter[chain].customBackfill ?? volumeAdapter[chain].fetch | ||
| try { | ||
| const startTimestamp = validStart[chain][1] | ||
| const result: FetchResultGeneric = await fetchFunction(cleanCurrentDayTimestamp - 1, chainBlocks); | ||
| if (id) | ||
| console.log("Result before cleaning", id, version, cleanCurrentDayTimestamp, chain, result, JSON.stringify(chainBlocks ?? {})) | ||
| cleanResult(result) | ||
| return Promise.resolve({ | ||
| chain, | ||
| startTimestamp, | ||
| ...result | ||
| }) | ||
| } catch (e) { | ||
| console.error(`Failed to get value on ${chain}: ${e}`) | ||
| return Promise.reject({ chain, error: e, timestamp: cleanPreviousDayTimestamp }); | ||
| } | ||
| })) as Promise<PromiseResult<IRunAdapterResponseFulfilled, IRunAdapterResponseRejected>[]> | ||
| } | ||
| return Promise.all(chains.filter(chain => validStart[chain]?.canRun).map(getChainResult)) | ||
|
|
||
| const cleanResult = (obj: any) => { | ||
| Object.keys(obj).forEach(key => { | ||
| if (typeof obj[key] === 'object') cleanResult(obj[key]) | ||
| else if (!okAttribute(obj[key])) { | ||
| console.log("Wrong value", obj[key], "with key", key) | ||
| delete obj[key] | ||
| async function getChainResult(chain: string) { | ||
| const fetchFunction = volumeAdapter[chain].customBackfill ?? volumeAdapter[chain].fetch | ||
| try { | ||
| const options = await getOptionsObject(cleanCurrentDayTimestamp, chain, chainBlocks) | ||
| let result: any | ||
| if (adapterVersion === 1) { | ||
| result = await (fetchFunction as Fetch)(options.toTimestamp, chainBlocks, options); | ||
| } else if (adapterVersion === 2) { | ||
| result = await (fetchFunction as FetchV2)(options); | ||
| result.timestamp = options.toTimestamp | ||
| } else { | ||
| throw new Error(`Adapter version ${adapterVersion} not supported`) | ||
| } | ||
| const ignoreKeys = ['timestamp', 'block'] | ||
| // if (id) | ||
| // console.log("Result before cleaning", id, version, cleanCurrentDayTimestamp, chain, result, JSON.stringify(chainBlocks ?? {})) | ||
| for (const [key, value] of Object.entries(result)) { | ||
| if (ignoreKeys.includes(key)) continue; | ||
| if (value === undefined || value === null) { // dont store undefined or null values | ||
| delete result[key] | ||
| continue; | ||
| } | ||
| }) | ||
| } | ||
| // if (value === undefined || value === null) throw new Error(`Value: ${value} ${key} is undefined or null`) | ||
| if (value instanceof Balances) result[key] = await value.getUSDString() | ||
| result[key] = +Number(result[key]).toFixed(0) | ||
| if (isNaN(result[key] as number)) throw new Error(`[${chain}]Value: ${value} ${key} is NaN`) | ||
| } | ||
| return { | ||
| chain, | ||
| startTimestamp: validStart[chain].startTimestamp, | ||
| ...result | ||
| } | ||
| } catch (error) { | ||
| try { (error as any).chain = chain } catch { } | ||
| throw error | ||
| } | ||
| } | ||
|
|
||
| const okAttribute = (value: any) => { | ||
| return !(value && Number.isNaN(+value)) | ||
| } | ||
| async function getOptionsObject(timestamp: number, chain: string, chainBlocks: ChainBlocks): Promise<FetchOptions> { | ||
| const withinTwoHours = Math.trunc(Date.now() / 1000) - timestamp < 2 * 60 * 60 // 2 hours | ||
| const createBalances: () => Balances = () => { | ||
| return new Balances({ timestamp: closeToCurrentTime ? undefined : timestamp, chain }) | ||
| } | ||
| const toTimestamp = timestamp - 1 | ||
| const fromTimestamp = toTimestamp - ONE_DAY_IN_SECONDS | ||
| const fromChainBlocks = {} | ||
| const getFromBlock = async () => await getBlock(fromTimestamp, chain, fromChainBlocks) | ||
| const getToBlock = async () => await getBlock(toTimestamp, chain, chainBlocks) | ||
| const getLogs = async ({ target, targets, onlyArgs = true, fromBlock, toBlock, flatten = true, eventAbi, topics, topic, cacheInCloud = false, skipCacheRead = false, entireLog = false, }: FetchGetLogsOptions) => { | ||
| fromBlock = fromBlock ?? await getFromBlock() | ||
| toBlock = toBlock ?? await getToBlock() | ||
|
|
||
| return getEventLogs({ fromBlock, toBlock, chain, target, targets, onlyArgs, flatten, eventAbi, topics, topic, cacheInCloud, skipCacheRead, entireLog, }) | ||
| } | ||
|
|
||
| const isFulfilled = <T,>(p: PromiseResult<T>): p is PromiseResolution<T> => p.status === 'fulfilled'; | ||
| const isRejected = <T, E>(p: PromiseResult<T, E>): p is PromiseRejection<E> => p.status === 'rejected'; | ||
| export const getFulfilledResults = <T,>(results: PromiseResult<T>[]) => results.filter(isFulfilled).map(r => r.value) | ||
| export const getRejectedResults = <T, E>(results: PromiseResult<T, E>[]) => results.filter(isRejected).map(r => r.reason) | ||
| // we intentionally add a delay to avoid fetching the same block before it is cached | ||
| await randomDelay() | ||
|
|
||
| let fromBlock, toBlock | ||
| // we fetch current block and previous blocks only for evm chains/ chains we have RPC for | ||
| if (getProvider(chain)) { | ||
| fromBlock = await getFromBlock() | ||
| toBlock = await getToBlock() | ||
| } | ||
| const fromApi = new ChainApi({ chain, timestamp: fromTimestamp, block: fromBlock }) | ||
| const api = new ChainApi({ chain, timestamp: withinTwoHours ? undefined : timestamp, block: toBlock }) | ||
| const startOfDay = getUniqStartOfTodayTimestamp(new Date(toTimestamp * 1000)) | ||
| const startTimestamp = fromTimestamp | ||
| const endTimestamp = toTimestamp | ||
| const getStartBlock = getFromBlock | ||
| const getEndBlock = getToBlock | ||
| const toApi = api | ||
|
|
||
| return { | ||
| createBalances, | ||
| getBlock, | ||
| toTimestamp, | ||
| fromTimestamp, | ||
| getFromBlock, | ||
| getToBlock, | ||
| getLogs, | ||
| chain, | ||
| fromApi, | ||
| toApi, | ||
| api, | ||
| startOfDay, | ||
| startTimestamp, | ||
| endTimestamp, | ||
| getStartBlock, | ||
| getEndBlock, | ||
| } | ||
| } | ||
|
|
||
| // code for random 1-4 second delay | ||
| async function randomDelay() { | ||
| const delay = Math.floor(Math.random() * 4) + 1 | ||
| return new Promise((resolve) => setTimeout(resolve, delay * 1000)) | ||
| } | ||
|
|
||
| async function setChainValidStart(chain: string) { | ||
| const cleanPreviousDayTimestamp = cleanCurrentDayTimestamp - ONE_DAY_IN_SECONDS | ||
| const _start = volumeAdapter[chain]?.start | ||
| if (_start === undefined) return; | ||
| if (typeof _start === 'number') { | ||
| validStart[chain] = { | ||
| canRun: _start <= cleanPreviousDayTimestamp, | ||
| startTimestamp: _start | ||
| } | ||
| } else if (_start) { | ||
| const defaultStart = Math.trunc(Date.now() / 1000) | ||
| if (closeToCurrentTime) {// intentionally set to true to allow for backfilling | ||
| validStart[chain] = { | ||
| canRun: true, | ||
| startTimestamp: defaultStart | ||
| } | ||
| return; | ||
| } | ||
| const start = await (_start as any)().catch(() => { | ||
| console.error(`Failed to get start time for ${id} ${version} ${chain}`) | ||
| return defaultStart | ||
| }) | ||
| validStart[chain] = { | ||
| canRun: typeof start === 'number' && start <= cleanPreviousDayTimestamp, | ||
| startTimestamp: start | ||
| } | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.