Skip to content

Commit

Permalink
morning bugs
Browse files Browse the repository at this point in the history
  • Loading branch information
@mikecot
mikecot committed Sep 2, 2024
1 parent 729472b commit 54030e5
Show file tree
Hide file tree
Showing 15 changed files with 102 additions and 94 deletions.
35 changes: 4 additions & 31 deletions drizzle/0028_great_war_machine.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ CREATE TABLE IF NOT EXISTS "unique_visitors" (
"timestamp" timestamp NOT NULL,
"value" integer
);

CREATE UNIQUE INDEX timestamp_unique_idx ON unique_visitors(timestamp);

--> statement-breakpoint
CREATE UNIQUE INDEX IF NOT EXISTS "ph2idx" ON "provider_health2" ("provider","spec","geolocation","interface","guid");--> statement-breakpoint
CREATE INDEX IF NOT EXISTS "provider_health2_provider_idx" ON "provider_health2" ("provider");--> statement-breakpoint
Expand All @@ -25,34 +28,4 @@ EXCEPTION
WHEN duplicate_object THEN null;
END $$;

INSERT INTO "unique_visitors" ("timestamp", "value") VALUES
('2023-07-21', 74899),
('2023-07-22', 123629),
('2023-07-23', 91242),
('2023-07-24', 80877),
('2023-07-25', 78870),
('2023-07-26', 90323),
('2023-07-27', 72563),
('2023-07-28', 64798),
('2023-07-29', 72809),
('2023-07-30', 124049),
('2023-07-31', 129330),
('2023-08-01', 135893),
('2023-08-02', 127403),
('2023-08-03', 110376),
('2023-08-04', 107232),
('2023-08-05', 128450),
('2023-08-06', 132260),
('2023-08-07', 136603),
('2023-08-08', 150081),
('2023-08-09', 160409),
('2023-08-10', 147368),
('2023-08-11', 139764),
('2023-08-12', 158252),
('2023-08-13', 155363),
('2023-08-14', 172072),
('2023-08-15', 163546),
('2023-08-16', 152826),
('2023-08-17', 139213),
('2023-08-18', 140847),
('2023-08-19', 156530);

27 changes: 20 additions & 7 deletions src/indexer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -302,21 +302,21 @@ const fillUp = async (db: PostgresJsDatabase, rpcConnection: RpcConnection) => {
return;
}

if (latestDbBlock.length != 0) {
const tHeight = latestDbBlock[0].height;
try {
const tHeight = latestDbBlock[0]?.height;
if (tHeight != null) {
dbHeight = tHeight;
logger.info(`fillUp: DB height set to latest DB block height: ${dbHeight}`);
}
} catch (error) {
logger.error(`Error accessing height: ${error}`);
logger.error(`Type of latestDbBlock[0]: ${typeof latestDbBlock}`);
logger.error(`Value of latestDbBlock[0]: ${JSON.stringify(latestDbBlock)}`);
}

// await doBatch(db, rpcConnection.client, rpcConnection.clientTm, dbHeight, latestHeight);

// if (latestHeight > dbHeight) {
logger.info(`fillUp: Starting batch process for DB height ${dbHeight} and blockchain height ${latestHeight}`);
await doBatch(db, rpcConnection.client, rpcConnection.clientTm, dbHeight, latestHeight);
logger.info('fillUp: Batch process completed');
// }

let latestDbBlock2;
try {
Expand All @@ -330,7 +330,20 @@ const fillUp = async (db: PostgresJsDatabase, rpcConnection: RpcConnection) => {
return;
}

if (latestHeight != latestDbBlock2[0].height) {
let dbHeight2 = consts.JSINFO_INDEXER_START_BLOCK;
try {
const tHeight2 = latestDbBlock2[0]?.height;
if (tHeight2 != null) {
dbHeight2 = tHeight2;
logger.info(`fillUp: DB height set to latest DB block height: ${dbHeight2}`);
}
} catch (error) {
logger.error(`Error accessing height: ${error}`);
logger.error(`Type of latestDbBlock2[0]: ${typeof latestDbBlock2}`);
logger.error(`Value of latestDbBlock2[0]: ${JSON.stringify(latestDbBlock2)}`);
}

if (latestHeight != dbHeight2) {
logger.error(`fillUp: latestHeight ${latestHeight} != latestDbBlock2[0].height ${latestDbBlock2[0].height}`);
db = await GetJsinfoDb();
fillUpBackoffRetryWTimeout(db, rpcConnection);
Expand Down
5 changes: 3 additions & 2 deletions src/indexer/agregators/consumerRelayPayments/aggConsumerAllTimeRelayPayments.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// src/indexer/agregators/aggConsumerAllTimeRelayPayments.ts

import { PostgresJsDatabase } from 'drizzle-orm/postgres-js';
import { sql } from "drizzle-orm";
import { sql, ne } from "drizzle-orm";
import * as JsinfoConsumerAgrSchema from '../../../schemas/jsinfoSchema/consumerRelayPaymentsAgregation';
import { logger } from "../../../utils/utils";
import { PgColumn } from 'drizzle-orm/pg-core';
Expand All @@ -26,7 +26,8 @@ export async function aggConsumerAllTimeRelayPayments(db: PostgresJsDatabase) {
.groupBy(
JsinfoConsumerAgrSchema.aggConsumerDailyRelayPayments.consumer,
JsinfoConsumerAgrSchema.aggConsumerDailyRelayPayments.specId
)
).where(ne(JsinfoConsumerAgrSchema.aggConsumerDailyRelayPayments.relaySum, 0))

if (aggResults.length === 0) {
logger.error("aggConsumerDailyRelayPayments: no agg results found")
return;
Expand Down
5 changes: 3 additions & 2 deletions src/indexer/agregators/consumerRelayPayments/aggConsumerDailyRelayPayments.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// src/indexer/agregators/aggConsumerDailyRelayPayments.ts

import { PostgresJsDatabase } from 'drizzle-orm/postgres-js';
import { isNotNull, sql, and } from "drizzle-orm";
import { isNotNull, sql, and, ne } from "drizzle-orm";
import * as JsinfoConsumerAgrSchema from '../../../schemas/jsinfoSchema/consumerRelayPaymentsAgregation';
import { DoInChunks, logger } from "../../../utils/utils";
import { PgColumn } from 'drizzle-orm/pg-core';
Expand Down Expand Up @@ -66,7 +66,8 @@ export async function aggConsumerDailyRelayPayments(db: PostgresJsDatabase) {
.where(
and(
sql`${JsinfoConsumerAgrSchema.aggConsumerHourlyRelayPayments.datehour} >= ${startTime}`,
isNotNull(JsinfoConsumerAgrSchema.aggConsumerHourlyRelayPayments.consumer)
isNotNull(JsinfoConsumerAgrSchema.aggConsumerHourlyRelayPayments.consumer),
ne(JsinfoConsumerAgrSchema.aggConsumerHourlyRelayPayments.relaySum, 0)
)
)
.groupBy(
Expand Down
5 changes: 3 additions & 2 deletions src/indexer/agregators/consumerRelayPayments/aggConsumerHourlyRelayPayments.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// src/indexer/agregators/aggConsumerHourlyRelayPayments.ts

import { PostgresJsDatabase } from 'drizzle-orm/postgres-js';
import { isNotNull, sql, and } from "drizzle-orm";
import { isNotNull, sql, and, ne } from "drizzle-orm";
import * as JsinfoSchema from "../../../schemas/jsinfoSchema/jsinfoSchema";
import * as JsinfoConsumerAgrSchema from '../../../schemas/jsinfoSchema/consumerRelayPaymentsAgregation';
import { DoInChunks, logger } from "../../../utils/utils";
Expand Down Expand Up @@ -67,7 +67,8 @@ export async function aggConsumerHourlyRelayPayments(db: PostgresJsDatabase) {
.where(
and(
sql`${JsinfoSchema.relayPayments.datetime} >= ${startTime}`,
isNotNull(JsinfoSchema.relayPayments.consumer)
isNotNull(JsinfoSchema.relayPayments.consumer),
ne(JsinfoSchema.relayPayments.relays, 0)
)
)
.groupBy(
Expand Down
2 changes: 1 addition & 1 deletion src/indexer/agregators/consumerRelayPayments/aggConsumerRelayPayments.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ export const aggConsumerRelayPayments = async (db: PostgresJsDatabase) => {
const executionTime = Date.now() - start;
logger.info(`Successfully executed aggConsumerAllTimeRelayPayments. Execution time: ${executionTime} ms`);
} catch (e) {
logger.error(`Failed to update aggConsumerAllTimeRelayPayments. Error: ${(e as Error).message}`, { stack: (e as Error).stack });
logger.error(`Failed to update aggConsumerAllTimeRelayPayments. Error: ${(e as Error).message}`, { stack: (e as Error).stack }, e);
isRunning = false;
return;
}
Expand Down
5 changes: 3 additions & 2 deletions src/indexer/agregators/providerRelayPayments/aggProviderAllTimeRelayPayments.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// src/indexer/agregators/aggProviderAllTimeRelayPayments.ts

import { PostgresJsDatabase } from 'drizzle-orm/postgres-js';
import { sql } from "drizzle-orm";
import { sql, ne } from "drizzle-orm";
import * as JsinfoProviderAgrSchema from '../../../schemas/jsinfoSchema/providerRelayPaymentsAgregation';
import { logger } from "../../../utils/utils";
import { PgColumn } from 'drizzle-orm/pg-core';
Expand All @@ -26,7 +26,8 @@ export async function aggProviderAllTimeRelayPayments(db: PostgresJsDatabase) {
.groupBy(
JsinfoProviderAgrSchema.aggDailyRelayPayments.provider,
JsinfoProviderAgrSchema.aggDailyRelayPayments.specId
)
).where(ne(JsinfoProviderAgrSchema.aggDailyRelayPayments.relaySum, 0))

if (aggResults.length === 0) {
logger.error("aggDailyRelayPayments: no agg results found")
return;
Expand Down
5 changes: 3 additions & 2 deletions src/indexer/agregators/providerRelayPayments/aggProviderDailyRelayPayments.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// src/indexer/agregators/aggProviderDailyRelayPayments.ts

import { PostgresJsDatabase } from 'drizzle-orm/postgres-js';
import { isNotNull, sql, and } from "drizzle-orm";
import { isNotNull, sql, and, ne } from "drizzle-orm";
import * as JsinfoProviderAgrSchema from '../../../schemas/jsinfoSchema/providerRelayPaymentsAgregation';
import { DoInChunks, logger } from "../../../utils/utils";
import { PgColumn } from 'drizzle-orm/pg-core';
Expand Down Expand Up @@ -66,7 +66,8 @@ export async function aggProviderDailyRelayPayments(db: PostgresJsDatabase) {
.where(
and(
sql`${JsinfoProviderAgrSchema.aggHourlyRelayPayments.datehour} >= ${startTime}`,
isNotNull(JsinfoProviderAgrSchema.aggHourlyRelayPayments.provider)
isNotNull(JsinfoProviderAgrSchema.aggHourlyRelayPayments.provider),
ne(JsinfoProviderAgrSchema.aggHourlyRelayPayments.relaySum, 0)
)
)
.groupBy(
Expand Down
5 changes: 3 additions & 2 deletions src/indexer/agregators/providerRelayPayments/aggProviderHourlyRelayPayments.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// src/indexer/agregators/aggProviderHourlyRelayPayments.ts

import { PostgresJsDatabase } from 'drizzle-orm/postgres-js';
import { isNotNull, sql, and } from "drizzle-orm";
import { isNotNull, sql, and, ne } from "drizzle-orm";
import * as JsinfoSchema from "../../../schemas/jsinfoSchema/jsinfoSchema";
import * as JsinfoProviderAgrSchema from '../../../schemas/jsinfoSchema/providerRelayPaymentsAgregation';
import { DoInChunks, logger } from "../../../utils/utils";
Expand Down Expand Up @@ -67,7 +67,8 @@ export async function aggProviderHourlyRelayPayments(db: PostgresJsDatabase) {
.where(
and(
sql`${JsinfoSchema.relayPayments.datetime} >= ${startTime}`,
isNotNull(JsinfoSchema.relayPayments.provider)
isNotNull(JsinfoSchema.relayPayments.provider),
ne(JsinfoSchema.relayPayments.relays, 0)
)
)
.groupBy(
Expand Down
4 changes: 1 addition & 3 deletions src/indexer/blockchainEntities/blockchainEntitiesGettersAndSetters.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,7 @@ export function GetOrSetProvider(
moniker: moniker,
} as JsinfoSchema.Provider

if (IsMeaningfulText(address) && IsMeaningfulText(moniker)) {
dbProviders.set(address, provider)
}
dbProviders.set(address, provider)

return provider
}
Expand Down
86 changes: 55 additions & 31 deletions src/indexer/blockchainEntities/blockchainEntitiesSync.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,10 @@
import * as JsinfoSchema from '../../schemas/jsinfoSchema/jsinfoSchema';
import { PostgresJsDatabase } from 'drizzle-orm/postgres-js';
import { and, eq, ne } from "drizzle-orm";
import { DoInChunks, logger } from "../../utils/utils";
import { DoInChunks, IsMeaningfulText, logger } from "../../utils/utils";
import { JSINFO_INDEXER_DO_IN_CHUNKS_CHUNK_SIZE } from '../indexerConsts';
import { LavaClient } from '../types';
import { UpdateStakeInformation } from './blockchainEntitiesStakeUpdater';
import { PgColumn } from 'drizzle-orm/pg-core';

// async function getLatestPlans(client: LavaClient, dbPlans: Map<string, JsinfoSchema.Plan>) {
// try {
Expand Down Expand Up @@ -52,6 +51,20 @@ export async function SyncBlockchainEntities(
.onConflictDoNothing();
})

let provider = {
address: 'lava@18fgqsc9qfcy5z77zg25l3wdel74wymqwe9hvdp',
moniker: '',
} as JsinfoSchema.Provider

blockchainEntitiesProviders.set('lava@18fgqsc9qfcy5z77zg25l3wdel74wymqwe9hvdp', provider)

let provider2 = {
address: 'lava@12nsvmz8jy6t9ll0fm5fknj35jxuc9cwduf6mmk',
moniker: 'nothing',
} as JsinfoSchema.Provider

blockchainEntitiesProviders.set('lava@12nsvmz8jy6t9ll0fm5fknj35jxuc9cwduf6mmk', provider2)

// Find / create all providers
const arrProviders = Array.from(blockchainEntitiesProviders.values())
// console.log("SyncBlockchainEntities: Processing", arrProviders.length, "providers");
Expand All @@ -70,31 +83,46 @@ export async function SyncBlockchainEntities(
})
})

// Insert all stakes
await Promise.all(Array.from(blockchainEntitiesStakes.values()).map(async (stakes) => {
return stakes.map(async (stake) => {
if (stake.specId == null || stake.specId == "") return;
return await tx.insert(JsinfoSchema.providerStakes)
.values(stake)
.onConflictDoUpdate(
{
target: [JsinfoSchema.providerStakes.provider, JsinfoSchema.providerStakes.specId],
set: {
stake: stake.stake,
appliedHeight: stake.appliedHeight,
blockId: height,
geolocation: stake.geolocation,
addons: stake.addons,
extensions: stake.extensions,
status: stake.status,
delegateCommission: stake.delegateCommission,
delegateLimit: stake.delegateLimit,
delegateTotal: stake.delegateTotal,
},
}
);
})
}))
const uniqueStakesMap = new Map<string, any>();

for (const stakes of blockchainEntitiesStakes.values()) {
stakes.forEach(stake => {
if (!IsMeaningfulText(stake.provider) || !IsMeaningfulText(stake.specId)) return;

const key = `${stake.provider}-${stake.specId}`;

if (uniqueStakesMap.has(key)) {
const existingStake = uniqueStakesMap.get(key);
if (stake.stake && existingStake.stake && stake.stake > existingStake.stake) {
uniqueStakesMap.set(key, stake);
}
} else {
uniqueStakesMap.set(key, stake);
}
});
}

const uniqueStakesArray = Array.from(uniqueStakesMap.values());

await Promise.all(uniqueStakesArray.map(async (stake) => {
return await tx.insert(JsinfoSchema.providerStakes)
.values(stake)
.onConflictDoUpdate({
target: [JsinfoSchema.providerStakes.provider, JsinfoSchema.providerStakes.specId],
set: {
stake: stake.stake,
appliedHeight: stake.appliedHeight,
blockId: height,
geolocation: stake.geolocation,
addons: stake.addons,
extensions: stake.extensions,
status: stake.status,
delegateCommission: stake.delegateCommission,
delegateLimit: stake.delegateLimit,
delegateTotal: stake.delegateTotal,
},
});
}));

// Update old stakes
// console.log("SyncBlockchainEntities: Updating old stakes to inactive status");
Expand All @@ -112,7 +140,3 @@ export async function SyncBlockchainEntities(
logger.info("SyncBlockchainEntities: SyncBlockchainEntities completed in", (endTime - startTime) / 1000, "seconds with stakes");
})
}

function neq(status: PgColumn<{ name: "status"; tableName: "provider_stakes"; dataType: "number"; columnType: "PgInteger"; data: number; driverParam: string | number; notNull: false; hasDefault: false; enumValues: undefined; baseColumn: never; }, {}, {}>, Inactive: JsinfoSchema.LavaProviderStakeStatus): import("drizzle-orm").SQLWrapper | undefined {
throw new Error('Function not implemented.');
}
2 changes: 1 addition & 1 deletion src/indexer/eventProcessor.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -367,7 +367,7 @@ export const ProcessOneEvent = (

default:
ParseEventUnidentified(evt, height, txHash, lavaBlock, blockchainEntitiesProviders, blockchainEntitiesSpecs, blockchainEntitiesStakes)
console.log(`ProcessOneEvent: Unknown event detected at height ${height} with event type ${evt.type} and event details:`, evt);
logger.error(`ProcessOneEvent: Unknown event detected at height ${height} with event type ${evt.type} and event details:`, evt);
break
}
}
2 changes: 1 addition & 1 deletion src/indexer/eventUtils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ export function EventProcessAttributes(
evt.attributes.forEach((attr: Attribute) => {
keysCounter++;
if (keysCounter > JSINFO_INDEXER_EVENT_ATTRIBUTE_KEY_COUNT_MAX) {
console.log(`EventProcessAttributes: Too many keys: ${evt.attributes.length}`);
logger.error(`EventProcessAttributes: Too many keys: ${evt.attributes.length}`);
return;
}

Expand Down
6 changes: 0 additions & 6 deletions src/query/handlers/index/indexTopChainsHandler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,6 @@ export async function IndexTopChainsHandler(request: FastifyRequest, reply: Fast
groupBy(sql`${JsinfoProviderAgrSchema.aggDailyRelayPayments.specId}`).
where(gt(JsinfoProviderAgrSchema.aggDailyRelayPayments.dateday, sql<Date>`now() - interval '30 day'`)).
orderBy(sql`relaySum DESC`)
let getChains: string[] = []
topSpecs.map((chain) => {
if (getChains.length < 8) {
getChains.push(chain.chainId!)
}
})

return {
allSpecs: topSpecs,
Expand Down
2 changes: 1 addition & 1 deletion src/utils/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ export function JSONStringifySpaced(obj: any): string {
);
}

export const IsMeaningfulText = (text: string): boolean => {
export const IsMeaningfulText = (text: string | null | undefined): boolean => {
if (!text) {
return false;
}
Expand Down

0 comments on commit 54030e5

Please sign in to comment.