Skip to content

Commit

Permalink
Merge pull request #226 from DestinyItemManager/sync-apps
Browse files Browse the repository at this point in the history
Implement list/sync for apps
  • Loading branch information
bhollis authored Sep 6, 2024
2 parents a80f392 + 25d517d commit 75649c6
Show file tree
Hide file tree
Showing 10 changed files with 324 additions and 105 deletions.
76 changes: 57 additions & 19 deletions api/apps/index.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
import * as Sentry from '@sentry/node';
import { ListToken } from '@stately-cloud/client';
import { RequestHandler } from 'express';
import _ from 'lodash';
import { getAllApps } from '../db/apps-queries.js';
import { getAllApps as getAllAppsPostgres } from '../db/apps-queries.js';
import { pool } from '../db/index.js';
import { metrics } from '../metrics/index.js';
import { ApiApp } from '../shapes/app.js';
import { getAllApps, updateApps } from '../stately/apps-queries.js';

/**
* Express middleware that requires an API key be provided in a header
Expand Down Expand Up @@ -38,10 +40,11 @@ export const apiKey: RequestHandler = (req, res, next) => {
}
};

let apps: ApiApp[];
let apps: ApiApp[] = [];
let appsByApiKey: { [apiKey: string]: ApiApp };
let origins = new Set<string>();
let appsInterval: NodeJS.Timeout | null = null;
let token: ListToken | undefined;

export function stopAppsRefresh() {
if (appsInterval) {
Expand All @@ -65,32 +68,67 @@ export function isAppOrigin(origin: string) {
return origins.has(origin);
}

export async function refreshApps() {
export async function refreshApps(): Promise<void> {
stopAppsRefresh();

try {
const client = await pool.connect();
try {
apps = await getAllApps(client);
appsByApiKey = _.keyBy(apps, (a) => a.dimApiKey.toLowerCase());
origins = new Set<string>();
for (const app of apps) {
origins.add(app.origin);
if (apps.length === 0) {
// Start off with a copy from postgres, just in case StatelyDB is having
// problems.
await fetchAppsFromPostgres();
digestApps();
}

if (!token) {
// First time, get 'em all
const [appsFromStately, newToken] = await getAllApps();
if (appsFromStately.length > 0) {
apps = appsFromStately;
digestApps();
token = newToken;
}
} else {
// After that, use a sync to update them
const [appsFromStately, newToken] = await updateApps(token, apps);
if (appsFromStately.length > 0) {
apps = appsFromStately;
digestApps();
token = newToken;
}
metrics.increment('apps.refresh.success.count');
return apps;
} catch (e) {
metrics.increment('apps.refresh.error.count');
console.error('Error refreshing apps', e);
Sentry.captureException(e);
throw e;
} finally {
client.release();
}
metrics.increment('apps.refresh.success.count');
} catch (e) {
metrics.increment('apps.refresh.error.count');
console.error('Error refreshing apps', e);
Sentry.captureException(e);
throw e;
} finally {
// Refresh again every minute or so
if (!appsInterval) {
appsInterval = setTimeout(refreshApps, 60000 + Math.random() * 10000);
}
}
}

async function fetchAppsFromPostgres() {
const client = await pool.connect();
try {
apps = await getAllAppsPostgres(client);
appsByApiKey = _.keyBy(apps, (a) => a.dimApiKey.toLowerCase());
origins = new Set<string>();
for (const app of apps) {
origins.add(app.origin);
}
return apps;
} finally {
client.release();
}
}

function digestApps() {
appsByApiKey = _.keyBy(apps, (a) => a.dimApiKey.toLowerCase());
origins = new Set<string>();
for (const app of apps) {
origins.add(app.origin);
}
}
10 changes: 7 additions & 3 deletions api/routes/create-app.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import asyncHandler from 'express-async-handler';
import { DatabaseError } from 'pg-protocol';
import { v4 as uuid } from 'uuid';
import { getAppById, insertApp } from '../db/apps-queries.js';
import { insertApp as insertAppPostgres } from '../db/apps-queries.js';
import { transaction } from '../db/index.js';
import { ApiApp, CreateAppRequest } from '../shapes/app.js';
import { insertApp } from '../stately/apps-queries.js';
import { badRequest } from '../utils.js';

const localHosts =
Expand Down Expand Up @@ -48,14 +49,17 @@ export const createAppHandler = asyncHandler(async (req, res) => {
dimApiKey: uuid(),
};

// Put it in StatelyDB
app = await insertApp(app);

// Also put it in Postgres, for now!
await transaction(async (client) => {
try {
await insertApp(client, app);
await insertAppPostgres(client, app);
} catch (e) {
// This is a unique constraint violation, so just get the app!
if (e instanceof DatabaseError && e.code === '23505') {
await client.query('ROLLBACK');
app = (await getAppById(client, request.id))!;
} else {
throw e;
}
Expand Down
126 changes: 126 additions & 0 deletions api/stately/apps-queries.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
import { keyPath, ListToken } from '@stately-cloud/client';
import { ApiApp } from '../shapes/app.js';
import { client } from './client.js';
import { ApiApp as StatelyApiApp } from './generated/index.js';

/**
* Get all registered apps.
*/
export async function getAllApps(): Promise<[ApiApp[], ListToken]> {
let apps = client.withAllowStale(true).beginList('/apps-1');
const allApps: ApiApp[] = [];
let token: ListToken | undefined = undefined;

while (true) {
for await (const app of apps) {
if (client.isType(app, 'ApiApp')) {
allApps.push(convertToApiApp(app));
}
}

token = apps.token!;
if (token.canContinue) {
apps = client.continueList(token);
} else {
break;
}
}

return [allApps, token] as const;
}

/**
* Update the list of apps with changes from the server.
*/
export async function updateApps(token: ListToken, apps: ApiApp[]): Promise<[ApiApp[], ListToken]> {
const updates = client.syncList(token);
for await (const update of updates) {
switch (update.type) {
case 'changed': {
const item = update.item;
if (client.isType(item, 'ApiApp')) {
const existingIndex = apps.findIndex((app) => app.id === item.id);
if (existingIndex >= 0) {
apps[existingIndex] = convertToApiApp(item);
} else {
apps.push(convertToApiApp(item));
}
}
break;
}
case 'deleted':
case 'updatedOutsideWindow': {
const item = update.keyPath;
// TODO: This could be easier!
const id = /^\/apps\/app-([^/]+)$/.exec(item)?.[1];
const existingIndex = apps.findIndex((app) => app.id === id);
if (existingIndex >= 0) {
apps.splice(existingIndex, 1);
}
break;
}
case 'reset': {
apps = [];
break;
}
}
}
return [apps, updates.token!] as const;
}

/**
* Get an app by its ID.
*/
export async function getAppById(id: string): Promise<ApiApp | undefined> {
const result = await client.get('ApiApp', keyPathFor(id));
if (result) {
return convertToApiApp(result);
}
return undefined;
}

/**
* Insert a new app into the list of registered apps, or update an existing app.
*/
export async function insertApp(app: ApiApp): Promise<ApiApp> {
let resultApp: ApiApp | undefined;
// TODO: wish I could set an if-not-exists condition here, to avoid
// accidentally updating an app. Instead I got a transaction.
const result = await client.transaction(async (txn) => {
const getResult = await txn.get('ApiApp', keyPathFor(app.id));
if (getResult) {
resultApp = convertToApiApp(getResult);
return;
}
await txn.put(client.create('ApiApp', { ...app, partition: 1n }));
});

if (resultApp) {
return resultApp;
}

if (result.committed && result.puts.length === 1) {
const put = result.puts[0];
if (client.isType(put, 'ApiApp')) {
return convertToApiApp(put);
}
}

throw new Error('No ApiApp in result!');
}

function keyPathFor(id: string) {
return keyPath`/apps-1/app-${id}`;
}

// This mostly serves to remove the partition field, which we don't need. It
// would cause problems serializing to JSON, since it's a bigint. It'd be nice
// if I could've used a 32-bit int, but that isn't in the standard schema types...
function convertToApiApp(app: StatelyApiApp): ApiApp {
return {
id: app.id,
bungieApiKey: app.bungieApiKey,
dimApiKey: app.dimApiKey,
origin: app.origin,
};
}
4 changes: 2 additions & 2 deletions api/stately/generated/index.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Code generated by Stately. DO NOT EDIT.

export * from './stately_item_types.js';
export * from './stately_pb.js';
export * from "./stately_pb.js";
export * from "./stately_item_types.js";
62 changes: 31 additions & 31 deletions api/stately/generated/stately_item_types.js
Original file line number Diff line number Diff line change
@@ -1,55 +1,55 @@
// Code generated by Stately. DO NOT EDIT.

import { createClient as createGenericClient } from '@stately-cloud/client';
import { createClient as createGenericClient } from "@stately-cloud/client";
import {
ApiAppSchema,
GlobalSettingsSchema,
ItemAnnotationSchema,
ItemHashTagSchema,
LoadoutSchema,
LoadoutShareSchema,
SearchSchema,
SettingsSchema,
TriumphSchema,
ArtifactUnlocksSchema,
CollapsedSectionSchema,
CustomStatDefSchema,
CustomStatWeightsEntrySchema,
CustomStatsEntrySchema,
GlobalSettingsSchema,
InGameLoadoutIdentifiersSchema,
ItemAnnotationSchema,
ItemHashTagSchema,
LoadoutItemSchema,
LoadoutParametersSchema,
LoadoutSchema,
LoadoutShareSchema,
ModsByBucketEntrySchema,
SearchSchema,
SettingsSchema,
SocketOverrideSchema,
StatConstraintSchema,
StatConstraintsEntrySchema,
TriumphSchema,
} from './stately_pb.js';
} from "./stately_pb.js";

export const typeToSchema = {
// itemTypes
ApiApp: ApiAppSchema,
GlobalSettings: GlobalSettingsSchema,
ItemAnnotation: ItemAnnotationSchema,
ItemHashTag: ItemHashTagSchema,
Loadout: LoadoutSchema,
LoadoutShare: LoadoutShareSchema,
Search: SearchSchema,
Settings: SettingsSchema,
Triumph: TriumphSchema,
"ApiApp": ApiAppSchema,
"GlobalSettings": GlobalSettingsSchema,
"ItemAnnotation": ItemAnnotationSchema,
"ItemHashTag": ItemHashTagSchema,
"Loadout": LoadoutSchema,
"LoadoutShare": LoadoutShareSchema,
"Search": SearchSchema,
"Settings": SettingsSchema,
"Triumph": TriumphSchema,

// objectTypes
ArtifactUnlocks: ArtifactUnlocksSchema,
CollapsedSection: CollapsedSectionSchema,
CustomStatDef: CustomStatDefSchema,
CustomStatWeightsEntry: CustomStatWeightsEntrySchema,
CustomStatsEntry: CustomStatsEntrySchema,
InGameLoadoutIdentifiers: InGameLoadoutIdentifiersSchema,
LoadoutItem: LoadoutItemSchema,
LoadoutParameters: LoadoutParametersSchema,
ModsByBucketEntry: ModsByBucketEntrySchema,
SocketOverride: SocketOverrideSchema,
StatConstraint: StatConstraintSchema,
StatConstraintsEntry: StatConstraintsEntrySchema,
"ArtifactUnlocks": ArtifactUnlocksSchema,
"CollapsedSection": CollapsedSectionSchema,
"CustomStatDef": CustomStatDefSchema,
"CustomStatWeightsEntry": CustomStatWeightsEntrySchema,
"CustomStatsEntry": CustomStatsEntrySchema,
"InGameLoadoutIdentifiers": InGameLoadoutIdentifiersSchema,
"LoadoutItem": LoadoutItemSchema,
"LoadoutParameters": LoadoutParametersSchema,
"ModsByBucketEntry": ModsByBucketEntrySchema,
"SocketOverride": SocketOverrideSchema,
"StatConstraint": StatConstraintSchema,
"StatConstraintsEntry": StatConstraintsEntrySchema,
};

export function createClient(storeId, opts) {
Expand Down
5 changes: 5 additions & 0 deletions api/stately/generated/stately_pb.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ export declare type ApiApp = Message<"stately.generated.ApiApp"> & {
* @generated from field: string origin = 4;
*/
origin: string;

/**
* @generated from field: uint64 partition = 5;
*/
partition: bigint;
};

/**
Expand Down
133 changes: 86 additions & 47 deletions api/stately/generated/stately_pb.js

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions api/stately/init/global-settings.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,5 @@ const prodSettings = client.create('GlobalSettings', {
await client.putBatch(devSettings, betaSettings, prodSettings);

console.log('Global settings initialized');
// This is due to a bug in Connect!
process.exit(0);
6 changes: 5 additions & 1 deletion api/stately/schema/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ These are the objects that DIM stores in its own service - most data comes from

Key paths are laid out like this:

- `/app-:id`: `ApiApp`
- `/apps/app-:id`: `ApiApp`
- `/gs-:stage`: `GlobalSettings`
- `/loadoutShare-:id`: `LoadoutShare`
- `/member-:memberId/settings`: `Settings`
Expand All @@ -20,3 +20,7 @@ The goal with this modeling is to allow for syncing all of a user's info in two

- `List("/p-:profileId/d-:destinyVersion")` - get all saved data for a particular game profile + destiny version (each profile can be associated with Destiny 1 and Destiny 2)
- `List("/member-:memberId")` - get settings for a whole Bungie.net account. It's a List instead of a Get so we can sync it!

Plus some fun stuff such as:

- `List("/apps/")` to get all registered apps, and `SyncList()` to keep them up to date.
Loading

0 comments on commit 75649c6

Please sign in to comment.