From 9143a34819d25d55026c0bd66bb91964b7a2192a Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Tue, 5 Mar 2024 17:11:47 +0100 Subject: [PATCH 01/12] Add retry for API operations that may be eventually consistent --- src/commands/database/mysql/create.tsx | 120 ++++++++++++++++--------- src/lib/api_retry.ts | 77 ++++++++++++++++ 2 files changed, 155 insertions(+), 42 deletions(-) create mode 100644 src/lib/api_retry.ts diff --git a/src/commands/database/mysql/create.tsx b/src/commands/database/mysql/create.tsx index 5ba032b9..c1cc9a46 100644 --- a/src/commands/database/mysql/create.tsx +++ b/src/commands/database/mysql/create.tsx @@ -11,6 +11,11 @@ import { Text } from "ink"; import { assertStatus } from "@mittwald/api-client-commons"; import { Success } from "../../../rendering/react/components/Success.js"; import { Value } from "../../../rendering/react/components/Value.js"; +import { withAttemptsToSuccess } from "../../../lib/api_retry.js"; +import type { MittwaldAPIV2 } from "@mittwald/api-client"; + +type Database = MittwaldAPIV2.Components.Schemas.DatabaseMySqlDatabase; +type User = MittwaldAPIV2.Components.Schemas.DatabaseMySqlUser; type Result = { databaseId: string; @@ -71,59 +76,75 @@ export class Create extends ExecRenderBaseCommand { const password = await this.getPassword(p); - const db = await p.runStep("creating MySQL database", async () => { - const r = await this.apiClient.database.createMysqlDatabase({ - projectId, - data: { - database: { - projectId, - description, - version, - characterSettings: { - collation, - characterSet, - }, - }, - user: { - password, - externalAccess, - accessLevel: accessLevel as "full" | "readonly", - }, - }, - }); - - assertStatus(r, 201); - return r.data; - }); + const db = await this.createMySQLDatabase( + p, + projectId, + description, + version, + collation, + characterSet, + password, + externalAccess, + accessLevel, + ); const database = await p.runStep("fetching database", async () => { - const r = await this.apiClient.database.getMysqlDatabase({ - mysqlDatabaseId: db.id, - }); - assertStatus(r, 200); - - return r.data; + const getWithRetry = withAttemptsToSuccess( + this.apiClient.database.getMysqlDatabase, + ); + return (await getWithRetry({ mysqlDatabaseId: db.id })).data; }); const user = await p.runStep("fetching user", async () => { - const r = await this.apiClient.database.getMysqlUser({ - mysqlUserId: db.userId, - }); - assertStatus(r, 200); - - return r.data; + const getWithRetry = withAttemptsToSuccess( + this.apiClient.database.getMysqlUser, + ); + return (await getWithRetry({ mysqlUserId: db.userId })).data; }); - p.complete( - - The database {database.name} and the user{" "} - {user.name} were successfully created. - , - ); + await p.complete(); return { databaseId: db.id, userId: db.userId }; } + private async createMySQLDatabase( + p: ProcessRenderer, + projectId: string, + description: string, + version: string, + collation: string, + characterSet: string, + password: string, + externalAccess: boolean, + accessLevel: string, + ) { + return await p.runStep("creating MySQL database", async () => { + const characterSettings = { collation, characterSet }; + const database = { + projectId, + description, + version, + characterSettings, + }; + const user = { + password, + externalAccess, + accessLevel: accessLevel as "full" | "readonly", + }; + + const r = await this.apiClient.database.createMysqlDatabase({ + projectId, + data: { + database, + user, + }, + }); + + assertStatus(r, 201); + return r.data; + }); + } + private async getPassword(p: ProcessRenderer): Promise { if (this.flags["user-password"]) { return this.flags["user-password"]; @@ -138,3 +159,18 @@ export class Create extends ExecRenderBaseCommand { } } } + +function DatabaseCreateSuccess({ + database, + user, +}: { + database: Database; + user: User; +}) { + return ( + + The database {database.name} and the user{" "} + {user.name} were successfully created. + + ); +} diff --git a/src/lib/api_retry.ts b/src/lib/api_retry.ts new file mode 100644 index 00000000..70d554e4 --- /dev/null +++ b/src/lib/api_retry.ts @@ -0,0 +1,77 @@ +import { Response } from "@mittwald/api-client-commons"; +import debug from "debug"; + +const d = debug("mw:api-retry"); + +type BackoffFunction = (attempt: number) => number; + +type RetryOptions = { + attempts: number; + backoff: BackoffFunction; +}; + +/** + * Contains a collection of backoff strategies for use with + * `withAttemptsToSuccess`. + */ +export const backoffStrategies = { + exponential: (initial: number, base: number) => (attempt: number) => + initial * base ** attempt, + linear: (step: number) => (attempt: number) => step * attempt, + max: (inner: BackoffFunction, max: number) => (attempt: number) => + Math.min(inner(attempt), max), +}; + +/** + * Wraps an API call function and retries it until it returns a successful + * response. + * + * On non-successful responses, it waits for a (configurable) backoff time + * before retrying. + * + * Usage: + * + * const getMysqlDatabase = withAttemptsToSuccess( + * this.apiClient.database.getMysqlDatabase, + * ); + * return (await getWithRetry({ mysqlDatabaseId })).data; + */ +export function withAttemptsToSuccess( + fn: (req: TReq) => Promise, + { + attempts = 50, + backoff = backoffStrategies.max( + backoffStrategies.exponential(100, 1.2), + 2000, + ), + }: Partial = {}, +) { + return async function (req: TReq) { + let response: TRes | undefined; + for (let i = 0; i < attempts; i++) { + response = await fn(req); + const waitFor = backoff(i); + + if (isStatus(response, 200)) { + return response; + } + + d("received status %d, waiting for %d ms", response.status, waitFor); + + await new Promise((resolve) => setTimeout(resolve, backoff(i))); + } + + throw new Error( + `received status ${response?.status} after ${attempts} attempts`, + ); + }; +} + +function isStatus( + response: T, + expectedStatus: S, +): response is T & { + status: S; +} { + return response.status === expectedStatus; +} From c85d56c23b2561ddd65c01732ed3e80bdb9b20f2 Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Tue, 5 Mar 2024 17:19:20 +0100 Subject: [PATCH 02/12] Also use "if-event-reached" mechanic --- src/commands/database/mysql/create.tsx | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/commands/database/mysql/create.tsx b/src/commands/database/mysql/create.tsx index c1cc9a46..80bed309 100644 --- a/src/commands/database/mysql/create.tsx +++ b/src/commands/database/mysql/create.tsx @@ -76,7 +76,7 @@ export class Create extends ExecRenderBaseCommand { const password = await this.getPassword(p); - const db = await this.createMySQLDatabase( + const [db, eventId] = await this.createMySQLDatabase( p, projectId, description, @@ -92,14 +92,24 @@ export class Create extends ExecRenderBaseCommand { const getWithRetry = withAttemptsToSuccess( this.apiClient.database.getMysqlDatabase, ); - return (await getWithRetry({ mysqlDatabaseId: db.id })).data; + return ( + await getWithRetry({ + mysqlDatabaseId: db.id, + headers: { "if-event-reached": eventId }, + }) + ).data; }); const user = await p.runStep("fetching user", async () => { const getWithRetry = withAttemptsToSuccess( this.apiClient.database.getMysqlUser, ); - return (await getWithRetry({ mysqlUserId: db.userId })).data; + return ( + await getWithRetry({ + mysqlUserId: db.userId, + headers: { "if-event-reached": eventId }, + }) + ).data; }); await p.complete(); @@ -141,7 +151,7 @@ export class Create extends ExecRenderBaseCommand { }); assertStatus(r, 201); - return r.data; + return [r.data, r.headers["etag"]]; }); } From 30f6b51bc4150120dc1dc7f4a71bfff5a0273e15 Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Wed, 6 Mar 2024 09:06:43 +0100 Subject: [PATCH 03/12] Refactor parameters of "createMySQLDatabase" function --- src/commands/database/mysql/create.tsx | 55 +++++++++++++------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/src/commands/database/mysql/create.tsx b/src/commands/database/mysql/create.tsx index 80bed309..ccfd81e0 100644 --- a/src/commands/database/mysql/create.tsx +++ b/src/commands/database/mysql/create.tsx @@ -79,13 +79,19 @@ export class Create extends ExecRenderBaseCommand { const [db, eventId] = await this.createMySQLDatabase( p, projectId, - description, - version, - collation, - characterSet, - password, - externalAccess, - accessLevel, + { + description, + version, + characterSettings: { + collation, + characterSet, + }, + }, + { + password, + externalAccess, + accessLevel: accessLevel as "full" | "readonly", + }, ); const database = await p.runStep("fetching database", async () => { @@ -120,32 +126,25 @@ export class Create extends ExecRenderBaseCommand { private async createMySQLDatabase( p: ProcessRenderer, projectId: string, - description: string, - version: string, - collation: string, - characterSet: string, - password: string, - externalAccess: boolean, - accessLevel: string, + database: { + description: string; + version: string; + characterSettings: { + collation: string; + characterSet: string; + }; + }, + user: { + password: string; + externalAccess: boolean; + accessLevel: "full" | "readonly"; + }, ) { return await p.runStep("creating MySQL database", async () => { - const characterSettings = { collation, characterSet }; - const database = { - projectId, - description, - version, - characterSettings, - }; - const user = { - password, - externalAccess, - accessLevel: accessLevel as "full" | "readonly", - }; - const r = await this.apiClient.database.createMysqlDatabase({ projectId, data: { - database, + database: { projectId, ...database }, user, }, }); From 94c2b197c2411747e8f02248b1202f203b72a789 Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Wed, 6 Mar 2024 09:55:00 +0100 Subject: [PATCH 04/12] Add test case for request retrying --- src/commands/database/mysql/create.test.ts | 118 +++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 src/commands/database/mysql/create.test.ts diff --git a/src/commands/database/mysql/create.test.ts b/src/commands/database/mysql/create.test.ts new file mode 100644 index 00000000..1981b887 --- /dev/null +++ b/src/commands/database/mysql/create.test.ts @@ -0,0 +1,118 @@ +import { expect, test } from "@oclif/test"; + +describe("database:mysql:create", () => { + const projectId = "339d6458-839f-4809-a03d-78700069690c"; + const databaseId = "83e0cb85-dcf7-4968-8646-87a63980ae91"; + const userId = "a8c1eb2a-aa4d-4daf-8e21-9d91d56559ca"; + test + .nock("https://api.mittwald.de", (api) => { + api.get(`/v2/projects/${projectId}`).reply(200, { + id: projectId, + }); + api + .post(`/v2/projects/${projectId}/mysql-databases`, { + database: { + projectId, + description: "Test", + version: "8.0", + characterSettings: { + collation: "utf8mb4_unicode_ci", + characterSet: "utf8mb4", + }, + }, + user: { + password: "secret", + externalAccess: false, + accessLevel: "full", + }, + }) + .reply(201, { id: databaseId, userId }); + + api.get(`/v2/mysql-databases/${databaseId}`).reply(200, { + id: databaseId, + name: "mysql_xxxxxx", + }); + + api.get(`/v2/mysql-users/${userId}`).reply(200, { + id: userId, + name: "dbu_xxxxxx", + }); + }) + .env({ MITTWALD_API_TOKEN: "foo" }) + .stdout() + .command([ + "database mysql create", + "--project-id", + projectId, + "--version", + "8.0", + "--description", + "Test", + "--user-password", + "secret", + ]) + .it("creates a database and prints database and user name", (ctx) => { + // cannot match on exact output, because linebreaks + expect(ctx.stdout).to.contain("The database mysql_xxxxxx"); + expect(ctx.stdout).to.contain("the user dbu_xxxxxx"); + }); + + test + .nock("https://api.mittwald.de", (api) => { + api.get(`/v2/projects/${projectId}`).reply(200, { + id: projectId, + }); + api + .post(`/v2/projects/${projectId}/mysql-databases`, { + database: { + projectId, + description: "Test", + version: "8.0", + characterSettings: { + collation: "utf8mb4_unicode_ci", + characterSet: "utf8mb4", + }, + }, + user: { + password: "secret", + externalAccess: false, + accessLevel: "full", + }, + }) + .reply(201, { id: databaseId, userId }); + + api.get(`/v2/mysql-databases/${databaseId}`).reply(200, { + id: databaseId, + name: "mysql_xxxxxx", + }); + + api.get(`/v2/mysql-users/${userId}`).reply(403, { + id: userId, + times: 3, + name: "dbu_xxxxxx", + }); + + api.get(`/v2/mysql-users/${userId}`).reply(200, { + id: userId, + name: "dbu_xxxxxx", + }); + }) + .env({ MITTWALD_API_TOKEN: "foo" }) + .stdout() + .command([ + "database mysql create", + "--project-id", + projectId, + "--version", + "8.0", + "--description", + "Test", + "--user-password", + "secret", + ]) + .it("retries fetching user until successful", (ctx) => { + // cannot match on exact output, because linebreaks + expect(ctx.stdout).to.contain("The database mysql_xxxxxx"); + expect(ctx.stdout).to.contain("the user dbu_xxxxxx"); + }); +}); From cf583200f6f8de1246f792630fbe9e101d750c6b Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Wed, 6 Mar 2024 13:49:51 +0100 Subject: [PATCH 05/12] Simplify handling of mysql flags --- README.md | 12 ++++----- src/commands/database/mysql/delete.ts | 1 - src/commands/database/mysql/dump.tsx | 7 +---- src/commands/database/mysql/get.ts | 1 - src/commands/database/mysql/phpmyadmin.ts | 7 +---- src/commands/database/mysql/port-forward.tsx | 7 +---- src/commands/database/mysql/shell.tsx | 7 +---- src/lib/database/mysql/flags.ts | 28 +++++--------------- 8 files changed, 17 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 6d230b2b..db48afba 100644 --- a/README.md +++ b/README.md @@ -2280,7 +2280,7 @@ USAGE $ mw database mysql delete DATABASE-ID [-q] [-f] ARGUMENTS - DATABASE-ID The ID of the database (when a project context is set, you can also use the name) + DATABASE-ID The ID or name of the database FLAGS -f, --force Do not ask for confirmation @@ -2305,7 +2305,7 @@ USAGE $ mw database mysql dump DATABASE-ID -o [-q] [-p ] [--ssh-user ] [--temporary-user] [--gzip] ARGUMENTS - DATABASE-ID The ID of the database (when a project context is set, you can also use the name) + DATABASE-ID The ID or name of the database FLAGS -o, --output= (required) the output file to write the dump to ("-" for stdout) @@ -2365,7 +2365,7 @@ USAGE $ mw database mysql get DATABASE-ID [-o json|yaml | | ] ARGUMENTS - DATABASE-ID The ID of the database (when a project context is set, you can also use the name) + DATABASE-ID The ID or name of the database FLAGS -o, --output=