From d41cde68a393e302852f5055a1e3830922551dd6 Mon Sep 17 00:00:00 2001 From: Harrison Borges Date: Mon, 29 Jan 2024 18:40:44 -0500 Subject: [PATCH 1/6] add budgets to schema --- .../20240129234021_add_budgets/migration.sql | 17 +++++++++++++ prisma/schema.prisma | 25 ++++++++++++++++--- 2 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 prisma/migrations/20240129234021_add_budgets/migration.sql diff --git a/prisma/migrations/20240129234021_add_budgets/migration.sql b/prisma/migrations/20240129234021_add_budgets/migration.sql new file mode 100644 index 0000000..5adc54c --- /dev/null +++ b/prisma/migrations/20240129234021_add_budgets/migration.sql @@ -0,0 +1,17 @@ +-- CreateTable +CREATE TABLE "Budget" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "name" TEXT NOT NULL, + "amount" REAL NOT NULL, + "range" INTEGER NOT NULL +); + +-- CreateTable +CREATE TABLE "BudgetRule" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "budgetId" INTEGER NOT NULL, + "column" TEXT NOT NULL, + "operator" TEXT NOT NULL DEFAULT 'CONTAINS', + "value" TEXT NOT NULL, + CONSTRAINT "BudgetRule_budgetId_fkey" FOREIGN KEY ("budgetId") REFERENCES "Budget" ("id") ON DELETE RESTRICT ON UPDATE CASCADE +); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index a428b7e..5e5532f 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -63,12 +63,12 @@ model Account { availableBalance Float? isoCurrencyCode String? transactions Transaction[] - Institution Institution? @relation(fields: [institutionPlaidId], references: [plaidId]) + institution Institution? @relation(fields: [institutionPlaidId], references: [plaidId]) institutionPlaidId String? importedAt DateTime @default(now()) importLog ImportLog? @relation(fields: [importLogId], references: [id]) importLogId Int? - AccountBalance AccountBalance[] + accountBalances AccountBalance[] } model ImportLog { @@ -76,8 +76,8 @@ model ImportLog { syncStartedAt DateTime @default(now()) syncCompletedAt DateTime? @updatedAt transactionsCount Int? - Transaction Transaction[] - Account Account[] + transactions Transaction[] + accounts Account[] } model AccountBalance { @@ -89,3 +89,20 @@ model AccountBalance { accountPlaidId String createdAt DateTime @default(now()) } + +model Budget { + id Int @id @default(autoincrement()) + name String + amount Float + range Int + budgetRules BudgetRule[] +} + +model BudgetRule { + id Int @id @default(autoincrement()) + budget Budget @relation(fields: [budgetId], references: [id]) + budgetId Int + column String + operator String @default("CONTAINS") + value String +} From 6c4b74bd74264495935d268a1e2c48c0fb3d30a4 Mon Sep 17 00:00:00 2001 From: Harrison Borges Date: Thu, 1 Feb 2024 12:40:24 -0500 Subject: [PATCH 2/6] update cli to use new schema --- cli/internal/db/models.go | 17 +++++++++++++++++ cli/internal/db/query.sql.go | 17 +++++++++++------ cli/internal/db/schema.sql | 16 +++++++++++++++- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/cli/internal/db/models.go b/cli/internal/db/models.go index 88490b1..7b161fa 100644 --- a/cli/internal/db/models.go +++ b/cli/internal/db/models.go @@ -34,6 +34,21 @@ type AccountBalance struct { CreatedAt time.Time } +type Budget struct { + ID int64 + Name string + Amount float64 + Range int64 +} + +type BudgetRule struct { + ID int64 + BudgetId int64 + Column string + Operator string + Value string +} + type Config struct { PlaidClientId string PlaidSecret string @@ -50,6 +65,8 @@ type Institution struct { PlaidId string Name string PlaidAccessToken string + Color sql.NullString + Logo sql.NullString } type PrismaMigrations struct { diff --git a/cli/internal/db/query.sql.go b/cli/internal/db/query.sql.go index 9a1df00..63224f3 100644 --- a/cli/internal/db/query.sql.go +++ b/cli/internal/db/query.sql.go @@ -157,15 +157,21 @@ SELECT "plaidId", from "Institution" ` -func (q *Queries) InstitutionList(ctx context.Context) ([]Institution, error) { +type InstitutionListRow struct { + PlaidId string + Name string + PlaidAccessToken string +} + +func (q *Queries) InstitutionList(ctx context.Context) ([]InstitutionListRow, error) { rows, err := q.db.QueryContext(ctx, institutionList) if err != nil { return nil, err } defer rows.Close() - var items []Institution + var items []InstitutionListRow for rows.Next() { - var i Institution + var i InstitutionListRow if err := rows.Scan(&i.PlaidId, &i.Name, &i.PlaidAccessToken); err != nil { return nil, err } @@ -182,9 +188,8 @@ func (q *Queries) InstitutionList(ctx context.Context) ([]Institution, error) { const transactionCreate = `-- name: TransactionCreate :exec INSERT INTO "Transaction"("plaidId", - "plaid - -date", + "plaidAccountId", + "date", "name", "amount", "category", diff --git a/cli/internal/db/schema.sql b/cli/internal/db/schema.sql index 0aadb5e..12b0803 100644 --- a/cli/internal/db/schema.sql +++ b/cli/internal/db/schema.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS "Institution" ( "plaidId" TEXT NOT NULL PRIMARY KEY, "name" TEXT NOT NULL, "plaidAccessToken" TEXT NOT NULL -); +, "color" TEXT, "logo" TEXT); CREATE TABLE sqlite_sequence(name,seq); CREATE TABLE IF NOT EXISTS "Transaction" ( "plaidId" TEXT NOT NULL PRIMARY KEY, @@ -74,3 +74,17 @@ CREATE TABLE IF NOT EXISTS "AccountBalance" ( "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, CONSTRAINT "AccountBalance_accountPlaidId_fkey" FOREIGN KEY ("accountPlaidId") REFERENCES "Account" ("plaidId") ON DELETE RESTRICT ON UPDATE CASCADE ); +CREATE TABLE IF NOT EXISTS "Budget" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "name" TEXT NOT NULL, + "amount" REAL NOT NULL, + "range" INTEGER NOT NULL +); +CREATE TABLE IF NOT EXISTS "BudgetRule" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "budgetId" INTEGER NOT NULL, + "column" TEXT NOT NULL, + "operator" TEXT NOT NULL DEFAULT 'CONTAINS', + "value" TEXT NOT NULL, + CONSTRAINT "BudgetRule_budgetId_fkey" FOREIGN KEY ("budgetId") REFERENCES "Budget" ("id") ON DELETE RESTRICT ON UPDATE CASCADE +); From 9badb3ded7b02d1d42d84ea644acbcae05927748 Mon Sep 17 00:00:00 2001 From: Harrison Borges Date: Thu, 1 Feb 2024 22:03:55 -0500 Subject: [PATCH 3/6] update cron script to dynamically build path --- bin/budgeted-crontab.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/bin/budgeted-crontab.sh b/bin/budgeted-crontab.sh index f1a4baf..0d13a17 100644 --- a/bin/budgeted-crontab.sh +++ b/bin/budgeted-crontab.sh @@ -1,11 +1,14 @@ -#/bin/sh +#/bin/bash date=$(date +%Y-%m-%d) -echo "----- [$date] ETL: start -----" >> budgeted-cron-log.log +path="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +echo "The script is located in: $path" -bin/budgeted-cli load plaid-data >> budgeted-cron-log.log -bin/budgeted-cli load csv >> budgeted-cron-log.log -bin/budgeted-cli load sqlite >> budgeted-cron-log.log +echo "----- [$date] ETL: start -----" >> budgeted-cron-log.log +$path/budgeted-cli load plaid-data >> budgeted-cron-log.log +$path/budgeted-cli load csv >> budgeted-cron-log.log +$path/budgeted-cli load sqlite >> budgeted-cron-log.log echo "----- [$date] ETL: complete -----" >> budgeted-cron-log.log From ec7fd5b352c6b081457e455afe7b93b09d1956f4 Mon Sep 17 00:00:00 2001 From: Harrison Borges Date: Thu, 1 Feb 2024 22:04:32 -0500 Subject: [PATCH 4/6] dynamically build path to budgeted cli --- electron/lib/crontab/crontab.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/electron/lib/crontab/crontab.ts b/electron/lib/crontab/crontab.ts index be27d5f..cbd979f 100644 --- a/electron/lib/crontab/crontab.ts +++ b/electron/lib/crontab/crontab.ts @@ -1,7 +1,11 @@ import { exec, execSync } from 'node:child_process' +import path from 'node:path' -const morningSyncEntry = '0 0 * * * bin/budgeted-crontab.sh' -const eveningSyncEntry = '0 12 * * * bin/budgeted-crontab.sh' +const pwd = execSync('pwd').toString().trim() +const scriptPath = path.join(pwd, 'bin/budgeted-crontab.sh') + +const morningSyncEntry = `0 0 * * * ${scriptPath}` +const eveningSyncEntry = `0 12 * * * ${scriptPath}` /** * Writes the crontab entries for the morning and evening ETL jobs From c9c0e169380068f7899d5827a82c50befa38e2ea Mon Sep 17 00:00:00 2001 From: quinn Date: Thu, 1 Feb 2024 23:20:57 -0500 Subject: [PATCH 5/6] separate by account --- cli/internal/cmd/load.plaid-data.go | 4 ++-- cli/internal/plaid/accounts.go | 5 +++-- cli/internal/plaid/cache.go | 18 ++++++++++++++---- cli/internal/plaid/transactions.go | 8 +++++--- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/cli/internal/cmd/load.plaid-data.go b/cli/internal/cmd/load.plaid-data.go index 3e450cf..0e32529 100644 --- a/cli/internal/cmd/load.plaid-data.go +++ b/cli/internal/cmd/load.plaid-data.go @@ -44,13 +44,13 @@ func LoadPlaidDataCmd(ctx context.Context) *cobra.Command { } for _, institution := range institutions { - err = pc.LoadTransactions(ctx, institution.PlaidAccessToken) + err = pc.LoadTransactions(ctx, institution.PlaidId, institution.PlaidAccessToken) if err != nil { return errors.Wrap(err, "failed to load transactions") } - err = pc.LoadAccounts(ctx, institution.PlaidAccessToken) + err = pc.LoadAccounts(ctx, institution.PlaidId, institution.PlaidAccessToken) if err != nil { return errors.Wrap(err, "failed to load accounts") } diff --git a/cli/internal/plaid/accounts.go b/cli/internal/plaid/accounts.go index c106ca8..c2206d0 100644 --- a/cli/internal/plaid/accounts.go +++ b/cli/internal/plaid/accounts.go @@ -4,11 +4,12 @@ import ( "context" "errors" "io" + "path/filepath" "github.com/plaid/plaid-go/v20/plaid" ) -func (pc *APIClient) LoadAccounts(ctx context.Context, accessToken string) error { +func (pc *APIClient) LoadAccounts(ctx context.Context, institutionId string, accessToken string) error { accountsGetRequest := plaid.NewAccountsGetRequest(accessToken) accountsGetRequest.SetOptions(plaid.AccountsGetRequestOptions{}) @@ -32,7 +33,7 @@ func (pc *APIClient) LoadAccounts(ctx context.Context, accessToken string) error return err } - if err = pc.SetCache(ctx, "accounts", "", body); err != nil { + if err = pc.SetCache(ctx, filepath.Join("transactions", institutionId), "", body); err != nil { return err } diff --git a/cli/internal/plaid/cache.go b/cli/internal/plaid/cache.go index fcfb56b..febf7a8 100644 --- a/cli/internal/plaid/cache.go +++ b/cli/internal/plaid/cache.go @@ -31,9 +31,13 @@ func (pc *APIClient) GetNextCursor(ctx context.Context, prefix string) (string, return syncResponse.GetNextCursor(), nil } -func (pc *APIClient) GetCache(ctx context.Context, path string) ([]byte, error) { +func (pc *APIClient) GetCache(ctx context.Context, prefix string) ([]byte, error) { var lastEntry os.DirEntry - cachePath := pc.cacheDir + "/" + path + cachePath := filepath.Join(pc.cacheDir, prefix) + err := os.MkdirAll(cachePath, 0755) + if err != nil { + return nil, err + } entries, err := os.ReadDir(cachePath) if err != nil { @@ -58,9 +62,15 @@ func (pc *APIClient) GetCache(ctx context.Context, path string) ([]byte, error) return nil, nil } -func (pc *APIClient) SetCache(ctx context.Context, path string, cursor string, bytes []byte) error { +func (pc *APIClient) SetCache(ctx context.Context, prefix string, cursor string, bytes []byte) error { + cachePath := filepath.Join(pc.cacheDir, prefix) + err := os.MkdirAll(cachePath, 0755) + if err != nil { + return err + } + timestamp := strings.Replace(time.Now().Format(time.RFC3339Nano), ":", "X", -1) - fileName := filepath.Join(pc.cacheDir, path, fmt.Sprintf("%s_%s.json", timestamp, strings.Replace(cursor, "/", "_", -1))) + fileName := filepath.Join(cachePath, fmt.Sprintf("%s_%s.json", timestamp, strings.Replace(cursor, "/", "_", -1))) log.Println("writing", fileName) if err := os.WriteFile(fileName, bytes, 0644); err != nil { diff --git a/cli/internal/plaid/transactions.go b/cli/internal/plaid/transactions.go index 9ff70f3..095455f 100644 --- a/cli/internal/plaid/transactions.go +++ b/cli/internal/plaid/transactions.go @@ -5,15 +5,17 @@ import ( "errors" "io" "log" + "path/filepath" "github.com/plaid/plaid-go/v20/plaid" ) -func (pc *APIClient) LoadTransactions(ctx context.Context, accessToken string) error { +func (pc *APIClient) LoadTransactions(ctx context.Context, institutionId string, accessToken string) error { var hasMore bool = true + prefix := filepath.Join("transactions", institutionId) // Get previous cursor from the latest cached response - cursor, err := pc.GetNextCursor(ctx, "transactions") + cursor, err := pc.GetNextCursor(ctx, prefix) if err != nil { return err } @@ -64,7 +66,7 @@ func (pc *APIClient) LoadTransactions(ctx context.Context, accessToken string) e nextCursor := resp.GetNextCursor() cursor = nextCursor - if err := pc.SetCache(ctx, "transactions", cursor, body); err != nil { + if err := pc.SetCache(ctx, prefix, cursor, body); err != nil { return err } } From 41c8a214f1ca137e827b577c4175703cedef6f8e Mon Sep 17 00:00:00 2001 From: quinn Date: Fri, 2 Feb 2024 16:30:24 -0500 Subject: [PATCH 6/6] fix bug --- cli/internal/plaid/accounts.go | 2 +- cli/internal/plaid/cache.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/internal/plaid/accounts.go b/cli/internal/plaid/accounts.go index c2206d0..05b1a14 100644 --- a/cli/internal/plaid/accounts.go +++ b/cli/internal/plaid/accounts.go @@ -33,7 +33,7 @@ func (pc *APIClient) LoadAccounts(ctx context.Context, institutionId string, acc return err } - if err = pc.SetCache(ctx, filepath.Join("transactions", institutionId), "", body); err != nil { + if err = pc.SetCache(ctx, filepath.Join("accounts", institutionId), "", body); err != nil { return err } diff --git a/cli/internal/plaid/cache.go b/cli/internal/plaid/cache.go index febf7a8..181d6e3 100644 --- a/cli/internal/plaid/cache.go +++ b/cli/internal/plaid/cache.go @@ -28,6 +28,7 @@ func (pc *APIClient) GetNextCursor(ctx context.Context, prefix string) (string, if err := json.Unmarshal(data, syncResponse); err != nil { return "", err } + return syncResponse.GetNextCursor(), nil }