Merge pull request #700 from seb-oss/feat/spanner-migrate

Update Spanner migrate to use SQL migrations files

.changeset/tall-jobs-float.md

@@ -0,0 +1,5 @@
+---
+"@sebspark/spanner-migrate": minor
+---
+
+switched from .ts to .sql migrations files

packages/spanner-migrate/README.md

@@ -48,6 +48,26 @@ Create a new migration file.
 
 ```zsh
 spanner-migrate create add users table
+```
+
+**Result:**
+
+Example:
+
+`./migrations/20250120145638000_create_table_users.sql`
+
+```sql
+-- Created: 2025-01-20T14:56:38.000Z
+-- Description: create table users
+
+---- UP ----
+
+
+
+---- DOWN ----
+
+
+
 ```
 
 #### `up`

packages/spanner-migrate/src/__tests__/apply.spec.ts

@@ -6,7 +6,7 @@ import type {
 import { applyDown, applyUp } from '../apply'
 import type { Migration } from '../types'
 
-describe('apply.ts', () => {
+describe('apply', () => {
   let db: jest.Mocked<Database>
 
   beforeEach(() => {

packages/spanner-migrate/src/__tests__/e2e/migrations/20250120145638000_create_table_users.sql

@@ -0,0 +1,14 @@
+-- Created: 2025-01-20T14:56:38.000Z
+-- Description: create table users
+
+---- UP ----
+
+CREATE TABLE users (
+  id INT64 NOT NULL,
+  username STRING(50) NOT NULL,
+  email STRING(100) NOT NULL,
+) PRIMARY KEY (id);
+
+---- DOWN ----
+
+DROP TABLE users;

packages/spanner-migrate/src/__tests__/e2e/migrations/20250121144738000_create_table_and_index_adresses.sql

@@ -0,0 +1,28 @@
+-- Created: 2025-01-21T14:48:38.000Z
+-- Description: create table and index adresses
+
+---- UP ----
+
+CREATE TABLE addresses (
+  id STRING(36) NOT NULL, -- Unique identifier for the address
+  user_id STRING(36) NOT NULL, -- ID of the user associated with the address
+  street STRING(256), -- Street name and number
+  city STRING(128), -- City name
+  state STRING(128), -- State or region
+  zip_code STRING(16), -- Postal code
+  country STRING(128), -- Country name
+  created_at TIMESTAMP NOT NULL OPTIONS (allow_commit_timestamp = true), -- Timestamp of creation
+  updated_at TIMESTAMP OPTIONS (allow_commit_timestamp = true) -- Timestamp of last update
+) PRIMARY KEY (id);
+
+-- Create an index on user_id for faster lookups by user
+CREATE INDEX idx_addresses_user_id ON addresses (user_id);
+
+
+---- DOWN ----
+
+-- Drop the index first (Spanner requires indexes to be dropped before dropping the table)
+DROP INDEX idx_addresses_user_id;
+
+-- Drop the table
+DROP TABLE addresses;

packages/spanner-migrate/src/__tests__/files.spec.ts

@@ -1,5 +1,5 @@
 import type { Dirent } from 'node:fs'
-import { access, mkdir, readdir, writeFile } from 'node:fs/promises'
+import { access, mkdir, readFile, readdir, writeFile } from 'node:fs/promises'
 import { resolve } from 'node:path'
 import {
   createMigration,
@@ -15,16 +15,18 @@ jest.mock('node:fs/promises', () => ({
   access: jest.fn(),
   readdir: jest.fn(),
   mkdir: jest.fn(),
+  readFile: jest.fn(),
   writeFile: jest.fn(),
 }))
 
 // Declare mocks
 const accessMock = access as jest.MockedFunction<typeof access>
 const mkdirMock = mkdir as jest.MockedFunction<typeof mkdir>
 const readdirMock = readdir as jest.MockedFunction<typeof readdir>
+const readFileMock = readFile as jest.MockedFunction<typeof readFile>
 const writeFileMock = writeFile as jest.MockedFunction<typeof writeFile>
 
-describe('files.ts', () => {
+describe('files', () => {
   const mockPath = './mock/migrations'
   const mockConfigPath = './mock/spanner-migrate.config.json'
 
@@ -35,8 +37,8 @@ describe('files.ts', () => {
   describe('getMigrationFiles', () => {
     it('returns migration file IDs', async () => {
       const mockFiles = [
-        '20250101T123456_add_users.ts',
-        '20250102T123456_add_roles.ts',
+        '20250101T123456_add_users.sql',
+        '20250102T123456_add_roles.sql',
       ]
       readdirMock.mockResolvedValue(mockFiles as unknown as Dirent[])
 
@@ -60,45 +62,45 @@ describe('files.ts', () => {
 
   describe('getMigration', () => {
     const mockMigrationId = '20250101T123456_add_users'
-    const mockMigrationPath = resolve(mockPath, `${mockMigrationId}.ts`)
-
-    beforeEach(() => {
-      jest.resetModules()
-    })
+    const mockMigrationPath = resolve(mockPath, `${mockMigrationId}.sql`)
 
     it('returns the migration object if valid', async () => {
-      const mockModule = {
-        up: 'CREATE TABLE users (id STRING(36))',
-        down: 'DROP TABLE users',
-      }
-      jest.mock(mockMigrationPath, () => mockModule, { virtual: true })
+      const migrationFile = `-- Created: 2025-01-20T14:56:38.000Z
+-- Description: Create table users
+
+---- UP ----
+
+CREATE TABLE users (id STRING(36))
+
+---- DOWN ----
+
+DROP TABLE users
+`
       accessMock.mockResolvedValue(undefined)
+      readFileMock.mockResolvedValue(migrationFile)
 
       const result = await getMigration(mockPath, mockMigrationId)
 
       expect(accessMock).toHaveBeenCalledWith(mockMigrationPath)
       expect(result).toEqual({
         id: mockMigrationId,
-        description: 'Add Users',
-        up: mockModule.up,
-        down: mockModule.down,
+        description: 'Create table users',
+        up: 'CREATE TABLE users (id STRING(36))',
+        down: 'DROP TABLE users',
       })
     })
 
     it('throws an error if migration file does not exist', async () => {
-      accessMock.mockImplementation(async () => {
-        throw new Error('File not found')
-      })
+      readFileMock.mockRejectedValue(new Error('File not found'))
 
       await expect(getMigration(mockPath, mockMigrationId)).rejects.toThrow(
-        `Migration file not found: ${mockMigrationPath}`
+        'Failed to get migration 20250101T123456_add_users: File not found'
       )
     })
 
     it('throws an error if migration file is invalid', async () => {
-      const invalidModule = { up: 'CREATE TABLE users (id STRING(36))' } // Missing down
-      jest.mock(mockMigrationPath, () => invalidModule, { virtual: true })
       accessMock.mockImplementation(async () => undefined)
+      readFileMock.mockResolvedValue('Herp derp')
 
       await expect(getMigration(mockPath, mockMigrationId)).rejects.toThrow(
         `Migration file ${mockMigrationPath} does not export required scripts (up, down).`
@@ -163,8 +165,8 @@ describe('files.ts', () => {
 
       expect(mkdirMock).toHaveBeenCalledWith(mockPath, { recursive: true })
       expect(writeFileMock).toHaveBeenCalledWith(
-        expect.stringMatching(/^mock\/migrations\/\d+_add_users_table\.ts$/),
-        expect.stringMatching(/-- SQL for migrate up/),
+        expect.stringMatching(/^mock\/migrations\/\d+_add_users_table\.sql$/),
+        expect.stringMatching(/^-- Created: /),
         'utf8'
       )
     })

packages/spanner-migrate/src/__tests__/index.spec.ts

@@ -12,7 +12,7 @@ const mockDb = db as jest.Mocked<typeof db>
 const mockFiles = files as jest.Mocked<typeof files>
 const mockApply = apply as jest.Mocked<typeof apply>
 
-describe('index.ts', () => {
+describe('index', () => {
   const mockConfig: Config = {
     migrationsPath: '/mock/migrations',
     instanceName: 'mock-instance',

packages/spanner-migrate/src/__tests__/spanner-migrate.e2e.ts

@@ -140,7 +140,7 @@ describe('spanner-migrate', () => {
 
       const files = await readdir(resolve(config.migrationsPath))
       expect(files).toHaveLength(1)
-      expect(files[0]).toMatch(/^\d{17}_this_is_a_test_migration.ts$/)
+      expect(files[0]).toMatch(/^\d{17}_this_is_a_test_migration.sql$/)
     })
   })
   describe('up/status/down', () => {

packages/spanner-migrate/src/apply.ts

@@ -92,13 +92,11 @@ const runScript = async (db: Database, script: string): Promise<void> => {
   for (const statement of statements) {
     console.log(`Executing statement: ${statement}`)
 
-    const sql = statement.replace(/--.*$/gm, '') // Remove comments
-
-    if (isSchemaChange(sql)) {
-      await db.updateSchema(sql)
+    if (isSchemaChange(statement)) {
+      await db.updateSchema(statement)
     } else {
       await db.runTransactionAsync(async (transaction) => {
-        await transaction.runUpdate(sql)
+        await transaction.runUpdate(statement)
         await transaction.commit()
       })
     }

packages/spanner-migrate/src/cli.ts

@@ -67,7 +67,7 @@ yargs(hideBin(process.argv))
       const fullDescription = args.description.join(' ')
       await create(config, fullDescription)
       console.log(
-        `Migration file created: '${join(config.migrationsPath, args.description.join('_'))}.ts'`
+        `Migration file created: '${join(config.migrationsPath, args.description.join('_'))}.sql'`
       )
     }
   )

packages/spanner-migrate/src/files.ts

@@ -1,4 +1,4 @@
-import { access, mkdir, readdir, writeFile } from 'node:fs/promises'
+import { access, mkdir, readFile, readdir, writeFile } from 'node:fs/promises'
 import { join, resolve } from 'node:path'
 import type { Config, Migration } from './types'
 
@@ -9,8 +9,8 @@ export const getMigrationFiles = async (path: string): Promise<string[]> => {
 
     // Filter and map files to extract the `id` (file name without extension)
     const migrationFileIds = files
-      .filter((file) => file.endsWith('.ts')) // Only include .ts files
-      .map((file) => file.replace(/\.ts$/, '')) // Remove file extension to use as `id`
+      .filter((file) => file.endsWith('.sql')) // Only include .sql files
+      .map((file) => file.replace(/\.sql$/, '')) // Remove file extension to use as `id`
 
     return migrationFileIds
   } catch (error) {
@@ -26,7 +26,7 @@ export const getMigration = async (
 ): Promise<Migration> => {
   try {
     // Construct the full file path
-    const filePath = resolve(process.cwd(), join(path, `${id}.ts`))
+    const filePath = resolve(process.cwd(), join(path, `${id}.sql`))
 
     // Check if the file exists
     try {
@@ -36,33 +36,39 @@ export const getMigration = async (
     }
 
     // Dynamically import the migration file
-    const migrationModule = await import(filePath)
+    const migrationText = await readFile(filePath, 'utf8')
+
+    const up = getSql(migrationText, 'up')
+    const down = getSql(migrationText, 'down')
+    const description = getDescription(migrationText)
 
     // Validate that the required exports (`up` and `down`) exist
-    if (!migrationModule.up || !migrationModule.down) {
+    if (!up || !down) {
       throw new Error(
         `Migration file ${filePath} does not export required scripts (up, down).`
       )
     }
 
     // Return the migration as a `Migration` object
-    return {
-      id,
-      description: id
-        .split('_')
-        .slice(1)
-        .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
-        .join(' '), // Generate a human-readable description
-      up: migrationModule.up,
-      down: migrationModule.down,
-    }
+    return { id, description, up, down }
   } catch (error) {
     throw new Error(
       `Failed to get migration ${id}: ${(error as Error).message}`
     )
   }
 }
 
+const getDescription = (text: string | undefined) =>
+  text?.match(/^--\s*Description:\s*(.+)$/m)?.[1]?.trim() || ''
+
+const getSql = (text: string | undefined, direction: 'up' | 'down') => {
+  const rx = {
+    up: /---- UP ----\n([\s\S]*?)\n---- DOWN ----/,
+    down: /---- DOWN ----\n([\s\S]*)$/,
+  }
+  return text?.match(rx[direction])?.[1]?.replace(/--.*$/gm, '').trim()
+}
+
 export const getNewMigrations = (
   applied: Migration[],
   files: string[]
@@ -95,22 +101,22 @@ export const createMigration = async (
   const timestamp = new Date().toISOString()
   const compactTimestamp = timestamp.replace(/[-:.TZ]/g, '')
   const parsedDescription = description.replace(/\s+/g, '_').toLowerCase()
-  const filename = `${compactTimestamp}_${parsedDescription}.ts`
+  const filename = `${compactTimestamp}_${parsedDescription}.sql`
 
   // Full file path
   const filePath = join(path, filename)
 
   // Template migration content
-  const template = `// ${timestamp}
-// ${description}
+  const template = `-- Created: ${timestamp}
+-- Description: ${description}
+
+---- UP ----
+
+
+
+---- DOWN ----
 
-export const up = \`
-  -- SQL for migrate up
-\`
 
-export const down = \`
-  -- SQL for migrate down
-\`
 `
 
   try {