feat: script in hasura/ to autogenerate table config (#861)

* feat: script in hasura/ to autogenerate table config

* genTables will crawl dbt directory for models and create a tracking
  config for the table

* feat: supabase migrations for api_keys and data_collective tables (#862)

* docs: starting a devops guide (#863)

* Remove bq prefix (#855)

* Rename model fields to be more explicit (#854)

* Rename model fields to be more explicit

* more fixes

* more fixes after testing

* Add readme

* add info

* fix artifact related fields

* more fixes

* big dbt rename

* update script

* fix update script

* Use macros for id creation

* clean up

* fix name

* docs: major updates (#859)

* docs: major updates

* fix: broken links in docs

* update: schema migrations

* add: npm library docs

* copy edits to schema pages

* Refresh eslint rules to inherit from root

---------

Co-authored-by: Reuven Gonzales <[email protected]>
Co-authored-by: Carl Cervone <[email protected]>

.eslintrc.js

@@ -1,73 +1,67 @@
 module.exports = {
-  "env": {
-    "browser": true,
-    "es2021": true,
-    "node": true
+  env: {
+    browser: true,
+    es2021: true,
+    node: true,
   },
-  "extends": [
+  extends: [
     "eslint:recommended",
-    "plugin:react/recommended",
     "plugin:@typescript-eslint/recommended",
+    "plugin:react/recommended",
     "prettier",
-    "next/core-web-vitals"
   ],
-  "ignorePatterns": [
+  ignorePatterns: [
     "**/vendor/*.js",
     "vendor/**/*.js",
     "**/jest.config.ts",
     "**/test.only/**",
     "**/utilities/**",
-    //"**/next.config.js",
+    "**/.eslintrc.js",
     "**/postcss.config.js",
+    //"**/next.config.js",
     //"**/tailwind.config.js",
-    "**/.env"
+    //"**/.env",
   ],
-  "parser": "@typescript-eslint/parser",
-  "parserOptions": {
-    "ecmaFeatures": {
-      "jsx": true
+  parser: "@typescript-eslint/parser",
+  parserOptions: {
+    ecmaFeatures: {
+      jsx: true,
     },
-    "ecmaVersion": "latest",
-    "sourceType": "module",
-    "project": "tsconfig.json",
-    "tsconfigRootDir": __dirname,
+    ecmaVersion: "latest",
+    sourceType: "module",
+    tsconfigRootDir: __dirname,
+    project: "./tsconfig.json",
   },
-  "plugins": [
-    "react",
-    "@typescript-eslint"
-  ],
-  "rules": {
-    "react-hooks/exhaustive-deps": "off",
+  plugins: ["@typescript-eslint", "react"],
+  settings: {
+    react: {
+      version: "detect",
+    },
+  },
+  rules: {
     "@typescript-eslint/no-misused-promises": "error",
     "@typescript-eslint/no-floating-promises": "error",
     "@typescript-eslint/no-explicit-any": "off",
     "@typescript-eslint/no-unused-vars": [
       "warn",
       {
-        "argsIgnorePattern": "^_"
-      }
+        argsIgnorePattern: "^_",
+      },
     ],
     "no-restricted-properties": [
       "error",
       {
-        "object": "console",
-        "property": "error",
-        "message": "Please use the logger instead."
-      }
+        object: "console",
+        property: "error",
+        message: "Please use the logger instead.",
+      },
     ],
     "no-restricted-globals": [
       "error",
       {
-        "name": "prompt",
-        "message": "Please use a React modal instead."
-      }
+        name: "prompt",
+        message: "Please use a React modal instead.",
+      },
     ],
-    "no-restricted-syntax": [
-      "error",
-      {
-        "selector": "Property[key.name='preview'][value.value=true]",
-        "message": "Make sure to turn 'preview' off before committing."
-      }
-    ]
-  }
-}
+  },
+};

cloudquery/github-resolve-repos/src/tables.ts

@@ -20,7 +20,6 @@ import readline from "readline";
 import { getReposFromUrls } from "./github/repositories.js";
 import { Octokit } from "octokit";
 
-/* eslint-disable import/no-named-as-default-member */
 dayjs.extend(utc);
 dayjs.extend(timezone);
 dayjs.extend(customParseFormat);

cloudquery/oss-directory/src/tables.ts

@@ -22,7 +22,6 @@ import timezone from "dayjs/plugin/timezone.js";
 import utc from "dayjs/plugin/utc.js";
 import { fetchData } from "oss-directory";
 
-/* eslint-disable import/no-named-as-default-member */
 dayjs.extend(utc);
 dayjs.extend(timezone);
 dayjs.extend(customParseFormat);

docs/.eslintrc.json

@@ -1,12 +1,7 @@
 {
-  "extends": ["../.eslintrc.js"],
   "root": false,
+  "extends": ["../.eslintrc.js"],
   "parserOptions": {
-    "ecmaFeatures": {
-      "jsx": true
-    },
-    "ecmaVersion": "latest",
-    "sourceType": "module",
     "project": ["./docs/tsconfig.json"]
   }
 }

docs/package.json

@@ -12,7 +12,10 @@
     "clear": "docusaurus clear",
     "serve": "docusaurus serve",
     "write-translations": "docusaurus write-translations",
-    "write-heading-ids": "docusaurus write-heading-ids"
+    "write-heading-ids": "docusaurus write-heading-ids",
+    "lint": "pnpm lint:eslint && pnpm lint:prettier",
+    "lint:eslint": "eslint --ignore-path ../.gitignore --ignore-path .gitignore --max-warnings 0 .",
+    "lint:prettier": "prettier --ignore-path ../.gitignore --ignore-path .gitignore --log-level warn --check **/*.{js,jsx,ts,tsx,sol,md,json}"
   },
   "dependencies": {
     "@docusaurus/core": "3.1.0",

docs/src/components/HomepageFeatures/index.tsx

@@ -1,3 +1,4 @@
+import React from "react";
 import clsx from "clsx";
 import Heading from "@theme/Heading";
 import styles from "./styles.module.css";

docs/src/pages/index-old.tsx

@@ -1,3 +1,4 @@
+import React from "react";
 import clsx from "clsx";
 import Link from "@docusaurus/Link";
 import useDocusaurusContext from "@docusaurus/useDocusaurusContext";

docs/tsconfig.json

@@ -2,6 +2,6 @@
   // This file is not used in compilation. It is here just for a nice editor experience.
   "extends": "@docusaurus/tsconfig",
   "compilerOptions": {
-    "baseUrl": "."
-  }
+    "baseUrl": ".",
+  },
 }

frontend/.eslintrc.json

@@ -1,12 +1,18 @@
 {
-  "extends": ["../.eslintrc.js"],
   "root": false,
+  "extends": ["../.eslintrc.js", "next/core-web-vitals"],
+  "parser": "@typescript-eslint/parser",
   "parserOptions": {
-    "ecmaFeatures": {
-      "jsx": true
-    },
-    "ecmaVersion": "latest",
-    "sourceType": "module",
     "project": ["./frontend/tsconfig.json"]
+  },
+  "rules": {
+    "react-hooks/exhaustive-deps": "off",
+    "no-restricted-syntax": [
+      "error",
+      {
+        "selector": "Property[key.name='preview'][value.value=true]",
+        "message": "Make sure to turn 'preview' off before committing."
+      }
+    ]
   }
 }

hasura/.eslintrc.json

@@ -0,0 +1,7 @@
+{
+  "extends": ["../.eslintrc.js"],
+  "root": false,
+  "parserOptions": {
+    "project": ["./hasura/tsconfig.json"]
+  }
+}

hasura/README.md

@@ -0,0 +1,37 @@
+# Hasura configuration
+
+This directory stores all configurations for the Hasura deployment.
+
+## Setup
+
+Copy `.env.example` to `.env` and set the environment variables as needed.
+
+## Configure
+
+You can modify any files in `./metadata` to update the Hasura configuration.
+See the
+[Hasura metadata reference](https://hasura.io/docs/latest/migrations-metadata-seeds/metadata-format/) on the schema.
+
+Note that anything in `./metadata/databases/cloudsql/tables/`
+will be overwritten by the build step.
+
+## Build
+
+This will generate table configurations for all tables found in `../dbt/`.
+This needs to be run every time the schema changes.
+For more information, see `./src/genTables.ts`.
+
+```bash
+pnpm build
+```
+
+## Deploy
+
+To reload the database schemas and apply the metadata configurations, run:
+
+```bash
+pnpm run deploy
+```
+
+For more information on how to manage metadata, see the
+[Hasura guide](https://hasura.io/docs/latest/migrations-metadata-seeds/manage-metadata/#reload-metadata).

hasura/package.json

@@ -5,6 +5,10 @@
   "author": "Kariba Labs",
   "license": "Apache-2.0",
   "private": true,
+  "bin": "./dist/src/genTables.js",
+  "main": "./dist/src/index.js",
+  "types": "./dist/src/index.d.ts",
+  "type": "module",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/opensource-observer/oso.git"
@@ -13,14 +17,25 @@
     "node": ">=20"
   },
   "scripts": {
+    "build": "tsc && pnpm metadata:genTables",
+    "lint": "tsc --noEmit && pnpm lint:eslint && pnpm lint:prettier",
+    "lint:eslint": "eslint --ignore-path ../.gitignore --max-warnings 0 .",
+    "lint:prettier": "prettier --ignore-path ../.gitignore --log-level warn --check **/*.{js,jsx,ts,tsx,sol,md,json}",
+    "metadata:genTables": "node --loader ts-node/esm src/genTables.ts",
     "metadata:pull": "hasura metadata export",
     "metadata:reload": "hasura metadata reload",
     "metadata:apply": "hasura metadata apply",
     "deploy": "pnpm metadata:reload && pnpm metadata:apply"
   },
   "keywords": [],
   "devDependencies": {
+    "@types/node": "^20.11.17",
     "dotenv-cli": "^7.3.0",
-    "hasura-cli": "^2.36.1"
+    "hasura-cli": "^2.36.1",
+    "ts-node": "^10.9.1",
+    "typescript": "^5.3.3"
+  },
+  "dependencies": {
+    "yaml": "^2.3.1"
   }
 }

hasura/src/genTables.ts

@@ -0,0 +1,97 @@
+import path from "node:path";
+import fs from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import * as yaml from "yaml";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+// YAML file extension
+const EXTENSION = ".yaml";
+// Recursively scan this directory for database tables
+const modelDir = path.resolve(__dirname, "../../dbt/models/marts/");
+// Where to store all table configs
+const tablesDir = path.resolve(
+  __dirname,
+  "../metadata/databases/cloudsql/tables/",
+);
+// Should map to the tables field in ./metadata/databases/databases.yaml
+const tablesListAbsPath = path.format({
+  dir: tablesDir,
+  name: "tables",
+  ext: EXTENSION,
+});
+
+// Schema for Hasura table configuration
+type TableConfig = {
+  table: {
+    name: string;
+    schema: string;
+  };
+  select_permissions: {
+    role: string;
+    permission: {
+      columns: string;
+      filter: Record<string, unknown>;
+      allow_aggregations: boolean;
+    };
+    comment: string;
+  }[];
+};
+
+// Create a table configuration object
+const createConfig = (name: string): TableConfig => ({
+  table: {
+    name: name,
+    schema: "public",
+  },
+  select_permissions: [
+    {
+      role: "anonymous",
+      permission: {
+        columns: "*",
+        filter: {},
+        allow_aggregations: false,
+      },
+      comment: "",
+    },
+  ],
+});
+
+async function main(): Promise<void> {
+  console.log(`Generating tables from ${modelDir}`);
+  // Recursively scan all files in the model directory
+  const allFiles = await fs.readdir(modelDir, { recursive: true });
+  // Get the basename as the table name
+  const tableNames = allFiles
+    .filter((f) => f.endsWith(".sql"))
+    .map((f) => path.basename(f, ".sql"));
+  console.log("Tables:");
+  console.log(tableNames);
+  // Write the list of tables
+  await fs.writeFile(
+    tablesListAbsPath,
+    yaml.stringify(
+      tableNames.map(
+        (n) => `!include ${path.format({ name: n, ext: EXTENSION })}`,
+      ),
+    ),
+  );
+  // Write a table config for each table
+  await Promise.all(
+    tableNames.map(async (n) => {
+      const config = createConfig(n);
+      const absPath = path.format({
+        dir: tablesDir,
+        name: n,
+        ext: EXTENSION,
+      });
+      await fs.writeFile(absPath, yaml.stringify(config));
+    }),
+  );
+}
+
+main()
+  .then(() => console.log("Done"))
+  .catch((e) => {
+    console.warn(e);
+  });