Merge pull request #103 from jserfun/feature/jsonschema-formatter

feature/jsonschema-formatter - fix: number types with wrong max and min, and add handling bigint in javascript Breaking changes: - Compact JSON format: width has been renamed to displayWidth, values are now different. - Compact JSON format: int datatypes ('tinyint', 'smallint', 'mediumint', 'int', 'bigint') names are preserved. - JSON Schema format: minimum and maximum values of some integer types have been changed.
duartealexf · Jan 13, 2025 · 1418554 · 1418554
2 parents 4dcba5c + 10a2cbb
commit 1418554
Show file tree

Hide file tree

Showing 44 changed files with 653 additions and 928 deletions.
diff --git a/README.md b/README.md
@@ -7,22 +7,23 @@
 
 Transforms SQL DDL statements into JSON format (JSON Schema and a compact format).
 
-- [Overview](#overview)
-- [Installation](#installation)
-- [Usage](#usage)
-  - [Shorthand](#shorthand)
-  - [Step by step](#step-by-step)
-- [Options for JSON Schema output](#options-for-json-schema-output)
-  - [`useRef`](#useref)
-- [Version compatibility table](#version-compatibility-table)
-- [What it is, what it is not](#what-it-is-what-it-is-not)
-- [About](#about)
-- [Contributing](#contributing)
-  - [Commiting](#commiting)
-  - [Understanding the internals](#understanding-the-internals)
-  - [Scripts at hand](#scripts-at-hand)
-    - [Visual Studio Code](#visual-studio-code)
-- [Links](#links)
+- [SQL DDL to JSON Schema converter](#sql-ddl-to-json-schema-converter)
+  - [Overview](#overview)
+  - [Installation](#installation)
+  - [Usage](#usage)
+    - [Shorthand](#shorthand)
+    - [Step by step](#step-by-step)
+  - [Options for JSON Schema output](#options-for-json-schema-output)
+    - [`useRef`](#useref)
+  - [Version compatibility table](#version-compatibility-table)
+  - [What it is, what it is not](#what-it-is-what-it-is-not)
+  - [About](#about)
+  - [Contributing](#contributing)
+    - [Commiting](#commiting)
+    - [Understanding the internals](#understanding-the-internals)
+    - [Scripts at hand](#scripts-at-hand)
+      - [Visual Studio Code](#visual-studio-code)
+  - [Links](#links)
 
 ## Overview
 
@@ -52,17 +53,13 @@ It parses and delivers an **array of JSON Schema documents** (one for each parse
     "title": "users",
     "description": "All system users",
     "type": "object",
-    "required": [
-      "id",
-      "nickname",
-      "created_at"
-    ],
+    "required": ["id", "nickname", "created_at"],
     "definitions": {
       "id": {
         "$comment": "primary key",
         "type": "integer",
         "minimum": 1,
-        "maximum": 1.5474250491067253e+26
+        "maximum": 2147483647
       },
       "nickname": {
         "type": "string",
@@ -184,7 +181,7 @@ And an array of tables in a compact JSON format:
 ]
 ```
 
-*Currently only DDL statements of MySQL and MariaDB dialects are supported.* - [Check out the roadmap](https://github.com/duartealexf/sql-ddl-to-json-schema/blob/master/ROADMAP.md)
+_Currently only DDL statements of MySQL and MariaDB dialects are supported._ - [Check out the roadmap](https://github.com/duartealexf/sql-ddl-to-json-schema/blob/master/ROADMAP.md)
 
 ## Installation
 
@@ -200,7 +197,7 @@ npm i sql-ddl-to-json-schema
 ```ts
 const { Parser } = require('sql-ddl-to-json-schema');
 // or:
-import { Parser } from 'sql-ddl-to-json-schema'
+import { Parser } from 'sql-ddl-to-json-schema';
 
 const parser = new Parser('mysql');
 
@@ -231,7 +228,6 @@ const compactJsonTablesArray = parser.feed(sql).toCompactJson(parser.results);
  * Or get the JSON Schema if you need to modify it...
  */
 const jsonSchemaDocuments = parser.feed(sql).toJsonSchemaArray(options, compactJsonTablesArray);
-
 ```
 
 ### Step by step
@@ -269,7 +265,7 @@ There are a few options when it comes to formatting the JSON Schema output:
 
 ### `useRef`
 
-Whether to add all properties to `definitions` and in `properties` only use $ref.
+Whether to add all properties to `definitions` and in `properties` only use \$ref.
 
 Default value: `true`.
 
@@ -314,26 +310,26 @@ To commit, use commitizen: `git cz` (you will need to have installed devDependen
 Folder structure:
 
 ```md
-|- lib/                   Compiled library folder, product of this project.
+|- lib/ Compiled library folder, product of this project.
 |
 |- src/
-|  |- typings/            Types used throughout project.
-|  |- shared/             Shared files used by dialects, parsers and formatters.
-|  |- mysql/
-|     |- formatter/       Formats the parsed JSON (output of parser) to other format.
-|        |- compact/      Formatter for compact JSON format.
-|        |- json-schema/  Formatter for JSON Schema format.
-|     |- language/
-|        |- dictionary/   TS files with array of keywords and symbols used in lexer.ne.
-|        |- rules/        Nearley files with grammar rules.
-|        |- lexer.ne      Entrypoint and first lines of the grammar.
+| |- typings/ Types used throughout project.
+| |- shared/ Shared files used by dialects, parsers and formatters.
+| |- mysql/
+| |- formatter/ Formats the parsed JSON (output of parser) to other format.
+| |- compact/ Formatter for compact JSON format.
+| |- json-schema/ Formatter for JSON Schema format.
+| |- language/
+| |- dictionary/ TS files with array of keywords and symbols used in lexer.ne.
+| |- rules/ Nearley files with grammar rules.
+| |- lexer.ne Entrypoint and first lines of the grammar.
 |
 |- tasks/
-|  |- mysql/
-|     |- assembly.ts      Script that concatenates all .ne files to grammar.ne to lib folder.
-|     |- formatters.ts    Script that sends a copy of formatters to lib folder.
+| |- mysql/
+| |- assembly.ts Script that concatenates all .ne files to grammar.ne to lib folder.
+| |- formatters.ts Script that sends a copy of formatters to lib folder.
 |
-|- test/                  Tests.
+|- test/ Tests.
 ```
 
 - There are naming rules for tokens in ne files, as stated in `lexer.ne`. They are prepended with:
@@ -347,7 +343,7 @@ S_ -> Symbol (not a keyword, but chars and other matches by RegExp's)
 
 ```
 
-1. The `dictionary/keywords.ts` file contains keywords, but they are prepended with K_ when used in .ne files. Take a look to make sure you understand how it is exported.
+1. The `dictionary/keywords.ts` file contains keywords, but they are prepended with K\_ when used in .ne files. Take a look to make sure you understand how it is exported.
 
 2. The compiled `grammar.ne` file comprises an assembly (concatenation) of `lexer.ne` and files in `language` folder. So don't worry about importing .ne files in other .ne files. This prevents circular dependency and grammar rules in `lexer.ne` are scoped to all files (thus not having to repeat them in every file).
 

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "sql-ddl-to-json-schema",
-  "version": "4.1.1",
+  "version": "5.0.0",
   "description": "Parse and convert SQL DDL statements to a JSON Schema.",
   "keywords": [
     "parse",
@@ -101,4 +101,4 @@
   "engines": {
     "node": ">=8.6.0 <=23"
   }
-}
+}
diff --git a/src/mysql/constants.ts b/src/mysql/constants.ts
@@ -0,0 +1,44 @@
+// https://learn.microsoft.com/en-us/sql/t-sql/data-types/int-bigint-smallint-and-tinyint-transact-sql?view=sql-server-ver16
+// https://dev.mysql.com/doc/refman/8.4/en/integer-types.html
+
+export const TINYINT_SIGNED_MIN = -128;
+export const TINYINT_SIGNED_MAX = 127;
+export const TINYINT_UNSIGNED_MIN = 0;
+export const TINYINT_UNSIGNED_MAX = 255;
+
+export const SMALLINT_SIGNED_MIN = -32768;
+export const SMALLINT_SIGNED_MAX = 32767;
+export const SMALLINT_UNSIGNED_MIN = 0;
+export const SMALLINT_UNSIGNED_MAX = 65535;
+
+export const MEDIUMINT_SIGNED_MIN = -8388608;
+export const MEDIUMINT_SIGNED_MAX = 8388607;
+export const MEDIUMINT_UNSIGNED_MIN = 0;
+export const MEDIUMINT_UNSIGNED_MAX = 16777215;
+
+export const INT_SIGNED_MIN = -2147483648;
+export const INT_SIGNED_MAX = 2147483647;
+export const INT_UNSIGNED_MIN = 0;
+export const INT_UNSIGNED_MAX = 4294967295;
+
+// -BigInt(1) * BigInt(2) ** BigInt(63)
+export const BIGINT_SIGNED_MIN = -9223372036854775808n;
+// BigInt(2) ** BigInt(63) - BigInt(1)
+export const BIGINT_SIGNED_MAX = 9223372036854775807n;
+//
+export const BIGINT_UNSIGNED_MIN = 0n;
+// BigInt(2) ** BigInt(64) - BigInt(1)
+export const BIGINT_UNSIGNED_MAX = 18446744073709551615n;
+
+// https://developer.mozilla.org/zh-CN/docs/Web/JavaScript/Reference/Global_Objects/BigInt
+
+/**  9007199254740991: 2 ^ 53 - 1 */
+export const MAX_SAFE_INTEGER = Number.MAX_SAFE_INTEGER;
+/** -9007199254740991: -1 * (2 ^ 53 - 1) */
+export const MIN_SAFE_INTEGER = Number.MIN_SAFE_INTEGER;
+
+// a proposal for javascript to handle big int numbers
+export const BIGINT_SIGNED_MIN_1 = MIN_SAFE_INTEGER;
+export const BIGINT_SIGNED_MAX_1 = MAX_SAFE_INTEGER;
+export const BIGINT_UNSIGNED_MIN_1 = 0;
+export const BIGINT_UNSIGNED_MAX_1 = MAX_SAFE_INTEGER;
diff --git a/src/mysql/formatter/compact/models/datatype.ts b/src/mysql/formatter/compact/models/datatype.ts
@@ -9,7 +9,7 @@ import { DatatypeModelInterface } from './typings';
 export class Datatype implements DatatypeModelInterface {
   datatype!: string;
 
-  width?: number;
+  displayWidth?: number;
 
   digits?: number;
 
@@ -53,18 +53,6 @@ export class Datatype implements DatatypeModelInterface {
     if (lowerTerm === 'integer') {
       return 'int';
     }
-    if (lowerTerm === 'tinyint') {
-      return 'int';
-    }
-    if (lowerTerm === 'smallint') {
-      return 'int';
-    }
-    if (lowerTerm === 'mediumint') {
-      return 'int';
-    }
-    if (lowerTerm === 'bigint') {
-      return 'int';
-    }
     if (lowerTerm === 'numeric') {
       return 'decimal';
     }
@@ -116,7 +104,11 @@ export class Datatype implements DatatypeModelInterface {
      */
     if (
       [
+        'tinyint',
+        'smallint',
+        'mediumint',
         'int',
+        'bigint',
         'decimal',
         'float',
         'double',
@@ -172,8 +164,8 @@ export class Datatype implements DatatypeModelInterface {
       datatype: this.datatype,
     };
 
-    if (isDefined(this.width)) {
-      json.width = this.width;
+    if (isDefined(this.displayWidth)) {
+      json.displayWidth = this.displayWidth;
     }
     if (isDefined(this.digits)) {
       json.digits = this.digits;
@@ -205,8 +197,8 @@ export class Datatype implements DatatypeModelInterface {
 
     datatype.datatype = this.datatype;
 
-    if (isDefined(this.width)) {
-      datatype.width = this.width;
+    if (isDefined(this.displayWidth)) {
+      datatype.displayWidth = this.displayWidth;
     }
     if (isDefined(this.digits)) {
       datatype.digits = this.digits;

diff --git a/src/mysql/formatter/json-schema/models/datatype.ts b/src/mysql/formatter/json-schema/models/datatype.ts
@@ -3,6 +3,29 @@ import { JSONSchema7TypeName, JSONSchema7 } from 'json-schema';
 import { DatatypeInterface } from '../../../../typings';
 import { isDefined } from '../../../../shared/utils';
 
+import {
+  TINYINT_SIGNED_MIN,
+  TINYINT_SIGNED_MAX,
+  TINYINT_UNSIGNED_MIN,
+  TINYINT_UNSIGNED_MAX,
+  SMALLINT_SIGNED_MIN,
+  SMALLINT_SIGNED_MAX,
+  SMALLINT_UNSIGNED_MIN,
+  SMALLINT_UNSIGNED_MAX,
+  MEDIUMINT_SIGNED_MIN,
+  MEDIUMINT_SIGNED_MAX,
+  MEDIUMINT_UNSIGNED_MIN,
+  MEDIUMINT_UNSIGNED_MAX,
+  INT_SIGNED_MIN,
+  INT_SIGNED_MAX,
+  INT_UNSIGNED_MIN,
+  INT_UNSIGNED_MAX,
+  BIGINT_SIGNED_MIN_1,
+  BIGINT_SIGNED_MAX_1,
+  BIGINT_UNSIGNED_MIN_1,
+  BIGINT_UNSIGNED_MAX_1,
+} from '../../../constants';
+
 /**
  * Data type.
  */
@@ -13,9 +36,9 @@ export class Datatype {
   datatype?: string;
 
   /**
-   * Width (in bytes) for integer types.
+   * Display width for integer types.
    */
-  width?: number;
+  displayWidth?: number;
 
   /**
    * Length of year type or length of a number
@@ -73,23 +96,27 @@ export class Datatype {
     if (isDefined(json.values)) {
       datatype.values = json.values;
     }
-    if (isDefined(json.width)) {
-      datatype.width = json.width;
+    if (isDefined(json.displayWidth)) {
+      datatype.displayWidth = json.displayWidth;
     }
 
     return datatype;
   }
 
+  static isIntegerDataType(type?: string): boolean | undefined {
+    return ['tinyint', 'smallint', 'mediumint', 'int', 'bigint'].includes(type ?? '');
+  }
+
   /**
    * Get standardized name for datatype, according to parsed compact JSON type.
    *
    * @param type Datatype parsed from compact JSON format.
    */
   static filterDatatype(type: string): JSONSchema7TypeName {
     /**
-     * Filters: int, integer, tinyint, smallint, mediumint, bigint
+     * Filters: int, tinyint, smallint, mediumint, bigint
      */
-    if (type === 'int') {
+    if (Datatype.isIntegerDataType(type)) {
       return 'integer';
     }
 
@@ -144,18 +171,45 @@ export class Datatype {
       type: Datatype.filterDatatype(this.datatype as string),
     };
 
-    if (this.datatype === 'int') {
-      /**
-       * Set minimum and maximum for int.
-       */
-      const width = 2 ** (8 * (this.width as number));
-
+    if (this.datatype === 'tinyint') {
       if (this.isUnsigned) {
-        json.minimum = 0;
-        json.maximum = width;
+        json.minimum = TINYINT_UNSIGNED_MIN;
+        json.maximum = TINYINT_UNSIGNED_MAX;
+      } else {
+        json.minimum = TINYINT_SIGNED_MIN;
+        json.maximum = TINYINT_SIGNED_MAX;
+      }
+    } else if (this.datatype === 'smallint') {
+      if (this.isUnsigned) {
+        json.minimum = SMALLINT_UNSIGNED_MIN;
+        json.maximum = SMALLINT_UNSIGNED_MAX;
+      } else {
+        json.minimum = SMALLINT_SIGNED_MIN;
+        json.maximum = SMALLINT_SIGNED_MAX;
+      }
+    } else if (this.datatype === 'mediumint') {
+      if (this.isUnsigned) {
+        json.minimum = MEDIUMINT_UNSIGNED_MIN;
+        json.maximum = MEDIUMINT_UNSIGNED_MAX;
+      } else {
+        json.minimum = MEDIUMINT_SIGNED_MIN;
+        json.maximum = MEDIUMINT_SIGNED_MAX;
+      }
+    } else if (this.datatype === 'int') {
+      if (this.isUnsigned) {
+        json.minimum = INT_UNSIGNED_MIN;
+        json.maximum = INT_UNSIGNED_MAX;
+      } else {
+        json.minimum = INT_SIGNED_MIN;
+        json.maximum = INT_SIGNED_MAX;
+      }
+    } else if (this.datatype === 'bigint') {
+      if (this.isUnsigned) {
+        json.minimum = BIGINT_UNSIGNED_MIN_1;
+        json.maximum = BIGINT_UNSIGNED_MAX_1;
       } else {
-        json.minimum = 0 - width / 2;
-        json.maximum = 0 - json.minimum - 1;
+        json.minimum = BIGINT_SIGNED_MIN_1;
+        json.maximum = BIGINT_SIGNED_MAX_1;
       }
     } else if (this.datatype === 'decimal' || this.datatype === 'float') {
       /**