refactor: factor out AST

CONTRIBUTING.md

@@ -119,7 +119,7 @@ Some CLI tests can be found in [.github/workflows/tact.yml](./.github/workflows/
 
 The [src/grammar/grammar.ohm](./src/grammar/grammar.ohm) file contains the Tact grammar expressed in the PEG-like language of the [Ohm.js](https://ohmjs.org) parser generator.
 
-The helper file [src/grammar/grammar.ts](./src/grammar/grammar.ts) contains the logic that transforms concrete syntax trees produced with the help of the Ohm.js-generated parser into abstract syntax trees (ASTs) defined in [src/grammar/ast.ts](./src/grammar/ast.ts). The grammar.ts file also does a bit of grammar validation, like checking that function or constant attributes are not duplicated or that user identifiers do not start with certain reserved prefixes.
+The helper file [src/grammar/grammar.ts](./src/grammar/grammar.ts) contains the logic that transforms concrete syntax trees produced with the help of the Ohm.js-generated parser into abstract syntax trees (ASTs) defined in [src/ast/ast.ts](./src/ast/ast.ts). The grammar.ts file also does a bit of grammar validation, like checking that function or constant attributes are not duplicated or that user identifiers do not start with certain reserved prefixes.
 
 The [src/grammar/test](./src/grammar/test) folder contains Tact files that are supposed to be parsed without any issues, and the [src/grammar/test-failed](./src/grammar/test-failed) folder contains grammatically incorrect test files which should result in parser errors. The parser error messages and the locations they point to are fixed in the [src/grammar/**snapshots**/grammar.spec.ts.snap](./src/grammar/__snapshots__/grammar.spec.ts.snap) Jest snapshot file.
 
@@ -177,6 +177,6 @@ Some other codegen tests are as follows:
 
 The entry point to the Tact AST pretty-printer is [src/prettyPrinter.ts](./src/prettyPrinter.ts). It is going to be used for the Tact source code formatter once the parser keeps comments and other relevant information.
 
-The AST comparator is defined in [src/grammar/compare.ts](./src/grammar/compare.ts). This is useful, for instance, for static analysis tools which can re-use the Tact TypeScript API.
+The AST comparator is defined in [src/ast/compare.ts](./src/ast/compare.ts). This is useful, for instance, for static analysis tools which can re-use the Tact TypeScript API.
 
 The corresponding test spec files can be found in [src/test](./src/test/) folder with the test contracts in [src/test/contracts](./src/test/contracts/) folder.

knip.json

@@ -3,7 +3,7 @@
   "entry": ["src/index.ts", "src/node.ts", "bin/tact.js", "bin/unboc.js"],
   "project": ["src/**/*.ts", "bin/tact.js", "bin/unboc.js"],
   "ignore": [
-    "src/grammar/ast.ts",
+    "src/ast/ast.ts",
     "src/prettyPrinter.ts",
     "src/error/display-to-json.ts",
     "src/grammar/src-info.ts",

src/abi/AbiFunction.ts

@@ -1,4 +1,4 @@
-import { AstExpression } from "../grammar/ast";
+import { AstExpression } from "../ast/ast";
 import { CompilerContext } from "../context/context";
 import { WriterContext } from "../generator/Writer";
 import { TypeRef } from "../types/types";

src/abi/global.ts

@@ -19,7 +19,7 @@ import {
     ensureString,
     interpretEscapeSequences,
 } from "../optimizer/interpreter";
-import { isLiteral } from "../grammar/ast";
+import { isLiteral } from "../ast/ast";
 
 export const GlobalFunctions: Map<string, AbiFunction> = new Map([
     [

src/abi/map.ts

@@ -7,7 +7,7 @@ import { writeExpression } from "../generator/writers/writeExpression";
 import { throwCompilationError } from "../error/errors";
 import { getType } from "../types/resolveDescriptors";
 import { AbiFunction } from "./AbiFunction";
-import { AstExpression } from "../grammar/ast";
+import { AstExpression } from "../ast/ast";
 
 // Helper functions to avoid redundancy
 function checkArgumentsLength(

src/grammar/ast.ts → src/ast/ast.ts

@@ -1,6 +1,6 @@
 import { Address, Cell, Slice } from "@ton/core";
 import { throwInternalCompilerError } from "../error/errors";
-import { dummySrcInfo, SrcInfo } from "./src-info";
+import { dummySrcInfo, SrcInfo } from "../grammar/src-info";
 
 export type AstModule = {
     kind: "module";
@@ -380,25 +380,28 @@ export type AstBouncedMessageType = {
 //
 
 export type AstExpression =
-    | AstExpressionPrimary
     | AstOpBinary
     | AstOpUnary
     | AstConditional
-    // AstLiteral could be added in AstExpressionPrimary,
-    // but AstExpressionPrimary is planned to be removed. See issue #1290 (https://github.com/tact-lang/tact/issues/1290).
-    | AstLiteral;
-
-export type AstExpressionPrimary =
     | AstMethodCall
     | AstFieldAccess
     | AstStaticCall
     | AstStructInstance
+    | AstId
+    | AstInitOf
+    | AstString
+    | AstLiteral;
+
+export type AstLiteral =
     | AstNumber
     | AstBoolean
-    | AstId
     | AstNull
-    | AstInitOf
-    | AstString;
+    | AstSimplifiedString
+    | AstAddress
+    | AstCell
+    | AstSlice
+    | AstCommentValue
+    | AstStructValue;
 
 export type AstBinaryOperation =
     | "+"
@@ -638,6 +641,20 @@ export type AstBoolean = {
     loc: SrcInfo;
 };
 
+// An AstSimplifiedString is a string in which escaping characters, like '\\' has been simplified, e.g., '\\' simplified to '\'.
+// An AstString is not a literal because it may contain escaping characters that have not been simplified, like '\\'.
+// AstSimplifiedString is always produced by the interpreter, never directly by the parser. The parser produces AstStrings, which
+// then get transformed into AstSimplifiedString by the interpreter.
+export type AstSimplifiedString = {
+    kind: "simplified_string";
+    value: string;
+    id: number;
+    loc: SrcInfo;
+};
+
+/**
+ * @deprecated AstSimplifiedString
+ */
 export type AstString = {
     kind: "string";
     value: string;
@@ -654,28 +671,6 @@ export type AstNull = {
     loc: SrcInfo;
 };
 
-export type AstLiteral =
-    | AstNumber
-    | AstBoolean
-    | AstNull
-    // An AstSimplifiedString is a string in which escaping characters, like '\\' has been simplified, e.g., '\\' simplified to '\'.
-    // An AstString is not a literal because it may contain escaping characters that have not been simplified, like '\\'.
-    // AstSimplifiedString is always produced by the interpreter, never directly by the parser. The parser produces AstStrings, which
-    // then get transformed into AstSimplifiedString by the interpreter.
-    | AstSimplifiedString
-    | AstAddress
-    | AstCell
-    | AstSlice
-    | AstCommentValue
-    | AstStructValue;
-
-export type AstSimplifiedString = {
-    kind: "simplified_string";
-    value: string;
-    id: number;
-    loc: SrcInfo;
-};
-
 export type AstAddress = {
     kind: "address";
     value: Address;

src/grammar/ast-typed.ts → src/ast/getAstSchema.ts

@@ -5,7 +5,7 @@
 
 import { Loc } from "@tonstudio/parser-runtime";
 import * as A from "./ast";
-import { SrcInfo } from "./src-info";
+import { SrcInfo } from "../grammar/src-info";
 
 export const getAstSchema = (
     factory: A.FactoryAst,

src/grammar/rename.ts → src/ast/rename.ts

@@ -19,7 +19,7 @@ import {
 } from "./ast";
 import { AstSorter } from "./sort";
 import { AstHasher, AstHash } from "./hash";
-import { dummySrcInfo } from "./src-info";
+import { dummySrcInfo } from "../grammar/src-info";
 
 type GivenName = string;
 

src/bindings/writeTypescript.ts

@@ -21,7 +21,7 @@ import {
     getAllocationOperationFromField,
 } from "../storage/allocator";
 import { serializers } from "./typescript/serializers";
-import { eqNames } from "../grammar/ast";
+import { eqNames } from "../ast/ast";
 
 function writeArguments(args: ABIArgument[]) {
     const res: string[] = [];

src/grammar/store.ts → src/context/store.ts

@@ -5,11 +5,11 @@ import {
     AstNativeFunctionDecl,
     AstTypeDecl,
     AstAsmFunctionDef,
-} from "./ast";
+} from "../ast/ast";
 import { throwInternalCompilerError } from "../error/errors";
-import { CompilerContext, createContextStore } from "../context/context";
-import { ItemOrigin } from "./src-info";
-import { Parser } from "./grammar";
+import { CompilerContext, createContextStore } from "./context";
+import { ItemOrigin } from "../grammar/src-info";
+import { Parser } from "../grammar/grammar";
 
 /**
  * @public

src/error/errors.ts

@@ -1,6 +1,6 @@
 import path from "path";
 import { cwd } from "process";
-import { AstFuncId, AstId, AstTypeId } from "../grammar/ast";
+import { AstFuncId, AstId, AstTypeId } from "../ast/ast";
 import { SrcInfo } from "../grammar";
 
 export class TactError extends Error {

src/generator/createABI.ts

@@ -1,7 +1,7 @@
 import { ABIGetter, ABIReceiver, ABIType, ContractABI } from "@ton/core";
 import { contractErrors } from "../abi/errors";
 import { CompilerContext } from "../context/context";
-import { idText } from "../grammar/ast";
+import { idText } from "../ast/ast";
 import { getSupportedInterfaces } from "../types/getSupportedInterfaces";
 import { createABITypeRefFromTypeRef } from "../types/resolveABITypeRef";
 import { getAllTypes } from "../types/resolveDescriptors";

src/generator/writeProgram.ts

@@ -18,7 +18,7 @@ import { writeAccessors } from "./writers/writeAccessors";
 import { ContractABI } from "@ton/core";
 import { writeFunction } from "./writers/writeFunction";
 import { calculateIPFSlink } from "../utils/calculateIPFSlink";
-import { getRawAST } from "../grammar/store";
+import { getRawAST } from "../context/store";
 import { emit } from "./emitter/emit";
 import {
     writeInit,

src/generator/writers/id.ts

@@ -1,4 +1,4 @@
-import { AstId, idText } from "../../grammar/ast";
+import { AstId, idText } from "../../ast/ast";
 
 export function funcIdOf(ident: AstId | string): string {
     if (typeof ident === "string") {

src/generator/writers/resolveFuncType.spec.ts

@@ -1,8 +1,8 @@
-import { getAstFactory } from "../../grammar/ast";
+import { getAstFactory } from "../../ast/ast";
 import { resolveDescriptors } from "../../types/resolveDescriptors";
 import { WriterContext } from "../Writer";
 import { resolveFuncType } from "./resolveFuncType";
-import { openContext } from "../../grammar/store";
+import { openContext } from "../../context/store";
 import { CompilerContext } from "../../context/context";
 import { getParser } from "../../grammar";
 import { defaultParser } from "../../grammar/grammar";

src/generator/writers/writeExpression.spec.ts

@@ -4,11 +4,11 @@ import {
 } from "../../types/resolveDescriptors";
 import { WriterContext } from "../Writer";
 import { writeExpression } from "./writeExpression";
-import { openContext } from "../../grammar/store";
+import { openContext } from "../../context/store";
 import { resolveStatements } from "../../types/resolveStatements";
 import { CompilerContext } from "../../context/context";
 import { getParser } from "../../grammar";
-import { getAstFactory } from "../../grammar/ast";
+import { getAstFactory } from "../../ast/ast";
 import { defaultParser } from "../../grammar/grammar";
 
 const code = `

src/generator/writers/writeExpression.ts

@@ -6,7 +6,7 @@ import {
     getAstFactory,
     idText,
     tryExtractPath,
-} from "../../grammar/ast";
+} from "../../ast/ast";
 import {
     idTextErr,
     TactConstEvalError,

src/generator/writers/writeFunction.ts

@@ -8,7 +8,7 @@ import {
     idText,
     isWildcard,
     tryExtractPath,
-} from "../../grammar/ast";
+} from "../../ast/ast";
 import { getType, resolveTypeRef } from "../../types/resolveDescriptors";
 import { getExpType } from "../../types/resolveExpression";
 import { FunctionDescription, TypeRef } from "../../types/types";

src/generator/writers/writeRouter.ts

@@ -7,7 +7,7 @@ import { ops } from "./ops";
 import { resolveFuncType } from "./resolveFuncType";
 import { resolveFuncTypeUnpack } from "./resolveFuncTypeUnpack";
 import { writeStatement } from "./writeFunction";
-import { AstNumber } from "../../grammar/ast";
+import { AstNumber } from "../../ast/ast";
 
 export function commentPseudoOpcode(comment: string): string {
     return beginCell()

src/generator/writers/writeSerialization.spec.ts

@@ -11,10 +11,10 @@ import {
 import { WriterContext } from "../Writer";
 import { writeParser, writeSerializer } from "./writeSerialization";
 import { writeStdlib } from "./writeStdlib";
-import { openContext } from "../../grammar/store";
+import { openContext } from "../../context/store";
 import { writeAccessors } from "./writeAccessors";
 import { getParser } from "../../grammar";
-import { getAstFactory } from "../../grammar/ast";
+import { getAstFactory } from "../../ast/ast";
 import { defaultParser } from "../../grammar/grammar";
 
 const code = `

src/grammar/grammar.ts

@@ -1,6 +1,6 @@
 import { getParser as getParserNext } from "./next";
 
-import { AstExpression, AstImport, AstModule, FactoryAst } from "./ast";
+import { AstExpression, AstImport, AstModule, FactoryAst } from "../ast/ast";
 
 import { getParser as getParserPrev } from "./prev/grammar";
 import { ItemOrigin } from "./src-info";

src/grammar/next/grammar.spec.ts

@@ -1,4 +1,4 @@
-import { getAstFactory } from "../ast";
+import { getAstFactory } from "../../ast/ast";
 import { loadCases } from "../../utils/loadCases";
 import { getParser } from "../grammar";
 import { SrcInfo, isSrcInfo } from "../src-info";

src/grammar/next/index.ts

@@ -1,13 +1,13 @@
 import * as $ from "@tonstudio/parser-runtime";
-import * as A from "../ast";
+import * as A from "../../ast/ast";
 import * as G from "./grammar";
 import type { $ast } from "./grammar";
 import {
     TactCompilationError,
     throwInternalCompilerError,
 } from "../../error/errors";
 import { SyntaxErrors, syntaxErrorSchema } from "../parser-error";
-import { AstSchema, getAstSchema } from "../ast-typed";
+import { AstSchema, getAstSchema } from "../../ast/getAstSchema";
 import { getSrcInfo, ItemOrigin } from "../src-info";
 import { displayToString } from "../../error/display-to-string";
 import { makeMakeVisitor } from "../../utils/tricks";

src/grammar/prev/grammar.spec.ts

@@ -1,4 +1,4 @@
-import { getAstFactory } from "../ast";
+import { getAstFactory } from "../../ast/ast";
 import { loadCases } from "../../utils/loadCases";
 import { getParser } from "../grammar";
 import { SrcInfo, isSrcInfo } from "../src-info";

src/grammar/prev/grammar.ts

@@ -17,7 +17,7 @@ import {
     AstNumberBase,
     AstId,
     FactoryAst,
-} from "../ast";
+} from "../../ast/ast";
 import { ItemOrigin, SrcInfo } from "../src-info";
 import { displayToString } from "../../error/display-to-string";
 import { ParserErrors, parserErrorSchema } from "./parser-error";

src/grammar/test/expr-equality.spec.ts

@@ -1,4 +1,4 @@
-import { eqExpressions, getAstFactory } from "../ast";
+import { eqExpressions, getAstFactory } from "../../ast/ast";
 import { getParser } from "../";
 import { defaultParser } from "../grammar";
 

src/grammar/test/expr-is-value.spec.ts

@@ -1,6 +1,6 @@
 //type Test = { expr: string; isValue: boolean };
 
-import { getAstFactory, isLiteral } from "../ast";
+import { getAstFactory, isLiteral } from "../../ast/ast";
 import { getParser } from "../";
 import { defaultParser } from "../grammar";
 

src/imports/resolveImports.spec.ts

@@ -2,7 +2,7 @@ import { resolveImports } from "./resolveImports";
 import { createNodeFileSystem } from "../vfs/createNodeFileSystem";
 import path from "path";
 import { getParser } from "../grammar";
-import { getAstFactory } from "../grammar/ast";
+import { getAstFactory } from "../ast/ast";
 import { defaultParser } from "../grammar/grammar";
 
 describe("resolveImports", () => {

src/index.ts

@@ -12,10 +12,10 @@ export {
     projectSchema,
     configSchema,
 } from "./config/parseConfig";
-export { AstSorter } from "./grammar/sort";
-export { AstRenamer } from "./grammar/rename";
-export { AstHasher } from "./grammar/hash";
-export { AstComparator } from "./grammar/compare";
+export { AstSorter } from "./ast/sort";
+export { AstRenamer } from "./ast/rename";
+export { AstHasher } from "./ast/hash";
+export { AstComparator } from "./ast/compare";
 
 export {
     Config,

src/optimizer/algebraic.ts

@@ -5,7 +5,7 @@ import {
     AstOpUnary,
     eqExpressions,
     isLiteral,
-} from "../grammar/ast";
+} from "../ast/ast";
 import { ExpressionTransformer, Rule } from "./types";
 import {
     checkIsBinaryOpNode,

src/optimizer/associative.ts

@@ -7,7 +7,7 @@ import {
     AstLiteral,
     AstOpBinary,
     isLiteral,
-} from "../grammar/ast";
+} from "../ast/ast";
 import * as iM from "./interpreter";
 import { ExpressionTransformer, Rule } from "./types";
 import {

src/optimizer/constEval.ts

@@ -5,7 +5,7 @@ import {
     AstUnaryOperation,
     isLiteral,
     AstLiteral,
-} from "../grammar/ast";
+} from "../ast/ast";
 import {
     TactConstEvalError,
     throwInternalCompilerError,