Merge pull request #5 from Exabyte-io/chore/SOF-7185

chore/SOF-7185 - pre-build PY modules, update names to @mat3ra, mat3ra-

.github/workflows/cicd.yml

@@ -48,7 +48,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.8.6]
+        python-version: [3.8.6, 3.9.x]
 
     steps:
       - name: Checkout this repository
@@ -73,7 +73,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [12.x, 14.x, 16.x]
+        node-version: [14.x, 20.x]
 
     steps:
       - name: Checkout this repository
@@ -99,55 +99,54 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
 
-#  publish-js-package:
-#    needs: [run-js-tests]
-#    runs-on: ubuntu-latest
-# yamllint disable-line rule:line-length
-#    if: (github.repository != 'Exabyte-io/template-definitions-js-py') && (github.ref_name == 'main')
-#
-#    steps:
-#      - name: Checkout this repository
-#        uses: actions/checkout@v3
-#
-#      - name: Checkout actions repository
-#        uses: actions/checkout@v3
-#        with:
-#          repository: Exabyte-io/actions
-#          token: ${{ secrets.BOT_GITHUB_TOKEN }}
-#          path: actions
-#
-#      - name: Publish JS release
-#        uses: ./actions/js/publish
-#        with:
-#          node-version: 14.19.x
-#          npm-token: ${{ secrets.NPM_TOKEN }}
-#          github-token: ${{ secrets.BOT_GITHUB_TOKEN }}
-#
- publish-py-package:
-   needs: publish-js-package
-   runs-on: ubuntu-latest
-# yamllint disable-line rule:line-length
-   if: (github.repository != 'Exabyte-io/template-definitions-js-py') && (github.ref_name == 'main')
-   publish-to-pypi: 'false'
-
-   steps:
-     -   name: Checkout this repository
-         uses: actions/checkout@v3
-         with:
-           lfs: true
-
-     -   name: Checkout actions repository
-         uses: actions/checkout@v3
-         with:
-           repository: Exabyte-io/actions
-           token: ${{ secrets.BOT_GITHUB_TOKEN }}
-           path: actions
-
-     -   name: Publish python release
-         uses: ./actions/py/publish
-         with:
-           python-version: 3.x
-           github-token: ${{ secrets.BOT_GITHUB_TOKEN }}
-           # pypi-username: ${{ secrets.PYPI_USERNAME }}
-           # pypi-password: ${{ secrets.PYPI_PASSWORD }}
-           publish-tag: 'true'
+  publish-js-package:
+    needs: [run-js-tests]
+    runs-on: ubuntu-latest
+    # yamllint disable-line rule:line-length
+    if: (github.repository != 'Exabyte-io/template-definitions-js-py') && (github.ref_name == 'main')
+
+    steps:
+      - name: Checkout this repository
+        uses: actions/checkout@v3
+
+      - name: Checkout actions repository
+        uses: actions/checkout@v3
+        with:
+          repository: Exabyte-io/actions
+          token: ${{ secrets.BOT_GITHUB_TOKEN }}
+          path: actions
+
+      - name: Publish JS release
+        uses: ./actions/js/publish
+        with:
+          node-version: 14.19.x
+          npm-token: ${{ secrets.NPM_TOKEN }}
+          github-token: ${{ secrets.BOT_GITHUB_TOKEN }}
+          publish-tag: 'true'
+
+  publish-py-package:
+    needs: [publish-js-package]
+    runs-on: ubuntu-latest
+    # yamllint disable-line rule:line-length
+    if: (github.repository != 'Exabyte-io/template-definitions-js-py') && (github.ref_name == 'main')
+    steps:
+      - name: Checkout this repository
+        uses: actions/checkout@v3
+        with:
+          lfs: true
+
+      - name: Checkout actions repository
+        uses: actions/checkout@v3
+        with:
+          repository: Exabyte-io/actions
+          token: ${{ secrets.BOT_GITHUB_TOKEN }}
+          path: actions
+
+      - name: Publish python release
+        uses: ./actions/py/publish
+        with:
+          python-version: 3.x
+          github-token: ${{ secrets.BOT_GITHUB_TOKEN }}
+          pypi-username: ${{ secrets.PYPI_USERNAME }}
+          pypi-password: ${{ secrets.PYPI_PASSWORD }}
+          publish-tag: 'false'

.pre-commit-config.yaml

@@ -2,8 +2,10 @@ repos:
   - repo: https://github.com/Exabyte-io/pre-commit-hooks
     rev: 2023.8.18
     hooks:
-      - id: ruff
-      - id: black
+# TB temporary disable to avoid firing for autogenerated files
+# TODO: figure out how to ignore the files properly and re-enable
+#       - id: ruff
+#       - id: black
       - id: isort
       - id: mypy
       - id: check-yaml

README.md

@@ -1,78 +1,105 @@
 # Standata
 
+Standard data for digital materials R&D entities in the [ESSE](https://github.com/Exabyte-io/esse) data format.
 
-Examples of entity data structures in the [ESSE](https://github.com/Exabyte-io/esse) data format (Essential Source
-of Schemas and Examples).
 
-## Installation
+## 1. Installation
 
-### Python
-
-The `standata` package is compatible with Python 3.8+. It can be installed as a Python package either via PyPI
-or as an editable local installation as below.
+### 1.1. Python
 
+The package is compatible with Python 3.8+. It can be installed as a Python package either via PyPI:
 ```shell
-pip install standata
+pip install mat3ra-standata
 ```
 
-Editable local installation in a virtual environment:
+Or as an editable local installation in a virtual environment after cloning the repository:
 ```shell
 virtualenv .venv
 source .venv/bin/activate
 pip install -e PATH_TO_STANDATA_REPOSITORY
 ```
 
-### Node
+### 1.2. JavaScript
 
 Standata can be installed as a Node.js package via NPM (node package manager).
 
 ```shell
-npm install @exabyte-io/standata
+npm install @mat3ra/standata
 ```
 
-#### Runtime Data
+
+## 2. Usage
+
+### 2.1. Python
+
+```python
+from mat3ra.standata.materials import materials_data
+# This returns a list of JSON configs for all materials.
+materialConfigs = materials_data["filesMapByName"].values();
+```
+
+
+### 2.2. JavaScript
+
+```javascript
+// Direct import can be used to avoid importing all data at once.
+import data from "@mat3ra/standata/lib/runtime_data/materials";
+// This creates a list of JSON configs for all materials.
+const materialConfigs = Object.values(data.filesMapByName);
+```
+
+
+## 3. Conventions
+
+#### 3.1. Runtime Modules
 
 To avoid file system calls on the client, the entity categories and data structures are made available at runtime via
 the files in `src/js/runtime_data`. These files are generated automatically using the following command:
 ```shell
 npm run build:runtime-data
 ```
 
-## CLI Script
+## 3.2. CLI Scripts for Creating Symlinks
+
+### 3.2.1. Python
 
-### Python
-The Python package adds a command line script `standata-symlinks` that creates a category-based file tree where
+The Python package adds a command line script `create-symlinks` that creates a category-based file tree where
 entity data files are symbolically linked in directories named after the categories associated with the entity.
 The resulting file tree will be contained in a directory names `by_category`.
 The script expects the (relative or absolute) path to an entity config file (`categories.yml`). The destination
 of the file tree can be modified by passing the `--destination`/`-d` option.
 ```shell
 # consult help page to view all options
-standata-symlinks --help
-
+create-symlinks --help
 # creates symbolic links in materials/by_category
-standata-symlinks materials/categories.yml
-
+create-symlinks materials/categories.yml
 # creates symbolic links for materials in tmp/by_category
-standata-symlinks materials/categories.yml
+create-symlinks materials/categories.yml -d tmp
 ```
 
-### Node
+### 3.2.1 JavaScript/Node
+
 Analogous to the command line script in Python, the repository also features a script in
 TypeScript (`src/js/cli.ts`) and (after transpiling) in JavaScript (`lib/cli.js`).
 The script takes the entity config file as a mandatory positional argument and the
 alternative location for the directory containing the symbolic links (`--destination`/`-d`).
 ```shell
 # creates symbolic links in materials/by_category (node)
 node lib/cli.js materials/categories.yml
-
 # creates symbolic links in materials/by_category (ts-node)
 ts-node src/js/cli.ts materials/categories.yml
-
 # creates symbolic links for materials in tmp/by_category
 ts-node src/js/cli.ts -d tmp materials/categories.yml
-
 # run via npm
 npm run build:categories -- materials/categories.yml
-
 ```
+
+
+## 4. Development
+
+See [ESSE](https://github.com/Exabyte-io/esse) for the notes about development and testing.
+
+
+## 5. Links
+
+To be added here.

build_runtime_data.js

@@ -9,7 +9,11 @@ const fs = require("fs");
 const path = require("path");
 const yaml = require("js-yaml");
 
-function buildAsset({ assetPath, targetPath }) {
+function buildAsset({
+    assetPath,
+    targetPath,
+    contentGenerator = (content) => `${JSON.stringify(content)}\n`,
+}) {
     const fileContent = fs.readFileSync(assetPath, { encoding: "utf-8" });
     const obj = {};
     obj.standataConfig = yaml.load(fileContent);
@@ -20,27 +24,52 @@ function buildAsset({ assetPath, targetPath }) {
         const content = fs.readFileSync(path.resolve(entityPath), { encoding: "utf-8" });
         obj.filesMapByName[entity.filename] = JSON.parse(content);
     });
-
-    fs.writeFileSync(targetPath, JSON.stringify(obj) + "\n", "utf8");
+    fs.writeFileSync(targetPath, contentGenerator(obj), "utf8");
     console.log(`Written entity category map to "${assetPath}" to "${targetPath}"`);
 }
 
+// JS Modules
+
 buildAsset({
     assetPath: "./materials/categories.yml",
     targetPath: "./src/js/runtime_data/materials.json",
 });
-
 buildAsset({
     assetPath: "./properties/categories.yml",
     targetPath: "./src/js/runtime_data/properties.json",
 });
-
 buildAsset({
     assetPath: "./applications/categories.yml",
     targetPath: "./src/js/runtime_data/applications.json",
 });
-
 buildAsset({
     assetPath: "./workflows/categories.yml",
     targetPath: "./src/js/runtime_data/workflows.json",
 });
+
+// Py Modules
+
+buildAsset({
+    assetPath: "./materials/categories.yml",
+    targetPath: "./src/py/mat3ra/standata/materials.py",
+    contentGenerator: (content) =>
+        `import json\n\nmaterials_data = json.loads(r'''${JSON.stringify(content)}''')\n`,
+});
+buildAsset({
+    assetPath: "./properties/categories.yml",
+    targetPath: "./src/py/mat3ra/standata/properties.py",
+    contentGenerator: (content) =>
+        `import json\n\nproperties_data = json.loads(r'''${JSON.stringify(content)}''')\n`,
+});
+buildAsset({
+    assetPath: "./applications/categories.yml",
+    targetPath: "./src/py/mat3ra/standata/applications.py",
+    contentGenerator: (content) =>
+        `import json\n\napplications_data = json.loads(r'''${JSON.stringify(content)}''')\n`,
+});
+buildAsset({
+    assetPath: "./workflows/categories.yml",
+    targetPath: "./src/py/mat3ra/standata/workflows.py",
+    contentGenerator: (content) =>
+        `import json\n\nworkflows_data = json.loads(r'''${JSON.stringify(content)}''')\n`,
+});

package.json

@@ -1,9 +1,9 @@
 {
-    "name": "@exabyte-io/standata",
+    "name": "@mat3ra/standata",
     "version": "0.0.0",
     "description": "Examples of entities following the Mat3ra data standard.",
-    "author": "Exabyte Inc.",
-    "license": "ISC",
+    "author": "Mat3ra.com",
+    "license": "Apache-2.0",
     "main": "lib/index.js",
     "files": [
         "/lib",
@@ -59,7 +59,6 @@
         "lint:fix": "eslint --fix --cache src/js tests/js && prettier --write src/js tests/js",
         "lint:staged": "lint-staged",
         "postinstall": "npm run transpile && cp -r src/js/runtime_data lib/",
-        "prepare": "node bootstrap.js",
         "prettier": "prettier --check src/js tests/js",
         "test": "mocha",
         "transpile": "tsc"