Merge pull request #18 from kmlbgn/feature/custom-pages

Feature/custom pages

README.md

@@ -25,6 +25,13 @@ Blocks listed under the Outline page can be of the following types:
 
 Docu-notion automatically identifies and removes blocks that are either child pages or links to pages located at the root level of the page. If you need to include such blocks within your content, they must be embedded within another block type, like a table or a column, or they should be accompanied by some text within the same block to trick this logic.
 
+# **Custom Pages**
+
+Docusaurus automatically generates custom pages from the `src/pages` directory, creating corresponding slugs and links. Pages located at the root but outside the 'Outline' are treated as custom pages, converted to markdown, and moved to `src/pages`. This setup supports both standard pages and links to database pages.
+
+**Note on Conflicts**: If the 'Outline' contains content, an `index.md` is generated. However, if there's also an `index.js` in `src/pages`, Docusaurus prioritizes the last processed page. Testing indicates that `src/pages` takes precedence over pages in the `docs` folder, therefore `index.md` will not be taken into account.
+
+
 # Custom parsing (Plugins)
 
 Custom parsing logic can be created using plugins. See the [plugin readme](src/plugins/README.md).

package.json

@@ -1,5 +1,6 @@
 {
   "scripts": {
+    "sync": "rsync -av --delete docs/ ../docs.kira.network/docs/",
     "test": "vitest",
     "build": "npm run test -- --run && tsc && cp ./src/css/*.css dist/ && echo Build successful",
     "build-only": "tsc && cp ./src/css/*.css dist/",
@@ -11,6 +12,7 @@
     "// typescript check": "",
     "tsc": "tsc",
     "// test out with a private sample notion db": "",
+    "pull": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_SAMPLE_ROOT_PAGE  --log-level verbose",
     "large-site-test": "npm run ts -- -n $SIL_BLOOM_DOCS_NOTION_TOKEN -r $SIL_BLOOM_DOCS_NOTION_ROOT_PAGE  --locales en,fr   --log-level debug",
     "pull-test-tagged": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_TEST_ROOT_PAGE_ID  --log-level debug --status-tag test",
     "pull-test-css": "npm run ts -- --css-output-directory ./test/css -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_TEST_ROOT_PAGE_ID  --log-level debug --status-tag test",

src/HierarchicalNamedLayoutStrategy.ts

@@ -1,7 +1,8 @@
 import * as fs from "fs-extra";
 import sanitize from "sanitize-filename";
 import { LayoutStrategy } from "./LayoutStrategy";
-import { NotionPage } from "./NotionPage";
+import { NotionPage, PageSubType } from "./NotionPage";
+import { warning } from "./log";
 
 // This strategy gives us a file tree that mirrors that of notion.
 // Each level in the outline becomes a directory, and each file bears the name of the Notion document.
@@ -36,13 +37,20 @@ export class HierarchicalNamedLayoutStrategy extends LayoutStrategy {
       .replaceAll("'", "")
       .replaceAll("?", "-");
 
+    let path;
+  if (page.subtype === PageSubType.Custom) {
+    // For Custom pages, store them directly in src/pages
+    path = this.rootDirectory +`/tmp/${sanitizedName}${extensionWithDot}`;
+  } else {
+
+    // For all other pages, use the existing structure for Docusaurus to parse
     const context = ("/" + page.layoutContext + "/").replaceAll("//", "/");
-    const path =
-      this.rootDirectory + context + sanitizedName + extensionWithDot;
-
-    return path;
+    path = this.rootDirectory + context + sanitizedName + extensionWithDot;
   }
 
+  return path;
+}
+
   //{
   //   "position": 2.5,
   //   "label": "Tutorial",

src/NotionPage.ts

@@ -9,26 +9,33 @@ import { ListBlockChildrenResponseResults } from "notion-to-md/build/types";
 // create pages for each node of the outline and then add links from those to the database pages. In this way, we get the benefits of database
 // pages (metadata, workflow, etc) and also normal pages (order, position in the outline).
 export enum PageType {
-  CategoryIndex,
   DatabasePage,
   Simple,
 }
+export enum PageSubType {
+  CategoryIndex,
+  Custom,
+  Content,
+}
 
 export class NotionPage {
   public metadata: GetPageResponse;
+  public parentId: string;
   public pageId: string;
   public order: number;
   public layoutContext: string; // where we found it in the hierarchy of the outline
   public foundDirectlyInOutline: boolean; // the page was found as a descendent of /outline instead of being linked to
 
   public constructor(args: {
     layoutContext: string;
+    parentId: string;
     pageId: string;
     order: number;
     metadata: GetPageResponse;
     foundDirectlyInOutline: boolean;
   }) {
     this.layoutContext = args.layoutContext;
+    this.parentId = args.parentId
     this.pageId = args.pageId;
     this.order = args.order;
     this.metadata = args.metadata;
@@ -59,31 +66,37 @@ export class NotionPage {
     {
         "object": "page",
         "parent": {
-            ("isCategory": "true")
             "type": "page_id",
             or
             "type": "database_id",
             ...
         },
     */
-
-    // Check IsCategory flag under parent for level pages with index content
-    if ((this.metadata as any).parent.IsCategory) {
-      return PageType.CategoryIndex;
-    }
     return (this.metadata as any).parent.type === "database_id"
       ? PageType.DatabasePage
       : PageType.Simple;
   }
 
+  public get subtype(): PageSubType {
+    // Check subtype flag under parent for level pages with index content or custom pages
+    let subtype = (this.metadata as any).parent?.subtype;
+    if (subtype === 'custom') {
+      return PageSubType.Custom;
+    } else if (subtype === 'categoryindex') {
+      return PageSubType.CategoryIndex;
+    } else {
+      return PageSubType.Content;
+    }
+  }
+
   // In Notion, pages from the Database have names and simple pages have titles.
   public get nameOrTitle(): string {
     return this.type === PageType.DatabasePage ? this.name : this.title;
   }
 
   public nameForFile(): string {
     // In Notion, pages from the Database have names and simple pages have titles. We use "index" by default for Level page with content.
-    if (this.type === PageType.CategoryIndex) {
+    if (this.subtype === PageSubType.CategoryIndex) {
       return "index";
     }
     return this.type === PageType.Simple

src/plugins/ColumnListTransformer.ts

@@ -30,9 +30,9 @@ async function notionColumnListToTabs(
         async child => await notionToMarkdown.blockToMarkdown(child)
       )
     );
-    const content = markdownContent.join("\n");
+    const content = markdownContent.join("\n\n");
 
-    return `<TabItem value="${label.toLowerCase()}" label="${label}">\n${content}\n</TabItem>`;
+    return `<TabItem value="${label.toLowerCase()}" label="${label}">\n\n${content}\n\n</TabItem>`;
   });
 
   const tabItems = await Promise.all(tabItemsPromises);

src/plugins/internalLinks.spec.ts

@@ -242,24 +242,61 @@ test("raw link to an existing page on this site that has a slug", async () => {
   });
 
   const results = await getMarkdown(
-    {
-      object: "block",
-      id: "2051d790-e527-4b4e-b145-ec0beee2addf",
-      parent: {
-        type: "page_id",
-        page_id: "333",
-      },
-      created_time: "2023-06-14T20:09:00.000Z",
-      last_edited_time: "2023-06-14T20:09:00.000Z",
-      has_children: false,
-      archived: false,
-      // TODO: mention has replaced link_to_page
-      type: "link_to_page",
-      link_to_page: {
-        type: "page_id",
-        page_id: targetPageId,
-      },
+{
+    "object": "block",
+    "id": "2051d790-e527-4b4e-b145-ec0beee2addf",
+    "parent": {
+        "type": "page_id",
+        "page_id": "333"
     },
+    "created_time": "2023-06-14T20:09:00.000Z",
+    "last_edited_time": "2023-06-14T20:09:00.000Z",
+    "has_children": false,
+    "archived": false,
+    "type": "paragraph",
+    "paragraph": {
+        "rich_text": [
+            {
+                "type": "mention",
+                "mention": {
+                    "type": "page",
+                    "page": {
+                        "id": targetPageId
+                    }
+                },
+                "annotations": {
+                    "bold": false,
+                    "italic": false,
+                    "strikethrough": false,
+                    "underline": false,
+                    "code": false,
+                    "color": "default"
+                },
+                "plain_text": "Link text",
+                "href": "https://www.notion.so/123"
+            },
+            {
+                "type": "text",
+                "text": {
+                    "content": " ",
+                    "link": null
+                },
+                "annotations": {
+                    "bold": false,
+                    "italic": false,
+                    "strikethrough": false,
+                    "underline": false,
+                    "code": false,
+                    "color": "default"
+                },
+                "plain_text": " ",
+                "href": null
+            }
+        ],
+        "color": "default"
+    }
+}
+,
     targetPage
   );
   expect(results.trim()).toBe("[Point to Me](/point-to-me)");

src/plugins/pluginTestRun.ts

@@ -223,6 +223,7 @@ export function makeSamplePageObject(options: {
 
   const p = new NotionPage({
     layoutContext: "/Second-Level/Third-Level",
+    parentId: "d20d8391-b365-42cb-8821-cf3c5382c6ed",
     pageId: id,
     order: 0,
     metadata: m,