Extend to directives

.changeset/pink-birds-join.md

@@ -0,0 +1,6 @@
+---
+"myst-directives": patch
+"myst-transforms": patch
+---
+
+Move QMD admonition recognition to a transform

.changeset/small-paws-suffer.md

@@ -0,0 +1,5 @@
+---
+"myst-directives": patch
+---
+
+div node does not require a body

.changeset/tough-terms-push.md

@@ -0,0 +1,5 @@
+---
+"myst-roles": patch
+---
+
+Introduce a span role

packages/markdown-it-myst/src/directives.ts

@@ -5,6 +5,7 @@ import type MarkdownIt from 'markdown-it/lib';
 import type StateCore from 'markdown-it/lib/rules_core/state_core.js';
 import { nestedPartToTokens } from './nestedParse.js';
 import { stateError, stateWarn } from './utils.js';
+import { inlineOptionsToTokens } from './inlineAttributes.js';
 
 const COLON_OPTION_REGEX = /^:(?<option>[^:\s]+?):(\s*(?<value>.*)){0,1}\s*$/;
 
@@ -26,7 +27,7 @@ function computeBlockTightness(
 function replaceFences(state: StateCore): boolean {
   for (const token of state.tokens) {
     if (token.type === 'fence' || token.type === 'colon_fence') {
-      const match = token.info.match(/^\s*\{\s*([^}\s]+)\s*\}\s*(.*)$/);
+      const match = token.info.match(/^\s*\{\s*([^}]+)\s*\}\s*(.*)$/);
       if (match) {
         token.type = 'directive';
         token.info = match[1].trim();
@@ -45,38 +46,49 @@ function runDirectives(state: StateCore): boolean {
       try {
         const { info, map } = token;
         const arg = token.meta.arg?.trim() || undefined;
+        const {
+          name = 'div',
+          tokens: inlineOptTokens,
+          options: inlineOptions,
+        } = inlineOptionsToTokens(info, map?.[0] ?? 0, state);
         const content = parseDirectiveContent(
           token.content.trim() ? token.content.split(/\r?\n/) : [],
-          info,
+          name,
           state,
         );
-        const { body, options } = content;
+        const { body, options, optionsLocation } = content;
         let { bodyOffset } = content;
         while (body.length && !body[0].trim()) {
           body.shift();
           bodyOffset++;
         }
         const bodyString = body.join('\n').trimEnd();
         const directiveOpen = new state.Token('parsed_directive_open', '', 1);
-        directiveOpen.info = info;
+        directiveOpen.info = name;
         directiveOpen.hidden = true;
         directiveOpen.content = bodyString;
         directiveOpen.map = map;
         directiveOpen.meta = {
           arg,
-          options: getDirectiveOptions(options),
+          options: getDirectiveOptions([...inlineOptions, ...(options ?? [])]),
           // Tightness is computed for all directives (are they separated by a newline before/after)
           tight: computeBlockTightness(state.src, token.map),
         };
         const startLineNumber = map ? map[0] : 0;
         const argTokens = directiveArgToTokens(arg, startLineNumber, state);
-        const optsTokens = directiveOptionsToTokens(options || [], startLineNumber + 1, state);
+        const optsTokens = directiveOptionsToTokens(
+          options || [],
+          startLineNumber + 1,
+          state,
+          optionsLocation,
+        );
         const bodyTokens = directiveBodyToTokens(bodyString, startLineNumber + bodyOffset, state);
         const directiveClose = new state.Token('parsed_directive_close', '', -1);
         directiveClose.info = info;
         directiveClose.hidden = true;
         const newTokens = [
           directiveOpen,
+          ...inlineOptTokens,
           ...argTokens,
           ...optsTokens,
           ...bodyTokens,
@@ -110,6 +122,7 @@ function parseDirectiveContent(
   body: string[];
   bodyOffset: number;
   options?: [string, string | true][];
+  optionsLocation?: 'yaml' | 'colon';
 } {
   let bodyOffset = 1;
   let yamlBlock: string[] | null = null;
@@ -136,7 +149,12 @@ function parseDirectiveContent(
     try {
       const options = yaml.load(yamlBlock.join('\n')) as Record<string, any>;
       if (options && typeof options === 'object') {
-        return { body: newContent, options: Object.entries(options), bodyOffset };
+        return {
+          body: newContent,
+          options: Object.entries(options),
+          bodyOffset,
+          optionsLocation: 'yaml',
+        };
       }
     } catch (err) {
       stateWarn(
@@ -162,7 +180,7 @@ function parseDirectiveContent(
         bodyOffset++;
       }
     }
-    return { body: newContent, options, bodyOffset };
+    return { body: newContent, options, bodyOffset, optionsLocation: 'colon' };
   }
   return { body: content, bodyOffset: 1 };
 }
@@ -172,9 +190,13 @@ function directiveArgToTokens(arg: string, lineNumber: number, state: StateCore)
 }
 
 function getDirectiveOptions(options?: [string, string | true][]) {
-  if (!options) return undefined;
+  if (!options || options.length === 0) return undefined;
   const simplified: Record<string, string | true> = {};
   options.forEach(([key, val]) => {
+    if (key === 'class' && simplified.class) {
+      simplified.class += ` ${val}`;
+      return;
+    }
     if (simplified[key] !== undefined) {
       return;
     }
@@ -187,28 +209,29 @@ function directiveOptionsToTokens(
   options: [string, string | true][],
   lineNumber: number,
   state: StateCore,
+  optionsLocation?: 'yaml' | 'colon',
 ) {
   const tokens = options.map(([key, value], index) => {
     // lineNumber mapping assumes each option is only one line;
     // not necessarily true for yaml options.
     const optTokens =
       typeof value === 'string'
         ? nestedPartToTokens(
-            'directive_option',
+            'myst_option',
             value,
             lineNumber + index,
             state,
             'run_directives',
             true,
           )
         : [
-            new state.Token('directive_option_open', '', 1),
-            new state.Token('directive_option_close', '', -1),
+            new state.Token('myst_option_open', '', 1),
+            new state.Token('myst_option_close', '', -1),
           ];
     if (optTokens.length) {
       optTokens[0].info = key;
       optTokens[0].content = typeof value === 'string' ? value : '';
-      optTokens[0].meta = { value };
+      optTokens[0].meta = { location: optionsLocation, value };
     }
     return optTokens;
   });

packages/markdown-it-myst/src/inlineAttributes.spec.ts

@@ -37,14 +37,14 @@ describe('parseRoleHeader', () => {
         { kind: 'attr', key: 'data', value: 'some "escaped" text' },
       ],
     ],
+    ['.className', [{ kind: 'class', value: 'className' }]],
   ])('parses valid header: %s', (header, expected) => {
     const result = tokenizeInlineAttributes(header);
     expect(result).toEqual(expected);
   });
 
   // Error test cases
   test.each([
-    ['Missing name', '.classOnly', 'Missing mandatory role name as the first token'],
     [
       'Extra bare token after name',
       'myRole anotherWord',

packages/markdown-it-myst/src/inlineAttributes.ts

@@ -56,24 +56,26 @@ export function inlineOptionsToTokens(
   header: string,
   lineNumber: number,
   state: StateCore,
-): { name: string; tokens: Token[] } {
-  let name = '';
-  // 1) Tokenize
+): { name?: string; tokens: Token[]; options: [string, string][] } {
+  // Tokenize
   const tokens = tokenizeInlineAttributes(header);
 
-  // 2) The first token must be a “bare” token => the role name
-  if (tokens.length === 0 || tokens[0].kind !== 'bare') {
-    throw new Error('Missing mandatory role name as the first token');
+  if (tokens.length === 0) {
+    throw new Error('No inline tokens found');
+  }
+
+  // The first token should be a “bare” token => the name
+  // If no bare token is included, then the name is undefined
+  let name = undefined;
+  if (tokens[0].kind === 'bare') {
+    name = tokens[0].value;
+    tokens.shift();
   }
-  name = tokens[0].value;
-  tokens.shift();
 
   if (tokens.filter(({ kind }) => kind === 'id').length > 1) {
-    // TODO: change this to a warning and take the last ID
     throw new Error('Cannot have more than one ID defined');
   }
   if (tokens.some(({ kind }) => kind === 'bare')) {
-    // TODO: Choose to open this up to boolean attributes
     throw new Error('No additional bare tokens allowed after the first token');
   }
 
@@ -93,8 +95,6 @@ export function inlineOptionsToTokens(
       return classTokens;
     }
 
-    // lineNumber mapping assumes each option is only one line;
-    // not necessarily true for yaml options.
     const optTokens = nestedPartToTokens(
       'myst_option',
       opt.value,
@@ -110,5 +110,9 @@ export function inlineOptionsToTokens(
     }
     return optTokens;
   });
-  return { name, tokens: markdownItTokens.flat() };
+  const options = tokens.map((t): [string, string] => [
+    t.kind === 'attr' ? t.key : t.kind === 'id' ? 'label' : t.kind,
+    t.value,
+  ]);
+  return { name, tokens: markdownItTokens.flat(), options };
 }

packages/markdown-it-myst/src/roles.ts

@@ -72,7 +72,7 @@ function runRoles(state: StateCore): boolean {
           try {
             const { map } = token;
             const { content, col } = child as any;
-            const { name, tokens: optTokens } = inlineOptionsToTokens(
+            const { name = 'span', tokens: optTokens } = inlineOptionsToTokens(
               child.info,
               map?.[0] ?? 0,
               state,

packages/markdown-it-myst/tests/directives.spec.ts

@@ -18,9 +18,9 @@ describe('parses directives', () => {
       'directive_arg_open',
       'inline',
       'directive_arg_close',
-      'directive_option_open',
+      'myst_option_open',
       'inline',
-      'directive_option_close',
+      'myst_option_close',
       'directive_body_open',
       'paragraph_open',
       'inline',
@@ -39,8 +39,8 @@ describe('parses directives', () => {
     const tokens = mdit.parse('```{abc}\n:flag:\n```', {});
     expect(tokens.map((t) => t.type)).toEqual([
       'parsed_directive_open',
-      'directive_option_open',
-      'directive_option_close',
+      'myst_option_open',
+      'myst_option_close',
       'parsed_directive_close',
     ]);
     expect(tokens[0].info).toEqual('abc');
@@ -53,8 +53,8 @@ describe('parses directives', () => {
     const tokens = mdit.parse('```{abc}\n:flag: \n```', {});
     expect(tokens.map((t) => t.type)).toEqual([
       'parsed_directive_open',
-      'directive_option_open',
-      'directive_option_close',
+      'myst_option_open',
+      'myst_option_close',
       'parsed_directive_close',
     ]);
     expect(tokens[0].info).toEqual('abc');
@@ -97,9 +97,9 @@ describe('parses directives', () => {
     const tokens = mdit.parse('```{abc}\n:key:val:val\n```', {});
     expect(tokens.map((t) => t.type)).toEqual([
       'parsed_directive_open',
-      'directive_option_open',
+      'myst_option_open',
       'inline',
-      'directive_option_close',
+      'myst_option_close',
       'parsed_directive_close',
     ]);
     expect(tokens[0].info).toEqual('abc');
@@ -125,12 +125,12 @@ describe('parses directives', () => {
     const tokens = mdit.parse('```{abc}\n---\na: x\nb: y\n---\n```', {});
     expect(tokens.map((t) => t.type)).toEqual([
       'parsed_directive_open',
-      'directive_option_open',
+      'myst_option_open',
       'inline',
-      'directive_option_close',
-      'directive_option_open',
+      'myst_option_close',
+      'myst_option_open',
       'inline',
-      'directive_option_close',
+      'myst_option_close',
       'parsed_directive_close',
     ]);
     expect(tokens[0].info).toEqual('abc');
@@ -167,13 +167,6 @@ describe('parses directives', () => {
     expect(tokens[0].info).toEqual('abc');
     expect(tokens[2].info).toEqual('xyz');
   });
-  it('directives cannot have spaces', () => {
-    // We may change this in the future, if we add pandoc support
-    const mdit = MarkdownIt().use(plugin);
-    const tokens = mdit.parse('```` { ab c }\n\n``` { xyz }\n```\n\n````', {});
-    expect(tokens.map((t) => t.type)).toEqual(['fence']);
-    expect(tokens[0].info).toEqual(' { ab c }');
-  });
   it.each([
     [false, 'Paragraph\n\n```{math}\nAx=b\n```\n\nAfter paragraph'],
     [false, '```{math}\nAx=b\n```'],
@@ -189,4 +182,29 @@ describe('parses directives', () => {
     const open = tokens.find((t) => t.type === 'parsed_directive_open');
     expect(open?.meta.tight).toBe(tight);
   });
+  it('directives can have inline options', () => {
+    // We may change this in the future, if we add pandoc support
+    const mdit = MarkdownIt().use(plugin);
+    const tokens = mdit.parse('```{ab .a} arg\n:class: b\n```', {});
+    expect(tokens.map((t) => t.type)).toEqual([
+      'parsed_directive_open',
+      'myst_option_open',
+      'myst_option_close',
+      'directive_arg_open',
+      'inline',
+      'directive_arg_close',
+      'myst_option_open',
+      'inline',
+      'myst_option_close',
+      'parsed_directive_close',
+    ]);
+    expect(tokens[0].info).toEqual('ab');
+    expect(tokens[0].meta.options).toEqual({ class: 'a b' });
+    expect(tokens[1].info).toEqual('class');
+    expect(tokens[1].meta.value).toEqual('a');
+    expect(tokens[1].content).toEqual('.a');
+    expect(tokens[6].info).toEqual('class');
+    expect(tokens[6].meta.value).toEqual('b');
+    expect(tokens[6].content).toEqual('b');
+  });
 });

packages/myst-cli/src/init/jupyter-book/syntax.ts

@@ -154,17 +154,20 @@ export async function upgradeContent(documentLines: string[]): Promise<string[]
 }
 
 const admonitionPattern =
-  /^(attention|caution|danger|error|important|hint|note|seealso|tip|warning|\.callout-note|\.callout-warning|\.callout-important|\.callout-tip|\.callout-caution)$/;
+  /^(attention|caution|danger|error|important|hint|note|seealso|tip|warning)$/;
 
 async function upgradeNotes(documentLines: string[]): Promise<string[] | undefined> {
   const data = documentLines.join('\n');
   const mdast = mystParse(data);
 
-  const caseInsenstivePattern = new RegExp(admonitionPattern.source, admonitionPattern.flags + 'i');
+  const caseInsensitivePattern = new RegExp(
+    admonitionPattern.source,
+    admonitionPattern.flags + 'i',
+  );
   const directiveNodes = selectAll('mystDirective', mdast);
   const mixedCaseAdmonitions = directiveNodes.filter((item) => {
     const name = (item as any).name as string;
-    return name.match(caseInsenstivePattern) && !name.match(admonitionPattern);
+    return name.match(caseInsensitivePattern) && !name.match(admonitionPattern);
   });
   mixedCaseAdmonitions.forEach((node) => {
     const start = node.position!.start.line;

packages/myst-cli/src/transforms/images.ts

@@ -196,7 +196,7 @@ export function transformImagesWithoutExt(
       const sortedExtensions = [
         // Valid extensions
         ...(opts?.imageExtensions ?? []),
-        // Convertable extensions
+        // Convertible extensions
         ...Object.keys(conversionFnLookup),
         // All known extensions
         ...KNOWN_IMAGE_EXTENSIONS,