xWidthAvg: Update character frequency weightings data source

.changeset/strong-kangaroos-tease.md

@@ -0,0 +1,16 @@
+---
+'@capsizecss/metrics': minor
+---
+
+xWidthAvg: Update character frequency weightings data source
+
+Character frequency weightings used to calculate the `xWidthAvg` metrics were previously hard coded internally, and were an adaption from a [frequency table] on Wikipedia.
+
+We now generate these weightings based on the abstracts of [WikiNews] articles, making it possible to add support for other languages that make use of non-latin [unicode subsets], e.g. Thai.
+
+The updated `xWidthAvg` metrics are very close to the original hard coded values.
+This results in either no or extremely minor changes to the generated fallback font CSS, meaning we don't expect any notable changes to consumers, with the benefit being this lays the ground work to support additional language subsets in the future.
+
+[frequency table]: https://en.wikipedia.org/wiki/Letter_frequency#Relative_frequencies_of_letters_in_other_languages
+[WikiNews]: https://wikinews.org/
+[unicode subsets]: https://www.utf8icons.com/subsets

.prettierignore

@@ -9,5 +9,6 @@ output
 packages/metrics/scripts/googleFontsApi.json
 packages/metrics/*.js
 packages/metrics/*.d.ts
+packages/unpack/src/weightings.ts
 CHANGELOG.md
 pnpm-lock.yaml

package.json

@@ -7,7 +7,8 @@
     "test": "jest",
     "format": "prettier --write .",
     "lint": "manypkg check && prettier --check . && tsc",
-    "build": "pnpm metrics:build && preconstruct build",
+    "build:prepare": "pnpm unpack:build && preconstruct build",
+    "build": "pnpm build:prepare && pnpm metrics:build",
     "copy-readme": "node scripts/copy-readme",
     "version": "changeset version && pnpm install --lockfile-only",
     "prepare-release": "pnpm copy-readme && pnpm build",
@@ -19,11 +20,12 @@
     "site:build": "pnpm --filter=./site build",
     "site:serve": "pnpm --filter=./site serve",
     "site:deploy": "pnpm --filter=./site run deploy",
-    "site:deploy-preview": "pnpm metrics:build && pnpm --filter=./site deploy-preview",
-    "metrics:build-system": "pnpm --filter=@capsizecss/metrics extract-system-metrics",
+    "site:deploy-preview": "pnpm build && pnpm --filter=./site deploy-preview",
+    "metrics:build-system": "pnpm build:prepare && pnpm --filter=@capsizecss/metrics extract-system-metrics",
     "metrics:build": "pnpm --filter=@capsizecss/metrics build",
     "metrics:clean": "pnpm --filter=@capsizecss/metrics clean",
     "metrics:download": "pnpm --filter=@capsizecss/metrics download",
+    "unpack:build": "pnpm --filter=@capsizecss/unpack build",
     "prepare": "preconstruct dev && (is-ci || husky install)"
   },
   "preconstruct": {

packages/metrics/README.md

@@ -41,20 +41,34 @@ const capsizeStyles = createStyleObject({
 
 The font metrics object returned contains the following properties if available:
 
-| Property   | Type   | Description                                                                                                                                                                                     |
-| ---------- | ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| familyName | string | The font family name as authored by font creator                                                                                                                                                |
-| category   | string | The style of the font: serif, sans-serif, monospace, display, or handwriting.                                                                                                                   |
-| capHeight  | number | The height of capital letters above the baseline                                                                                                                                                |
-| ascent     | number | The height of the ascenders above baseline                                                                                                                                                      |
-| descent    | number | The descent of the descenders below baseline                                                                                                                                                    |
-| lineGap    | number | The amount of space included between lines                                                                                                                                                      |
-| unitsPerEm | number | The size of the font’s internal coordinate grid                                                                                                                                                 |
-| xHeight    | number | The height of the main body of lower case letters above baseline                                                                                                                                |
-| xWidthAvg  | number | The average width of lowercase characters.<br/><br/>Currently derived from latin [character frequencies] in English language, falling back to the built in [xAvgCharWidth] from the OS/2 table. |
-
-[character frequencies]: https://en.wikipedia.org/wiki/Letter_frequency#Relative_frequencies_of_letters_in_other_languages
+| Property   | Type   | Description                                                                                                                                                                                   |
+| ---------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| familyName | string | The font family name as authored by font creator                                                                                                                                              |
+| category   | string | The style of the font: serif, sans-serif, monospace, display, or handwriting.                                                                                                                 |
+| capHeight  | number | The height of capital letters above the baseline                                                                                                                                              |
+| ascent     | number | The height of the ascenders above baseline                                                                                                                                                    |
+| descent    | number | The descent of the descenders below baseline                                                                                                                                                  |
+| lineGap    | number | The amount of space included between lines                                                                                                                                                    |
+| unitsPerEm | number | The size of the font’s internal coordinate grid                                                                                                                                               |
+| xHeight    | number | The height of the main body of lower case letters above baseline                                                                                                                              |
+| xWidthAvg  | number | The average width of character glyphs in the font. Calculated based on character frequencies in written text ([see below]), falling back to the built in [xAvgCharWidth] from the OS/2 table. |
+
+#### How `xWidthAvg` is calculated
+
+The `xWidthAvg` metric is derived from character frequencies in written language.
+The value takes a weighted average of character glyph widths in the font, falling back to the built in [xAvgCharWidth] from the OS/2 table if the glyph width is not available.
+
+The purpose of this metric is to support generating CSS metric overrides (e.g. [`ascent-override`], [`size-adjust`], etc) for fallback fonts, enabling inference of average line lengths so that a fallback font can be scaled to better align with a web font. This can be done either manually or using [`createFontStack`].
+
+For this technique to be effective, the metric factors in a character frequency weightings as observed in written language, using “abstracts” from [Wikinews] articles as a data source.
+Currently only supporting English ([source](https://en.wikinews.org/)).
+
+[see below]: #how-xwidthavg-is-calculated
 [xavgcharwidth]: https://learn.microsoft.com/en-us/typography/opentype/spec/os2#xavgcharwidth
+[`ascent-override`]: https://developer.mozilla.org/en-US/docs/Web/CSS/@font-face/ascent-override
+[`size-adjust`]: https://developer.mozilla.org/en-US/docs/Web/CSS/@font-face/size-adjust
+[`createfontstack`]: ../core/README.md#createfontstack
+[wikinews]: https://www.wikinews.org/
 
 ## Supporting APIs
 

packages/metrics/scripts/build.ts

@@ -4,18 +4,16 @@ import dedent from 'dedent';
 import PQueue from 'p-queue';
 import cliProgress from 'cli-progress';
 import sortKeys from 'sort-keys';
-import { Font, fromUrl } from '@capsizecss/unpack';
 
 import googleFonts from './googleFontsApi.json';
 import systemMetrics from './systemFonts.json';
 import { fontFamilyToCamelCase } from './../src';
+import { buildMetrics } from './buildMetrics';
 
 const writeFile = async (fileName: string, content: string) =>
   await fs.writeFile(path.join(__dirname, fileName), content, 'utf-8');
 
-interface MetricsFont extends Font {
-  category: string;
-}
+type MetricsFont = Awaited<ReturnType<typeof buildMetrics>>;
 
 const allMetrics: Record<string, MetricsFont> = {};
 
@@ -43,66 +41,63 @@ const buildFiles = async ({
     xWidthAvg,
   };
 
-  allMetrics[fileName] = data;
-
-  await writeFile(
-    path.join('..', `${fileName}.js`),
-    `module.exports = ${JSON.stringify(data, null, 2)
-      .replace(/"(.+)":/g, '$1:')
-      .replace(/"/g, `'`)};\n`,
-  );
-
   const typeName = `${fileName.charAt(0).toUpperCase()}${fileName.slice(
     1,
   )}Metrics`;
 
-  await writeFile(
-    path.join('..', `${fileName}.d.ts`),
-    dedent`
-      declare module '@capsizecss/metrics/${fileName}' {
-        interface ${typeName} {
-          familyName: string;
-          category: string;${
-            typeof capHeight === 'number' && capHeight > 0
-              ? `
-          capHeight: number;`
-              : ''
-          }${
-      typeof ascent === 'number' && ascent > 0
-        ? `
-          ascent: number;`
-        : ''
-    }${
-      typeof descent === 'number' && descent < 0
-        ? `
-          descent: number;`
-        : ''
-    }${
-      typeof lineGap === 'number'
-        ? `
-          lineGap: number;`
-        : ''
-    }${
-      typeof unitsPerEm === 'number' && unitsPerEm > 0
-        ? `
-          unitsPerEm: number;`
-        : ''
-    }${
-      typeof xHeight === 'number' && xHeight > 0
-        ? `
-          xHeight: number;`
-        : ''
-    }${
-      typeof xWidthAvg === 'number' && xWidthAvg > 0
-        ? `
-          xWidthAvg: number;`
-        : ''
-    }
-        }
-        export const fontMetrics: ${typeName};
-        export default fontMetrics;
-      }\n`,
-  );
+  allMetrics[fileName] = data;
+
+  const jsOutput = `module.exports = ${JSON.stringify(data, null, 2)
+    .replace(/"(.+)":/g, '$1:')
+    .replace(/"/g, `'`)};\n`;
+
+  const typesOutput = dedent`
+    declare module '@capsizecss/metrics/${fileName}' {
+      interface ${typeName} {
+        familyName: string;
+        category: string;${
+          typeof capHeight === 'number' && capHeight > 0
+            ? `
+        capHeight: number;`
+            : ''
+        }${
+    typeof ascent === 'number' && ascent > 0
+      ? `
+        ascent: number;`
+      : ''
+  }${
+    typeof descent === 'number' && descent < 0
+      ? `
+        descent: number;`
+      : ''
+  }${
+    typeof lineGap === 'number'
+      ? `
+        lineGap: number;`
+      : ''
+  }${
+    typeof unitsPerEm === 'number' && unitsPerEm > 0
+      ? `
+        unitsPerEm: number;`
+      : ''
+  }${
+    typeof xHeight === 'number' && xHeight > 0
+      ? `
+        xHeight: number;`
+      : ''
+  }${
+    typeof xWidthAvg === 'number' && xWidthAvg > 0
+      ? `
+        xWidthAvg: number;`
+      : ''
+  }
+      }
+      export const fontMetrics: ${typeName};
+      export default fontMetrics;
+    `;
+
+  await writeFile(path.join('..', `${fileName}.js`), jsOutput);
+  await writeFile(path.join('..', `${fileName}.d.ts`), `${typesOutput}}\n`);
 };
 
 (async () => {
@@ -126,7 +121,10 @@ const buildFiles = async ({
 
   const metricsForAnalysis: MetricsFont[] = [];
 
-  await queue.addAll(systemMetrics.map((m) => async () => await buildFiles(m)));
+  await queue.addAll(
+    systemMetrics.map((m) => async () => await buildFiles(m as MetricsFont)),
+  );
+
   await queue.addAll(
     googleFonts.items.map((font: (typeof googleFonts.items)[number]) => {
       const fontUrl =
@@ -135,10 +133,14 @@ const buildFiles = async ({
           : font.files[font.variants[0] as keyof typeof font.files];
 
       return async () => {
-        const m = await fromUrl(fontUrl as string);
-        const categorisedMetrics = { ...m, category: font.category };
-        metricsForAnalysis.push(categorisedMetrics);
-        await buildFiles(categorisedMetrics);
+        const m = await buildMetrics({
+          fontSource: fontUrl as string,
+          sourceType: 'url',
+          category: font.category as MetricsFont['category'],
+        });
+
+        metricsForAnalysis.push(m);
+        await buildFiles(m);
       };
     }),
   );

packages/metrics/scripts/buildMetrics.ts

@@ -0,0 +1,41 @@
+import { Font, fromFile, fromUrl } from '@capsizecss/unpack';
+
+type FontCategory =
+  | 'serif'
+  | 'sans-serif'
+  | 'monospace'
+  | 'display'
+  | 'handwriting';
+interface MetricsFont extends Font {
+  category: FontCategory;
+}
+
+interface Options {
+  fontSource: string;
+  sourceType: 'file' | 'url';
+  category: FontCategory;
+  overrides?: Partial<Font>;
+}
+
+const extractor: Record<
+  Options['sourceType'],
+  typeof fromFile | typeof fromUrl
+> = {
+  file: fromFile,
+  url: fromUrl,
+};
+
+export const buildMetrics = async ({
+  fontSource,
+  sourceType,
+  category,
+  overrides = {},
+}: Options): Promise<MetricsFont> => {
+  const metrics = await extractor[sourceType](fontSource);
+
+  return {
+    ...metrics,
+    ...overrides,
+    category,
+  };
+};