feat(cli, core): add option to download/update/clear full license texts

biome.json

@@ -7,7 +7,8 @@
   },
   "files": {
     "ignoreUnknown": false,
-    "ignore": ["dist", "build"]
+    "ignore": ["dist", "build"],
+    "maxSize": 10000000
   },
   "formatter": {
     "enabled": true,

packages/cli/src/commands/index.tsx

@@ -22,7 +22,7 @@ export const options = z.object({
   production: z
     .boolean()
     .describe(`Don't check licenses in development dependencies`),
-  defaultConfig: z // pacsalCase options are converted to kebab-case, so the flag is actually --default-config
+  defaultConfig: z // camelCase options are converted to kebab-case, so the flag is actually --default-config
     .boolean()
     .describe("Run audit with default whitelist/blacklist configuration"),
   filterRegex: z

packages/cli/src/commands/update-licenses.tsx

@@ -0,0 +1,39 @@
+import { deleteLicenses, updateLicenses } from "@license-auditor/data";
+import { Text, useApp } from "ink";
+import { useEffect, useState } from "react";
+import { z } from "zod";
+import { SpinnerWithLabel } from "../components/spinner-with-label.js";
+
+export const options = z.object({
+  clearCache: z.boolean().describe("Compress output"),
+});
+
+type Props = {
+  options: z.infer<typeof options>;
+};
+
+export default function UpdateLicenses({ options }: Props) {
+  const { exit } = useApp();
+  const [working, setWorking] = useState(false);
+  useEffect(() => {
+    setWorking(true);
+
+    const runUpdate = async () => {
+      if (options.clearCache) {
+        deleteLicenses();
+      } else {
+        await updateLicenses({ fetchAllLicenseTexts: true });
+      }
+      setWorking(false);
+      exit();
+    };
+
+    void runUpdate();
+  }, [options, exit]);
+
+  if (working) {
+    return <SpinnerWithLabel label="Updating licenses..." />;
+  }
+
+  return <Text>Licenses updated!</Text>;
+}

packages/core/package.json

@@ -26,6 +26,7 @@
     "@license-auditor/data": "*",
     "@total-typescript/ts-reset": "0.6.1",
     "detect-package-manager": "3.0.2",
+    "env-paths": "3.0.0",
     "fast-glob": "3.3.2",
     "lodash.flattendeep": "4.4.0",
     "spdx-expression-parse": "4.0.0",

packages/core/src/license-finder/detect-from-license-content.test.ts

@@ -10,13 +10,13 @@ describe("detectFromLicenseContent", () => {
     });
   });
   describe("detectFromLicenseContent", () => {
-    it("detects license from license content", () => {
+    it("detects license from license content", async () => {
       const licenseContents = licenseMap.get("MIT")?.licenseText;
 
       if (!licenseContents) {
         throw new Error("MIT doesn't have license text");
       }
-      expect(detectLicenses(licenseContents)[0]?.licenseId).toBe("MIT");
+      expect((await detectLicenses(licenseContents))[0]?.licenseId).toBe("MIT");
     });
   });
 });

packages/core/src/license-finder/detect-from-license-content.ts

@@ -1,4 +1,22 @@
-import { type LicenseId, licenses } from "@license-auditor/data";
+import { existsSync } from "node:fs";
+import {
+  type LicenseId,
+  licenses as defaultLicenses,
+} from "@license-auditor/data";
+import envPaths from "env-paths";
+
+const resolveLicenses = async (): Promise<typeof defaultLicenses> => {
+  const paths = envPaths("license-auditor");
+  const licensesPath = `${paths.cache}/licenses.js`;
+
+  if (existsSync(licensesPath)) {
+    const fullLicenses = (await import(licensesPath)) as typeof defaultLicenses;
+
+    return fullLicenses;
+  }
+
+  return defaultLicenses;
+};
 
 /**
  * Tokenizes text into words, removes punctuation and whitespaces
@@ -49,19 +67,22 @@ const createLibrary = (
   );
 };
 
-const licensesLibrary = createLibrary(
-  licenses.reduce<Record<string, string>>((acc, license) => {
-    if (license.licenseText) {
-      acc[license.licenseId as LicenseId] = license.licenseText;
-    }
-    return acc;
-  }, {}),
-);
-const calculateSimilarity = getCalculateSimilarity(licensesLibrary);
-
-export function detectLicenses(
+export async function detectLicenses(
   licenseContent: string,
-): { licenseId: LicenseId; similarity: number }[] {
+): Promise<{ licenseId: LicenseId; similarity: number }[]> {
+  const licenses = await resolveLicenses();
+
+  const licensesLibrary = createLibrary(
+    licenses.reduce<Record<string, string>>((acc, license) => {
+      if (license.licenseText) {
+        acc[license.licenseId as LicenseId] = license.licenseText;
+      }
+      return acc;
+    }, {}),
+  );
+
+  const calculateSimilarity = getCalculateSimilarity(licensesLibrary);
+
   const similarities = calculateSimilarity(licenseContent);
   return similarities.sort((a, b) => b.similarity - a.similarity);
 }

packages/core/src/license-finder/find-license-in-license-file.test.ts

@@ -3,19 +3,19 @@ import { describe, expect, it } from "vitest";
 import { retrieveLicenseFromLicenseFileContent } from "./find-license-in-license-file.js";
 
 describe("retrieveLicenseFromLicenseFileContent", () => {
-  it("should return an empty array when content does not match any licenses", () => {
+  it("should return an empty array when content does not match any licenses", async () => {
     const content = "This is some random content without any license keywords.";
-    const result = retrieveLicenseFromLicenseFileContent(
+    const result = await retrieveLicenseFromLicenseFileContent(
       content,
       "/path/to/LICENSE",
     );
     expect(result.licenses).toEqual([]);
   });
 
-  it("should return the correct license when content matches a license key", () => {
+  it("should return the correct license when content matches a license key", async () => {
     const content = "MIT";
     const expectedLicense = LicenseSchema.parse(licenseMap.get("MIT"));
-    const result = retrieveLicenseFromLicenseFileContent(
+    const result = await retrieveLicenseFromLicenseFileContent(
       content,
       "/path/to/LICENSE",
     );
@@ -28,10 +28,10 @@ describe("retrieveLicenseFromLicenseFileContent", () => {
     ]);
   });
 
-  it("should return the correct license when content matches a license name", () => {
+  it("should return the correct license when content matches a license name", async () => {
     const content = "MIT License";
     const expectedLicense = LicenseSchema.parse(licenseMap.get("MIT"));
-    const result = retrieveLicenseFromLicenseFileContent(
+    const result = await retrieveLicenseFromLicenseFileContent(
       content,
       "/path/to/LICENSE",
     );
@@ -44,7 +44,7 @@ describe("retrieveLicenseFromLicenseFileContent", () => {
     ]);
   });
 
-  it("should return multiple licenses when content matches multiple license keys or names", () => {
+  it("should return multiple licenses when content matches multiple license keys or names", async () => {
     const content = "MIT, Apache-2.0";
     const expectedLicenses = [
       {
@@ -58,7 +58,7 @@ describe("retrieveLicenseFromLicenseFileContent", () => {
         licensePath: "/path/to/LICENSE",
       },
     ].sort((a, b) => a.name.localeCompare(b.name));
-    const result = retrieveLicenseFromLicenseFileContent(
+    const result = await retrieveLicenseFromLicenseFileContent(
       content,
       "/path/to/LICENSE",
     );

packages/core/src/license-finder/find-license-in-license-file.ts

@@ -10,13 +10,13 @@ import { addLicenseSource } from "./add-license-source.js";
 import { detectLicenses } from "./detect-from-license-content.js";
 import type { LicensesWithPathAndStatus } from "./licenses-with-path.js";
 
-export function retrieveLicenseFromLicenseFileContent(
+export async function retrieveLicenseFromLicenseFileContent(
   content: string,
   licensePath: string,
-): {
+): Promise<{
   licenses: LicenseWithSource[];
-} {
-  const detectedLicenses = detectLicenses(content);
+}> {
+  const detectedLicenses = await detectLicenses(content);
   const detectedLicense = detectedLicenses[0];
   if (detectedLicense && (detectedLicense.similarity ?? 0) > 0.75) {
     // threshold selected empirically based on our tests
@@ -62,7 +62,10 @@ export async function findLicenseInLicenseFile(filePath: string): Promise<{
       };
     }
 
-    const result = retrieveLicenseFromLicenseFileContent(content, filePath);
+    const result = await retrieveLicenseFromLicenseFileContent(
+      content,
+      filePath,
+    );
 
     return {
       licenses: result.licenses,

tooling/data/package.json

@@ -13,6 +13,7 @@
     "update-licenses": "tsx src/licenses/update-licenses.ts"
   },
   "dependencies": {
+    "env-paths": "3.0.0",
     "zod": "3.23.8"
   },
   "devDependencies": {

tooling/data/src/licenses/index.ts

@@ -1,3 +1,4 @@
 export * from "./schemas.js";
 export * from "./types.js";
 export * from "./constants.js";
+export * from "./update-licenses.js";

tooling/data/src/licenses/update-licenses.ts

@@ -1,8 +1,11 @@
-import { writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, unlinkSync, writeFileSync } from "node:fs";
+import envPaths from "env-paths";
 
+const paths = envPaths("license-auditor");
+const licensesFilePath = `${paths.cache}/licenses.js`;
 const url =
   "https://raw.githubusercontent.com/spdx/license-list-data/main/json/licenses.json";
-const outputFile = "./src/licenses/licenses.ts";
+// const defaultOutputFile = "./src/licenses/licenses.ts";
 
 // licenses are chosen arbitrarily, based on their popularity in our projects
 const licensesToFetchContentFor = [
@@ -26,9 +29,13 @@ const licensesToFetchContentFor = [
   "MPL-2.0",
 ];
 
-// pulls the licenses from spdx and transforms them into an object
-// needed so TS can properly infer types in union types
-(async () => {
+export async function updateLicenses({
+  outputFile,
+  fetchAllLicenseTexts,
+}: {
+  outputFile?: string;
+  fetchAllLicenseTexts?: boolean;
+}) {
   try {
     console.log("Fetching license list...");
     const response = await fetch(url);
@@ -47,6 +54,7 @@ const licensesToFetchContentFor = [
       );
       try {
         if (
+          fetchAllLicenseTexts ||
           licensesToFetchContentFor.includes(
             // biome-ignore lint/style/noNonNullAssertion: we can be sure that the licenses field is a dense array
             licensesData.licenses[i]!.licenseId,
@@ -65,21 +73,43 @@ const licensesToFetchContentFor = [
         }
       } catch (error) {
         console.log(
-          // biome-ignore lint/style/noNonNullAssertion: we can be sure that the licenses field is a dense array
-          `Failed to fetch license contents for "${licensesData.licenses[i]!.licenseId}"`,
+          `Failed to fetch license contents for "${
+            // biome-ignore lint/style/noNonNullAssertion: we can be sure that the licenses field is a dense array
+            licensesData.licenses[i]!.licenseId
+          }"`,
         );
         failedFetches++;
       }
     }
 
-    const content = `export const licensesData = ${JSON.stringify(licensesData, null, 2)} as const;`;
+    const content = `export const licensesData = ${JSON.stringify(
+      licensesData,
+      null,
+      2,
+    )};`;
+
+    if (!(outputFile || existsSync(paths.cache))) {
+      mkdirSync(paths.cache, { recursive: true });
+    }
 
-    writeFileSync(outputFile, content);
+    writeFileSync(outputFile || licensesFilePath, content);
     console.log(
-      `licenses.ts has been updated.${failedFetches ? ` ${failedFetches} licenses failed to fetch.` : ""}`,
+      `licenses.ts has been updated.${
+        failedFetches ? ` ${failedFetches} licenses failed to fetch.` : ""
+      }`,
     );
   } catch (error) {
     console.error("Error fetching licenses:", error);
     process.exit(1);
   }
-})();
+}
+
+export function deleteLicenses() {
+  if (existsSync(licensesFilePath)) {
+    unlinkSync(licensesFilePath);
+  }
+}
+
+// pulls the licenses from spdx and transforms them into an object
+// needed so TS can properly infer types in union types
+// (async () => updateLicenses({ outputFile: defaultOutputFile }))();