Supports NEARFS urls (#116)

* working ipfs url

* add tests

* missing attributes

* server

* clean up

jest.config.ts

@@ -9,6 +9,7 @@ const config: Config = {
     "^@/(.*)$": "<rootDir>/$1",
   },
   testTimeout: 15000,
+  setupFiles: ['./jest.setup.ts'],
 };
 
 export default config;

jest.setup.ts

@@ -0,0 +1,4 @@
+import { TextEncoder, TextDecoder } from 'util';
+
+global.TextEncoder = TextEncoder;
+global.TextDecoder = TextDecoder;

lib/dev.ts

@@ -10,7 +10,6 @@ import { Network } from "./types";
 import { loopThroughFiles, readFile } from "./utils/fs";
 import { mergeDeep, substractDeep } from "./utils/objects";
 import { startFileWatcher } from "./watcher";
-import { optional } from "joi";
 
 const DEV_DIST_FOLDER = "build";
 

lib/gateway.ts

@@ -1,4 +1,7 @@
 import { DevOptions } from "./dev";
+import axios from "axios";
+
+import { JSDOM } from "jsdom";
 
 function renderAttribute(name, value) {
   return value !== undefined ? `${name}="${value}"` : "";
@@ -49,4 +52,29 @@ function injectHTML(html: string, injections: Record<string, any>) {
 
 function normalizeHtml(html) {
   return html.replace(/\s+/g, ' ').trim();
+}
+
+const contentCache = {};
+
+export async function fetchAndCacheContent(url) {
+  if (!contentCache[url]) {
+    const response = await axios.get(url);
+    contentCache[url] = response.data;
+  }
+  return contentCache[url];
+}
+
+export function modifyIndexHtml(content: string, opts: DevOptions) {
+  const dom = new JSDOM(content);
+  const document = dom.window.document;
+
+  const viewer = document.querySelector('near-social-viewer');
+
+  if (viewer) {
+    viewer.setAttribute('src', opts.index);
+    viewer.setAttribute('rpc', `http://127.0.0.1:${opts.port}/api/proxy-rpc`);
+    viewer.setAttribute('network', opts.network);
+  }
+
+  return dom.serialize();
 }

lib/server.ts

@@ -6,7 +6,7 @@ import express, { Request, Response } from 'express';
 import { existsSync, readJson, writeJson } from "fs-extra";
 import http from 'http';
 import path from "path";
-import { handleReplacements } from './gateway';
+import { fetchAndCacheContent, handleReplacements, modifyIndexHtml } from './gateway';
 import { readFile } from "./utils/fs";
 
 // the gateway dist path in node_modules
@@ -32,7 +32,7 @@ export function startDevServer(srcs: string[], dists: string[], devJsonPath: str
   const app = createApp(devJsonPath, opts);
   const server = http.createServer(app);
   startServer(server, opts, () => {
-    const postData = JSON.stringify({srcs: srcs.map((src) => path.resolve(src)), dists: dists.map((dist) => path.resolve(dist))});
+    const postData = JSON.stringify({ srcs: srcs.map((src) => path.resolve(src)), dists: dists.map((dist) => path.resolve(dist)) });
     const options = {
       hostname: '127.0.0.1',
       port: opts.port,
@@ -50,7 +50,7 @@ export function startDevServer(srcs: string[], dists: string[], devJsonPath: str
       let data = '';
 
       res.on('data', (chunk) => {
-          data += chunk;
+        data += chunk;
       });
 
       res.on('end', () => {
@@ -61,7 +61,7 @@ export function startDevServer(srcs: string[], dists: string[], devJsonPath: str
     req.on('error', (e) => {
       log.error(`problem with request: ${e.message}`);
     });
-    
+
     // Write data to request body
     req.write(postData);
     req.end();
@@ -198,37 +198,69 @@ export function createApp(devJsonPath: string, opts: DevOptions): Express.Applic
    */
   app.all('/api/proxy-rpc', proxyMiddleware(RPC_URL[opts.network]));
 
-  if (opts.gateway) {
-    // do things with gateway
-    const gatewayPath = typeof opts.gateway === "string" ? path.resolve(opts.gateway) : GATEWAY_PATH;
+  if (opts.gateway) { // Gateway setup, may be string or boolean
+
+    /**
+     * starts gateway from local path
+     */
+    const setupLocalGateway = (gatewayPath: string) => {
+      if (!existsSync(path.join(gatewayPath, "index.html"))) {
+        log.error("Gateway not found. Skipping...");
+        opts.gateway = false;
+        return;
+      }
 
-    // let's check if gateway/dist/index.html exists
-    if (!(existsSync(path.join(gatewayPath, "index.html")))) {
-      log.error("Gateway not found. Skipping...");
-      opts.gateway = false;
-    } else {
-      // everything else is redirected to the gateway/dist
       app.use((req, res, next) => {
-        if (req.path === "/") {
-          return next();
+        if (req.path !== "/") {
+          return express.static(gatewayPath)(req, res, next);
         }
-        express.static(gatewayPath)(req, res, next); // serve static files
+        next();
       });
+
       app.get("*", (_, res) => {
-        // Inject Gateway with Environment Variables
-        readFile(
-          path.join(gatewayPath, "index.html"),
-          "utf8",
-        ).then((data) => {
-          const modifiedDist = handleReplacements(data, opts);
-          res.send(modifiedDist);
-        }).catch((err) => {
-          log.error(err);
-          return res.status(404).send("Something went wrong.");
-        })
+        readFile(path.join(gatewayPath, "index.html"), "utf8")
+          .then(data => {
+            const modifiedDist = modifyIndexHtml(data, opts);
+            res.type('text/html').send(modifiedDist);
+          })
+          .catch(err => {
+            log.error(err);
+            res.status(404).send("Something went wrong.");
+          });
       });
-      log.success("Gateway setup successfully.");
+    };
+
+    if (typeof opts.gateway === "string") { // Gateway is a string, could be local path or remote url
+      if (opts.gateway.startsWith("http")) { // remote url (web4)
+        app.use(async (req, res) => {
+          try { // forward requests to the web4 bundle
+            const filePath = req.path;
+            const ext = path.extname(filePath);
+            let fullUrl = (opts.gateway as string).replace(/\/$/, ''); // remove trailing slash
+
+            if (ext === '.js' || ext === '.css') {
+              fullUrl += filePath;
+              const content = await fetchAndCacheContent(fullUrl);
+              res.type(ext === '.js' ? 'application/javascript' : 'text/css');
+              res.send(content);
+            } else {
+              fullUrl += "/index.html";
+              let content = await fetchAndCacheContent(fullUrl);
+              content = modifyIndexHtml(content, opts);
+              res.type('text/html').send(content);
+            }
+          } catch (error) {
+            log.error(`Error fetching content: ${error}`);
+            res.status(404).send('Not found');
+          }
+        });
+      } else { // local path
+        setupLocalGateway(path.resolve(opts.gateway));
+      }
+    } else { // Gateway is boolean, setup default gateway
+      setupLocalGateway(GATEWAY_PATH);
     }
+    log.success("Gateway setup successfully.");
   }
 
   return app;
@@ -287,4 +319,4 @@ export function startServer(server, opts, sendAddApps) {
         process.exit(1);
       }
     });
-}
+}

package.json

@@ -28,6 +28,7 @@
   "dependencies": {
     "@near-js/providers": "^0.2.1",
     "@near-js/types": "^0.2.0",
+    "axios": "^1.7.2",
     "body-parser": "^1.20.2",
     "commander": "^11.1.0",
     "crypto-js": "^4.2.0",
@@ -37,14 +38,16 @@
     "glob": "^10.3.10",
     "https": "^1.0.0",
     "joi": "^17.11.0",
+    "jsdom": "^24.1.0",
     "multilang-extract-comments": "^0.4.0",
     "mvdir": "^1.0.21",
     "prettier": "^2.8.8",
     "prompts": "^2.4.2",
     "replace-in-file": "^7.1.0",
     "slugify": "^1.6.6",
     "socket.io": "^4.7.3",
-    "sucrase": "^3.34.0"
+    "sucrase": "^3.34.0",
+    "web-encoding": "^1.1.5"
   },
   "devDependencies": {
     "@types/crypto-js": "^4.2.2",

tests/unit/server.ts

@@ -1,14 +1,18 @@
+import { DevOptions } from './../../lib/dev';
 import { Logger, LogLevel } from "@/lib/logger";
 import { createApp, RPC_URL } from '@/lib/server';
 import supertest from 'supertest';
 import { TextEncoder } from 'util';
 import { Network } from './../../lib/types';
 import { fetchJson } from "@near-js/providers";
+import * as gateway from '@/lib/gateway';
 
 import { vol } from 'memfs';
+import path from 'path';
 jest.mock('fs', () => require('memfs').fs);
 jest.mock('fs/promises', () => require('memfs').fs.promises);
 jest.mock("@near-js/providers");
+jest.mock('@/lib/gateway');
 
 Object.assign(global, { TextEncoder });
 
@@ -24,7 +28,7 @@ describe('createApp', () => {
   let app;
   const mockSrc = "/app_example_1";
   const devJsonPath = `${mockSrc}/build/bos-loader.json`;
-  const opts = {
+  const opts: DevOptions = {
     gateway: true,
     port: 3000,
     hot: true,
@@ -45,6 +49,65 @@ describe('createApp', () => {
     global.fetch = unmockedFetch;
   });
 
+  it('should set up the app correctly when opts.gateway is a valid local path', () => {
+    const mockGatewayPath = "/mock_gateway_1";
+    opts.gateway = `${mockGatewayPath}/dist`;
+    vol.mkdirSync(path.join(mockGatewayPath, 'dist'), { recursive: true });
+    vol.writeFileSync(path.join(mockGatewayPath, 'dist', 'index.html'), '<html></html>');
+
+    jest.spyOn(gateway, 'modifyIndexHtml').mockReturnValue('<html>modified</html>');
+
+    app = createApp(devJsonPath, opts);
+    expect(app).toBeDefined();
+
+    return supertest(app)
+      .get('/')
+      .expect(200)
+      .expect('Content-Type', /html/)
+      .expect('<html>modified</html>');
+  });
+
+  it('should log an error when opts.gateway is an invalid local path', () => {
+    const mockGatewayPath = '/invalid/gateway/path';
+    opts.gateway = mockGatewayPath;
+
+    const logSpy = jest.spyOn(global.log, 'error');
+
+    app = createApp(devJsonPath, opts);
+    expect(app).toBeDefined();
+    expect(logSpy).toHaveBeenCalledWith("Gateway not found. Skipping...");
+  });
+
+  it('should set up the app correctly when opts.gateway is a valid http URL', async () => {
+    const mockGatewayUrl = 'http://mock-gateway.com';
+    opts.gateway = mockGatewayUrl;
+
+    jest.spyOn(gateway, 'fetchAndCacheContent').mockResolvedValue('<html></html>');
+    jest.spyOn(gateway, 'modifyIndexHtml').mockReturnValue('<html>modified</html>');
+
+    app = createApp(devJsonPath, opts);
+    expect(app).toBeDefined();
+
+    const response = await supertest(app).get('/');
+    expect(response.status).toBe(200);
+    expect(response.headers['content-type']).toMatch(/html/);
+    expect(response.text).toBe('<html>modified</html>');
+  });
+
+  it('should handle errors when fetching content from http gateway', async () => {
+    const mockGatewayUrl = 'http://mock-gateway.com';
+    opts.gateway = mockGatewayUrl;
+
+    jest.spyOn(gateway, 'fetchAndCacheContent').mockRejectedValue(new Error('Fetch error'));
+
+    app = createApp(devJsonPath, opts);
+    expect(app).toBeDefined();
+
+    const response = await supertest(app).get('/');
+    expect(response.status).toBe(404);
+    expect(response.text).toBe('Not found');
+  });
+
   it('/api/loader should return devJson', async () => {
     const response = await supertest(app).get('/api/loader');
     expect(response.status).toBe(200);