fix(network): fetch content data as long as exists target session (#25)

closes #24

src/Plugin.ts

@@ -7,9 +7,13 @@ import {
   writeFile as writeFileCb
 } from 'fs';
 import { promisify } from 'util';
-import { ChromeRemoteInterface } from 'chrome-remote-interface';
-import { HarBuilder, NetworkObserver, NetworkRequest } from './network';
-import { Har } from 'har-format';
+import {
+  EntryBuilder,
+  HarBuilder,
+  NetworkObserver,
+  NetworkRequest
+} from './network';
+import { Entry, Har } from 'har-format';
 import { PluginOptions } from './PluginOptions';
 
 const access = promisify(accessCb);
@@ -18,7 +22,8 @@ const writeFile = promisify(writeFileCb);
 
 export class Plugin {
   private rdpPort?: number;
-  private readonly requests: NetworkRequest[] = [];
+  private readonly entries: Entry[] = [];
+  private connection?: CRIConnection;
 
   constructor(private readonly logger: Logger, private options: PluginOptions) {
     this.validatePluginOptions(options);
@@ -55,30 +60,24 @@ export class Plugin {
   }
 
   public async recordHar(): Promise<void> {
-    const factory: CRIConnection = new CRIConnection(
+    await this.closeConnection();
+
+    this.connection = new CRIConnection(
       { port: this.rdpPort },
       this.logger,
       new RetryStrategy(20, 5, 100)
     );
 
-    const chromeRemoteInterface: ChromeRemoteInterface = await factory.open();
-
-    const networkObservable: NetworkObserver = new NetworkObserver(
-      chromeRemoteInterface,
-      this.logger,
-      this.options
-    );
+    await this.connection.open();
 
-    await networkObservable.subscribe((request: NetworkRequest) =>
-      this.requests.push(request)
-    );
+    await this.subscribeToRequests();
 
     return null;
   }
 
   public async saveHar(): Promise<void> {
     try {
-      const har: Har = await new HarBuilder(this.requests).build();
+      const har: Har = new HarBuilder(this.entries).build();
       await writeFile(this.options.file, JSON.stringify(har, null, 2));
     } catch (e) {
       this.logger.err(`Failed to save HAR: ${e.message}`);
@@ -87,6 +86,25 @@ export class Plugin {
     return null;
   }
 
+  private async subscribeToRequests(): Promise<void> {
+    const networkObservable: NetworkObserver = new NetworkObserver(
+      this.connection,
+      this.logger,
+      this.options
+    );
+
+    await networkObservable.subscribe(async (request: NetworkRequest) =>
+      this.entries.push(await new EntryBuilder(request).build())
+    );
+  }
+
+  private async closeConnection(): Promise<void> {
+    if (this.connection) {
+      await this.connection.close();
+      delete this.connection;
+    }
+  }
+
   private validatePluginOptions(options: PluginOptions): void | never {
     this.stubPathIsDefined(options.stubPath);
     this.fileIsDefined(options.file);

src/cdp/CRIConnection.ts

@@ -1,20 +1,40 @@
 import connect, {
   ChromeRemoteInterface,
-  ChromeRemoteInterfaceOptions
+  ChromeRemoteInterfaceOptions,
+  Network,
+  Security
 } from 'chrome-remote-interface';
 import { RetryStrategy } from './RetryStrategy';
 import { Logger } from '../utils';
-import Timeout = NodeJS.Timeout;
 import * as CRIOutputMessages from './CRIOutputMessages';
+import Timeout = NodeJS.Timeout;
+import ProtocolMapping from 'devtools-protocol/types/protocol-mapping';
+
+export type ChromeRemoteInterfaceMethod = keyof ProtocolMapping.Events;
+
+export type ChromeRemoteInterfaceEvent = {
+  method: ChromeRemoteInterfaceMethod;
+  params?: ProtocolMapping.Events[ChromeRemoteInterfaceMethod][0];
+};
 
 export class CRIConnection {
+  private chromeRemoteInterface?: ChromeRemoteInterface;
+
   constructor(
     private readonly options: ChromeRemoteInterfaceOptions,
     private readonly logger: Logger,
     private readonly retryStrategy: RetryStrategy
   ) {}
 
-  public async open(): Promise<ChromeRemoteInterface> {
+  get network(): Network | undefined {
+    return this.chromeRemoteInterface?.Network;
+  }
+
+  get security(): Security | undefined {
+    return this.chromeRemoteInterface?.Security;
+  }
+
+  public async open(): Promise<void> {
     try {
       this.logger.debug(CRIOutputMessages.ATTEMPT_TO_CONNECT);
 
@@ -25,22 +45,13 @@ export class CRIConnection {
         port
       });
 
-      const { Security } = chromeRemoteInterface;
-
-      await Security.enable();
-      await Security.setOverrideCertificateErrors({ override: true });
-
-      Security.certificateError(({ eventId }) =>
-        Security.handleCertificateError({ eventId, action: 'continue' })
-      );
-
       this.logger.debug(CRIOutputMessages.CONNECTED);
 
       chromeRemoteInterface.once('disconnect', () =>
         this.logger.debug(CRIOutputMessages.DISCONNECTED)
       );
 
-      return chromeRemoteInterface;
+      this.chromeRemoteInterface = chromeRemoteInterface;
     } catch (e) {
       this.logger.debug(
         `${CRIOutputMessages.FAILED_ATTEMPT_TO_CONNECT}: ${e.message}`
@@ -50,7 +61,30 @@ export class CRIConnection {
     }
   }
 
-  private async scheduleReconnect(): Promise<ChromeRemoteInterface> {
+  public async close(): Promise<void> {
+    if (this.chromeRemoteInterface) {
+      await this.chromeRemoteInterface.close();
+      this.chromeRemoteInterface.removeAllListeners();
+      delete this.chromeRemoteInterface;
+      this.logger.debug(CRIOutputMessages.DISCONNECTED);
+    }
+  }
+
+  public async subscribe(
+    callback: (event: ChromeRemoteInterfaceEvent) => void
+  ): Promise<void> {
+    if (!this.chromeRemoteInterface) {
+      this.logger.debug(CRIOutputMessages.CONNECTION_IS_NOT_DEFINED);
+
+      return;
+    }
+
+    this.chromeRemoteInterface.on('event', callback);
+
+    await Promise.all([this.security.enable(), this.network.enable()]);
+  }
+
+  private async scheduleReconnect(): Promise<void> {
     const timeout: number | undefined = this.retryStrategy.getNextTime();
 
     if (!timeout) {

src/cdp/CRIOutputMessages.ts

@@ -2,6 +2,7 @@ export const FAILED_ATTEMPT_TO_CONNECT = `Failed to connect to Chrome Debugging
 export const ATTEMPT_TO_CONNECT = `Attempting to connect to Chrome Debugging Protocol`;
 export const CONNECTED = `Connected to Chrome Debugging Protocol`;
 export const DISCONNECTED = `Chrome Debugging Protocol disconnected`;
+export const CONNECTION_IS_NOT_DEFINED = `Connection to Chrome Debugging Protocol isn't defined yet.`;
 export const FAILED_TO_CONNECT = `${FAILED_ATTEMPT_TO_CONNECT}
 
 Common situations why this would fail:

src/cdp/index.ts

@@ -1,2 +1,6 @@
-export { CRIConnection } from './CRIConnection';
+export {
+  CRIConnection,
+  ChromeRemoteInterfaceEvent,
+  ChromeRemoteInterfaceMethod
+} from './CRIConnection';
 export { RetryStrategy } from './RetryStrategy';

src/network/EntryBuilder.ts

@@ -1,4 +1,4 @@
-import { NetworkRequest, WebSocket } from './NetworkRequest';
+import { ContentData, NetworkRequest, WebSocket } from './NetworkRequest';
 import {
   Content,
   Cookie,
@@ -123,11 +123,13 @@ export class EntryBuilder {
   }
 
   private async buildContent(): Promise<Content> {
+    const data: ContentData = await this.request.contentData();
+
     return {
       size: this.request.resourceSize,
       mimeType: this.request.mimeType || 'x-unknown',
-      ...(await this.request.contentData()),
-      compression: this.getResponseCompression() ?? undefined
+      compression: this.getResponseCompression() ?? undefined,
+      ...data
     };
   }
 

src/network/ExtraInfoBuilder.ts

@@ -0,0 +1,87 @@
+import { NetworkRequest } from './NetworkRequest';
+import { Header } from 'har-format';
+
+export interface RequestExtraInfo {
+  requestHeaders: Header[];
+}
+
+export interface ResponseExtraInfo {
+  responseHeaders: Header[];
+  responseHeadersText: string;
+}
+
+export class ExtraInfoBuilder {
+  private _hasExtraInfo: boolean = false;
+  private _finished: boolean = false;
+  private readonly _requests: NetworkRequest[] = [];
+  private readonly _requestExtraInfo: RequestExtraInfo[] = [];
+  private readonly _responseExtraInfo: ResponseExtraInfo[] = [];
+
+  constructor(private readonly deleteCallback: () => void) {}
+
+  public addRequestExtraInfo(info: RequestExtraInfo): void {
+    this._requestExtraInfo.push(info);
+    this.sync();
+  }
+
+  public addResponseExtraInfo(info: ResponseExtraInfo): void {
+    this._responseExtraInfo.push(info);
+    this.sync();
+  }
+
+  public finished(): void {
+    this._finished = true;
+    this.deleteIfComplete();
+  }
+
+  private deleteIfComplete(): void {
+    if (!this._finished) {
+      return;
+    }
+
+    if (this._hasExtraInfo) {
+      const lastRequest: NetworkRequest | undefined = this.getLastRequest();
+
+      if (!lastRequest?.hasExtraResponseInfo) {
+        return;
+      }
+    }
+
+    this.deleteCallback();
+  }
+
+  private getLastRequest(): NetworkRequest | undefined {
+    return this._requests[this._requests.length - 1];
+  }
+
+  private getRequestIndex(req: NetworkRequest): number | undefined {
+    return this._requests.indexOf(req);
+  }
+
+  private sync(): void {
+    const req: NetworkRequest | undefined = this.getLastRequest();
+
+    if (!req) {
+      return;
+    }
+
+    const index: number | undefined = this.getRequestIndex(req);
+
+    const requestExtraInfo: RequestExtraInfo | undefined = this
+      ._requestExtraInfo[index];
+
+    if (requestExtraInfo) {
+      req.addExtraRequestInfo(requestExtraInfo);
+      this._requestExtraInfo[index] = null;
+    }
+
+    const responseExtraInfo: ResponseExtraInfo | undefined = this
+      ._responseExtraInfo[index];
+    if (responseExtraInfo) {
+      req.addExtraResponseInfo(responseExtraInfo);
+      this._responseExtraInfo[index] = null;
+    }
+
+    this.deleteIfComplete();
+  }
+}

src/network/HarBuilder.ts

@@ -1,20 +1,12 @@
 import { Entry, Har } from 'har-format';
-import { NetworkRequest } from './NetworkRequest';
-import { EntryBuilder } from './EntryBuilder';
 
 export class HarBuilder {
-  constructor(private readonly chromeRequests: NetworkRequest[]) {}
+  constructor(private readonly entries: Entry[]) {}
 
-  public async build(): Promise<Har> {
+  public build(): Har {
     // eslint-disable-next-line @typescript-eslint/no-var-requires
     const { name, version, homepage: comment } = require('../../package.json');
 
-    const entries: Entry[] = await Promise.all(
-      this.chromeRequests.map((request: NetworkRequest) =>
-        new EntryBuilder(request).build()
-      )
-    );
-
     return {
       log: {
         version: '1.2',
@@ -24,7 +16,7 @@ export class HarBuilder {
           version,
           comment
         },
-        entries
+        entries: this.entries
       }
     };
   }