Packer (#172)

* introduce pako and add general packer interface

* add tests for packer

* use function API instead of class API for better tree shaking support

* refcatoring the rollup bundle config

package.json

@@ -59,6 +59,7 @@
   "dependencies": {
     "@types/smoothscroll-polyfill": "^0.3.0",
     "mitt": "^1.1.3",
+    "pako": "^1.0.11",
     "rrweb-snapshot": "^0.7.26",
     "smoothscroll-polyfill": "^0.4.3"
   }

rollup.config.js

@@ -11,176 +11,122 @@ function toRecordPath(path) {
     .replace('rrweb', 'rrweb-record');
 }
 
+function toPackPath(path) {
+  return path
+    .replace(/^([\w]+)\//, '$1/packer/')
+    .replace('rrweb', 'rrweb-pack');
+}
+
 function toMinPath(path) {
   return path.replace(/\.js$/, '.min.js');
 }
 
-let configs = [
+const namedExports = {
-  // browser(record only)
+  'pako/dist/pako_deflate': ['deflate'],
+  'pako/dist/pako_inflate': ['inflate'],
+  pako: ['deflate'],
+};
+
+const baseConfigs = [
   {
     input: './src/record/index.ts',
-    plugins: [resolve(), commonjs(), typescript()],
-    output: [
-      {
     name: 'rrwebRecord',
-        format: 'iife',
+    pathFn: toRecordPath,
-        file: toRecordPath(pkg.unpkg),
-      },
-    ],
   },
   {
-    input: './src/record/index.ts',
+    input: './src/packer/pack.ts',
-    plugins: [resolve(), commonjs(), typescript(), terser()],
+    name: 'rrwebPack',
-    output: [
+    pathFn: toPackPath,
-      {
-        name: 'rrwebRecord',
-        format: 'iife',
-        file: toMinPath(toRecordPath(pkg.unpkg)),
-        sourcemap: true,
   },
-    ],
-  },
-  // CommonJS(record only)
-  {
-    input: './src/record/index.ts',
-    plugins: [resolve(), commonjs(), typescript()],
-    output: [
-      {
-        format: 'cjs',
-        file: toRecordPath(pkg.main),
-      },
-    ],
-  },
-  // ES module(record only)
-  {
-    input: './src/record/index.ts',
-    plugins: [resolve(), commonjs(), typescript()],
-    output: [
-      {
-        format: 'esm',
-        file: toRecordPath(pkg.module),
-      },
-    ],
-  },
-  {
-    input: './src/record/index.ts',
-    plugins: [resolve(), commonjs(), typescript(), terser()],
-    output: [
-      {
-        format: 'esm',
-        file: toMinPath(toRecordPath(pkg.module)),
-        sourcemap: true,
-      },
-    ],
-  },
-  // browser
   {
     input: './src/index.ts',
-    plugins: [
-      resolve(),
-      commonjs(),
-      typescript(),
-      postcss({
-        extract: false,
-        inject: false,
-      }),
-    ],
-    output: [
-      {
     name: 'rrweb',
-        format: 'iife',
+    pathFn: (p) => p,
-        file: pkg.unpkg,
-      },
-    ],
-  },
-  {
-    input: './src/index.ts',
-    plugins: [
-      resolve(),
-      commonjs(),
-      typescript(),
-      postcss({
-        extract: true,
-        minimize: true,
-        sourceMap: true,
-      }),
-      terser(),
-    ],
-    output: [
-      {
-        name: 'rrweb',
-        format: 'iife',
-        file: toMinPath(pkg.unpkg),
-        sourcemap: true,
-      },
-    ],
-  },
-  // CommonJS
-  {
-    input: './src/index.ts',
-    plugins: [
-      resolve(),
-      commonjs(),
-      typescript(),
-      postcss({
-        extract: false,
-        inject: false,
-      }),
-    ],
-    output: [
-      {
-        format: 'cjs',
-        file: pkg.main,
-      },
-    ],
-  },
-  // ES module
-  {
-    input: './src/index.ts',
-    plugins: [
-      resolve(),
-      commonjs(),
-      typescript(),
-      postcss({
-        extract: false,
-        inject: false,
-      }),
-    ],
-    output: [
-      {
-        format: 'esm',
-        file: pkg.module,
-      },
-    ],
-  },
-  {
-    input: './src/index.ts',
-    plugins: [
-      resolve(),
-      commonjs(),
-      typescript(),
-      postcss({
-        extract: false,
-        inject: false,
-      }),
-      terser(),
-    ],
-    output: [
-      {
-        format: 'esm',
-        file: toMinPath(pkg.module),
-        sourcemap: true,
-      },
-    ],
   },
 ];
 
+let configs = [];
+
+for (const c of baseConfigs) {
+  const plugins = [
+    resolve(),
+    commonjs({ namedExports }),
+    typescript(),
+    postcss({
+      extract: false,
+      inject: false,
+    }),
+  ];
+  const minifyPlugins = plugins.concat(terser());
+  // browser
+  configs.push({
+    input: c.input,
+    plugins,
+    output: [
+      {
+        name: c.name,
+        format: 'iife',
+        file: c.pathFn(pkg.unpkg),
+      },
+    ],
+  });
+  // browser + minify
+  configs.push({
+    input: c.input,
+    plugins: minifyPlugins,
+    output: [
+      {
+        name: c.name,
+        format: 'iife',
+        file: toMinPath(c.pathFn(pkg.unpkg)),
+        sourcemap: true,
+      },
+    ],
+  });
+  // CommonJS
+  configs.push({
+    input: c.input,
+    plugins,
+    output: [
+      {
+        format: 'cjs',
+        file: c.pathFn(pkg.main),
+      },
+    ],
+  });
+  // ES module
+  configs.push({
+    input: c.input,
+    plugins,
+    output: [
+      {
+        format: 'esm',
+        file: c.pathFn(pkg.module),
+      },
+    ],
+  });
+  // ES module + minify
+  configs.push({
+    input: c.input,
+    plugins: minifyPlugins,
+    output: [
+      {
+        format: 'esm',
+        file: toMinPath(c.pathFn(pkg.module)),
+        sourcemap: true,
+      },
+    ],
+  });
+}
+
 if (process.env.BROWSER_ONLY) {
   configs = {
     input: './src/index.ts',
     plugins: [
       resolve(),
-      commonjs(),
+      commonjs({
+        namedExports,
+      }),
       typescript(),
       postcss({
         extract: true,

src/declarations/pako/index.d.ts

@@ -0,0 +1,12 @@
+declare module 'pako/dist/pako_deflate' {
+  export const deflate: any;
+}
+
+declare module 'pako/dist/pako_inflate' {
+  export const inflate: any;
+}
+
+declare module 'pako' {
+  export const deflate: any;
+  export const inflate: any;
+}

src/index.ts

@@ -8,6 +8,7 @@ export {
   MouseInteractions,
   ReplayerEvents,
 } from './types';
+export { pack, unpack } from './packer';
 
 const { addCustomEvent } = record;
 

src/packer/base.ts

@@ -0,0 +1,10 @@
+import { eventWithTime } from '../types';
+
+export type PackFn = (event: eventWithTime) => string;
+export type UnpackFn = (raw: string) => eventWithTime;
+
+export type eventWithTimeAndPacker = eventWithTime & {
+  v: string;
+};
+
+export const MARK = 'v1';

src/packer/index.ts

@@ -0,0 +1,2 @@
+export { pack } from './pack';
+export { unpack } from './unpack';

src/packer/pack.ts

@@ -0,0 +1,10 @@
+import { deflate } from 'pako/dist/pako_deflate';
+import { PackFn, MARK, eventWithTimeAndPacker } from './base';
+
+export const pack: PackFn = (event) => {
+  const _e: eventWithTimeAndPacker = {
+    ...event,
+    v: MARK,
+  };
+  return deflate(JSON.stringify(_e), { to: 'string' });
+};

src/packer/unpack.ts

@@ -0,0 +1,31 @@
+import { inflate } from 'pako/dist/pako_inflate';
+import { UnpackFn, eventWithTimeAndPacker, MARK } from './base';
+import { eventWithTime } from '../types';
+
+export const unpack: UnpackFn = (raw: string) => {
+  if (typeof raw !== 'string') {
+    return raw;
+  }
+  try {
+    const e: eventWithTime = JSON.parse(raw);
+    if (e.timestamp) {
+      return e;
+    }
+  } catch (error) {
+    // ignore and continue
+  }
+  try {
+    const e: eventWithTimeAndPacker = JSON.parse(
+      inflate(raw, { to: 'string' }),
+    );
+    if (e.v === MARK) {
+      return e;
+    }
+    throw new Error(
+      `These events were packed with packer ${e.v} which is incompatible with current packer ${MARK}.`,
+    );
+  } catch (error) {
+    console.error(error);
+    throw new Error('Unknown data format.');
+  }
+};

src/record/index.ts

@@ -25,7 +25,9 @@ function wrapEvent(e: event): eventWithTime {
 
 let wrappedEmit!: (e: eventWithTime, isCheckout?: boolean) => void;
 
-function record(options: recordOptions = {}): listenerHandler | undefined {
+function record<T = eventWithTime>(
+  options: recordOptions<T> = {},
+): listenerHandler | undefined {
   const {
     emit,
     checkoutEveryNms,
@@ -36,6 +38,7 @@ function record(options: recordOptions = {}): listenerHandler | undefined {
     maskAllInputs = false,
     hooks,
     mousemoveWait = 50,
+    packFn,
   } = options;
   // runtime checks for user options
   if (!emit) {
@@ -47,7 +50,7 @@ function record(options: recordOptions = {}): listenerHandler | undefined {
   let lastFullSnapshotEvent: eventWithTime;
   let incrementalSnapshotCount = 0;
   wrappedEmit = (e: eventWithTime, isCheckout?: boolean) => {
-    emit(e, isCheckout);
+    emit(((packFn ? packFn(e) : e) as unknown) as T, isCheckout);
     if (e.type === EventType.FullSnapshot) {
       lastFullSnapshotEvent = e;
       incrementalSnapshotCount = 0;
@@ -120,7 +123,7 @@ function record(options: recordOptions = {}): listenerHandler | undefined {
       handlers.push(
         initObservers(
           {
-            mutationCb: m =>
+            mutationCb: (m) =>
               wrappedEmit(
                 wrapEvent({
                   type: EventType.IncrementalSnapshot,
@@ -140,7 +143,7 @@ function record(options: recordOptions = {}): listenerHandler | undefined {
                   },
                 }),
               ),
-            mouseInteractionCb: d =>
+            mouseInteractionCb: (d) =>
               wrappedEmit(
                 wrapEvent({
                   type: EventType.IncrementalSnapshot,
@@ -150,7 +153,7 @@ function record(options: recordOptions = {}): listenerHandler | undefined {
                   },
                 }),
               ),
-            scrollCb: p =>
+            scrollCb: (p) =>
               wrappedEmit(
                 wrapEvent({
                   type: EventType.IncrementalSnapshot,
@@ -160,7 +163,7 @@ function record(options: recordOptions = {}): listenerHandler | undefined {
                   },
                 }),
               ),
-            viewportResizeCb: d =>
+            viewportResizeCb: (d) =>
               wrappedEmit(
                 wrapEvent({
                   type: EventType.IncrementalSnapshot,
@@ -170,7 +173,7 @@ function record(options: recordOptions = {}): listenerHandler | undefined {
                   },
                 }),
               ),
-            inputCb: v =>
+            inputCb: (v) =>
               wrappedEmit(
                 wrapEvent({
                   type: EventType.IncrementalSnapshot,
@@ -180,7 +183,7 @@ function record(options: recordOptions = {}): listenerHandler | undefined {
                   },
                 }),
               ),
-            mediaInteractionCb: p =>
+            mediaInteractionCb: (p) =>
               wrappedEmit(
                 wrapEvent({
                   type: EventType.IncrementalSnapshot,
@@ -190,7 +193,7 @@ function record(options: recordOptions = {}): listenerHandler | undefined {
                   },
                 }),
               ),
-            styleSheetRuleCb: r =>
+            styleSheetRuleCb: (r) =>
               wrappedEmit(
                 wrapEvent({
                   type: EventType.IncrementalSnapshot,
@@ -233,7 +236,7 @@ function record(options: recordOptions = {}): listenerHandler | undefined {
       );
     }
     return () => {
-      handlers.forEach(h => h());
+      handlers.forEach((h) => h());
     };
   } catch (error) {
     // TODO: handle internal error

src/replay/index.ts

@@ -59,11 +59,19 @@ export class Replayer {
 
   private playing: boolean = false;
 
-  constructor(events: eventWithTime[], config?: Partial<playerConfig>) {
+  constructor(
+    events: Array<eventWithTime | string>,
+    config?: Partial<playerConfig>,
+  ) {
     if (events.length < 2) {
       throw new Error('Replayer need at least 2 events.');
     }
-    this.events = events;
+    this.events = events.map((e) => {
+      if (config && config.unpackFn) {
+        return config.unpackFn(e as string);
+      }
+      return e as eventWithTime;
+    });
     this.handleResize = this.handleResize.bind(this);
 
     const defaultConfig: playerConfig = {
@@ -179,7 +187,10 @@ export class Replayer {
     this.emitter.emit(ReplayerEvents.Resume);
   }
 
-  public addEvent(event: eventWithTime) {
+  public addEvent(rawEvent: eventWithTime | string) {
+    const event = this.config.unpackFn
+      ? this.config.unpackFn(rawEvent as string)
+      : (rawEvent as eventWithTime);
     const castFn = this.getCastFn(event, true);
     castFn();
   }
@@ -364,7 +375,7 @@ export class Replayer {
     const { data: d } = e;
     switch (d.source) {
       case IncrementalSource.Mutation: {
-        d.removes.forEach(mutation => {
+        d.removes.forEach((mutation) => {
           const target = mirror.getNode(mutation.id);
           if (!target) {
             return this.warnNodeNotFound(d, mutation.id);
@@ -432,13 +443,13 @@ export class Replayer {
           }
         };
 
-        d.adds.forEach(mutation => {
+        d.adds.forEach((mutation) => {
           appendNode(mutation);
         });
 
         while (queue.length) {
-          if (queue.every(m => !Boolean(mirror.getNode(m.parentId)))) {
+          if (queue.every((m) => !Boolean(mirror.getNode(m.parentId)))) {
-            return queue.forEach(m => this.warnNodeNotFound(d, m.node.id));
+            return queue.forEach((m) => this.warnNodeNotFound(d, m.node.id));
           }
           const mutation = queue.shift()!;
           appendNode(mutation);
@@ -448,14 +459,14 @@ export class Replayer {
           Object.assign(this.missingNodeRetryMap, missingNodeMap);
         }
 
-        d.texts.forEach(mutation => {
+        d.texts.forEach((mutation) => {
           const target = mirror.getNode(mutation.id);
           if (!target) {
             return this.warnNodeNotFound(d, mutation.id);
           }
           target.textContent = mutation.value;
         });
-        d.attributes.forEach(mutation => {
+        d.attributes.forEach((mutation) => {
           const target = mirror.getNode(mutation.id);
           if (!target) {
             return this.warnNodeNotFound(d, mutation.id);
@@ -481,7 +492,7 @@ export class Replayer {
           const lastPosition = d.positions[d.positions.length - 1];
           this.moveAndHover(d, lastPosition.x, lastPosition.y, lastPosition.id);
         } else {
-          d.positions.forEach(p => {
+          d.positions.forEach((p) => {
             const action = {
               doAction: () => {
                 this.moveAndHover(d, p.x, p.y, p.id);
@@ -702,7 +713,7 @@ export class Replayer {
   private hoverElements(el: Element) {
     this.iframe
       .contentDocument!.querySelectorAll('.\\:hover')
-      .forEach(hoveredEl => {
+      .forEach((hoveredEl) => {
         hoveredEl.classList.remove(':hover');
       });
     let currentEl: Element | null = el;

src/types.ts

@@ -1,4 +1,5 @@
 import { serializedNodeWithId, idNodeMap, INode } from 'rrweb-snapshot';
+import { PackFn, UnpackFn } from './packer/base';
 
 export enum EventType {
   DomContentLoaded,
@@ -125,8 +126,8 @@ export type eventWithTime = event & {
 
 export type blockClass = string | RegExp;
 
-export type recordOptions = {
+export type recordOptions<T> = {
-  emit?: (e: eventWithTime, isCheckout?: boolean) => void;
+  emit?: (e: T, isCheckout?: boolean) => void;
   checkoutEveryNth?: number;
   checkoutEveryNms?: number;
   blockClass?: blockClass;
@@ -135,6 +136,7 @@ export type recordOptions = {
   inlineStylesheet?: boolean;
   hooks?: hooksParam;
   mousemoveWait?: number;
+  packFn?: PackFn;
 };
 
 export type observerParam = {
@@ -319,6 +321,7 @@ export type playerConfig = {
   liveMode: boolean;
   insertStyleRules: string[];
   triggerFocus: boolean;
+  unpackFn?: UnpackFn;
 };
 
 export type playerMetaData = {

test/packer.test.ts

@@ -0,0 +1,43 @@
+import { expect } from 'chai';
+import { matchSnapshot } from './utils';
+import { pack, unpack } from '../src/packer';
+import { eventWithTime, EventType } from '../src/types';
+import { MARK } from '../src/packer/base';
+
+const event: eventWithTime = {
+  type: EventType.DomContentLoaded,
+  data: {},
+  timestamp: new Date('2020-01-01').getTime(),
+};
+
+describe('pack', () => {
+  it('can pack event', () => {
+    const packedData = pack(event);
+    matchSnapshot(packedData, __filename, 'pack');
+  });
+});
+
+describe('unpack', () => {
+  it('is compatible with unpacked data 1', () => {
+    const result = unpack((event as unknown) as string);
+    expect(result).to.deep.equal(event);
+  });
+
+  it('is compatible with unpacked data 2', () => {
+    const result = unpack(JSON.stringify(event));
+    expect(result).to.deep.equal(event);
+  });
+
+  it('stop on unknown data format', () => {
+    expect(() => unpack('[""]')).to.throw('');
+  });
+
+  it('can unpack packed data', () => {
+    const packedData = pack(event);
+    const result = unpack(packedData);
+    expect(result).to.deep.equal({
+      ...event,
+      v: MARK,
+    });
+  });
+});

test/utils.ts

@@ -15,7 +15,11 @@ export async function launchPuppeteer() {
   });
 }
 
-function matchSnapshot(actual: string, testFile: string, testTitle: string) {
+export function matchSnapshot(
+  actual: string,
+  testFile: string,
+  testTitle: string,
+) {
   const snapshotState = new SnapshotState(testFile, {
     updateSnapshot: process.env.SNAPSHOT_UPDATE ? 'all' : 'new',
   });

tslint.json

@@ -16,7 +16,8 @@
       "allow-leading-underscore"
     ],
     "arrow-parens": false,
-    "only-arrow-functions": false
+    "only-arrow-functions": false,
+    "max-line-length": false
   },
   "rulesDirectory": []
 }

typings/index.d.ts

@@ -2,5 +2,6 @@ import record from './record';
 import { Replayer } from './replay';
 import { mirror } from './utils';
 export { EventType, IncrementalSource, MouseInteractions, ReplayerEvents, } from './types';
+export { pack, unpack } from './packer';
 declare const addCustomEvent: <T>(tag: string, payload: T) => void;
 export { record, addCustomEvent, Replayer, mirror };

typings/packer/base.d.ts

@@ -0,0 +1,7 @@
+import { eventWithTime } from '../types';
+export declare type PackFn = (event: eventWithTime) => string;
+export declare type UnpackFn = (raw: string) => eventWithTime;
+export declare type eventWithTimeAndPacker = eventWithTime & {
+    v: string;
+};
+export declare const MARK = "v1";

typings/packer/index.d.ts

@@ -0,0 +1,2 @@
+export { pack } from './pack';
+export { unpack } from './unpack';

typings/packer/pack.d.ts

@@ -0,0 +1,2 @@
+import { PackFn } from './base';
+export declare const pack: PackFn;

typings/packer/unpack.d.ts

@@ -0,0 +1,2 @@
+import { UnpackFn } from './base';
+export declare const unpack: UnpackFn;

typings/record/index.d.ts

@@ -1,5 +1,5 @@
-import { recordOptions, listenerHandler } from '../types';
+import { eventWithTime, recordOptions, listenerHandler } from '../types';
-declare function record(options?: recordOptions): listenerHandler | undefined;
+declare function record<T = eventWithTime>(options?: recordOptions<T>): listenerHandler | undefined;
 declare namespace record {
     var addCustomEvent: <T>(tag: string, payload: T) => void;
 }

typings/replay/index.d.ts

@@ -15,7 +15,10 @@ export declare class Replayer {
   private noramlSpeed;
   private missingNodeRetryMap;
   private playing;
-    constructor(events: eventWithTime[], config?: Partial<playerConfig>);
+  constructor(
+    events: Array<eventWithTime | string>,
+    config?: Partial<playerConfig>,
+  );
   on(event: string, handler: Handler): void;
   setConfig(config: Partial<playerConfig>): void;
   getMetaData(): playerMetaData;
@@ -24,7 +27,7 @@ export declare class Replayer {
   play(timeOffset?: number): void;
   pause(): void;
   resume(timeOffset?: number): void;
-    addEvent(event: eventWithTime): void;
+  addEvent(rawEvent: eventWithTime | string): void;
   private setupDom;
   private handleResize;
   private getDelay;

typings/types.d.ts

@@ -1,4 +1,5 @@
 import { serializedNodeWithId, idNodeMap, INode } from 'rrweb-snapshot';
+import { PackFn, UnpackFn } from './packer/base';
 export declare enum EventType {
     DomContentLoaded = 0,
     Load = 1,
@@ -89,8 +90,8 @@ export declare type eventWithTime = event & {
     delay?: number;
 };
 export declare type blockClass = string | RegExp;
-export declare type recordOptions = {
+export declare type recordOptions<T> = {
-    emit?: (e: eventWithTime, isCheckout?: boolean) => void;
+    emit?: (e: T, isCheckout?: boolean) => void;
     checkoutEveryNth?: number;
     checkoutEveryNms?: number;
     blockClass?: blockClass;
@@ -99,6 +100,7 @@ export declare type recordOptions = {
     inlineStylesheet?: boolean;
     hooks?: hooksParam;
     mousemoveWait?: number;
+    packFn?: PackFn;
 };
 export declare type observerParam = {
     mutationCb: mutationCallBack;
@@ -252,6 +254,7 @@ export declare type playerConfig = {
     liveMode: boolean;
     insertStyleRules: string[];
     triggerFocus: boolean;
+    unpackFn?: UnpackFn;
 };
 export declare type playerMetaData = {
     totalTime: number;