optimize the append queue algorithm

Loop the append queue has been proved to be very inefficient, and
some times lead to N^2 time complexity.

Especially when some abnormal data could not be appended into the
real DOM, will make a dead loop.

Previously we use a 5000ms time out to handle this, which is not
user-friendly and not explicitly.

In this patch, we transform the queue into a tree data structure,
which reflects the layout of real DOM. With the tree data structure,
we can find whether there are dangling nodes that need to be dropped.
Also, the iteration will be much more efficient.

There is still a 500ms time out to avoid a dead loop, but should not
be called in expected scenarios.

src/replay/index.ts

@@ -27,7 +27,13 @@ import {
   inputData,
   canvasMutationData,
 } from '../types';
-import { mirror, polyfill, TreeIndex } from '../utils';
+import {
+  mirror,
+  polyfill,
+  TreeIndex,
+  queueToResolveTrees,
+  iterateResolveTree,
+} from '../utils';
 import getInjectStyleRules from './styles/inject-style';
 import './styles/style.css';
 
@@ -746,17 +752,19 @@ export class Replayer {
         }
 
         const styleEl = (target as Node) as HTMLStyleElement;
-        const parent = ((target.parentNode as unknown) as INode);
+        const parent = (target.parentNode as unknown) as INode;
         const usingVirtualParent = this.fragmentParentMap.has(parent);
         let placeholderNode;
 
         if (usingVirtualParent) {
           /**
-           * styleEl.sheet is only accessible if the styleEl is part of the 
+           * styleEl.sheet is only accessible if the styleEl is part of the
            * dom. This doesn't work on DocumentFragments so we have to re-add
            * it to the dom temporarily.
            */
-          const domParent = this.fragmentParentMap.get((target.parentNode as unknown) as INode);
+          const domParent = this.fragmentParentMap.get(
+            (target.parentNode as unknown) as INode,
+          );
           placeholderNode = document.createTextNode('');
           parent.replaceChild(placeholderNode, target);
           domParent!.appendChild(target);
@@ -985,21 +993,29 @@ export class Replayer {
 
     let startTime = Date.now();
     while (queue.length) {
-      /**
-       * Looks like this check is killing the performance
-       */
-      // if (
-      //   queue.every(
-      //     (m) => !Boolean(mirror.getNode(m.parentId)) || nextNotInDOM(m),
-      //   )
-      // ) {
-      //   return queue.forEach((m) => this.warnNodeNotFound(d, m.node.id));
-      // }
-      if (Date.now() - startTime > 5000) {
-        return queue.forEach((m) => this.warnNodeNotFound(d, m.node.id));
+      // transform queue to resolve tree
+      const resolveTrees = queueToResolveTrees(queue);
+      queue.length = 0;
+      if (Date.now() - startTime > 500) {
+        this.warn(
+          'Timeout in the loop, please check the resolve tree data:',
+          resolveTrees,
+        );
+        break;
+      }
+      for (const tree of resolveTrees) {
+        let parent = mirror.getNode(tree.value.parentId);
+        if (!parent) {
+          this.debug(
+            'Drop resolve tree since there is no parent for the root node.',
+            tree,
+          );
+        } else {
+          iterateResolveTree(tree, (mutation) => {
+            appendNode(mutation);
+          });
+        }
       }
-      const mutation = queue.shift()!;
-      appendNode(mutation);
     }
 
     if (Object.keys(legacy_missingNodeMap).length) {
@@ -1200,10 +1216,7 @@ export class Replayer {
   }
 
   private warnNodeNotFound(d: incrementalData, id: number) {
-    if (!this.config.showWarning) {
-      return;
-    }
-    console.warn(REPLAY_CONSOLE_PREFIX, `Node with id '${id}' not found in`, d);
+    this.warn(`Node with id '${id}' not found in`, d);
   }
 
   private warnCanvasMutationFailed(
@@ -1211,12 +1224,7 @@ export class Replayer {
     id: number,
     error: unknown,
   ) {
-    console.warn(
-      REPLAY_CONSOLE_PREFIX,
-      `Has error on update canvas '${id}'`,
-      d,
-      error,
-    );
+    this.warn(`Has error on update canvas '${id}'`, d, error);
   }
 
   private debugNodeNotFound(d: incrementalData, id: number) {
@@ -1226,10 +1234,21 @@ export class Replayer {
      * is microtask, so events fired on a removed DOM may emit
      * snapshots in the reverse order.
      */
+    this.debug(REPLAY_CONSOLE_PREFIX, `Node with id '${id}' not found in`, d);
+  }
+
+  private warn(...args: Parameters<typeof console.warn>) {
+    if (!this.config.showWarning) {
+      return;
+    }
+    console.warn(REPLAY_CONSOLE_PREFIX, ...args);
+  }
+
+  private debug(...args: Parameters<typeof console.log>) {
     if (!this.config.showDebug) {
       return;
     }
     // tslint:disable-next-line: no-console
-    console.log(REPLAY_CONSOLE_PREFIX, `Node with id '${id}' not found in`, d);
+    console.log(REPLAY_CONSOLE_PREFIX, ...args);
   }
 }

src/utils.ts

@@ -453,3 +453,68 @@ export class TreeIndex {
     this.inputMap = new Map();
   }
 }
+
+type ResolveTree = {
+  value: addedNodeMutation;
+  children: ResolveTree[];
+  parent: ResolveTree | null;
+};
+
+export function queueToResolveTrees(queue: addedNodeMutation[]): ResolveTree[] {
+  const queueNodeMap: Record<number, ResolveTree> = {};
+  const putIntoMap = (
+    m: addedNodeMutation,
+    parent: ResolveTree | null,
+  ): ResolveTree => {
+    const nodeInTree: ResolveTree = {
+      value: m,
+      parent,
+      children: [],
+    };
+    queueNodeMap[m.node.id] = nodeInTree;
+    return nodeInTree;
+  };
+
+  const queueNodeTrees: ResolveTree[] = [];
+  for (const mutation of queue) {
+    const { nextId, parentId } = mutation;
+    if (nextId && nextId in queueNodeMap) {
+      const nextInTree = queueNodeMap[nextId];
+      if (nextInTree.parent) {
+        const idx = nextInTree.parent.children.indexOf(nextInTree);
+        nextInTree.parent.children.splice(
+          idx,
+          0,
+          putIntoMap(mutation, nextInTree.parent),
+        );
+      } else {
+        const idx = queueNodeTrees.indexOf(nextInTree);
+        queueNodeTrees.splice(idx, 0, putIntoMap(mutation, null));
+      }
+      continue;
+    }
+    if (parentId in queueNodeMap) {
+      const parentInTree = queueNodeMap[parentId];
+      parentInTree.children.push(putIntoMap(mutation, parentInTree));
+      continue;
+    }
+    queueNodeTrees.push(putIntoMap(mutation, null));
+  }
+
+  return queueNodeTrees;
+}
+
+export function iterateResolveTree(
+  tree: ResolveTree,
+  cb: (mutation: addedNodeMutation) => unknown,
+) {
+  cb(tree.value);
+  /**
+   * The resolve tree was designed to reflect the DOM layout,
+   * but we need append next sibling first, so we do a reverse
+   * loop here.
+   */
+  for (let i = tree.children.length - 1; i >= 0; i--) {
+    iterateResolveTree(tree.children[i], cb);
+  }
+}

test/record.test.ts

@@ -157,7 +157,7 @@ describe('record', function (this: ISuite) {
     expect(this.events[35].type).to.equal(EventType.FullSnapshot);
   });
 
-  it.only('is safe to checkout during async callbacks', async () => {
+  it('is safe to checkout during async callbacks', async () => {
     await this.page.evaluate(() => {
       const { record } = ((window as unknown) as IWindow).rrweb;
       record({