fix: canvas data in iframe wasn't applied in the fast-forward mode (#944)

* fix: canvas data in iframe wasn't applied in the fastforward mode

* add more comments

* Update packages/rrdom/src/diff.ts

Co-authored-by: Justin Halsall <Juice10@users.noreply.github.com>

* apply Juice10's suggestion

Co-authored-by: Justin Halsall <Juice10@users.noreply.github.com>

packages/rrdom/src/diff.ts

@@ -136,14 +136,27 @@ export function diff(
           break;
         }
         case 'CANVAS':
-          (newTree as RRCanvasElement).canvasMutations.forEach(
+          {
-            (canvasMutation) =>
+            const rrCanvasElement = newTree as RRCanvasElement;
+            // This canvas element is created with initial data in an iframe element. https://github.com/rrweb-io/rrweb/pull/944
+            if (rrCanvasElement.rr_dataURL !== null) {
+              const image = document.createElement('img');
+              image.onload = () => {
+                const ctx = (oldElement as HTMLCanvasElement).getContext('2d');
+                if (ctx) {
+                  ctx.drawImage(image, 0, 0, image.width, image.height);
+                }
+              };
+              image.src = rrCanvasElement.rr_dataURL;
+            }
+            rrCanvasElement.canvasMutations.forEach((canvasMutation) =>
               replayer.applyCanvas(
                 canvasMutation.event,
                 canvasMutation.mutation,
                 oldTree as HTMLCanvasElement,
               ),
-          );
+            );
+          }
           break;
         case 'STYLE':
           applyVirtualStyleRulesToNode(

packages/rrdom/src/index.ts

@@ -149,6 +149,7 @@ export class RRElement extends BaseRRElementImpl(RRNode) {
 export class RRMediaElement extends BaseRRMediaElementImpl(RRElement) {}
 
 export class RRCanvasElement extends RRElement implements IRRElement {
+  public rr_dataURL: string | null = null;
   public canvasMutations: {
     event: canvasEventWithTime;
     mutation: canvasMutationData;

packages/rrweb-snapshot/src/rebuild.ts

@@ -228,13 +228,20 @@ function buildNode(
           // handle internal attributes
           if (tagName === 'canvas' && name === 'rr_dataURL') {
             const image = document.createElement('img');
-            image.src = value;
             image.onload = () => {
               const ctx = (node as HTMLCanvasElement).getContext('2d');
               if (ctx) {
                 ctx.drawImage(image, 0, 0, image.width, image.height);
               }
             };
+            image.src = value;
+            type RRCanvasElement = {
+              RRNodeType: NodeType;
+              rr_dataURL: string;
+            };
+            // If the canvas element is created in RRDom runtime (seeking to a time point), the canvas context isn't supported. So the data has to be stored and not handled until diff process. https://github.com/rrweb-io/rrweb/pull/944
+            if (((node as unknown) as RRCanvasElement).RRNodeType)
+              ((node as unknown) as RRCanvasElement).rr_dataURL = value;
           } else if (tagName === 'img' && name === 'rr_dataURL') {
             const image = node as HTMLImageElement;
             if (!image.currentSrc.startsWith('data:')) {

packages/rrweb/test/events/canvas-in-iframe.ts

@@ -0,0 +1,181 @@
+import { EventType, eventWithTime, IncrementalSource } from '../../src/types';
+
+const now = Date.now();
+
+const events: eventWithTime[] = [
+  {
+    type: EventType.DomContentLoaded,
+    data: {},
+    timestamp: now,
+  },
+  {
+    type: EventType.Load,
+    data: {},
+    timestamp: now + 100,
+  },
+  {
+    type: EventType.Meta,
+    data: {
+      href: 'http://localhost',
+      width: 1200,
+      height: 500,
+    },
+    timestamp: now + 100,
+  },
+  {
+    type: EventType.FullSnapshot,
+    data: {
+      node: {
+        type: 0,
+        childNodes: [
+          {
+            type: 2,
+            tagName: 'html',
+            attributes: {},
+            childNodes: [
+              {
+                type: 2,
+                tagName: 'head',
+                attributes: {},
+                childNodes: [
+                  { type: 3, textContent: '\n    ', id: 4 },
+                  {
+                    type: 2,
+                    tagName: 'meta',
+                    attributes: { charset: 'utf-8' },
+                    childNodes: [],
+                    id: 5,
+                  },
+                  { type: 3, textContent: '    \n  ', id: 6 },
+                ],
+                id: 3,
+              },
+              { type: 3, textContent: '\n  ', id: 7 },
+              {
+                type: 2,
+                tagName: 'body',
+                attributes: {},
+                childNodes: [
+                  { type: 3, textContent: '\n    ', id: 9 },
+                  {
+                    type: 2,
+                    tagName: 'iframe',
+                    attributes: { id: 'target' },
+                    childNodes: [],
+                    id: 19,
+                  },
+                  { type: 3, textContent: '\n\n', id: 27 },
+                ],
+                id: 8,
+              },
+            ],
+            id: 2,
+          },
+        ],
+        compatMode: 'BackCompat',
+        id: 1,
+      },
+      initialOffset: { left: 0, top: 0 },
+    },
+    timestamp: now + 200,
+  },
+  // add an iframe
+  {
+    type: EventType.IncrementalSnapshot,
+    data: {
+      source: IncrementalSource.Mutation,
+      adds: [
+        {
+          parentId: 19,
+          nextId: null,
+          node: {
+            type: 0,
+            childNodes: [
+              {
+                type: 2,
+                tagName: 'html',
+                attributes: {},
+                childNodes: [
+                  {
+                    type: 2,
+                    tagName: 'head',
+                    attributes: {},
+                    childNodes: [],
+                    rootId: 30,
+                    id: 32,
+                  },
+                  {
+                    type: 2,
+                    tagName: 'body',
+                    attributes: {},
+                    childNodes: [],
+                    rootId: 30,
+                    id: 33,
+                  },
+                ],
+                rootId: 30,
+                id: 31,
+              },
+            ],
+            compatMode: 'BackCompat',
+            id: 30,
+          },
+        },
+      ],
+      removes: [],
+      texts: [],
+      attributes: [],
+      isAttachIframe: true,
+    },
+    timestamp: now + 500,
+  },
+  // add two canvas, one is blank ans the other is filled with data
+  {
+    type: EventType.IncrementalSnapshot,
+    data: {
+      source: 0,
+      texts: [],
+      attributes: [],
+      removes: [],
+      adds: [
+        {
+          parentId: 33,
+          nextId: null,
+          node: {
+            type: 2,
+            tagName: 'canvas',
+            attributes: {
+              width: '10',
+              height: '10',
+              id: 'blank_canvas',
+            },
+            childNodes: [],
+            rootId: 30,
+            id: 34,
+          },
+        },
+        {
+          parentId: 33,
+          nextId: null,
+          node: {
+            type: 2,
+            tagName: 'canvas',
+            attributes: {
+              width: '10',
+              height: '10',
+              rr_dataURL:
+                'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAAXNSR0IArs4c6QAAAB5JREFUKFNjZCASMBKpjmEQKvzPwIDqrEHoRozgBQC/ZQELU4DiXAAAAABJRU5ErkJggg==',
+              id: 'canvas_with_data',
+            },
+            childNodes: [],
+            rootId: 30,
+            id: 35,
+          },
+        },
+      ],
+    },
+    timestamp: now + 500,
+  },
+];
+
+export default events;

packages/rrweb/test/replayer.test.ts

@@ -15,6 +15,7 @@ import inputEvents from './events/input';
 import iframeEvents from './events/iframe';
 import shadowDomEvents from './events/shadow-dom';
 import StyleSheetTextMutation from './events/style-sheet-text-mutation';
+import canvasInIframe from './events/canvas-in-iframe';
 
 interface ISuite {
   code: string;
@@ -613,6 +614,31 @@ describe('replayer', function () {
     ).toEqual('shadow dom two');
   });
 
+  it('can fast-forward mutation events containing painted canvas in iframe', async () => {
+    await page.evaluate(`
+      events = ${JSON.stringify(canvasInIframe)};
+      const { Replayer } = rrweb;
+      var replayer = new Replayer(events,{showDebug:true});
+      replayer.pause(550);            
+    `);
+    const replayerIframe = await page.$('iframe');
+    const contentDocument = await replayerIframe!.contentFrame()!;
+    const iframe = await contentDocument!.$('iframe');
+    expect(iframe).not.toBeNull();
+    const docInIFrame = await iframe?.contentFrame();
+    expect(docInIFrame).not.toBeNull();
+    const canvasElements = await docInIFrame!.$$('canvas');
+    // The first canvas is a blank one and the second is a painted one.
+    expect(canvasElements.length).toEqual(2);
+
+    const dataUrls = await docInIFrame?.$$eval('canvas', (elements) =>
+      elements.map((element) => (element as HTMLCanvasElement).toDataURL()),
+    );
+    expect(dataUrls?.length).toEqual(2);
+    // The painted canvas's data should not be empty.
+    expect(dataUrls![1]).not.toEqual(dataUrls![0]);
+  });
+
   it('can stream events in live mode', async () => {
     const status = await page.evaluate(`
       const { Replayer } = rrweb;