From 7cd03662a49f9056ff99562155a9705ce4deb5a9 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 1 Apr 2026 12:00:00 +0800 Subject: [PATCH] Add WebGL support (#756) * Add very basic webgl support * document the default * only capture rr_dataURL in 2d canvas contexts * rr_dataURL no longer part of webgl snapshot * ignore __diff_output__ from jest-image-snapshot * Rename generic "Monorepo" to "RRWeb Monorepo" * Serialize WebGL variables * Move rrweb test port number to unique port rrweb-snapshot uses 3030, rrweb uses 3031 * Prepare for WebGL2 * Split up canvas replay and record webgl vars * fix typo * fix typo part 2 * fix typo * Handle non-variables too * provide correct context for warning * (De)Serialize a lot of different objects * monorepo root should be the first in the list * Upgrade puppeteer to 11.x * Correctly de-serialize webgl variables * Encode arrayBuffers contents to base64 * rename contents to base64 * add webgl2 support and serialize HTMLImageElements * Support serializing ImageData * Correctly classify WebGL2 events * Serialize format changed * check if canvas has contents before we save the dataURL * Remove blank dataURL * reference original file not type defintion file * update types * rename code worspace * update dependencies * add spector to inspect webgl * remove live server settings from code workspace * Save canvas context in the node Prevents from saving webgl canvases as 2d dataUrls * remove extra braces * add ICanvas type * use ICanvas from rrweb-snapshot in rrweb instead of OgmentedCanvas * add snapshots and webgl 2 tests * Upgrade to puppeteer 12.0.1 * Revert back to puppeteer 9.1.1 * Keep index order consistent between replay and record * keep correct index order in webgl2 * fixed forgotten import * buffer up pending canvas mutations * unify the way webgl and webgl2 get patched * fix parsing error * Add types for serialize-args * Add debugging for webgl replay * Move start-server to utils * turn off debug mode by default * Move pendingCanvasMutations to local object and fix if/else statement * Always save pending mutations * only use assert snapshot as it's clearer whats going on * Ugly fix for now * Making the tests more DRY * flush at the end of each request animation frame * Looks like the promise made this test more predictable * add waitForRAF * Make nested iframe recording robust no matter the test speed * mute noisy error in test * force a requestAnimationFrame * Bundle events within one frame together as much as possible WebGL events need to be bundled together as much as possible so they don't accidentally get split over multiple animation frames. `newFrame: true` is used to indicate the start of an new animation frame in the recording, and that the event shouldn't be bundled with the previous events. * Rename RafStamps * Override event.delay * cleanup * Add tests for addDelay * Add webgl e2e test * Remove settimeout * DRY-up test * Preload images in webgl * Add e2e test for webgl image preloading * don't turn on devtools by default! * Remove spector * close server after use * Add imageMap parameter * Make e2e image test more robust * document debug mode * cleanup * WebGL recording in iframes & Safari 14 support * fix tests * don't save null objects as WebGLVar * group (de)serialized webgl variables by context * Fix test * fix tests * bundle webgl mutations on request animation frame Instead of fixing it on the replay side we buffer up webgl canvas mutations and wait for a new RAF to flush them. This allows us to remove `newFrame` from the events and simplify things a little * Add canvas element to mutation observer file * Add Canvas (Mutation) Manager Allows you to do `record.freezePage()` and canvas events will get paused. Based on https://github.com/rrweb-io/rrweb/pull/756#issuecomment-1007566907 * cleanup * Make sure the correct gets replaced * Perf: Speed up check to see if canvas is blank * Access unpatched getImageData * Use is2DCanvasBlank only for 2d context --- .gitignore | 2 +- ...orkspace => rrweb-monorepo.code-workspace} | 11 +- packages/rrweb-snapshot/src/snapshot.ts | 34 +- packages/rrweb-snapshot/src/types.ts | 4 + packages/rrweb-snapshot/src/utils.ts | 37 + packages/rrweb-snapshot/typings/types.d.ts | 3 + packages/rrweb-snapshot/typings/utils.d.ts | 1 + packages/rrweb/.gitignore | 1 + packages/rrweb/jest.config.js | 3 + packages/rrweb/package.json | 6 +- packages/rrweb/src/record/index.ts | 32 +- packages/rrweb/src/record/mutation.ts | 16 +- packages/rrweb/src/record/observer.ts | 96 +- .../rrweb/src/record/observers/canvas/2d.ts | 94 ++ .../record/observers/canvas/canvas-manager.ts | 152 ++ .../src/record/observers/canvas/canvas.ts | 35 + .../record/observers/canvas/serialize-args.ts | 166 ++ .../src/record/observers/canvas/webgl.ts | 106 ++ .../rrweb/src/record/shadow-dom-manager.ts | 3 + packages/rrweb/src/replay/canvas/2d.ts | 48 + packages/rrweb/src/replay/canvas/index.ts | 51 + packages/rrweb/src/replay/canvas/webgl.ts | 175 ++ packages/rrweb/src/replay/index.ts | 117 +- packages/rrweb/src/replay/machine.ts | 1 + packages/rrweb/src/replay/timer.ts | 2 + packages/rrweb/src/types.ts | 58 +- .../__snapshots__/integration.test.ts.snap | 452 ++++- ...record-and-replay-a-webgl-image-1-snap.png | Bin 0 -> 10913 bytes ...ecord-and-replay-a-webgl-square-1-snap.png | Bin 0 -> 10812 bytes packages/rrweb/test/e2e/webgl.test.ts | 175 ++ packages/rrweb/test/events/webgl.ts | 118 ++ .../rrweb/test/html/assets/webgl-utils.js | 1496 +++++++++++++++++ .../rrweb/test/html/canvas-webgl-image.html | 149 ++ .../rrweb/test/html/canvas-webgl-square.html | 110 ++ packages/rrweb/test/html/canvas-webgl.html | 27 + .../rrweb/test/html/mutation-observer.html | 3 +- packages/rrweb/test/integration.test.ts | 85 +- packages/rrweb/test/packer.test.ts | 7 + .../record/__snapshots__/webgl.test.ts.snap | 814 +++++++++ .../rrweb/test/record/serialize-args.test.ts | 179 ++ packages/rrweb/test/record/webgl.test.ts | 260 +++ ...ould-output-simple-webgl-object-1-snap.png | Bin 0 -> 10796 bytes .../test/replay/deserialize-args.test.ts | 133 ++ .../test/replay/preload-all-images.test.ts | 125 ++ .../rrweb/test/replay/webgl-mutation.test.ts | 47 + packages/rrweb/test/replay/webgl.test.ts | 66 + packages/rrweb/test/utils.ts | 151 ++ packages/rrweb/tsconfig.json | 6 +- packages/rrweb/typings/record/mutation.d.ts | 5 +- packages/rrweb/typings/record/observer.d.ts | 3 +- .../typings/record/observers/canvas/2d.d.ts | 2 + .../observers/canvas/canvas-manager.d.ts | 31 + .../record/observers/canvas/canvas.d.ts | 2 + .../observers/canvas/serialize-args.d.ts | 6 + .../record/observers/canvas/webgl.d.ts | 2 + .../record/observers/canvas/webgl2.d.ts | 2 + .../typings/record/shadow-dom-manager.d.ts | 2 + packages/rrweb/typings/replay/canvas/2d.d.ts | 9 + .../rrweb/typings/replay/canvas/index.d.ts | 9 + .../rrweb/typings/replay/canvas/webgl.d.ts | 11 + packages/rrweb/typings/replay/index.d.ts | 3 + packages/rrweb/typings/types.d.ts | 37 +- yarn.lock | 242 ++- 63 files changed, 5695 insertions(+), 328 deletions(-) rename .vscode/{monorepo.code-workspace => rrweb-monorepo.code-workspace} (51%) create mode 100644 packages/rrweb/src/record/observers/canvas/2d.ts create mode 100644 packages/rrweb/src/record/observers/canvas/canvas-manager.ts create mode 100644 packages/rrweb/src/record/observers/canvas/canvas.ts create mode 100644 packages/rrweb/src/record/observers/canvas/serialize-args.ts create mode 100644 packages/rrweb/src/record/observers/canvas/webgl.ts create mode 100644 packages/rrweb/src/replay/canvas/2d.ts create mode 100644 packages/rrweb/src/replay/canvas/index.ts create mode 100644 packages/rrweb/src/replay/canvas/webgl.ts create mode 100644 packages/rrweb/test/e2e/__image_snapshots__/webgl-test-ts-e-2-e-webgl-will-record-and-replay-a-webgl-image-1-snap.png create mode 100644 packages/rrweb/test/e2e/__image_snapshots__/webgl-test-ts-e-2-e-webgl-will-record-and-replay-a-webgl-square-1-snap.png create mode 100644 packages/rrweb/test/e2e/webgl.test.ts create mode 100644 packages/rrweb/test/events/webgl.ts create mode 100644 packages/rrweb/test/html/assets/webgl-utils.js create mode 100644 packages/rrweb/test/html/canvas-webgl-image.html create mode 100644 packages/rrweb/test/html/canvas-webgl-square.html create mode 100644 packages/rrweb/test/html/canvas-webgl.html create mode 100644 packages/rrweb/test/record/__snapshots__/webgl.test.ts.snap create mode 100644 packages/rrweb/test/record/serialize-args.test.ts create mode 100644 packages/rrweb/test/record/webgl.test.ts create mode 100644 packages/rrweb/test/replay/__image_snapshots__/webgl-test-ts-replayer-webgl-should-output-simple-webgl-object-1-snap.png create mode 100644 packages/rrweb/test/replay/deserialize-args.test.ts create mode 100644 packages/rrweb/test/replay/preload-all-images.test.ts create mode 100644 packages/rrweb/test/replay/webgl-mutation.test.ts create mode 100644 packages/rrweb/test/replay/webgl.test.ts create mode 100644 packages/rrweb/typings/record/observers/canvas/2d.d.ts create mode 100644 packages/rrweb/typings/record/observers/canvas/canvas-manager.d.ts create mode 100644 packages/rrweb/typings/record/observers/canvas/canvas.d.ts create mode 100644 packages/rrweb/typings/record/observers/canvas/serialize-args.d.ts create mode 100644 packages/rrweb/typings/record/observers/canvas/webgl.d.ts create mode 100644 packages/rrweb/typings/record/observers/canvas/webgl2.d.ts create mode 100644 packages/rrweb/typings/replay/canvas/2d.d.ts create mode 100644 packages/rrweb/typings/replay/canvas/index.d.ts create mode 100644 packages/rrweb/typings/replay/canvas/webgl.d.ts diff --git a/.gitignore b/.gitignore index c70cf42b..895e1115 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ .vscode/* -!/.vscode/monorepo.code-workspace +!/.vscode/rrweb-monorepo.code-workspace .idea node_modules package-lock.json diff --git a/.vscode/monorepo.code-workspace b/.vscode/rrweb-monorepo.code-workspace similarity index 51% rename from .vscode/monorepo.code-workspace rename to .vscode/rrweb-monorepo.code-workspace index 4caba947..79849c8f 100644 --- a/.vscode/monorepo.code-workspace +++ b/.vscode/rrweb-monorepo.code-workspace @@ -1,26 +1,31 @@ { "folders": [ { - "name": "Monorepo", + "name": " rrweb monorepo", // added a space to bump it to the top "path": ".." }, { + "name": "rrdom (package)", "path": "../packages/rrdom" }, { + "name": "rrweb (package)", "path": "../packages/rrweb" }, { + "name": "rrweb-player (package)", "path": "../packages/rrweb-player" }, { + "name": "rrweb-snapshot (package)", "path": "../packages/rrweb-snapshot" } ], "settings": { "jest.disabledWorkspaceFolders": [ - "Monorepo", - "rrweb-player" + " rrweb monorepo", + "rrweb-player (package)", + "rrdom (package)" ] } } diff --git a/packages/rrweb-snapshot/src/snapshot.ts b/packages/rrweb-snapshot/src/snapshot.ts index 1134301e..0f8de04a 100644 --- a/packages/rrweb-snapshot/src/snapshot.ts +++ b/packages/rrweb-snapshot/src/snapshot.ts @@ -10,8 +10,14 @@ import { MaskTextFn, MaskInputFn, KeepIframeSrcFn, + ICanvas, } from './types'; -import { isElement, isShadowRoot, maskInputValue } from './utils'; +import { + is2DCanvasBlank, + isElement, + isShadowRoot, + maskInputValue, +} from './utils'; let _id = 1; const tagNameRegex = new RegExp('[^a-z0-9-_:]'); @@ -504,7 +510,26 @@ function serializeNode( } // canvas image data if (tagName === 'canvas' && recordCanvas) { - attributes.rr_dataURL = (n as HTMLCanvasElement).toDataURL(); + if ((n as ICanvas).__context === '2d') { + // only record this on 2d canvas + if (!is2DCanvasBlank(n as HTMLCanvasElement)) { + attributes.rr_dataURL = (n as HTMLCanvasElement).toDataURL(); + } + } else if (!('__context' in n)) { + // context is unknown, better not call getContext to trigger it + const canvasDataURL = (n as HTMLCanvasElement).toDataURL(); + + // create blank canvas of same dimensions + const blankCanvas = document.createElement('canvas'); + blankCanvas.width = (n as HTMLCanvasElement).width; + blankCanvas.height = (n as HTMLCanvasElement).height; + const blankCanvasDataURL = blankCanvas.toDataURL(); + + // no need to save dataURL if it's the same as blank canvas + if (canvasDataURL !== blankCanvasDataURL) { + attributes.rr_dataURL = canvasDataURL; + } + } } // save image offline if (tagName === 'img' && inlineImages) { @@ -592,7 +617,10 @@ function serializeNode( ); } } catch (err) { - console.warn(`Cannot get CSS styles from text's parentNode. Error: ${err}`, n); + console.warn( + `Cannot get CSS styles from text's parentNode. Error: ${err}`, + n, + ); } textContent = absoluteToStylesheet(textContent, getHref()); } diff --git a/packages/rrweb-snapshot/src/types.ts b/packages/rrweb-snapshot/src/types.ts index 8720783e..f239213d 100644 --- a/packages/rrweb-snapshot/src/types.ts +++ b/packages/rrweb-snapshot/src/types.ts @@ -71,6 +71,10 @@ export interface INode extends Node { __sn: serializedNodeWithId; } +export interface ICanvas extends HTMLCanvasElement { + __context: string; +} + export type idNodeMap = { [key: number]: INode; }; diff --git a/packages/rrweb-snapshot/src/utils.ts b/packages/rrweb-snapshot/src/utils.ts index ed5c5f75..2b059765 100644 --- a/packages/rrweb-snapshot/src/utils.ts +++ b/packages/rrweb-snapshot/src/utils.ts @@ -35,3 +35,40 @@ export function maskInputValue({ } return text; } + +const ORIGINAL_ATTRIBUTE_NAME = '__rrweb_original__'; +type PatchedGetImageData = { + [ORIGINAL_ATTRIBUTE_NAME]: CanvasImageData['getImageData']; +} & CanvasImageData['getImageData']; + +export function is2DCanvasBlank(canvas: HTMLCanvasElement): boolean { + const ctx = canvas.getContext('2d'); + if (!ctx) return true; + + const chunkSize = 50; + + // get chunks of the canvas and check if it is blank + for (let x = 0; x < canvas.width; x += chunkSize) { + for (let y = 0; y < canvas.height; y += chunkSize) { + const getImageData = ctx.getImageData as PatchedGetImageData; + const originalGetImageData = + ORIGINAL_ATTRIBUTE_NAME in getImageData + ? getImageData[ORIGINAL_ATTRIBUTE_NAME] + : getImageData; + // by getting the canvas in chunks we avoid an expensive + // `getImageData` call that retrieves everything + // even if we can already tell from the first chunk(s) that + // the canvas isn't blank + const pixelBuffer = new Uint32Array( + originalGetImageData( + x, + y, + Math.min(chunkSize, canvas.width - x), + Math.min(chunkSize, canvas.height - y), + ).data.buffer, + ); + if (pixelBuffer.some((pixel) => pixel !== 0)) return false; + } + } + return true; +} diff --git a/packages/rrweb-snapshot/typings/types.d.ts b/packages/rrweb-snapshot/typings/types.d.ts index 5831c7b5..262d3f9a 100644 --- a/packages/rrweb-snapshot/typings/types.d.ts +++ b/packages/rrweb-snapshot/typings/types.d.ts @@ -55,6 +55,9 @@ export declare type tagMap = { export interface INode extends Node { __sn: serializedNodeWithId; } +export interface ICanvas extends HTMLCanvasElement { + __context: string; +} export declare type idNodeMap = { [key: number]: INode; }; diff --git a/packages/rrweb-snapshot/typings/utils.d.ts b/packages/rrweb-snapshot/typings/utils.d.ts index dfb1b70a..6572ab82 100644 --- a/packages/rrweb-snapshot/typings/utils.d.ts +++ b/packages/rrweb-snapshot/typings/utils.d.ts @@ -8,3 +8,4 @@ export declare function maskInputValue({ maskInputOptions, tagName, type, value, value: string | null; maskInputFn?: MaskInputFn; }): string; +export declare function is2DCanvasBlank(canvas: HTMLCanvasElement): boolean; diff --git a/packages/rrweb/.gitignore b/packages/rrweb/.gitignore index afbda82b..4875c32f 100644 --- a/packages/rrweb/.gitignore +++ b/packages/rrweb/.gitignore @@ -13,3 +13,4 @@ temp *.log .env +__diff_output__ \ No newline at end of file diff --git a/packages/rrweb/jest.config.js b/packages/rrweb/jest.config.js index 46cb05c3..29db4e7f 100644 --- a/packages/rrweb/jest.config.js +++ b/packages/rrweb/jest.config.js @@ -3,4 +3,7 @@ module.exports = { preset: 'ts-jest', testEnvironment: 'node', testMatch: ['**/**.test.ts'], + moduleNameMapper: { + '\\.css$': 'identity-obj-proxy', + }, }; diff --git a/packages/rrweb/package.json b/packages/rrweb/package.json index 4a4be361..61a1fcb8 100644 --- a/packages/rrweb/package.json +++ b/packages/rrweb/package.json @@ -46,15 +46,18 @@ "@types/chai": "^4.1.6", "@types/inquirer": "0.0.43", "@types/jest": "^27.0.2", + "@types/jest-image-snapshot": "^4.3.1", "@types/jsdom": "^16.2.12", "@types/node": "^12.20.16", "@types/prettier": "^2.3.2", - "@types/puppeteer": "^5.4.3", + "@types/puppeteer": "^5.4.4", "cross-env": "^5.2.0", "fast-mhtml": "^1.1.9", + "identity-obj-proxy": "^3.0.0", "ignore-styles": "^5.0.1", "inquirer": "^6.2.1", "jest": "^27.2.4", + "jest-image-snapshot": "^4.5.1", "jest-snapshot": "^23.6.0", "jsdom": "^17.0.0", "jsdom-global": "^3.0.2", @@ -73,6 +76,7 @@ "dependencies": { "@types/css-font-loading-module": "0.0.7", "@xstate/fsm": "^1.4.0", + "base64-arraybuffer": "^1.0.1", "fflate": "^0.4.4", "mitt": "^1.1.3", "rrweb-snapshot": "^1.1.12" diff --git a/packages/rrweb/src/record/index.ts b/packages/rrweb/src/record/index.ts index e9189c16..e4c92365 100644 --- a/packages/rrweb/src/record/index.ts +++ b/packages/rrweb/src/record/index.ts @@ -18,9 +18,11 @@ import { listenerHandler, mutationCallbackParam, scrollCallback, + canvasMutationParam, } from '../types'; import { IframeManager } from './iframe-manager'; import { ShadowDomManager } from './shadow-dom-manager'; +import { CanvasManager } from './observers/canvas/canvas-manager'; function wrapEvent(e: event): eventWithTime { return { @@ -180,11 +182,28 @@ function record( }, }), ); + const wrappedCanvasMutationEmit = (p: canvasMutationParam) => + wrappedEmit( + wrapEvent({ + type: EventType.IncrementalSnapshot, + data: { + source: IncrementalSource.CanvasMutation, + ...p, + }, + }), + ); const iframeManager = new IframeManager({ mutationCb: wrappedMutationEmit, }); + const canvasManager = new CanvasManager({ + mutationCb: wrappedCanvasMutationEmit, + win: window, + blockClass, + mirror, + }); + const shadowDomManager = new ShadowDomManager({ mutationCb: wrappedMutationEmit, scrollCb: wrappedScrollEmit, @@ -202,6 +221,7 @@ function record( sampling, slimDOMOptions, iframeManager, + canvasManager, }, mirror, }); @@ -365,16 +385,7 @@ function record( }, }), ), - canvasMutationCb: (p) => - wrappedEmit( - wrapEvent({ - type: EventType.IncrementalSnapshot, - data: { - source: IncrementalSource.CanvasMutation, - ...p, - }, - }), - ), + canvasMutationCb: wrappedCanvasMutationEmit, fontCb: (p) => wrappedEmit( wrapEvent({ @@ -404,6 +415,7 @@ function record( mirror, iframeManager, shadowDomManager, + canvasManager, plugins: plugins?.map((p) => ({ observer: p.observer, diff --git a/packages/rrweb/src/record/mutation.ts b/packages/rrweb/src/record/mutation.ts index 275843fb..842a98b8 100644 --- a/packages/rrweb/src/record/mutation.ts +++ b/packages/rrweb/src/record/mutation.ts @@ -31,6 +31,7 @@ import { hasShadowRoot, } from '../utils'; import { IframeManager } from './iframe-manager'; +import { CanvasManager } from './observers/canvas/canvas-manager'; import { ShadowDomManager } from './shadow-dom-manager'; type DoubleLinkedListNode = { @@ -179,6 +180,7 @@ export default class MutationBuffer { private mirror: Mirror; private iframeManager: IframeManager; private shadowDomManager: ShadowDomManager; + private canvasManager: CanvasManager; public init( cb: mutationCallBack, @@ -197,6 +199,7 @@ export default class MutationBuffer { mirror: Mirror, iframeManager: IframeManager, shadowDomManager: ShadowDomManager, + canvasManager: CanvasManager, ) { this.blockClass = blockClass; this.blockSelector = blockSelector; @@ -214,14 +217,17 @@ export default class MutationBuffer { this.mirror = mirror; this.iframeManager = iframeManager; this.shadowDomManager = shadowDomManager; + this.canvasManager = canvasManager; } public freeze() { this.frozen = true; + this.canvasManager.freeze(); } public unfreeze() { this.frozen = false; + this.canvasManager.unfreeze(); this.emit(); } @@ -231,16 +237,22 @@ export default class MutationBuffer { public lock() { this.locked = true; + this.canvasManager.lock(); } public unlock() { this.locked = false; + this.canvasManager.unlock(); this.emit(); } + public reset() { + this.canvasManager.reset(); + } + public processMutations = (mutations: mutationRecord[]) => { - mutations.forEach(this.processMutation); - this.emit(); + mutations.forEach(this.processMutation); // adds mutations to the buffer + this.emit(); // clears buffer if not locked/frozen }; public emit = () => { diff --git a/packages/rrweb/src/record/observer.ts b/packages/rrweb/src/record/observer.ts index f69e81e9..7a08416f 100644 --- a/packages/rrweb/src/record/observer.ts +++ b/packages/rrweb/src/record/observer.ts @@ -49,6 +49,7 @@ import { import MutationBuffer from './mutation'; import { IframeManager } from './iframe-manager'; import { ShadowDomManager } from './shadow-dom-manager'; +import { CanvasManager } from './observers/canvas/canvas-manager'; type WindowWithStoredMutationObserver = IWindow & { __rrMutationObserver?: MutationObserver; @@ -102,6 +103,7 @@ export function initMutationObserver( mirror: Mirror, iframeManager: IframeManager, shadowDomManager: ShadowDomManager, + canvasManager: CanvasManager, rootEl: Node, ): MutationObserver { const mutationBuffer = new MutationBuffer(); @@ -124,6 +126,7 @@ export function initMutationObserver( mirror, iframeManager, shadowDomManager, + canvasManager, ); let mutationObserverCtor = window.MutationObserver || @@ -721,88 +724,6 @@ function initMediaInteractionObserver( }; } -function initCanvasMutationObserver( - cb: canvasMutationCallback, - win: IWindow, - blockClass: blockClass, - mirror: Mirror, -): listenerHandler { - const props = Object.getOwnPropertyNames( - win.CanvasRenderingContext2D.prototype, - ); - const handlers: listenerHandler[] = []; - for (const prop of props) { - try { - if ( - typeof win.CanvasRenderingContext2D.prototype[ - prop as keyof CanvasRenderingContext2D - ] !== 'function' - ) { - continue; - } - const restoreHandler = patch( - win.CanvasRenderingContext2D.prototype, - prop, - function (original) { - return function ( - this: CanvasRenderingContext2D, - ...args: Array - ) { - if (!isBlocked(this.canvas, blockClass)) { - setTimeout(() => { - const recordArgs = [...args]; - if (prop === 'drawImage') { - if ( - recordArgs[0] && - recordArgs[0] instanceof HTMLCanvasElement - ) { - const canvas = recordArgs[0]; - const ctx = canvas.getContext('2d'); - let imgd = ctx?.getImageData( - 0, - 0, - canvas.width, - canvas.height, - ); - let pix = imgd?.data; - recordArgs[0] = JSON.stringify(pix); - } - } - cb({ - id: mirror.getId((this.canvas as unknown) as INode), - property: prop, - args: recordArgs, - }); - }, 0); - } - return original.apply(this, args); - }; - }, - ); - handlers.push(restoreHandler); - } catch { - const hookHandler = hookSetter( - win.CanvasRenderingContext2D.prototype, - prop, - { - set(v) { - cb({ - id: mirror.getId((this.canvas as unknown) as INode), - property: prop, - args: [v], - setter: true, - }); - }, - }, - ); - handlers.push(hookHandler); - } - } - return () => { - handlers.forEach((h) => h()); - }; -} - function initFontObserver(cb: fontCallback, doc: Document): listenerHandler { const win = doc.defaultView as IWindow; if (!win) { @@ -965,6 +886,7 @@ export function initObservers( o.mirror, o.iframeManager, o.shadowDomManager, + o.canvasManager, o.doc, ); const mousemoveHandler = initMoveObserver( @@ -1016,14 +938,6 @@ export function initObservers( currentWindow, o.mirror, ); - const canvasMutationObserver = o.recordCanvas - ? initCanvasMutationObserver( - o.canvasMutationCb, - currentWindow, - o.blockClass, - o.mirror, - ) - : () => {}; const fontObserver = o.collectFonts ? initFontObserver(o.fontCb, o.doc) : () => {}; @@ -1036,6 +950,7 @@ export function initObservers( } return () => { + mutationBuffers.forEach((b) => b.reset()); mutationObserver.disconnect(); mousemoveHandler(); mouseInteractionHandler(); @@ -1045,7 +960,6 @@ export function initObservers( mediaInteractionHandler(); styleSheetObserver(); styleDeclarationObserver(); - canvasMutationObserver(); fontObserver(); pluginHandlers.forEach((h) => h()); }; diff --git a/packages/rrweb/src/record/observers/canvas/2d.ts b/packages/rrweb/src/record/observers/canvas/2d.ts new file mode 100644 index 00000000..dd63469b --- /dev/null +++ b/packages/rrweb/src/record/observers/canvas/2d.ts @@ -0,0 +1,94 @@ +import { INode } from 'rrweb-snapshot'; +import { + blockClass, + CanvasContext, + canvasManagerMutationCallback, + IWindow, + listenerHandler, + Mirror, +} from '../../../types'; +import { hookSetter, isBlocked, patch } from '../../../utils'; + +export default function initCanvas2DMutationObserver( + cb: canvasManagerMutationCallback, + win: IWindow, + blockClass: blockClass, + mirror: Mirror, +): listenerHandler { + const handlers: listenerHandler[] = []; + const props2D = Object.getOwnPropertyNames( + win.CanvasRenderingContext2D.prototype, + ); + for (const prop of props2D) { + try { + if ( + typeof win.CanvasRenderingContext2D.prototype[ + prop as keyof CanvasRenderingContext2D + ] !== 'function' + ) { + continue; + } + const restoreHandler = patch( + win.CanvasRenderingContext2D.prototype, + prop, + function (original) { + return function ( + this: CanvasRenderingContext2D, + ...args: Array + ) { + if (!isBlocked((this.canvas as unknown) as INode, blockClass)) { + // Using setTimeout as getImageData + JSON.stringify can be heavy + // and we'd rather not block the main thread + setTimeout(() => { + const recordArgs = [...args]; + if (prop === 'drawImage') { + if ( + recordArgs[0] && + recordArgs[0] instanceof HTMLCanvasElement + ) { + const canvas = recordArgs[0]; + const ctx = canvas.getContext('2d'); + let imgd = ctx?.getImageData( + 0, + 0, + canvas.width, + canvas.height, + ); + let pix = imgd?.data; + recordArgs[0] = JSON.stringify(pix); + } + } + cb(this.canvas, { + type: CanvasContext['2D'], + property: prop, + args: recordArgs, + }); + }, 0); + } + return original.apply(this, args); + }; + }, + ); + handlers.push(restoreHandler); + } catch { + const hookHandler = hookSetter( + win.CanvasRenderingContext2D.prototype, + prop, + { + set(v) { + cb(this.canvas, { + type: CanvasContext['2D'], + property: prop, + args: [v], + setter: true, + }); + }, + }, + ); + handlers.push(hookHandler); + } + } + return () => { + handlers.forEach((h) => h()); + }; +} diff --git a/packages/rrweb/src/record/observers/canvas/canvas-manager.ts b/packages/rrweb/src/record/observers/canvas/canvas-manager.ts new file mode 100644 index 00000000..28baaa85 --- /dev/null +++ b/packages/rrweb/src/record/observers/canvas/canvas-manager.ts @@ -0,0 +1,152 @@ +import { INode } from 'rrweb-snapshot'; +import { + blockClass, + canvasManagerMutationCallback, + canvasMutationCallback, + canvasMutationCommand, + canvasMutationWithType, + IWindow, + listenerHandler, + Mirror, +} from '../../../types'; +import initCanvas2DMutationObserver from './2d'; +import initCanvasContextObserver from './canvas'; +import initCanvasWebGLMutationObserver from './webgl'; + +export type RafStamps = { latestId: number; invokeId: number | null }; + +type pendingCanvasMutationsMap = Map< + HTMLCanvasElement, + canvasMutationWithType[] +>; + +export class CanvasManager { + private pendingCanvasMutations: pendingCanvasMutationsMap = new Map(); + private rafStamps: RafStamps = { latestId: 0, invokeId: null }; + private mirror: Mirror; + + private mutationCb: canvasMutationCallback; + private resetObservers: listenerHandler; + private frozen: boolean = false; + private locked: boolean = false; + + public reset() { + this.pendingCanvasMutations.clear(); + this.resetObservers(); + } + + public freeze() { + this.frozen = true; + } + + public unfreeze() { + this.frozen = false; + } + + public lock() { + this.locked = true; + } + + public unlock() { + this.locked = false; + } + + constructor(options: { + mutationCb: canvasMutationCallback; + win: IWindow; + blockClass: blockClass; + mirror: Mirror; + }) { + this.mutationCb = options.mutationCb; + this.mirror = options.mirror; + + this.initCanvasMutationObserver(options.win, options.blockClass); + } + + private processMutation: canvasManagerMutationCallback = function ( + target, + mutation, + ) { + const newFrame = + this.rafStamps.invokeId && + this.rafStamps.latestId !== this.rafStamps.invokeId; + if (newFrame || !this.rafStamps.invokeId) + this.rafStamps.invokeId = this.rafStamps.latestId; + + if (!this.pendingCanvasMutations.has(target)) { + this.pendingCanvasMutations.set(target, []); + } + + this.pendingCanvasMutations.get(target)!.push(mutation); + }; + + private initCanvasMutationObserver( + win: IWindow, + blockClass: blockClass, + ): void { + this.startRAFTimestamping(); + this.startPendingCanvasMutationFlusher(); + + const canvasContextReset = initCanvasContextObserver(win, blockClass); + const canvas2DReset = initCanvas2DMutationObserver( + this.processMutation.bind(this), + win, + blockClass, + this.mirror, + ); + + const canvasWebGL1and2Reset = initCanvasWebGLMutationObserver( + this.processMutation.bind(this), + win, + blockClass, + this.mirror, + ); + + this.resetObservers = () => { + canvasContextReset(); + canvas2DReset(); + canvasWebGL1and2Reset(); + }; + } + + private startPendingCanvasMutationFlusher() { + requestAnimationFrame(() => this.flushPendingCanvasMutations()); + } + + private startRAFTimestamping() { + const setLatestRAFTimestamp = (timestamp: DOMHighResTimeStamp) => { + this.rafStamps.latestId = timestamp; + requestAnimationFrame(setLatestRAFTimestamp); + }; + requestAnimationFrame(setLatestRAFTimestamp); + } + + flushPendingCanvasMutations() { + this.pendingCanvasMutations.forEach( + (values: canvasMutationCommand[], canvas: HTMLCanvasElement) => { + const id = this.mirror.getId((canvas as unknown) as INode); + this.flushPendingCanvasMutationFor(canvas, id); + }, + ); + requestAnimationFrame(() => this.flushPendingCanvasMutations()); + } + + flushPendingCanvasMutationFor(canvas: HTMLCanvasElement, id: number) { + if (this.frozen || this.locked) { + return; + } + + const valuesWithType = this.pendingCanvasMutations.get(canvas); + if (!valuesWithType || id === -1) return; + + const values = valuesWithType.map((value) => { + const { type, ...rest } = value; + return rest; + }); + const { type } = valuesWithType[0]; + + this.mutationCb({ id, type, commands: values }); + + this.pendingCanvasMutations.delete(canvas); + } +} diff --git a/packages/rrweb/src/record/observers/canvas/canvas.ts b/packages/rrweb/src/record/observers/canvas/canvas.ts new file mode 100644 index 00000000..437af7d5 --- /dev/null +++ b/packages/rrweb/src/record/observers/canvas/canvas.ts @@ -0,0 +1,35 @@ +import { INode, ICanvas } from 'rrweb-snapshot'; +import { blockClass, IWindow, listenerHandler } from '../../../types'; +import { isBlocked, patch } from '../../../utils'; + +export default function initCanvasContextObserver( + win: IWindow, + blockClass: blockClass, +): listenerHandler { + const handlers: listenerHandler[] = []; + try { + const restoreHandler = patch( + win.HTMLCanvasElement.prototype, + 'getContext', + function (original) { + return function ( + this: ICanvas, + contextType: string, + ...args: Array + ) { + if (!isBlocked((this as unknown) as INode, blockClass)) { + if (!('__context' in this)) + (this as ICanvas).__context = contextType; + } + return original.apply(this, [contextType, ...args]); + }; + }, + ); + handlers.push(restoreHandler); + } catch { + console.error('failed to patch HTMLCanvasElement.prototype.getContext'); + } + return () => { + handlers.forEach((h) => h()); + }; +} diff --git a/packages/rrweb/src/record/observers/canvas/serialize-args.ts b/packages/rrweb/src/record/observers/canvas/serialize-args.ts new file mode 100644 index 00000000..e245ee41 --- /dev/null +++ b/packages/rrweb/src/record/observers/canvas/serialize-args.ts @@ -0,0 +1,166 @@ +import { encode } from 'base64-arraybuffer'; +import { IWindow, SerializedWebGlArg } from '../../../types'; + +// TODO: unify with `replay/webgl.ts` +type GLVarMap = Map; +const webGLVarMap: Map< + WebGLRenderingContext | WebGL2RenderingContext, + GLVarMap +> = new Map(); +export function variableListFor( + ctx: WebGLRenderingContext | WebGL2RenderingContext, + ctor: string, +) { + let contextMap = webGLVarMap.get(ctx); + if (!contextMap) { + contextMap = new Map(); + webGLVarMap.set(ctx, contextMap); + } + if (!contextMap.has(ctor)) { + contextMap.set(ctor, []); + } + return contextMap.get(ctor) as any[]; +} + +export const saveWebGLVar = ( + value: any, + win: IWindow, + ctx: WebGL2RenderingContext | WebGLRenderingContext, +): number | void => { + if ( + !value || + !(isInstanceOfWebGLObject(value, win) || typeof value === 'object') + ) + return; + + const name = value.constructor.name; + const list = variableListFor(ctx, name); + let index = list.indexOf(value); + + if (index === -1) { + index = list.length; + list.push(value); + } + return index; +}; + +// from webgl-recorder: https://github.com/evanw/webgl-recorder/blob/bef0e65596e981ee382126587e2dcbe0fc7748e2/webgl-recorder.js#L50-L77 +export function serializeArg( + value: any, + win: IWindow, + ctx: WebGL2RenderingContext | WebGLRenderingContext, +): SerializedWebGlArg { + if (value instanceof Array) { + return value.map((arg) => serializeArg(arg, win, ctx)); + } else if (value === null) { + return value; + } else if ( + value instanceof Float32Array || + value instanceof Float64Array || + value instanceof Int32Array || + value instanceof Uint32Array || + value instanceof Uint8Array || + value instanceof Uint16Array || + value instanceof Int16Array || + value instanceof Int8Array || + value instanceof Uint8ClampedArray + ) { + const name = value.constructor.name; + return { + rr_type: name, + args: [Object.values(value)], + }; + } else if ( + // SharedArrayBuffer disabled on most browsers due to spectre. + // More info: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer/SharedArrayBuffer + // value instanceof SharedArrayBuffer || + value instanceof ArrayBuffer + ) { + const name = value.constructor.name as 'ArrayBuffer'; + const base64 = encode(value); + + return { + rr_type: name, + base64, + }; + } else if (value instanceof DataView) { + const name = value.constructor.name; + return { + rr_type: name, + args: [ + serializeArg(value.buffer, win, ctx), + value.byteOffset, + value.byteLength, + ], + }; + } else if (value instanceof HTMLImageElement) { + const name = value.constructor.name; + const { src } = value; + return { + rr_type: name, + src, + }; + } else if (value instanceof ImageData) { + const name = value.constructor.name; + return { + rr_type: name, + args: [serializeArg(value.data, win, ctx), value.width, value.height], + }; + } else if (isInstanceOfWebGLObject(value, win) || typeof value === 'object') { + const name = value.constructor.name; + const index = saveWebGLVar(value, win, ctx) as number; + + return { + rr_type: name, + index: index, + }; + } + + return value; +} + +export const serializeArgs = ( + args: Array, + win: IWindow, + ctx: WebGLRenderingContext | WebGL2RenderingContext, +) => { + return [...args].map((arg) => serializeArg(arg, win, ctx)); +}; + +export const isInstanceOfWebGLObject = ( + value: any, + win: IWindow, +): value is + | WebGLActiveInfo + | WebGLBuffer + | WebGLFramebuffer + | WebGLProgram + | WebGLRenderbuffer + | WebGLShader + | WebGLShaderPrecisionFormat + | WebGLTexture + | WebGLUniformLocation + | WebGLVertexArrayObject => { + const webGLConstructorNames: string[] = [ + 'WebGLActiveInfo', + 'WebGLBuffer', + 'WebGLFramebuffer', + 'WebGLProgram', + 'WebGLRenderbuffer', + 'WebGLShader', + 'WebGLShaderPrecisionFormat', + 'WebGLTexture', + 'WebGLUniformLocation', + 'WebGLVertexArrayObject', + // In old Chrome versions, value won't be an instanceof WebGLVertexArrayObject. + 'WebGLVertexArrayObjectOES', + ]; + const supportedWebGLConstructorNames = webGLConstructorNames.filter( + (name: string) => typeof win[name as keyof Window] === 'function', + ); + return Boolean( + supportedWebGLConstructorNames.find( + (name: string) => value instanceof win[name as keyof Window], + ), + ); +}; diff --git a/packages/rrweb/src/record/observers/canvas/webgl.ts b/packages/rrweb/src/record/observers/canvas/webgl.ts new file mode 100644 index 00000000..45b03928 --- /dev/null +++ b/packages/rrweb/src/record/observers/canvas/webgl.ts @@ -0,0 +1,106 @@ +import { INode } from 'rrweb-snapshot'; +import { + blockClass, + CanvasContext, + canvasManagerMutationCallback, + canvasMutationWithType, + IWindow, + listenerHandler, + Mirror, +} from '../../../types'; +import { hookSetter, isBlocked, patch } from '../../../utils'; +import { saveWebGLVar, serializeArgs } from './serialize-args'; + +function patchGLPrototype( + prototype: WebGLRenderingContext | WebGL2RenderingContext, + type: CanvasContext, + cb: canvasManagerMutationCallback, + blockClass: blockClass, + mirror: Mirror, + win: IWindow, +): listenerHandler[] { + const handlers: listenerHandler[] = []; + + const props = Object.getOwnPropertyNames(prototype); + + for (const prop of props) { + try { + if (typeof prototype[prop as keyof typeof prototype] !== 'function') { + continue; + } + const restoreHandler = patch(prototype, prop, function (original) { + return function (this: typeof prototype, ...args: Array) { + const result = original.apply(this, args); + saveWebGLVar(result, win, prototype); + if (!isBlocked((this.canvas as unknown) as INode, blockClass)) { + const id = mirror.getId((this.canvas as unknown) as INode); + + const recordArgs = serializeArgs([...args], win, prototype); + const mutation: canvasMutationWithType = { + type, + property: prop, + args: recordArgs, + }; + // TODO: this could potentially also be an OffscreenCanvas as well as HTMLCanvasElement + cb(this.canvas as HTMLCanvasElement, mutation); + } + + return result; + }; + }); + handlers.push(restoreHandler); + } catch { + const hookHandler = hookSetter(prototype, prop, { + set(v) { + // TODO: this could potentially also be an OffscreenCanvas as well as HTMLCanvasElement + cb(this.canvas as HTMLCanvasElement, { + type, + property: prop, + args: [v], + setter: true, + }); + }, + }); + handlers.push(hookHandler); + } + } + + return handlers; +} + +export default function initCanvasWebGLMutationObserver( + cb: canvasManagerMutationCallback, + win: IWindow, + blockClass: blockClass, + mirror: Mirror, +): listenerHandler { + const handlers: listenerHandler[] = []; + + handlers.push( + ...patchGLPrototype( + win.WebGLRenderingContext.prototype, + CanvasContext.WebGL, + cb, + blockClass, + mirror, + win, + ), + ); + + if (typeof win.WebGL2RenderingContext !== 'undefined') { + handlers.push( + ...patchGLPrototype( + win.WebGL2RenderingContext.prototype, + CanvasContext.WebGL2, + cb, + blockClass, + mirror, + win, + ), + ); + } + + return () => { + handlers.forEach((h) => h()); + }; +} diff --git a/packages/rrweb/src/record/shadow-dom-manager.ts b/packages/rrweb/src/record/shadow-dom-manager.ts index 6f2ed26a..2ddaf535 100644 --- a/packages/rrweb/src/record/shadow-dom-manager.ts +++ b/packages/rrweb/src/record/shadow-dom-manager.ts @@ -14,6 +14,7 @@ import { } from 'rrweb-snapshot'; import { IframeManager } from './iframe-manager'; import { initMutationObserver, initScrollObserver } from './observer'; +import { CanvasManager } from './observers/canvas/canvas-manager'; type BypassOptions = { blockClass: blockClass; @@ -29,6 +30,7 @@ type BypassOptions = { sampling: SamplingStrategy; slimDOMOptions: SlimDOMOptions; iframeManager: IframeManager; + canvasManager: CanvasManager; }; export class ShadowDomManager { @@ -67,6 +69,7 @@ export class ShadowDomManager { this.mirror, this.bypassOptions.iframeManager, this, + this.bypassOptions.canvasManager, shadowRoot, ); initScrollObserver( diff --git a/packages/rrweb/src/replay/canvas/2d.ts b/packages/rrweb/src/replay/canvas/2d.ts new file mode 100644 index 00000000..b9fde639 --- /dev/null +++ b/packages/rrweb/src/replay/canvas/2d.ts @@ -0,0 +1,48 @@ +import { Replayer } from '../'; +import { canvasMutationCommand } from '../../types'; + +export default function canvasMutation({ + event, + mutation, + target, + imageMap, + errorHandler, +}: { + event: Parameters[0]; + mutation: canvasMutationCommand; + target: HTMLCanvasElement; + imageMap: Replayer['imageMap']; + errorHandler: Replayer['warnCanvasMutationFailed']; +}): void { + try { + const ctx = ((target as unknown) as HTMLCanvasElement).getContext('2d')!; + + if (mutation.setter) { + // skip some read-only type checks + // tslint:disable-next-line:no-any + (ctx as any)[mutation.property] = mutation.args[0]; + return; + } + const original = ctx[ + mutation.property as Exclude + ] as Function; + + /** + * We have serialized the image source into base64 string during recording, + * which has been preloaded before replay. + * So we can get call drawImage SYNCHRONOUSLY which avoid some fragile cast. + */ + if ( + mutation.property === 'drawImage' && + typeof mutation.args[0] === 'string' + ) { + const image = imageMap.get(event); + mutation.args[0] = image; + original.apply(ctx, mutation.args); + } else { + original.apply(ctx, mutation.args); + } + } catch (error) { + errorHandler(mutation, error); + } +} diff --git a/packages/rrweb/src/replay/canvas/index.ts b/packages/rrweb/src/replay/canvas/index.ts new file mode 100644 index 00000000..73411a2b --- /dev/null +++ b/packages/rrweb/src/replay/canvas/index.ts @@ -0,0 +1,51 @@ +import { Replayer } from '..'; +import { + CanvasContext, + canvasMutationCommand, + canvasMutationData, +} from '../../types'; +import webglMutation from './webgl'; +import canvas2DMutation from './2d'; + +export default function canvasMutation({ + event, + mutation, + target, + imageMap, + errorHandler, +}: { + event: Parameters[0]; + mutation: canvasMutationData; + target: HTMLCanvasElement; + imageMap: Replayer['imageMap']; + errorHandler: Replayer['warnCanvasMutationFailed']; +}): void { + try { + const mutations: canvasMutationCommand[] = + 'commands' in mutation ? mutation.commands : [mutation]; + + if ([CanvasContext.WebGL, CanvasContext.WebGL2].includes(mutation.type)) { + return mutations.forEach((command) => { + webglMutation({ + mutation: command, + type: mutation.type, + target, + imageMap, + errorHandler, + }); + }); + } + // default is '2d' for backwards compatibility (rrweb below 1.1.x) + return mutations.forEach((command) => { + canvas2DMutation({ + event, + mutation: command, + target, + imageMap, + errorHandler, + }); + }); + } catch (error) { + errorHandler(mutation, error); + } +} diff --git a/packages/rrweb/src/replay/canvas/webgl.ts b/packages/rrweb/src/replay/canvas/webgl.ts new file mode 100644 index 00000000..58d323dc --- /dev/null +++ b/packages/rrweb/src/replay/canvas/webgl.ts @@ -0,0 +1,175 @@ +import { decode } from 'base64-arraybuffer'; +import { Replayer } from '../'; +import { + CanvasContext, + canvasMutationCommand, + SerializedWebGlArg, +} from '../../types'; + +// TODO: add ability to wipe this list +type GLVarMap = Map; +const webGLVarMap: Map< + WebGLRenderingContext | WebGL2RenderingContext, + GLVarMap +> = new Map(); +export function variableListFor( + ctx: WebGLRenderingContext | WebGL2RenderingContext, + ctor: string, +) { + let contextMap = webGLVarMap.get(ctx); + if (!contextMap) { + contextMap = new Map(); + webGLVarMap.set(ctx, contextMap); + } + if (!contextMap.has(ctor)) { + contextMap.set(ctor, []); + } + return contextMap.get(ctor) as any[]; +} + +function getContext( + target: HTMLCanvasElement, + type: CanvasContext, +): WebGLRenderingContext | WebGL2RenderingContext | null { + // Note to whomever is going to implement support for `contextAttributes`: + // if `preserveDrawingBuffer` is set to true, + // you might have to do `ctx.flush()` before every webgl canvas event + try { + if (type === CanvasContext.WebGL) { + return ( + target.getContext('webgl')! || target.getContext('experimental-webgl') + ); + } + return target.getContext('webgl2')!; + } catch (e) { + return null; + } +} + +const WebGLVariableConstructorsNames = [ + 'WebGLActiveInfo', + 'WebGLBuffer', + 'WebGLFramebuffer', + 'WebGLProgram', + 'WebGLRenderbuffer', + 'WebGLShader', + 'WebGLShaderPrecisionFormat', + 'WebGLTexture', + 'WebGLUniformLocation', + 'WebGLVertexArrayObject', +]; + +function saveToWebGLVarMap( + ctx: WebGLRenderingContext | WebGL2RenderingContext, + result: any, +) { + if (!result?.constructor) return; // probably null or undefined + + const { name } = result.constructor; + if (!WebGLVariableConstructorsNames.includes(name)) return; // not a WebGL variable + + const variables = variableListFor(ctx, name); + if (!variables.includes(result)) variables.push(result); +} + +export function deserializeArg( + imageMap: Replayer['imageMap'], + ctx: WebGLRenderingContext | WebGL2RenderingContext, +): (arg: SerializedWebGlArg) => any { + return (arg: SerializedWebGlArg): any => { + if (arg && typeof arg === 'object' && 'rr_type' in arg) { + if ('index' in arg) { + const { rr_type: name, index } = arg; + return variableListFor(ctx, name)[index]; + } else if ('args' in arg) { + const { rr_type: name, args } = arg; + const ctor = window[name as keyof Window]; + + return new ctor(...args.map(deserializeArg(imageMap, ctx))); + } else if ('base64' in arg) { + return decode(arg.base64); + } else if ('src' in arg) { + const image = imageMap.get(arg.src); + if (image) { + return image; + } else { + const image = new Image(); + image.src = arg.src; + imageMap.set(arg.src, image); + return image; + } + } + } else if (Array.isArray(arg)) { + return arg.map(deserializeArg(imageMap, ctx)); + } + return arg; + }; +} + +export default function webglMutation({ + mutation, + target, + type, + imageMap, + errorHandler, +}: { + mutation: canvasMutationCommand; + target: HTMLCanvasElement; + type: CanvasContext; + imageMap: Replayer['imageMap']; + errorHandler: Replayer['warnCanvasMutationFailed']; +}): void { + try { + const ctx = getContext(target, type); + if (!ctx) return; + + // NOTE: if `preserveDrawingBuffer` is set to true, + // we must flush the buffers on every new canvas event + // if (mutation.newFrame) ctx.flush(); + + if (mutation.setter) { + // skip some read-only type checks + // tslint:disable-next-line:no-any + (ctx as any)[mutation.property] = mutation.args[0]; + return; + } + const original = ctx[ + mutation.property as Exclude + ] as Function; + + const args = mutation.args.map(deserializeArg(imageMap, ctx)); + const result = original.apply(ctx, args); + saveToWebGLVarMap(ctx, result); + + // Slows down replay considerably, only use for debugging + const debugMode = false; + if (debugMode) { + if (mutation.property === 'compileShader') { + if (!ctx.getShaderParameter(args[0], ctx.COMPILE_STATUS)) + console.warn( + 'something went wrong in replay', + ctx.getShaderInfoLog(args[0]), + ); + } else if (mutation.property === 'linkProgram') { + ctx.validateProgram(args[0]); + if (!ctx.getProgramParameter(args[0], ctx.LINK_STATUS)) + console.warn( + 'something went wrong in replay', + ctx.getProgramInfoLog(args[0]), + ); + } + const webglError = ctx.getError(); + if (webglError !== ctx.NO_ERROR) { + console.warn( + 'WEBGL ERROR', + webglError, + 'on command:', + mutation.property, + ...args, + ); + } + } + } catch (error) { + errorHandler(mutation, error); + } +} diff --git a/packages/rrweb/src/replay/index.ts b/packages/rrweb/src/replay/index.ts index ed01a271..66e727b1 100644 --- a/packages/rrweb/src/replay/index.ts +++ b/packages/rrweb/src/replay/index.ts @@ -39,6 +39,7 @@ import { styleValueWithPriority, mouseMovePos, IWindow, + canvasMutationCommand, } from '../types'; import { createMirror, @@ -62,6 +63,7 @@ import { getNestedRule, getPositionsAndIndex, } from './virtual-styles'; +import canvasMutation from './canvas'; const SKIP_TIME_THRESHOLD = 10 * 1000; const SKIP_TIME_INTERVAL = 5 * 1000; @@ -120,7 +122,7 @@ export class Replayer { // The replayer uses the cache to speed up replay and scrubbing. private cache: BuildCache = createCache(); - private imageMap: Map = new Map(); + private imageMap: Map = new Map(); private mirror: Mirror = createMirror(); @@ -810,6 +812,37 @@ export class Replayer { } } + private hasImageArg(args: any[]): boolean { + for (const arg of args) { + if (!arg || typeof arg !== 'object') { + // do nothing + } else if ('rr_type' in arg && 'args' in arg) { + if (this.hasImageArg(arg.args)) return true; + } else if ('rr_type' in arg && arg.rr_type === 'HTMLImageElement') { + return true; // has image! + } else if (arg instanceof Array) { + if (this.hasImageArg(arg)) return true; + } + } + return false; + } + + private getImageArgs(args: any[]): string[] { + const images: string[] = []; + for (const arg of args) { + if (!arg || typeof arg !== 'object') { + // do nothing + } else if ('rr_type' in arg && 'args' in arg) { + images.push(...this.getImageArgs(arg.args)); + } else if ('rr_type' in arg && arg.rr_type === 'HTMLImageElement') { + images.push(arg.src); + } else if (arg instanceof Array) { + images.push(...this.getImageArgs(arg)); + } + } + return images; + } + /** * pause when there are some canvas drawImage args need to be loaded */ @@ -823,18 +856,34 @@ export class Replayer { for (const event of this.service.state.context.events) { if ( event.type === EventType.IncrementalSnapshot && - event.data.source === IncrementalSource.CanvasMutation && - event.data.property === 'drawImage' && - typeof event.data.args[0] === 'string' && - !this.imageMap.has(event) - ) { - const canvas = document.createElement('canvas'); - const ctx = canvas.getContext('2d'); - const imgd = ctx?.createImageData(canvas.width, canvas.height); - let d = imgd?.data; - d = JSON.parse(event.data.args[0]); - ctx?.putImageData(imgd!, 0, 0); - } + event.data.source === IncrementalSource.CanvasMutation + ) + if ('commands' in event.data) { + event.data.commands.forEach((c) => this.preloadImages(c, event)); + } else { + this.preloadImages(event.data, event); + } + } + } + + private preloadImages(data: canvasMutationCommand, event: eventWithTime) { + if ( + data.property === 'drawImage' && + typeof data.args[0] === 'string' && + !this.imageMap.has(event) + ) { + const canvas = document.createElement('canvas'); + const ctx = canvas.getContext('2d'); + const imgd = ctx?.createImageData(canvas.width, canvas.height); + let d = imgd?.data; + d = JSON.parse(data.args[0]); + ctx?.putImageData(imgd!, 0, 0); + } else if (this.hasImageArg(data.args)) { + this.getImageArgs(data.args).forEach((url) => { + const image = new Image(); + image.src = url; // this preloads the image + this.imageMap.set(url, image); + }); } } @@ -1210,34 +1259,15 @@ export class Replayer { if (!target) { return this.debugNodeNotFound(d, d.id); } - try { - const ctx = ((target as unknown) as HTMLCanvasElement).getContext( - '2d', - )!; - if (d.setter) { - // skip some read-only type checks - // tslint:disable-next-line:no-any - (ctx as any)[d.property] = d.args[0]; - return; - } - const original = ctx[ - d.property as keyof CanvasRenderingContext2D - ] as Function; - /** - * We have serialized the image source into base64 string during recording, - * which has been preloaded before replay. - * So we can get call drawImage SYNCHRONOUSLY which avoid some fragile cast. - */ - if (d.property === 'drawImage' && typeof d.args[0] === 'string') { - const image = this.imageMap.get(e); - d.args[0] = image; - original.apply(ctx, d.args); - } else { - original.apply(ctx, d.args); - } - } catch (error) { - this.warnCanvasMutationFailed(d, d.id, error); - } + + canvasMutation({ + event: e, + mutation: d, + target: (target as unknown) as HTMLCanvasElement, + imageMap: this.imageMap, + errorHandler: this.warnCanvasMutationFailed.bind(this), + }); + break; } case IncrementalSource.Font: { @@ -1866,11 +1896,10 @@ export class Replayer { } private warnCanvasMutationFailed( - d: canvasMutationData, - id: number, + d: canvasMutationData | canvasMutationCommand, error: unknown, ) { - this.warn(`Has error on update canvas '${id}'`, d, error); + this.warn(`Has error on canvas update`, error, 'canvas mutation:', d); } private debugNodeNotFound(d: incrementalData, id: number) { diff --git a/packages/rrweb/src/replay/machine.ts b/packages/rrweb/src/replay/machine.ts index 59ec3db1..8c5ca33f 100644 --- a/packages/rrweb/src/replay/machine.ts +++ b/packages/rrweb/src/replay/machine.ts @@ -167,6 +167,7 @@ export function createPlayerService( play(ctx) { const { timer, events, baselineTime, lastPlayedEvent } = ctx; timer.clear(); + for (const event of events) { // TODO: improve this API addDelay(event, baselineTime); diff --git a/packages/rrweb/src/replay/timer.ts b/packages/rrweb/src/replay/timer.ts index 684c3ff0..f6e7f826 100644 --- a/packages/rrweb/src/replay/timer.ts +++ b/packages/rrweb/src/replay/timer.ts @@ -44,6 +44,7 @@ export class Timer { lastTimestamp = time; while (actions.length) { const action = actions[0]; + if (self.timeOffset >= action.delay) { actions.shift(); action.doAction(); @@ -111,6 +112,7 @@ export function addDelay(event: eventWithTime, baselineTime: number): number { event.delay = firstTimestamp - baselineTime; return firstTimestamp - baselineTime; } + event.delay = event.timestamp - baselineTime; return event.delay; } diff --git a/packages/rrweb/src/types.ts b/packages/rrweb/src/types.ts index 78ed5f8c..74f8a52f 100644 --- a/packages/rrweb/src/types.ts +++ b/packages/rrweb/src/types.ts @@ -11,6 +11,7 @@ import { PackFn, UnpackFn } from './packer/base'; import { IframeManager } from './record/iframe-manager'; import { ShadowDomManager } from './record/shadow-dom-manager'; import type { Replayer } from './replay'; +import { CanvasManager } from './record/observers/canvas/canvas-manager'; export enum EventType { DomContentLoaded, @@ -267,6 +268,7 @@ export type observerParam = { mirror: Mirror; iframeManager: IframeManager; shadowDomManager: ShadowDomManager; + canvasManager: CanvasManager; plugins: Array<{ observer: Function; callback: Function; @@ -386,6 +388,35 @@ export enum MouseInteractions { TouchCancel, } +export enum CanvasContext { + '2D', + WebGL, + WebGL2, +} + +export type SerializedWebGlArg = + | { + rr_type: 'ArrayBuffer'; + base64: string; // base64 + } + | { + rr_type: string; + src: string; // url of image + } + | { + rr_type: string; + args: Array; + } + | { + rr_type: string; + index: number; + } + | string + | number + | boolean + | null + | SerializedWebGlArg[]; + type mouseInteractionParam = { type: MouseInteractions; id: number; @@ -435,15 +466,34 @@ export type styleDeclarationParam = { export type styleDeclarationCallback = (s: styleDeclarationParam) => void; -export type canvasMutationCallback = (p: canvasMutationParam) => void; - -export type canvasMutationParam = { - id: number; +export type canvasMutationCommand = { property: string; args: Array; setter?: true; }; +export type canvasMutationParam = + | { + id: number; + type: CanvasContext; + commands: canvasMutationCommand[]; + } + | ({ + id: number; + type: CanvasContext; + } & canvasMutationCommand); + +export type canvasMutationWithType = { + type: CanvasContext; +} & canvasMutationCommand; + +export type canvasMutationCallback = (p: canvasMutationParam) => void; + +export type canvasManagerMutationCallback = ( + target: HTMLCanvasElement, + p: canvasMutationWithType, +) => void; + export type fontParam = { family: string; fontSource: string; diff --git a/packages/rrweb/test/__snapshots__/integration.test.ts.snap b/packages/rrweb/test/__snapshots__/integration.test.ts.snap index 983788b7..bea952c4 100644 --- a/packages/rrweb/test/__snapshots__/integration.test.ts.snap +++ b/packages/rrweb/test/__snapshots__/integration.test.ts.snap @@ -98,9 +98,21 @@ exports[`record integration tests can freeze mutations 1`] = ` }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n\\\\n \\", + \\"textContent\\": \\"\\\\n \\", \\"id\\": 14 }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 15 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n\\\\n \\", + \\"id\\": 16 + }, { \\"type\\": 2, \\"tagName\\": \\"script\\", @@ -109,15 +121,15 @@ exports[`record integration tests can freeze mutations 1`] = ` { \\"type\\": 3, \\"textContent\\": \\"SCRIPT_PLACEHOLDER\\", - \\"id\\": 16 + \\"id\\": 18 } ], - \\"id\\": 15 + \\"id\\": 17 }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n \\\\n \\", - \\"id\\": 17 + \\"textContent\\": \\"\\\\n \\\\n \\\\n\\", + \\"id\\": 19 } ], \\"id\\": 5 @@ -141,7 +153,7 @@ exports[`record integration tests can freeze mutations 1`] = ` \\"texts\\": [], \\"attributes\\": [ { - \\"id\\": 18, + \\"id\\": 20, \\"attributes\\": { \\"foo\\": \\"bar\\" } @@ -165,7 +177,7 @@ exports[`record integration tests can freeze mutations 1`] = ` \\"foo\\": \\"bar\\" }, \\"childNodes\\": [], - \\"id\\": 18 + \\"id\\": 20 } } ] @@ -272,9 +284,21 @@ exports[`record integration tests can mask character data mutations 1`] = ` }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n\\\\n \\", + \\"textContent\\": \\"\\\\n \\", \\"id\\": 14 }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 15 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n\\\\n \\", + \\"id\\": 16 + }, { \\"type\\": 2, \\"tagName\\": \\"script\\", @@ -283,15 +307,15 @@ exports[`record integration tests can mask character data mutations 1`] = ` { \\"type\\": 3, \\"textContent\\": \\"SCRIPT_PLACEHOLDER\\", - \\"id\\": 16 + \\"id\\": 18 } ], - \\"id\\": 15 + \\"id\\": 17 }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n \\\\n \\", - \\"id\\": 17 + \\"textContent\\": \\"\\\\n \\\\n \\\\n\\", + \\"id\\": 19 } ], \\"id\\": 5 @@ -338,16 +362,16 @@ exports[`record integration tests can mask character data mutations 1`] = ` \\"class\\": \\"rr-mask\\" }, \\"childNodes\\": [], - \\"id\\": 18 + \\"id\\": 20 } }, { - \\"parentId\\": 18, + \\"parentId\\": 20, \\"nextId\\": null, \\"node\\": { \\"type\\": 3, \\"textContent\\": \\"*** **** ****\\", - \\"id\\": 19 + \\"id\\": 21 } }, { @@ -356,7 +380,7 @@ exports[`record integration tests can mask character data mutations 1`] = ` \\"node\\": { \\"type\\": 3, \\"textContent\\": \\"*******\\", - \\"id\\": 20 + \\"id\\": 22 } } ] @@ -463,9 +487,21 @@ exports[`record integration tests can record attribute mutation 1`] = ` }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n\\\\n \\", + \\"textContent\\": \\"\\\\n \\", \\"id\\": 14 }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 15 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n\\\\n \\", + \\"id\\": 16 + }, { \\"type\\": 2, \\"tagName\\": \\"script\\", @@ -474,15 +510,15 @@ exports[`record integration tests can record attribute mutation 1`] = ` { \\"type\\": 3, \\"textContent\\": \\"SCRIPT_PLACEHOLDER\\", - \\"id\\": 16 + \\"id\\": 18 } ], - \\"id\\": 15 + \\"id\\": 17 }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n \\\\n \\", - \\"id\\": 17 + \\"textContent\\": \\"\\\\n \\\\n \\\\n\\", + \\"id\\": 19 } ], \\"id\\": 5 @@ -622,9 +658,21 @@ exports[`record integration tests can record character data muatations 1`] = ` }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n\\\\n \\", + \\"textContent\\": \\"\\\\n \\", \\"id\\": 14 }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 15 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n\\\\n \\", + \\"id\\": 16 + }, { \\"type\\": 2, \\"tagName\\": \\"script\\", @@ -633,15 +681,15 @@ exports[`record integration tests can record character data muatations 1`] = ` { \\"type\\": 3, \\"textContent\\": \\"SCRIPT_PLACEHOLDER\\", - \\"id\\": 16 + \\"id\\": 18 } ], - \\"id\\": 15 + \\"id\\": 17 }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n \\\\n \\", - \\"id\\": 17 + \\"textContent\\": \\"\\\\n \\\\n \\\\n\\", + \\"id\\": 19 } ], \\"id\\": 5 @@ -681,7 +729,7 @@ exports[`record integration tests can record character data muatations 1`] = ` \\"node\\": { \\"type\\": 3, \\"textContent\\": \\"mutated\\", - \\"id\\": 18 + \\"id\\": 20 } } ] @@ -788,9 +836,21 @@ exports[`record integration tests can record childList mutations 1`] = ` }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n\\\\n \\", + \\"textContent\\": \\"\\\\n \\", \\"id\\": 14 }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 15 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n\\\\n \\", + \\"id\\": 16 + }, { \\"type\\": 2, \\"tagName\\": \\"script\\", @@ -799,15 +859,15 @@ exports[`record integration tests can record childList mutations 1`] = ` { \\"type\\": 3, \\"textContent\\": \\"SCRIPT_PLACEHOLDER\\", - \\"id\\": 16 + \\"id\\": 18 } ], - \\"id\\": 15 + \\"id\\": 17 }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n \\\\n \\", - \\"id\\": 17 + \\"textContent\\": \\"\\\\n \\\\n \\\\n\\", + \\"id\\": 19 } ], \\"id\\": 5 @@ -845,7 +905,7 @@ exports[`record integration tests can record childList mutations 1`] = ` \\"tagName\\": \\"span\\", \\"attributes\\": {}, \\"childNodes\\": [], - \\"id\\": 18 + \\"id\\": 20 } } ] @@ -7315,33 +7375,28 @@ exports[`record integration tests should record canvas mutations 1`] = ` \\"data\\": { \\"source\\": 9, \\"id\\": 16, - \\"property\\": \\"moveTo\\", - \\"args\\": [ - 0, - 0 + \\"type\\": 0, + \\"commands\\": [ + { + \\"property\\": \\"moveTo\\", + \\"args\\": [ + 0, + 0 + ] + }, + { + \\"property\\": \\"lineTo\\", + \\"args\\": [ + 200, + 100 + ] + }, + { + \\"property\\": \\"stroke\\", + \\"args\\": [] + } ] } - }, - { - \\"type\\": 3, - \\"data\\": { - \\"source\\": 9, - \\"id\\": 16, - \\"property\\": \\"lineTo\\", - \\"args\\": [ - 200, - 100 - ] - } - }, - { - \\"type\\": 3, - \\"data\\": { - \\"source\\": 9, - \\"id\\": 16, - \\"property\\": \\"stroke\\", - \\"args\\": [] - } } ]" `; @@ -9522,6 +9577,221 @@ exports[`record integration tests should record shadow DOM 1`] = ` ]" `; +exports[`record integration tests should record webgl canvas mutations 1`] = ` +"[ + { + \\"type\\": 0, + \\"data\\": {} + }, + { + \\"type\\": 1, + \\"data\\": {} + }, + { + \\"type\\": 4, + \\"data\\": { + \\"href\\": \\"about:blank\\", + \\"width\\": 1920, + \\"height\\": 1080 + } + }, + { + \\"type\\": 2, + \\"data\\": { + \\"node\\": { + \\"type\\": 0, + \\"childNodes\\": [ + { + \\"type\\": 1, + \\"name\\": \\"html\\", + \\"publicId\\": \\"\\", + \\"systemId\\": \\"\\", + \\"id\\": 2 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"html\\", + \\"attributes\\": { + \\"lang\\": \\"en\\" + }, + \\"childNodes\\": [ + { + \\"type\\": 2, + \\"tagName\\": \\"head\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 5 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"meta\\", + \\"attributes\\": { + \\"charset\\": \\"UTF-8\\" + }, + \\"childNodes\\": [], + \\"id\\": 6 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 7 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"meta\\", + \\"attributes\\": { + \\"name\\": \\"viewport\\", + \\"content\\": \\"width=device-width, initial-scale=1.0\\" + }, + \\"childNodes\\": [], + \\"id\\": 8 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 9 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"title\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 3, + \\"textContent\\": \\"canvas\\", + \\"id\\": 11 + } + ], + \\"id\\": 10 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 12 + } + ], + \\"id\\": 4 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 13 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"body\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 15 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": { + \\"id\\": \\"myCanvas\\", + \\"width\\": \\"200\\", + \\"height\\": \\"100\\", + \\"style\\": \\"border: 1px solid #000000\\" + }, + \\"childNodes\\": [ + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 17 + } + ], + \\"id\\": 16 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\", + \\"id\\": 18 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"script\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 3, + \\"textContent\\": \\"SCRIPT_PLACEHOLDER\\", + \\"id\\": 20 + } + ], + \\"id\\": 19 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\\\n \\", + \\"id\\": 21 + }, + { + \\"type\\": 2, + \\"tagName\\": \\"script\\", + \\"attributes\\": {}, + \\"childNodes\\": [ + { + \\"type\\": 3, + \\"textContent\\": \\"SCRIPT_PLACEHOLDER\\", + \\"id\\": 23 + } + ], + \\"id\\": 22 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n \\\\n \\\\n\\\\n\\", + \\"id\\": 24 + } + ], + \\"id\\": 14 + } + ], + \\"id\\": 3 + } + ], + \\"id\\": 1 + }, + \\"initialOffset\\": { + \\"left\\": 0, + \\"top\\": 0 + } + } + }, + { + \\"type\\": 3, + \\"data\\": { + \\"source\\": 9, + \\"id\\": 16, + \\"type\\": 1, + \\"commands\\": [ + { + \\"property\\": \\"clearColor\\", + \\"args\\": [ + 1, + 0, + 0, + 1 + ] + }, + { + \\"property\\": \\"clear\\", + \\"args\\": [ + 16384 + ] + } + ] + } + } +]" +`; + exports[`record integration tests will serialize node before record 1`] = ` "[ { @@ -9620,9 +9890,21 @@ exports[`record integration tests will serialize node before record 1`] = ` }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n\\\\n \\", + \\"textContent\\": \\"\\\\n \\", \\"id\\": 14 }, + { + \\"type\\": 2, + \\"tagName\\": \\"canvas\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 15 + }, + { + \\"type\\": 3, + \\"textContent\\": \\"\\\\n\\\\n \\", + \\"id\\": 16 + }, { \\"type\\": 2, \\"tagName\\": \\"script\\", @@ -9631,15 +9913,15 @@ exports[`record integration tests will serialize node before record 1`] = ` { \\"type\\": 3, \\"textContent\\": \\"SCRIPT_PLACEHOLDER\\", - \\"id\\": 16 + \\"id\\": 18 } ], - \\"id\\": 15 + \\"id\\": 17 }, { \\"type\\": 3, - \\"textContent\\": \\"\\\\n \\\\n \\", - \\"id\\": 17 + \\"textContent\\": \\"\\\\n \\\\n \\\\n\\", + \\"id\\": 19 } ], \\"id\\": 5 @@ -9667,28 +9949,6 @@ exports[`record integration tests will serialize node before record 1`] = ` { \\"parentId\\": 10, \\"nextId\\": null, - \\"node\\": { - \\"type\\": 2, - \\"tagName\\": \\"li\\", - \\"attributes\\": {}, - \\"childNodes\\": [], - \\"id\\": 18 - } - }, - { - \\"parentId\\": 10, - \\"nextId\\": 18, - \\"node\\": { - \\"type\\": 2, - \\"tagName\\": \\"li\\", - \\"attributes\\": {}, - \\"childNodes\\": [], - \\"id\\": 19 - } - }, - { - \\"parentId\\": 10, - \\"nextId\\": 19, \\"node\\": { \\"type\\": 2, \\"tagName\\": \\"li\\", @@ -9696,6 +9956,28 @@ exports[`record integration tests will serialize node before record 1`] = ` \\"childNodes\\": [], \\"id\\": 20 } + }, + { + \\"parentId\\": 10, + \\"nextId\\": 20, + \\"node\\": { + \\"type\\": 2, + \\"tagName\\": \\"li\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 21 + } + }, + { + \\"parentId\\": 10, + \\"nextId\\": 21, + \\"node\\": { + \\"type\\": 2, + \\"tagName\\": \\"li\\", + \\"attributes\\": {}, + \\"childNodes\\": [], + \\"id\\": 22 + } } ] } diff --git a/packages/rrweb/test/e2e/__image_snapshots__/webgl-test-ts-e-2-e-webgl-will-record-and-replay-a-webgl-image-1-snap.png b/packages/rrweb/test/e2e/__image_snapshots__/webgl-test-ts-e-2-e-webgl-will-record-and-replay-a-webgl-image-1-snap.png new file mode 100644 index 0000000000000000000000000000000000000000..731898701b69eeca46729aa8eef2af872597d9de GIT binary patch literal 10913 zcmeAS@N?(olHy`uVBq!ia0y~yU~geyV6ov~1Bz6wRMrPljKx9jP7LeL$-HD>P+;(M zaSW-L^XB%(TqQ?_w!mjwuI5PQ9ho@0fKlXOk6Xq`)ibW*b9sf>p1r$Vt*Bh~zt{fk zj~_o~umcSOfuL`-^Y@4KLpTf%_I<0hfAhQU#=k!wzJ1$P#19oO|Mc@`rG*S1yLk&E zL}(F9B81VP;V1!NDR>EhTnPjnLZixn!9YoXY`e;wXL>B+*FVeWm%cNDLzN-sahm~z zbG_t>0NuXEpEt z%jeKQu$d)ITqqz6!V*|p*UA;FzWnh|yuCcc%?~b8CtSWZxL4NJ|GfAg7&LRIL8EKU zL+V6poXzt+6?VsVfI_6cRNH~lZ4&ZPY@hyjvr5__07`XgFYXL?JiqT3FwjyA( zMGeXgqb+K1nJ`*uLW;)GN)sFoBfipvq=wPLVYF}mdKI;Vst0Q@>>_5c6? literal 0 HcmV?d00001 diff --git a/packages/rrweb/test/e2e/__image_snapshots__/webgl-test-ts-e-2-e-webgl-will-record-and-replay-a-webgl-square-1-snap.png b/packages/rrweb/test/e2e/__image_snapshots__/webgl-test-ts-e-2-e-webgl-will-record-and-replay-a-webgl-square-1-snap.png new file mode 100644 index 0000000000000000000000000000000000000000..e22bc48831104120ac0c4a19cd3b9f41dc62b426 GIT binary patch literal 10812 zcmeAS@N?(olHy`uVBq!ia0y~yU~geyV6ov~1Bz6wRMrPljKx9jP7LeL$-HD>P+;(M zaSW-L^XAUR+`|S8u8vwCOxA3k^!vd>vQoAdb7--lIu=j~*Nh&If%k?S`+Ch<=I$!Ol9Sr{Z><@wFZ*LC!v-SLD_78`D z-()Te+9?l_PN^`GBF1PUT`;N)9GE1>+-SIsh8zC8jX#Cx?YFA6&S3q2`}j>}NOoTE z@~Og}YgR?jVur!`bBYN>#$k#A4}>M4$^ljF!g&ZH#1PnIFsh7!VKhL%sbDmPfP-N) z%RtJ6(ZT^74x@#`XyGtgQ9_D@(Y!F47e@2KXkK6#Z6iSv!Dt%^91f%P!bq(bK3ttA zJB5LVg^?%5>*|{N`1FNpT7C00}Tl=sjP@{zdC_#)C4&ZDtTJ3^^VYIam z4hBkw0$$CXdGoI2^E-Laa^`{i`FH2u-FpWynBhUXylc0-o#vIc*B}v3S3j3^P6 { + let code: ISuite['code']; + let page: ISuite['page']; + let browser: ISuite['browser']; + let server: ISuite['server']; + let serverURL: ISuite['serverURL']; + + beforeAll(async () => { + server = await startServer(); + serverURL = getServerURL(server); + browser = await launchPuppeteer(); + + const bundlePath = path.resolve(__dirname, '../../dist/rrweb.min.js'); + code = fs.readFileSync(bundlePath, 'utf8'); + }); + + afterEach(async () => { + await page.close(); + }); + + afterAll(async () => { + await server.close(); + await browser.close(); + }); + + const getHtml = ( + fileName: string, + options: recordOptions = {}, + ): string => { + const filePath = path.resolve(__dirname, `../html/${fileName}`); + const html = fs.readFileSync(filePath, 'utf8'); + return replaceLast( + html, + '', + ` + + + `, + ); + }; + + const fakeGoto = async (page: puppeteer.Page, url: string) => { + const intercept = async (request: puppeteer.HTTPRequest) => { + await request.respond({ + status: 200, + contentType: 'text/html', + body: ' ', // non-empty string or page will load indefinitely + }); + }; + await page.setRequestInterception(true); + page.on('request', intercept); + await page.goto(url); + page.off('request', intercept); + await page.setRequestInterception(false); + }; + + const hideMouseAnimation = async (page: puppeteer.Page) => { + await page.addStyleTag({ + content: '.replayer-mouse-tail{display: none !important;}', + }); + }; + + it('will record and replay a webgl square', async () => { + page = await browser.newPage(); + await fakeGoto(page, `${serverURL}/html/canvas-webgl-square.html`); + + await page.setContent( + getHtml.call(this, 'canvas-webgl-square.html', { recordCanvas: true }), + ); + + await waitForRAF(page); + + const snapshots: eventWithTime[] = await page.evaluate('window.snapshots'); + + page = await browser.newPage(); + + await page.goto('about:blank'); + await page.evaluate(code); + + await hideMouseAnimation(page); + await page.evaluate(`let events = ${JSON.stringify(snapshots)}`); + await page.evaluate(` + const { Replayer } = rrweb; + const replayer = new Replayer(events, { + UNSAFE_replayCanvas: true, + }); + replayer.play(500); + `); + await page.waitForTimeout(50); + + const element = await page.$('iframe'); + const frameImage = await element!.screenshot(); + + expect(frameImage).toMatchImageSnapshot(); + }); + + it('will record and replay a webgl image', async () => { + page = await browser.newPage(); + await fakeGoto(page, `${serverURL}/html/canvas-webgl-image.html`); + + await page.setContent( + getHtml.call(this, 'canvas-webgl-image.html', { recordCanvas: true }), + ); + + await page.waitForTimeout(100); + const snapshots: eventWithTime[] = await page.evaluate('window.snapshots'); + + page = await browser.newPage(); + + await page.goto('about:blank'); + await page.evaluate(code); + + await hideMouseAnimation(page); + await page.evaluate(`let events = ${JSON.stringify(snapshots)}`); + await page.evaluate(` + const { Replayer } = rrweb; + const replayer = new Replayer(events, { + UNSAFE_replayCanvas: true, + }); + `); + // wait for iframe to get added and `preloadAllImages` to ge called + await page.waitForSelector('iframe'); + await page.evaluate(`replayer.play(500);`); + await page.waitForTimeout(50); + + const element = await page.$('iframe'); + const frameImage = await element!.screenshot(); + + expect(frameImage).toMatchImageSnapshot(); + }); +}); diff --git a/packages/rrweb/test/events/webgl.ts b/packages/rrweb/test/events/webgl.ts new file mode 100644 index 00000000..513c1ade --- /dev/null +++ b/packages/rrweb/test/events/webgl.ts @@ -0,0 +1,118 @@ +export default [ + { + type: 4, + data: { + href: '', + width: 1600, + height: 900, + }, + timestamp: 1636379531385, + }, + { + type: 2, + data: { + node: { + type: 0, + childNodes: [ + { type: 1, name: 'html', publicId: '', systemId: '', id: 2 }, + { + type: 2, + tagName: 'html', + attributes: { lang: 'en' }, + childNodes: [ + { + type: 2, + tagName: 'head', + attributes: {}, + childNodes: [ + { type: 3, textContent: '\n ', id: 5 }, + { + type: 2, + tagName: 'meta', + attributes: { charset: 'UTF-8' }, + childNodes: [], + id: 6, + }, + { type: 3, textContent: '\n ', id: 7 }, + { + type: 2, + tagName: 'meta', + attributes: { + name: 'viewport', + content: 'width=device-width, initial-scale=1.0', + }, + childNodes: [], + id: 8, + }, + { type: 3, textContent: '\n ', id: 9 }, + { + type: 2, + tagName: 'title', + attributes: {}, + childNodes: [{ type: 3, textContent: 'canvas', id: 11 }], + id: 10, + }, + { type: 3, textContent: '\n ', id: 12 }, + ], + id: 4, + }, + { type: 3, textContent: '\n ', id: 13 }, + { + type: 2, + tagName: 'body', + attributes: {}, + childNodes: [ + { type: 3, textContent: '\n ', id: 15 }, + { + type: 2, + tagName: 'canvas', + attributes: { + id: 'myCanvas', + width: '200', + height: '100', + style: 'border: 1px solid #000000', + }, + childNodes: [{ type: 3, textContent: '\n ', id: 17 }], + id: 16, + }, + { type: 3, textContent: '\n ', id: 18 }, + { + type: 2, + tagName: 'script', + attributes: {}, + childNodes: [ + { type: 3, textContent: 'SCRIPT_PLACEHOLDER', id: 20 }, + ], + id: 19, + }, + { type: 3, textContent: '\n \n\n', id: 21 }, + ], + id: 14, + }, + ], + id: 3, + }, + ], + id: 1, + }, + initialOffset: { left: 0, top: 0 }, + }, + timestamp: 1636379531389, + }, + { + type: 3, + data: { + source: 9, + id: 16, + type: 1, + property: 'clearColor', + args: [1, 0, 0, 1], + }, + timestamp: 1636379532355, + }, + { + type: 3, + data: { source: 9, id: 16, type: 1, property: 'clear', args: [16384] }, + timestamp: 1636379532356, + }, +]; diff --git a/packages/rrweb/test/html/assets/webgl-utils.js b/packages/rrweb/test/html/assets/webgl-utils.js new file mode 100644 index 00000000..183a5acf --- /dev/null +++ b/packages/rrweb/test/html/assets/webgl-utils.js @@ -0,0 +1,1496 @@ +/* + * Copyright 2021 GFXFundamentals. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of GFXFundamentals. nor the names of his + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +(function (root, factory) { + // eslint-disable-line + if (typeof define === 'function' && define.amd) { + // AMD. Register as an anonymous module. + define([], function () { + return factory.call(root); + }); + } else { + // Browser globals + root.webglUtils = factory.call(root); + } +})(this, function () { + 'use strict'; + + const topWindow = this; + + /** @module webgl-utils */ + + function isInIFrame(w) { + w = w || topWindow; + return w !== w.top; + } + + if (!isInIFrame()) { + console.log( + '%c%s', + 'color:blue;font-weight:bold;', + 'for more about webgl-utils.js see:', + ); // eslint-disable-line + console.log( + '%c%s', + 'color:blue;font-weight:bold;', + 'https://webglfundamentals.org/webgl/lessons/webgl-boilerplate.html', + ); // eslint-disable-line + } + + /** + * Wrapped logging function. + * @param {string} msg The message to log. + */ + function error(msg) { + if (topWindow.console) { + if (topWindow.console.error) { + topWindow.console.error(msg); + } else if (topWindow.console.log) { + topWindow.console.log(msg); + } + } + } + + /** + * Error Callback + * @callback ErrorCallback + * @param {string} msg error message. + * @memberOf module:webgl-utils + */ + + /** + * Loads a shader. + * @param {WebGLRenderingContext} gl The WebGLRenderingContext to use. + * @param {string} shaderSource The shader source. + * @param {number} shaderType The type of shader. + * @param {module:webgl-utils.ErrorCallback} opt_errorCallback callback for errors. + * @return {WebGLShader} The created shader. + */ + function loadShader(gl, shaderSource, shaderType, opt_errorCallback) { + const errFn = opt_errorCallback || error; + // Create the shader object + const shader = gl.createShader(shaderType); + + // Load the shader source + gl.shaderSource(shader, shaderSource); + + // Compile the shader + gl.compileShader(shader); + + // Check the compile status + const compiled = gl.getShaderParameter(shader, gl.COMPILE_STATUS); + if (!compiled) { + // Something went wrong during compilation; get the error + const lastError = gl.getShaderInfoLog(shader); + errFn( + "*** Error compiling shader '" + + shader + + "':" + + lastError + + `\n` + + shaderSource + .split('\n') + .map((l, i) => `${i + 1}: ${l}`) + .join('\n'), + ); + gl.deleteShader(shader); + return null; + } + + return shader; + } + + /** + * Creates a program, attaches shaders, binds attrib locations, links the + * program and calls useProgram. + * @param {WebGLShader[]} shaders The shaders to attach + * @param {string[]} [opt_attribs] An array of attribs names. Locations will be assigned by index if not passed in + * @param {number[]} [opt_locations] The locations for the. A parallel array to opt_attribs letting you assign locations. + * @param {module:webgl-utils.ErrorCallback} opt_errorCallback callback for errors. By default it just prints an error to the console + * on error. If you want something else pass an callback. It's passed an error message. + * @memberOf module:webgl-utils + */ + function createProgram( + gl, + shaders, + opt_attribs, + opt_locations, + opt_errorCallback, + ) { + const errFn = opt_errorCallback || error; + const program = gl.createProgram(); + shaders.forEach(function (shader) { + gl.attachShader(program, shader); + }); + if (opt_attribs) { + opt_attribs.forEach(function (attrib, ndx) { + gl.bindAttribLocation( + program, + opt_locations ? opt_locations[ndx] : ndx, + attrib, + ); + }); + } + gl.linkProgram(program); + + // Check the link status + const linked = gl.getProgramParameter(program, gl.LINK_STATUS); + if (!linked) { + // something went wrong with the link + const lastError = gl.getProgramInfoLog(program); + errFn('Error in program linking:' + lastError); + + gl.deleteProgram(program); + return null; + } + return program; + } + + /** + * Loads a shader from a script tag. + * @param {WebGLRenderingContext} gl The WebGLRenderingContext to use. + * @param {string} scriptId The id of the script tag. + * @param {number} opt_shaderType The type of shader. If not passed in it will + * be derived from the type of the script tag. + * @param {module:webgl-utils.ErrorCallback} opt_errorCallback callback for errors. + * @return {WebGLShader} The created shader. + */ + function createShaderFromScript( + gl, + scriptId, + opt_shaderType, + opt_errorCallback, + ) { + let shaderSource = ''; + let shaderType; + const shaderScript = document.getElementById(scriptId); + if (!shaderScript) { + throw '*** Error: unknown script element' + scriptId; + } + shaderSource = shaderScript.text; + + if (!opt_shaderType) { + if (shaderScript.type === 'x-shader/x-vertex') { + shaderType = gl.VERTEX_SHADER; + } else if (shaderScript.type === 'x-shader/x-fragment') { + shaderType = gl.FRAGMENT_SHADER; + } else if ( + shaderType !== gl.VERTEX_SHADER && + shaderType !== gl.FRAGMENT_SHADER + ) { + throw '*** Error: unknown shader type'; + } + } + + return loadShader( + gl, + shaderSource, + opt_shaderType ? opt_shaderType : shaderType, + opt_errorCallback, + ); + } + + const defaultShaderType = ['VERTEX_SHADER', 'FRAGMENT_SHADER']; + + /** + * Creates a program from 2 script tags. + * + * @param {WebGLRenderingContext} gl The WebGLRenderingContext + * to use. + * @param {string[]} shaderScriptIds Array of ids of the script + * tags for the shaders. The first is assumed to be the + * vertex shader, the second the fragment shader. + * @param {string[]} [opt_attribs] An array of attribs names. Locations will be assigned by index if not passed in + * @param {number[]} [opt_locations] The locations for the. A parallel array to opt_attribs letting you assign locations. + * @param {module:webgl-utils.ErrorCallback} opt_errorCallback callback for errors. By default it just prints an error to the console + * on error. If you want something else pass an callback. It's passed an error message. + * @return {WebGLProgram} The created program. + * @memberOf module:webgl-utils + */ + function createProgramFromScripts( + gl, + shaderScriptIds, + opt_attribs, + opt_locations, + opt_errorCallback, + ) { + const shaders = []; + for (let ii = 0; ii < shaderScriptIds.length; ++ii) { + shaders.push( + createShaderFromScript( + gl, + shaderScriptIds[ii], + gl[defaultShaderType[ii]], + opt_errorCallback, + ), + ); + } + return createProgram( + gl, + shaders, + opt_attribs, + opt_locations, + opt_errorCallback, + ); + } + + /** + * Creates a program from 2 sources. + * + * @param {WebGLRenderingContext} gl The WebGLRenderingContext + * to use. + * @param {string[]} shaderSourcess Array of sources for the + * shaders. The first is assumed to be the vertex shader, + * the second the fragment shader. + * @param {string[]} [opt_attribs] An array of attribs names. Locations will be assigned by index if not passed in + * @param {number[]} [opt_locations] The locations for the. A parallel array to opt_attribs letting you assign locations. + * @param {module:webgl-utils.ErrorCallback} opt_errorCallback callback for errors. By default it just prints an error to the console + * on error. If you want something else pass an callback. It's passed an error message. + * @return {WebGLProgram} The created program. + * @memberOf module:webgl-utils + */ + function createProgramFromSources( + gl, + shaderSources, + opt_attribs, + opt_locations, + opt_errorCallback, + ) { + const shaders = []; + for (let ii = 0; ii < shaderSources.length; ++ii) { + shaders.push( + loadShader( + gl, + shaderSources[ii], + gl[defaultShaderType[ii]], + opt_errorCallback, + ), + ); + } + return createProgram( + gl, + shaders, + opt_attribs, + opt_locations, + opt_errorCallback, + ); + } + + /** + * Returns the corresponding bind point for a given sampler type + */ + function getBindPointForSamplerType(gl, type) { + if (type === gl.SAMPLER_2D) return gl.TEXTURE_2D; // eslint-disable-line + if (type === gl.SAMPLER_CUBE) return gl.TEXTURE_CUBE_MAP; // eslint-disable-line + return undefined; + } + + /** + * @typedef {Object.} Setters + */ + + /** + * Creates setter functions for all uniforms of a shader + * program. + * + * @see {@link module:webgl-utils.setUniforms} + * + * @param {WebGLProgram} program the program to create setters for. + * @returns {Object.} an object with a setter by name for each uniform + * @memberOf module:webgl-utils + */ + function createUniformSetters(gl, program) { + let textureUnit = 0; + + /** + * Creates a setter for a uniform of the given program with it's + * location embedded in the setter. + * @param {WebGLProgram} program + * @param {WebGLUniformInfo} uniformInfo + * @returns {function} the created setter. + */ + function createUniformSetter(program, uniformInfo) { + const location = gl.getUniformLocation(program, uniformInfo.name); + const type = uniformInfo.type; + // Check if this uniform is an array + const isArray = + uniformInfo.size > 1 && uniformInfo.name.substr(-3) === '[0]'; + if (type === gl.FLOAT && isArray) { + return function (v) { + gl.uniform1fv(location, v); + }; + } + if (type === gl.FLOAT) { + return function (v) { + gl.uniform1f(location, v); + }; + } + if (type === gl.FLOAT_VEC2) { + return function (v) { + gl.uniform2fv(location, v); + }; + } + if (type === gl.FLOAT_VEC3) { + return function (v) { + gl.uniform3fv(location, v); + }; + } + if (type === gl.FLOAT_VEC4) { + return function (v) { + gl.uniform4fv(location, v); + }; + } + if (type === gl.INT && isArray) { + return function (v) { + gl.uniform1iv(location, v); + }; + } + if (type === gl.INT) { + return function (v) { + gl.uniform1i(location, v); + }; + } + if (type === gl.INT_VEC2) { + return function (v) { + gl.uniform2iv(location, v); + }; + } + if (type === gl.INT_VEC3) { + return function (v) { + gl.uniform3iv(location, v); + }; + } + if (type === gl.INT_VEC4) { + return function (v) { + gl.uniform4iv(location, v); + }; + } + if (type === gl.BOOL) { + return function (v) { + gl.uniform1iv(location, v); + }; + } + if (type === gl.BOOL_VEC2) { + return function (v) { + gl.uniform2iv(location, v); + }; + } + if (type === gl.BOOL_VEC3) { + return function (v) { + gl.uniform3iv(location, v); + }; + } + if (type === gl.BOOL_VEC4) { + return function (v) { + gl.uniform4iv(location, v); + }; + } + if (type === gl.FLOAT_MAT2) { + return function (v) { + gl.uniformMatrix2fv(location, false, v); + }; + } + if (type === gl.FLOAT_MAT3) { + return function (v) { + gl.uniformMatrix3fv(location, false, v); + }; + } + if (type === gl.FLOAT_MAT4) { + return function (v) { + gl.uniformMatrix4fv(location, false, v); + }; + } + if ((type === gl.SAMPLER_2D || type === gl.SAMPLER_CUBE) && isArray) { + const units = []; + for (let ii = 0; ii < info.size; ++ii) { + units.push(textureUnit++); + } + return (function (bindPoint, units) { + return function (textures) { + gl.uniform1iv(location, units); + textures.forEach(function (texture, index) { + gl.activeTexture(gl.TEXTURE0 + units[index]); + gl.bindTexture(bindPoint, texture); + }); + }; + })(getBindPointForSamplerType(gl, type), units); + } + if (type === gl.SAMPLER_2D || type === gl.SAMPLER_CUBE) { + return (function (bindPoint, unit) { + return function (texture) { + gl.uniform1i(location, unit); + gl.activeTexture(gl.TEXTURE0 + unit); + gl.bindTexture(bindPoint, texture); + }; + })(getBindPointForSamplerType(gl, type), textureUnit++); + } + throw 'unknown type: 0x' + type.toString(16); // we should never get here. + } + + const uniformSetters = {}; + const numUniforms = gl.getProgramParameter(program, gl.ACTIVE_UNIFORMS); + + for (let ii = 0; ii < numUniforms; ++ii) { + const uniformInfo = gl.getActiveUniform(program, ii); + if (!uniformInfo) { + break; + } + let name = uniformInfo.name; + // remove the array suffix. + if (name.substr(-3) === '[0]') { + name = name.substr(0, name.length - 3); + } + const setter = createUniformSetter(program, uniformInfo); + uniformSetters[name] = setter; + } + return uniformSetters; + } + + /** + * Set uniforms and binds related textures. + * + * Example: + * + * let programInfo = createProgramInfo( + * gl, ["some-vs", "some-fs"]); + * + * let tex1 = gl.createTexture(); + * let tex2 = gl.createTexture(); + * + * ... assume we setup the textures with data ... + * + * let uniforms = { + * u_someSampler: tex1, + * u_someOtherSampler: tex2, + * u_someColor: [1,0,0,1], + * u_somePosition: [0,1,1], + * u_someMatrix: [ + * 1,0,0,0, + * 0,1,0,0, + * 0,0,1,0, + * 0,0,0,0, + * ], + * }; + * + * gl.useProgram(program); + * + * This will automatically bind the textures AND set the + * uniforms. + * + * setUniforms(programInfo.uniformSetters, uniforms); + * + * For the example above it is equivalent to + * + * let texUnit = 0; + * gl.activeTexture(gl.TEXTURE0 + texUnit); + * gl.bindTexture(gl.TEXTURE_2D, tex1); + * gl.uniform1i(u_someSamplerLocation, texUnit++); + * gl.activeTexture(gl.TEXTURE0 + texUnit); + * gl.bindTexture(gl.TEXTURE_2D, tex2); + * gl.uniform1i(u_someSamplerLocation, texUnit++); + * gl.uniform4fv(u_someColorLocation, [1, 0, 0, 1]); + * gl.uniform3fv(u_somePositionLocation, [0, 1, 1]); + * gl.uniformMatrix4fv(u_someMatrix, false, [ + * 1,0,0,0, + * 0,1,0,0, + * 0,0,1,0, + * 0,0,0,0, + * ]); + * + * Note it is perfectly reasonable to call `setUniforms` multiple times. For example + * + * let uniforms = { + * u_someSampler: tex1, + * u_someOtherSampler: tex2, + * }; + * + * let moreUniforms { + * u_someColor: [1,0,0,1], + * u_somePosition: [0,1,1], + * u_someMatrix: [ + * 1,0,0,0, + * 0,1,0,0, + * 0,0,1,0, + * 0,0,0,0, + * ], + * }; + * + * setUniforms(programInfo.uniformSetters, uniforms); + * setUniforms(programInfo.uniformSetters, moreUniforms); + * + * @param {Object.|module:webgl-utils.ProgramInfo} setters the setters returned from + * `createUniformSetters` or a ProgramInfo from {@link module:webgl-utils.createProgramInfo}. + * @param {Object.} an object with values for the + * uniforms. + * @memberOf module:webgl-utils + */ + function setUniforms(setters, ...values) { + setters = setters.uniformSetters || setters; + for (const uniforms of values) { + Object.keys(uniforms).forEach(function (name) { + const setter = setters[name]; + if (setter) { + setter(uniforms[name]); + } + }); + } + } + + /** + * Creates setter functions for all attributes of a shader + * program. You can pass this to {@link module:webgl-utils.setBuffersAndAttributes} to set all your buffers and attributes. + * + * @see {@link module:webgl-utils.setAttributes} for example + * @param {WebGLProgram} program the program to create setters for. + * @return {Object.} an object with a setter for each attribute by name. + * @memberOf module:webgl-utils + */ + function createAttributeSetters(gl, program) { + const attribSetters = {}; + + function createAttribSetter(index) { + return function (b) { + if (b.value) { + gl.disableVertexAttribArray(index); + switch (b.value.length) { + case 4: + gl.vertexAttrib4fv(index, b.value); + break; + case 3: + gl.vertexAttrib3fv(index, b.value); + break; + case 2: + gl.vertexAttrib2fv(index, b.value); + break; + case 1: + gl.vertexAttrib1fv(index, b.value); + break; + default: + throw new Error( + 'the length of a float constant value must be between 1 and 4!', + ); + } + } else { + gl.bindBuffer(gl.ARRAY_BUFFER, b.buffer); + gl.enableVertexAttribArray(index); + gl.vertexAttribPointer( + index, + b.numComponents || b.size, + b.type || gl.FLOAT, + b.normalize || false, + b.stride || 0, + b.offset || 0, + ); + } + }; + } + + const numAttribs = gl.getProgramParameter(program, gl.ACTIVE_ATTRIBUTES); + for (let ii = 0; ii < numAttribs; ++ii) { + const attribInfo = gl.getActiveAttrib(program, ii); + if (!attribInfo) { + break; + } + const index = gl.getAttribLocation(program, attribInfo.name); + attribSetters[attribInfo.name] = createAttribSetter(index); + } + + return attribSetters; + } + + /** + * Sets attributes and binds buffers (deprecated... use {@link module:webgl-utils.setBuffersAndAttributes}) + * + * Example: + * + * let program = createProgramFromScripts( + * gl, ["some-vs", "some-fs"]); + * + * let attribSetters = createAttributeSetters(program); + * + * let positionBuffer = gl.createBuffer(); + * let texcoordBuffer = gl.createBuffer(); + * + * let attribs = { + * a_position: {buffer: positionBuffer, numComponents: 3}, + * a_texcoord: {buffer: texcoordBuffer, numComponents: 2}, + * }; + * + * gl.useProgram(program); + * + * This will automatically bind the buffers AND set the + * attributes. + * + * setAttributes(attribSetters, attribs); + * + * Properties of attribs. For each attrib you can add + * properties: + * + * * type: the type of data in the buffer. Default = gl.FLOAT + * * normalize: whether or not to normalize the data. Default = false + * * stride: the stride. Default = 0 + * * offset: offset into the buffer. Default = 0 + * + * For example if you had 3 value float positions, 2 value + * float texcoord and 4 value uint8 colors you'd setup your + * attribs like this + * + * let attribs = { + * a_position: {buffer: positionBuffer, numComponents: 3}, + * a_texcoord: {buffer: texcoordBuffer, numComponents: 2}, + * a_color: { + * buffer: colorBuffer, + * numComponents: 4, + * type: gl.UNSIGNED_BYTE, + * normalize: true, + * }, + * }; + * + * @param {Object.|model:webgl-utils.ProgramInfo} setters Attribute setters as returned from createAttributeSetters or a ProgramInfo as returned {@link module:webgl-utils.createProgramInfo} + * @param {Object.} attribs AttribInfos mapped by attribute name. + * @memberOf module:webgl-utils + * @deprecated use {@link module:webgl-utils.setBuffersAndAttributes} + */ + function setAttributes(setters, attribs) { + setters = setters.attribSetters || setters; + Object.keys(attribs).forEach(function (name) { + const setter = setters[name]; + if (setter) { + setter(attribs[name]); + } + }); + } + + /** + * Creates a vertex array object and then sets the attributes + * on it + * + * @param {WebGLRenderingContext} gl The WebGLRenderingContext + * to use. + * @param {Object.} setters Attribute setters as returned from createAttributeSetters + * @param {Object.} attribs AttribInfos mapped by attribute name. + * @param {WebGLBuffer} [indices] an optional ELEMENT_ARRAY_BUFFER of indices + */ + function createVAOAndSetAttributes(gl, setters, attribs, indices) { + const vao = gl.createVertexArray(); + gl.bindVertexArray(vao); + setAttributes(setters, attribs); + if (indices) { + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, indices); + } + // We unbind this because otherwise any change to ELEMENT_ARRAY_BUFFER + // like when creating buffers for other stuff will mess up this VAO's binding + gl.bindVertexArray(null); + return vao; + } + + /** + * Creates a vertex array object and then sets the attributes + * on it + * + * @param {WebGLRenderingContext} gl The WebGLRenderingContext + * to use. + * @param {Object.| module:webgl-utils.ProgramInfo} programInfo as returned from createProgramInfo or Attribute setters as returned from createAttributeSetters + * @param {module:webgl-utils:BufferInfo} bufferInfo BufferInfo as returned from createBufferInfoFromArrays etc... + * @param {WebGLBuffer} [indices] an optional ELEMENT_ARRAY_BUFFER of indices + */ + function createVAOFromBufferInfo(gl, programInfo, bufferInfo) { + return createVAOAndSetAttributes( + gl, + programInfo.attribSetters || programInfo, + bufferInfo.attribs, + bufferInfo.indices, + ); + } + + /** + * @typedef {Object} ProgramInfo + * @property {WebGLProgram} program A shader program + * @property {Object} uniformSetters: object of setters as returned from createUniformSetters, + * @property {Object} attribSetters: object of setters as returned from createAttribSetters, + * @memberOf module:webgl-utils + */ + + /** + * Creates a ProgramInfo from 2 sources. + * + * A ProgramInfo contains + * + * programInfo = { + * program: WebGLProgram, + * uniformSetters: object of setters as returned from createUniformSetters, + * attribSetters: object of setters as returned from createAttribSetters, + * } + * + * @param {WebGLRenderingContext} gl The WebGLRenderingContext + * to use. + * @param {string[]} shaderSourcess Array of sources for the + * shaders or ids. The first is assumed to be the vertex shader, + * the second the fragment shader. + * @param {string[]} [opt_attribs] An array of attribs names. Locations will be assigned by index if not passed in + * @param {number[]} [opt_locations] The locations for the. A parallel array to opt_attribs letting you assign locations. + * @param {module:webgl-utils.ErrorCallback} opt_errorCallback callback for errors. By default it just prints an error to the console + * on error. If you want something else pass an callback. It's passed an error message. + * @return {module:webgl-utils.ProgramInfo} The created program. + * @memberOf module:webgl-utils + */ + function createProgramInfo( + gl, + shaderSources, + opt_attribs, + opt_locations, + opt_errorCallback, + ) { + shaderSources = shaderSources.map(function (source) { + const script = document.getElementById(source); + return script ? script.text : source; + }); + const program = webglUtils.createProgramFromSources( + gl, + shaderSources, + opt_attribs, + opt_locations, + opt_errorCallback, + ); + if (!program) { + return null; + } + const uniformSetters = createUniformSetters(gl, program); + const attribSetters = createAttributeSetters(gl, program); + return { + program: program, + uniformSetters: uniformSetters, + attribSetters: attribSetters, + }; + } + + /** + * Sets attributes and buffers including the `ELEMENT_ARRAY_BUFFER` if appropriate + * + * Example: + * + * let programInfo = createProgramInfo( + * gl, ["some-vs", "some-fs"]); + * + * let arrays = { + * position: { numComponents: 3, data: [0, 0, 0, 10, 0, 0, 0, 10, 0, 10, 10, 0], }, + * texcoord: { numComponents: 2, data: [0, 0, 0, 1, 1, 0, 1, 1], }, + * }; + * + * let bufferInfo = createBufferInfoFromArrays(gl, arrays); + * + * gl.useProgram(programInfo.program); + * + * This will automatically bind the buffers AND set the + * attributes. + * + * setBuffersAndAttributes(programInfo.attribSetters, bufferInfo); + * + * For the example above it is equivilent to + * + * gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); + * gl.enableVertexAttribArray(a_positionLocation); + * gl.vertexAttribPointer(a_positionLocation, 3, gl.FLOAT, false, 0, 0); + * gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer); + * gl.enableVertexAttribArray(a_texcoordLocation); + * gl.vertexAttribPointer(a_texcoordLocation, 4, gl.FLOAT, false, 0, 0); + * + * @param {WebGLRenderingContext} gl A WebGLRenderingContext. + * @param {Object.} setters Attribute setters as returned from `createAttributeSetters` + * @param {module:webgl-utils.BufferInfo} buffers a BufferInfo as returned from `createBufferInfoFromArrays`. + * @memberOf module:webgl-utils + */ + function setBuffersAndAttributes(gl, setters, buffers) { + setAttributes(setters, buffers.attribs); + if (buffers.indices) { + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, buffers.indices); + } + } + + // Add your prefix here. + const browserPrefixes = ['', 'MOZ_', 'OP_', 'WEBKIT_']; + + /** + * Given an extension name like WEBGL_compressed_texture_s3tc + * returns the supported version extension, like + * WEBKIT_WEBGL_compressed_teture_s3tc + * @param {string} name Name of extension to look for + * @return {WebGLExtension} The extension or undefined if not + * found. + * @memberOf module:webgl-utils + */ + function getExtensionWithKnownPrefixes(gl, name) { + for (let ii = 0; ii < browserPrefixes.length; ++ii) { + const prefixedName = browserPrefixes[ii] + name; + const ext = gl.getExtension(prefixedName); + if (ext) { + return ext; + } + } + return undefined; + } + + /** + * Resize a canvas to match the size its displayed. + * @param {HTMLCanvasElement} canvas The canvas to resize. + * @param {number} [multiplier] amount to multiply by. + * Pass in window.devicePixelRatio for native pixels. + * @return {boolean} true if the canvas was resized. + * @memberOf module:webgl-utils + */ + function resizeCanvasToDisplaySize(canvas, multiplier) { + multiplier = multiplier || 1; + const width = (canvas.clientWidth * multiplier) | 0; + const height = (canvas.clientHeight * multiplier) | 0; + if (canvas.width !== width || canvas.height !== height) { + canvas.width = width; + canvas.height = height; + return true; + } + return false; + } + + // Add `push` to a typed array. It just keeps a 'cursor' + // and allows use to `push` values into the array so we + // don't have to manually compute offsets + function augmentTypedArray(typedArray, numComponents) { + let cursor = 0; + typedArray.push = function () { + for (let ii = 0; ii < arguments.length; ++ii) { + const value = arguments[ii]; + if ( + value instanceof Array || + (value.buffer && value.buffer instanceof ArrayBuffer) + ) { + for (let jj = 0; jj < value.length; ++jj) { + typedArray[cursor++] = value[jj]; + } + } else { + typedArray[cursor++] = value; + } + } + }; + typedArray.reset = function (opt_index) { + cursor = opt_index || 0; + }; + typedArray.numComponents = numComponents; + Object.defineProperty(typedArray, 'numElements', { + get: function () { + return (this.length / this.numComponents) | 0; + }, + }); + return typedArray; + } + + /** + * creates a typed array with a `push` function attached + * so that you can easily *push* values. + * + * `push` can take multiple arguments. If an argument is an array each element + * of the array will be added to the typed array. + * + * Example: + * + * let array = createAugmentedTypedArray(3, 2); // creates a Float32Array with 6 values + * array.push(1, 2, 3); + * array.push([4, 5, 6]); + * // array now contains [1, 2, 3, 4, 5, 6] + * + * Also has `numComponents` and `numElements` properties. + * + * @param {number} numComponents number of components + * @param {number} numElements number of elements. The total size of the array will be `numComponents * numElements`. + * @param {constructor} opt_type A constructor for the type. Default = `Float32Array`. + * @return {ArrayBuffer} A typed array. + * @memberOf module:webgl-utils + */ + function createAugmentedTypedArray(numComponents, numElements, opt_type) { + const Type = opt_type || Float32Array; + return augmentTypedArray( + new Type(numComponents * numElements), + numComponents, + ); + } + + function createBufferFromTypedArray(gl, array, type, drawType) { + type = type || gl.ARRAY_BUFFER; + const buffer = gl.createBuffer(); + gl.bindBuffer(type, buffer); + gl.bufferData(type, array, drawType || gl.STATIC_DRAW); + return buffer; + } + + function allButIndices(name) { + return name !== 'indices'; + } + + function createMapping(obj) { + const mapping = {}; + Object.keys(obj) + .filter(allButIndices) + .forEach(function (key) { + mapping['a_' + key] = key; + }); + return mapping; + } + + function getGLTypeForTypedArray(gl, typedArray) { + if (typedArray instanceof Int8Array) { + return gl.BYTE; + } // eslint-disable-line + if (typedArray instanceof Uint8Array) { + return gl.UNSIGNED_BYTE; + } // eslint-disable-line + if (typedArray instanceof Int16Array) { + return gl.SHORT; + } // eslint-disable-line + if (typedArray instanceof Uint16Array) { + return gl.UNSIGNED_SHORT; + } // eslint-disable-line + if (typedArray instanceof Int32Array) { + return gl.INT; + } // eslint-disable-line + if (typedArray instanceof Uint32Array) { + return gl.UNSIGNED_INT; + } // eslint-disable-line + if (typedArray instanceof Float32Array) { + return gl.FLOAT; + } // eslint-disable-line + throw 'unsupported typed array type'; + } + + // This is really just a guess. Though I can't really imagine using + // anything else? Maybe for some compression? + function getNormalizationForTypedArray(typedArray) { + if (typedArray instanceof Int8Array) { + return true; + } // eslint-disable-line + if (typedArray instanceof Uint8Array) { + return true; + } // eslint-disable-line + return false; + } + + function isArrayBuffer(a) { + return a.buffer && a.buffer instanceof ArrayBuffer; + } + + function guessNumComponentsFromName(name, length) { + let numComponents; + if (name.indexOf('coord') >= 0) { + numComponents = 2; + } else if (name.indexOf('color') >= 0) { + numComponents = 4; + } else { + numComponents = 3; // position, normals, indices ... + } + + if (length % numComponents > 0) { + throw 'can not guess numComponents. You should specify it.'; + } + + return numComponents; + } + + function makeTypedArray(array, name) { + if (isArrayBuffer(array)) { + return array; + } + + if (array.data && isArrayBuffer(array.data)) { + return array.data; + } + + if (Array.isArray(array)) { + array = { + data: array, + }; + } + + if (!array.numComponents) { + array.numComponents = guessNumComponentsFromName(name, array.length); + } + + let type = array.type; + if (!type) { + if (name === 'indices') { + type = Uint16Array; + } + } + const typedArray = createAugmentedTypedArray( + array.numComponents, + (array.data.length / array.numComponents) | 0, + type, + ); + typedArray.push(array.data); + return typedArray; + } + + /** + * @typedef {Object} AttribInfo + * @property {number} [numComponents] the number of components for this attribute. + * @property {number} [size] the number of components for this attribute. + * @property {number} [type] the type of the attribute (eg. `gl.FLOAT`, `gl.UNSIGNED_BYTE`, etc...) Default = `gl.FLOAT` + * @property {boolean} [normalized] whether or not to normalize the data. Default = false + * @property {number} [offset] offset into buffer in bytes. Default = 0 + * @property {number} [stride] the stride in bytes per element. Default = 0 + * @property {WebGLBuffer} buffer the buffer that contains the data for this attribute + * @memberOf module:webgl-utils + */ + + /** + * Creates a set of attribute data and WebGLBuffers from set of arrays + * + * Given + * + * let arrays = { + * position: { numComponents: 3, data: [0, 0, 0, 10, 0, 0, 0, 10, 0, 10, 10, 0], }, + * texcoord: { numComponents: 2, data: [0, 0, 0, 1, 1, 0, 1, 1], }, + * normal: { numComponents: 3, data: [0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1], }, + * color: { numComponents: 4, data: [255, 255, 255, 255, 255, 0, 0, 255, 0, 0, 255, 255], type: Uint8Array, }, + * indices: { numComponents: 3, data: [0, 1, 2, 1, 2, 3], }, + * }; + * + * returns something like + * + * let attribs = { + * a_position: { numComponents: 3, type: gl.FLOAT, normalize: false, buffer: WebGLBuffer, }, + * a_texcoord: { numComponents: 2, type: gl.FLOAT, normalize: false, buffer: WebGLBuffer, }, + * a_normal: { numComponents: 3, type: gl.FLOAT, normalize: false, buffer: WebGLBuffer, }, + * a_color: { numComponents: 4, type: gl.UNSIGNED_BYTE, normalize: true, buffer: WebGLBuffer, }, + * }; + * + * @param {WebGLRenderingContext} gl The webgl rendering context. + * @param {Object.} arrays The arrays + * @param {Object.} [opt_mapping] mapping from attribute name to array name. + * if not specified defaults to "a_name" -> "name". + * @return {Object.} the attribs + * @memberOf module:webgl-utils + */ + function createAttribsFromArrays(gl, arrays, opt_mapping) { + const mapping = opt_mapping || createMapping(arrays); + const attribs = {}; + Object.keys(mapping).forEach(function (attribName) { + const bufferName = mapping[attribName]; + const origArray = arrays[bufferName]; + if (origArray.value) { + attribs[attribName] = { + value: origArray.value, + }; + } else { + const array = makeTypedArray(origArray, bufferName); + attribs[attribName] = { + buffer: createBufferFromTypedArray(gl, array), + numComponents: + origArray.numComponents || + array.numComponents || + guessNumComponentsFromName(bufferName), + type: getGLTypeForTypedArray(gl, array), + normalize: getNormalizationForTypedArray(array), + }; + } + }); + return attribs; + } + + function getArray(array) { + return array.length ? array : array.data; + } + + const texcoordRE = /coord|texture/i; + const colorRE = /color|colour/i; + + function guessNumComponentsFromName(name, length) { + let numComponents; + if (texcoordRE.test(name)) { + numComponents = 2; + } else if (colorRE.test(name)) { + numComponents = 4; + } else { + numComponents = 3; // position, normals, indices ... + } + + if (length % numComponents > 0) { + throw new Error( + `Can not guess numComponents for attribute '${name}'. Tried ${numComponents} but ${length} values is not evenly divisible by ${numComponents}. You should specify it.`, + ); + } + + return numComponents; + } + + function getNumComponents(array, arrayName) { + return ( + array.numComponents || + array.size || + guessNumComponentsFromName(arrayName, getArray(array).length) + ); + } + + /** + * tries to get the number of elements from a set of arrays. + */ + const positionKeys = ['position', 'positions', 'a_position']; + function getNumElementsFromNonIndexedArrays(arrays) { + let key; + for (const k of positionKeys) { + if (k in arrays) { + key = k; + break; + } + } + key = key || Object.keys(arrays)[0]; + const array = arrays[key]; + const length = getArray(array).length; + const numComponents = getNumComponents(array, key); + const numElements = length / numComponents; + if (length % numComponents > 0) { + throw new Error( + `numComponents ${numComponents} not correct for length ${length}`, + ); + } + return numElements; + } + + /** + * @typedef {Object} BufferInfo + * @property {number} numElements The number of elements to pass to `gl.drawArrays` or `gl.drawElements`. + * @property {WebGLBuffer} [indices] The indices `ELEMENT_ARRAY_BUFFER` if any indices exist. + * @property {Object.} attribs The attribs approriate to call `setAttributes` + * @memberOf module:webgl-utils + */ + + /** + * Creates a BufferInfo from an object of arrays. + * + * This can be passed to {@link module:webgl-utils.setBuffersAndAttributes} and to + * {@link module:webgl-utils:drawBufferInfo}. + * + * Given an object like + * + * let arrays = { + * position: { numComponents: 3, data: [0, 0, 0, 10, 0, 0, 0, 10, 0, 10, 10, 0], }, + * texcoord: { numComponents: 2, data: [0, 0, 0, 1, 1, 0, 1, 1], }, + * normal: { numComponents: 3, data: [0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1], }, + * indices: { numComponents: 3, data: [0, 1, 2, 1, 2, 3], }, + * }; + * + * Creates an BufferInfo like this + * + * bufferInfo = { + * numElements: 4, // or whatever the number of elements is + * indices: WebGLBuffer, // this property will not exist if there are no indices + * attribs: { + * a_position: { buffer: WebGLBuffer, numComponents: 3, }, + * a_normal: { buffer: WebGLBuffer, numComponents: 3, }, + * a_texcoord: { buffer: WebGLBuffer, numComponents: 2, }, + * }, + * }; + * + * The properties of arrays can be JavaScript arrays in which case the number of components + * will be guessed. + * + * let arrays = { + * position: [0, 0, 0, 10, 0, 0, 0, 10, 0, 10, 10, 0], + * texcoord: [0, 0, 0, 1, 1, 0, 1, 1], + * normal: [0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1], + * indices: [0, 1, 2, 1, 2, 3], + * }; + * + * They can also by TypedArrays + * + * let arrays = { + * position: new Float32Array([0, 0, 0, 10, 0, 0, 0, 10, 0, 10, 10, 0]), + * texcoord: new Float32Array([0, 0, 0, 1, 1, 0, 1, 1]), + * normal: new Float32Array([0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1]), + * indices: new Uint16Array([0, 1, 2, 1, 2, 3]), + * }; + * + * Or augmentedTypedArrays + * + * let positions = createAugmentedTypedArray(3, 4); + * let texcoords = createAugmentedTypedArray(2, 4); + * let normals = createAugmentedTypedArray(3, 4); + * let indices = createAugmentedTypedArray(3, 2, Uint16Array); + * + * positions.push([0, 0, 0, 10, 0, 0, 0, 10, 0, 10, 10, 0]); + * texcoords.push([0, 0, 0, 1, 1, 0, 1, 1]); + * normals.push([0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1]); + * indices.push([0, 1, 2, 1, 2, 3]); + * + * let arrays = { + * position: positions, + * texcoord: texcoords, + * normal: normals, + * indices: indices, + * }; + * + * For the last example it is equivalent to + * + * let bufferInfo = { + * attribs: { + * a_position: { numComponents: 3, buffer: gl.createBuffer(), }, + * a_texcoods: { numComponents: 2, buffer: gl.createBuffer(), }, + * a_normals: { numComponents: 3, buffer: gl.createBuffer(), }, + * }, + * indices: gl.createBuffer(), + * numElements: 6, + * }; + * + * gl.bindBuffer(gl.ARRAY_BUFFER, bufferInfo.attribs.a_position.buffer); + * gl.bufferData(gl.ARRAY_BUFFER, arrays.position, gl.STATIC_DRAW); + * gl.bindBuffer(gl.ARRAY_BUFFER, bufferInfo.attribs.a_texcoord.buffer); + * gl.bufferData(gl.ARRAY_BUFFER, arrays.texcoord, gl.STATIC_DRAW); + * gl.bindBuffer(gl.ARRAY_BUFFER, bufferInfo.attribs.a_normal.buffer); + * gl.bufferData(gl.ARRAY_BUFFER, arrays.normal, gl.STATIC_DRAW); + * gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, bufferInfo.indices); + * gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, arrays.indices, gl.STATIC_DRAW); + * + * @param {WebGLRenderingContext} gl A WebGLRenderingContext + * @param {Object.} arrays Your data + * @param {Object.} [opt_mapping] an optional mapping of attribute to array name. + * If not passed in it's assumed the array names will be mapped to an attribute + * of the same name with "a_" prefixed to it. An other words. + * + * let arrays = { + * position: ..., + * texcoord: ..., + * normal: ..., + * indices: ..., + * }; + * + * bufferInfo = createBufferInfoFromArrays(gl, arrays); + * + * Is the same as + * + * let arrays = { + * position: ..., + * texcoord: ..., + * normal: ..., + * indices: ..., + * }; + * + * let mapping = { + * a_position: "position", + * a_texcoord: "texcoord", + * a_normal: "normal", + * }; + * + * bufferInfo = createBufferInfoFromArrays(gl, arrays, mapping); + * + * @return {module:webgl-utils.BufferInfo} A BufferInfo + * @memberOf module:webgl-utils + */ + function createBufferInfoFromArrays(gl, arrays, opt_mapping) { + const bufferInfo = { + attribs: createAttribsFromArrays(gl, arrays, opt_mapping), + }; + let indices = arrays.indices; + if (indices) { + indices = makeTypedArray(indices, 'indices'); + bufferInfo.indices = createBufferFromTypedArray( + gl, + indices, + gl.ELEMENT_ARRAY_BUFFER, + ); + bufferInfo.numElements = indices.length; + } else { + bufferInfo.numElements = getNumElementsFromNonIndexedArrays(arrays); + } + + return bufferInfo; + } + + /** + * Creates buffers from typed arrays + * + * Given something like this + * + * let arrays = { + * positions: [1, 2, 3], + * normals: [0, 0, 1], + * } + * + * returns something like + * + * buffers = { + * positions: WebGLBuffer, + * normals: WebGLBuffer, + * } + * + * If the buffer is named 'indices' it will be made an ELEMENT_ARRAY_BUFFER. + * + * @param {WebGLRenderingContext} gl A WebGLRenderingContext. + * @param {Object} arrays + * @return {Object} returns an object with one WebGLBuffer per array + * @memberOf module:webgl-utils + */ + function createBuffersFromArrays(gl, arrays) { + const buffers = {}; + Object.keys(arrays).forEach(function (key) { + const type = + key === 'indices' ? gl.ELEMENT_ARRAY_BUFFER : gl.ARRAY_BUFFER; + const array = makeTypedArray(arrays[key], name); + buffers[key] = createBufferFromTypedArray(gl, array, type); + }); + + // hrm + if (arrays.indices) { + buffers.numElements = arrays.indices.length; + } else if (arrays.position) { + buffers.numElements = arrays.position.length / 3; + } + + return buffers; + } + + /** + * Calls `gl.drawElements` or `gl.drawArrays`, whichever is appropriate + * + * normally you'd call `gl.drawElements` or `gl.drawArrays` yourself + * but calling this means if you switch from indexed data to non-indexed + * data you don't have to remember to update your draw call. + * + * @param {WebGLRenderingContext} gl A WebGLRenderingContext + * @param {module:webgl-utils.BufferInfo} bufferInfo as returned from createBufferInfoFromArrays + * @param {enum} [primitiveType] eg (gl.TRIANGLES, gl.LINES, gl.POINTS, gl.TRIANGLE_STRIP, ...) + * @param {number} [count] An optional count. Defaults to bufferInfo.numElements + * @param {number} [offset] An optional offset. Defaults to 0. + * @memberOf module:webgl-utils + */ + function drawBufferInfo(gl, bufferInfo, primitiveType, count, offset) { + const indices = bufferInfo.indices; + primitiveType = primitiveType === undefined ? gl.TRIANGLES : primitiveType; + const numElements = count === undefined ? bufferInfo.numElements : count; + offset = offset === undefined ? 0 : offset; + if (indices) { + gl.drawElements(primitiveType, numElements, gl.UNSIGNED_SHORT, offset); + } else { + gl.drawArrays(primitiveType, offset, numElements); + } + } + + /** + * @typedef {Object} DrawObject + * @property {module:webgl-utils.ProgramInfo} programInfo A ProgramInfo as returned from createProgramInfo + * @property {module:webgl-utils.BufferInfo} bufferInfo A BufferInfo as returned from createBufferInfoFromArrays + * @property {Object} uniforms The values for the uniforms + * @memberOf module:webgl-utils + */ + + /** + * Draws a list of objects + * @param {WebGLRenderingContext} gl A WebGLRenderingContext + * @param {DrawObject[]} objectsToDraw an array of objects to draw. + * @memberOf module:webgl-utils + */ + function drawObjectList(gl, objectsToDraw) { + let lastUsedProgramInfo = null; + let lastUsedBufferInfo = null; + + objectsToDraw.forEach(function (object) { + const programInfo = object.programInfo; + const bufferInfo = object.bufferInfo; + let bindBuffers = false; + + if (programInfo !== lastUsedProgramInfo) { + lastUsedProgramInfo = programInfo; + gl.useProgram(programInfo.program); + bindBuffers = true; + } + + // Setup all the needed attributes. + if (bindBuffers || bufferInfo !== lastUsedBufferInfo) { + lastUsedBufferInfo = bufferInfo; + setBuffersAndAttributes(gl, programInfo.attribSetters, bufferInfo); + } + + // Set the uniforms. + setUniforms(programInfo.uniformSetters, object.uniforms); + + // Draw + drawBufferInfo(gl, bufferInfo); + }); + } + + function glEnumToString(gl, v) { + const results = []; + for (const key in gl) { + if (gl[key] === v) { + results.push(key); + } + } + return results.length ? results.join(' | ') : `0x${v.toString(16)}`; + } + + const isIE = /*@cc_on!@*/ false || !!document.documentMode; + // Edge 20+ + const isEdge = !isIE && !!window.StyleMedia; + if (isEdge) { + // Hack for Edge. Edge's WebGL implmentation is crap still and so they + // only respond to "experimental-webgl". I don't want to clutter the + // examples with that so his hack works around it + HTMLCanvasElement.prototype.getContext = (function (origFn) { + return function () { + let args = arguments; + const type = args[0]; + if (type === 'webgl') { + args = [].slice.call(arguments); + args[0] = 'experimental-webgl'; + } + return origFn.apply(this, args); + }; + })(HTMLCanvasElement.prototype.getContext); + } + + return { + createAugmentedTypedArray: createAugmentedTypedArray, + createAttribsFromArrays: createAttribsFromArrays, + createBuffersFromArrays: createBuffersFromArrays, + createBufferInfoFromArrays: createBufferInfoFromArrays, + createAttributeSetters: createAttributeSetters, + createProgram: createProgram, + createProgramFromScripts: createProgramFromScripts, + createProgramFromSources: createProgramFromSources, + createProgramInfo: createProgramInfo, + createUniformSetters: createUniformSetters, + createVAOAndSetAttributes: createVAOAndSetAttributes, + createVAOFromBufferInfo: createVAOFromBufferInfo, + drawBufferInfo: drawBufferInfo, + drawObjectList: drawObjectList, + glEnumToString: glEnumToString, + getExtensionWithKnownPrefixes: getExtensionWithKnownPrefixes, + resizeCanvasToDisplaySize: resizeCanvasToDisplaySize, + setAttributes: setAttributes, + setBuffersAndAttributes: setBuffersAndAttributes, + setUniforms: setUniforms, + }; +}); diff --git a/packages/rrweb/test/html/canvas-webgl-image.html b/packages/rrweb/test/html/canvas-webgl-image.html new file mode 100644 index 00000000..96bc3103 --- /dev/null +++ b/packages/rrweb/test/html/canvas-webgl-image.html @@ -0,0 +1,149 @@ + + + + + + + Document + + + + + + + + + + + + diff --git a/packages/rrweb/test/html/canvas-webgl-square.html b/packages/rrweb/test/html/canvas-webgl-square.html new file mode 100644 index 00000000..cbdc7ec6 --- /dev/null +++ b/packages/rrweb/test/html/canvas-webgl-square.html @@ -0,0 +1,110 @@ + + + + + + canvas webgl square + + + + + + + + + diff --git a/packages/rrweb/test/html/canvas-webgl.html b/packages/rrweb/test/html/canvas-webgl.html new file mode 100644 index 00000000..52aacabc --- /dev/null +++ b/packages/rrweb/test/html/canvas-webgl.html @@ -0,0 +1,27 @@ + + + + + + canvas + + + + + + + diff --git a/packages/rrweb/test/html/mutation-observer.html b/packages/rrweb/test/html/mutation-observer.html index 0175a319..9d149c5a 100644 --- a/packages/rrweb/test/html/mutation-observer.html +++ b/packages/rrweb/test/html/mutation-observer.html @@ -4,4 +4,5 @@
- \ No newline at end of file + + diff --git a/packages/rrweb/test/integration.test.ts b/packages/rrweb/test/integration.test.ts index acddfdf0..b0f3b826 100644 --- a/packages/rrweb/test/integration.test.ts +++ b/packages/rrweb/test/integration.test.ts @@ -1,14 +1,21 @@ import * as fs from 'fs'; import * as path from 'path'; import * as http from 'http'; -import * as url from 'url'; import * as puppeteer from 'puppeteer'; -import { assertSnapshot, launchPuppeteer } from './utils'; +import { + assertSnapshot, + startServer, + getServerURL, + launchPuppeteer, + waitForRAF, + replaceLast, +} from './utils'; import { recordOptions, eventWithTime, EventType } from '../src/types'; import { visitSnapshot, NodeType } from 'rrweb-snapshot'; interface ISuite { server: http.Server; + serverURL: string; code: string; browser: puppeteer.Browser; } @@ -17,39 +24,6 @@ interface IMimeType { [key: string]: string; } -const startServer = () => - new Promise((resolve) => { - const mimeType: IMimeType = { - '.html': 'text/html', - '.js': 'text/javascript', - '.css': 'text/css', - }; - const s = http.createServer((req, res) => { - const parsedUrl = url.parse(req.url!); - const sanitizePath = path - .normalize(parsedUrl.pathname!) - .replace(/^(\.\.[\/\\])+/, ''); - let pathname = path.join(__dirname, sanitizePath); - try { - const data = fs.readFileSync(pathname); - const ext = path.parse(pathname).ext; - res.setHeader('Content-type', mimeType[ext] || 'text/plain'); - res.setHeader('Access-Control-Allow-Origin', '*'); - res.setHeader('Access-Control-Allow-Methods', 'GET'); - res.setHeader('Access-Control-Allow-Headers', 'Content-type'); - setTimeout(() => { - res.end(data); - // mock delay - }, 100); - } catch (error) { - res.end(); - } - }); - s.listen(3030).on('listening', () => { - resolve(s); - }); - }); - describe('record integration tests', function (this: ISuite) { jest.setTimeout(10_000); @@ -59,7 +33,8 @@ describe('record integration tests', function (this: ISuite) { ): string => { const filePath = path.resolve(__dirname, `./html/${fileName}`); const html = fs.readFileSync(filePath, 'utf8'); - return html.replace( + return replaceLast( + html, '', `