From 7247bc21e39f4e1a8edcf81fee25fecdd0b972a4 Mon Sep 17 00:00:00 2001 From: Justin Halsall Date: Wed, 1 Apr 2026 12:00:00 +0800 Subject: [PATCH] [Plugin] Live stream canvas via webrtc (#976) * inline stylesheets when loaded * set empty link elements to loaded by default * Clean up stylesheet manager * Remove attribute mutation code * Update packages/rrweb/test/record.test.ts * Update packages/rrweb/test/record.test.ts * Update packages/rrweb/test/record.test.ts * Update packages/rrweb/scripts/repl.js * Update packages/rrweb/test/record.test.ts * Update packages/rrweb/src/record/index.ts * Add todo * Move require out of time sensitive assert * Add waitForRAF, its more reliable than waitForTimeout * Remove flaky tests * Add recording stylesheets in iframes * Remove variability from flaky test * Make test more robust * Fix naming * Add test cases for inlineImages * Add test cases for inlineImages * Record iframe mutations cross page * Test: should record images inside iframe with blob url after iframe was reloaded * Handle negative ids in rrdom correctly When iframes get inserted they create untracked elements, both on the dom and rrdom side. Because they are untracked they generate negative numbers when fetching the id from mirror. This creates a problem when comparing and fetching ids across mirrors. This commit tries to get away from using negative ids as much as possible in rrdom's comparisons * Update packages/rrdom/src/diff.ts Co-authored-by: Yun Feng * Start unserialized nodes at -2 This way we don't accidentally think of them as mirror misses * Set unserialized id starting number at -2 * Remove duplication * Use turbo instead of lerna * Skip benchmark as it is unreliable when executed in parallel * Strip port number from serialization, it can vary * Add settimeout to virtual dom test * Remove console.log and refactor blob:url serialization * Include references in tsconfig to indicate which monorepo packages are being used * Add stream setup * Migrate project to es module * Add reference to rrweb from rrdom * Move jest config to ESM * Setup basic WebRTC canvas streaming * Cleanup and refactor WebRTC streaming * Remove ? which isn't propper javascript * Yarn lock * Remove webrtc code from rrweb * Add plugin hooks Record/Replay plugins `.getMirror` exposes the mirror to plugins Replay plugins `.onBuild` called whenever a node was added to the dom * Expose plugins with server * Use unminified version for tests * Don't include simple-peer in rrweb main project * Add canvas webrtc plugin Streams contents of canvas via webrtc * ignore tsconfig.tsbuildinfo * Cleanup unused code * type definition files are no longer committed * Devtools off by default * Extract .css into its own file * Refactor plugin apis and fix multi canvas streaming support * Add readme to rrweb canvas webrtc plugin * Reference canvas-webrtc plugin in documentation * Forbidden non-null assertion * Remove linting of each project, yarn lint:report will do this * Remove test code * Cut down line length * fix CI failure and improve the zh_CN doc * Update packages/rrweb/src/plugins/canvas-webrtc/replay/index.ts Co-authored-by: Yun Feng * Cleaner styling of replay Co-authored-by: Yun Feng * Clean up stream.js based on @Mark-Fenng's feedback * Remove duplicate send Co-authored-by: Yun Feng Co-authored-by: Yun Feng --- .gitignore | 1 + .travis.yml | 2 +- docs/recipes/canvas.md | 6 +- docs/recipes/canvas.zh_CN.md | 3 + package.json | 6 +- packages/rrdom-nodejs/tsconfig.json | 11 +- packages/rrdom/test/virtual-dom.test.ts | 1 + packages/rrdom/tsconfig.json | 6 + packages/rrweb-player/tsconfig.json | 8 + packages/rrweb-snapshot/src/rebuild.ts | 6 +- packages/rrweb-snapshot/test/rebuild.test.ts | 3 +- packages/rrweb-snapshot/tsconfig.json | 4 +- packages/rrweb/.gitignore | 1 + packages/rrweb/jest.config.js | 2 +- packages/rrweb/package.json | 10 +- packages/rrweb/rollup.config.js | 40 +- packages/rrweb/scripts/repl.js | 22 +- packages/rrweb/scripts/stream.js | 253 +++++++++ packages/rrweb/scripts/utils.js | 62 +++ .../rrweb/src/plugins/canvas-webrtc/Readme.md | 74 +++ .../src/plugins/canvas-webrtc/record/index.ts | 94 ++++ .../src/plugins/canvas-webrtc/replay/index.ts | 167 ++++++ .../canvas-webrtc/simple-peer-light.d.ts | 308 ++++++++++ .../rrweb/src/plugins/canvas-webrtc/types.ts | 4 + packages/rrweb/src/record/index.ts | 11 +- packages/rrweb/src/replay/index.ts | 33 +- packages/rrweb/src/types.ts | 8 +- .../__snapshots__/integration.test.ts.snap | 6 +- packages/rrweb/test/e2e/webgl.test.ts | 2 +- packages/rrweb/test/integration.test.ts | 4 +- packages/rrweb/test/record.test.ts | 2 +- packages/rrweb/test/record/webgl.test.ts | 2 +- packages/rrweb/test/replay/webgl.test.ts | 2 +- packages/rrweb/test/replayer.test.ts | 2 +- packages/rrweb/test/utils.ts | 32 +- packages/rrweb/tsconfig.json | 14 +- tsconfig.json | 26 +- yarn.lock | 525 ++++++++++-------- 38 files changed, 1465 insertions(+), 298 deletions(-) create mode 100644 packages/rrweb/scripts/stream.js create mode 100644 packages/rrweb/scripts/utils.js create mode 100644 packages/rrweb/src/plugins/canvas-webrtc/Readme.md create mode 100644 packages/rrweb/src/plugins/canvas-webrtc/record/index.ts create mode 100644 packages/rrweb/src/plugins/canvas-webrtc/replay/index.ts create mode 100644 packages/rrweb/src/plugins/canvas-webrtc/simple-peer-light.d.ts create mode 100644 packages/rrweb/src/plugins/canvas-webrtc/types.ts diff --git a/.gitignore b/.gitignore index af311e67..02a1c32f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ .idea node_modules package-lock.json +tsconfig.tsbuildinfo temp diff --git a/.travis.yml b/.travis.yml index 89d0ef2b..b122c43a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,4 +13,4 @@ install: script: - yarn build:all - yarn turbo run check-types - - xvfb-run --server-args="-screen 0 1920x1080x24" yarn lerna run test + - xvfb-run --server-args="-screen 0 1920x1080x24" yarn test diff --git a/docs/recipes/canvas.md b/docs/recipes/canvas.md index 934461ee..145adabb 100644 --- a/docs/recipes/canvas.md +++ b/docs/recipes/canvas.md @@ -1,6 +1,7 @@ # Canvas -Canvas is a special HTML element, which will not be recorded by rrweb by default. There are some options for recording and replaying Canvas. +Canvas is a special HTML element, and will not be recorded by rrweb by default. +There are some options for recording and replaying Canvas. Enable recording Canvas： @@ -33,3 +34,6 @@ replayer.play(); ``` **Enable replaying Canvas will remove the sandbox, which may cause a potential security issue.** + +Alternatively you can stream canvas elements via webrtc with the canvas-webrtc plugin. +For more information see [canvas-webrtc documentation](../../packages/rrweb/src/plugins/canvas-webrtc/Readme.md) diff --git a/docs/recipes/canvas.zh_CN.md b/docs/recipes/canvas.zh_CN.md index 59ba5072..e02e899e 100644 --- a/docs/recipes/canvas.zh_CN.md +++ b/docs/recipes/canvas.zh_CN.md @@ -34,3 +34,6 @@ replayer.play(); ``` **回放 Canvas 将会关闭沙盒策略，导致一定风险**。 + +另外，您可以使用 canvas-webrtc 插件通过 WEBRTC 流式传输 Canvas 元素。 +有关更多信息，请参考[canvas-webrtc 文档](../../packages/rrweb/src/plugins/canvas-webrtc/readme.md) diff --git a/package.json b/package.json index 452d5f18..9a1d39b9 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "packages/*" ], "devDependencies": { + "@monorepo-utils/workspaces-to-typescript-project-references": "^2.8.2", "@typescript-eslint/eslint-plugin": "^5.25.0", "@typescript-eslint/parser": "^5.25.0", "concurrently": "^7.1.0", @@ -32,11 +33,12 @@ }, "scripts": { "lerna": "lerna", - "build:all": "yarn turbo run prepublish", - "test": "yarn turbo run test", + "build:all": "yarn run concurrently --success=all -r -m=1 'yarn workspaces-to-typescript-project-references' 'yarn turbo run prepublish'", + "test": "yarn run concurrently --success=all -r -m=1 'yarn workspaces-to-typescript-project-references --check' 'yarn turbo run test'", "test:watch": "yarn turbo run test:watch", "dev": "yarn turbo run dev", "repl": "cd packages/rrweb && npm run repl", + "live-stream": "cd packages/rrweb && yarn live-stream", "lint": "yarn run concurrently --success=all -r -m=1 'yarn run markdownlint docs' 'yarn eslint packages/*/src --ext .ts,.tsx,.js,.jsx,.svelte'", "lint:report": "yarn eslint --output-file eslint_report.json --format json packages/*/src --ext .ts,.tsx,.js,.jsx" }, diff --git a/packages/rrdom-nodejs/tsconfig.json b/packages/rrdom-nodejs/tsconfig.json index 4a4f18a0..4497eff2 100644 --- a/packages/rrdom-nodejs/tsconfig.json +++ b/packages/rrdom-nodejs/tsconfig.json @@ -1,5 +1,6 @@ { "compilerOptions": { + "composite": true, "target": "ES6", "module": "commonjs", "noImplicitAny": true, @@ -16,5 +17,13 @@ }, "compileOnSave": true, "exclude": ["test"], - "include": ["src", "test.d.ts", "../rrweb/src/record/workers/workers.d.ts"] + "include": ["src", "test.d.ts", "../rrweb/src/record/workers/workers.d.ts"], + "references": [ + { + "path": "../rrdom" + }, + { + "path": "../rrweb-snapshot" + } + ] } diff --git a/packages/rrdom/test/virtual-dom.test.ts b/packages/rrdom/test/virtual-dom.test.ts index 57aed3e7..e414e230 100644 --- a/packages/rrdom/test/virtual-dom.test.ts +++ b/packages/rrdom/test/virtual-dom.test.ts @@ -53,6 +53,7 @@ function walk(node, mirror, blankSpace) { `; describe('RRDocument for browser environment', () => { + jest.setTimeout(60_000); let mirror: Mirror; beforeEach(() => { mirror = new Mirror(); diff --git a/packages/rrdom/tsconfig.json b/packages/rrdom/tsconfig.json index c5d366ad..44af127b 100644 --- a/packages/rrdom/tsconfig.json +++ b/packages/rrdom/tsconfig.json @@ -1,5 +1,6 @@ { "compilerOptions": { + "composite": true, "target": "ES6", "module": "commonjs", "noImplicitAny": true, @@ -14,6 +15,11 @@ "declaration": true, "importsNotUsedAsValues": "error" }, + "references": [ + { + "path": "../rrweb-snapshot" + } + ], "compileOnSave": true, "exclude": ["test"], "include": ["src", "../rrweb/src/record/workers/workers.d.ts"] diff --git a/packages/rrweb-player/tsconfig.json b/packages/rrweb-player/tsconfig.json index b049c15d..63866506 100644 --- a/packages/rrweb-player/tsconfig.json +++ b/packages/rrweb-player/tsconfig.json @@ -6,5 +6,13 @@ "__sapper__/*", "public/*", "../rrweb/src/record/workers/workers.d.ts" + ], + "compilerOptions": { + "composite": true + }, + "references": [ + { + "path": "../rrweb" + } ] } diff --git a/packages/rrweb-snapshot/src/rebuild.ts b/packages/rrweb-snapshot/src/rebuild.ts index 60e3669b..978aa87d 100644 --- a/packages/rrweb-snapshot/src/rebuild.ts +++ b/packages/rrweb-snapshot/src/rebuild.ts @@ -320,7 +320,7 @@ export function buildNodeWithSN( mirror: Mirror; skipChild?: boolean; hackCss: boolean; - afterAppend?: (n: Node) => unknown; + afterAppend?: (n: Node, id: number) => unknown; cache: BuildCache; }, ): Node | null { @@ -398,7 +398,7 @@ export function buildNodeWithSN( node.appendChild(childNode); } if (afterAppend) { - afterAppend(childNode); + afterAppend(childNode, childN.id); } } } @@ -449,7 +449,7 @@ function rebuild( doc: Document; onVisit?: (node: Node) => unknown; hackCss?: boolean; - afterAppend?: (n: Node) => unknown; + afterAppend?: (n: Node, id: number) => unknown; cache: BuildCache; mirror: Mirror; }, diff --git a/packages/rrweb-snapshot/test/rebuild.test.ts b/packages/rrweb-snapshot/test/rebuild.test.ts index e669a29a..f7bbebb4 100644 --- a/packages/rrweb-snapshot/test/rebuild.test.ts +++ b/packages/rrweb-snapshot/test/rebuild.test.ts @@ -59,7 +59,8 @@ describe('add hover class to hover selector related rules', function () { expect(addHoverClass(cssText, cache)).toEqual(cssText); }); - it('benchmark', () => { + // this benchmark is unreliable when run in parallel with other tests + it.skip('benchmark', () => { const cssText = fs.readFileSync( path.resolve(__dirname, './css/benchmark.css'), 'utf8', diff --git a/packages/rrweb-snapshot/tsconfig.json b/packages/rrweb-snapshot/tsconfig.json index 2b6d7032..879f459a 100644 --- a/packages/rrweb-snapshot/tsconfig.json +++ b/packages/rrweb-snapshot/tsconfig.json @@ -1,5 +1,6 @@ { "compilerOptions": { + "composite": true, "module": "ESNext", "moduleResolution": "Node", "noImplicitAny": true, @@ -11,5 +12,6 @@ "lib": ["es6", "dom"] }, "exclude": ["test"], - "include": ["src"] + "include": ["src"], + "references": [] } diff --git a/packages/rrweb/.gitignore b/packages/rrweb/.gitignore index 42374e6e..083fd631 100644 --- a/packages/rrweb/.gitignore +++ b/packages/rrweb/.gitignore @@ -3,6 +3,7 @@ node_modules package-lock.json # yarn.lock +tsconfig.tsbuildinfo build dist es diff --git a/packages/rrweb/jest.config.js b/packages/rrweb/jest.config.js index 29db4e7f..8ebc7349 100644 --- a/packages/rrweb/jest.config.js +++ b/packages/rrweb/jest.config.js @@ -1,5 +1,5 @@ /** @type {import('ts-jest/dist/types').InitialOptionsTsJest} */ -module.exports = { +export default { preset: 'ts-jest', testEnvironment: 'node', testMatch: ['**/**.test.ts'], diff --git a/packages/rrweb/package.json b/packages/rrweb/package.json index 2c19de3e..f3e5c296 100644 --- a/packages/rrweb/package.json +++ b/packages/rrweb/package.json @@ -9,6 +9,7 @@ "test:headless": "PUPPETEER_HEADLESS=true npm run test", "test:watch": "PUPPETEER_HEADLESS=true npm run test -- --watch", "repl": "npm run bundle:browser && node scripts/repl.js", + "live-stream": "yarn bundle:browser && node scripts/stream.js", "dev": "yarn bundle:browser --watch", "bundle:browser": "cross-env BROWSER_ONLY=true rollup --config", "bundle": "rollup --config", @@ -18,6 +19,7 @@ "lint": "yarn eslint src", "benchmark": "jest test/benchmark" }, + "type": "module", "repository": { "type": "git", "url": "git+ssh://git@github.com/rrweb-io/rrweb.git" @@ -45,7 +47,8 @@ "devDependencies": { "@rollup/plugin-node-resolve": "^13.1.3", "@types/chai": "^4.1.6", - "@types/inquirer": "0.0.43", + "@types/dom-mediacapture-transform": "^0.1.3", + "@types/inquirer": "^8.2.1", "@types/jest": "^27.4.1", "@types/jest-image-snapshot": "^4.3.1", "@types/node": "^17.0.21", @@ -57,7 +60,7 @@ "fast-mhtml": "^1.1.9", "identity-obj-proxy": "^3.0.0", "ignore-styles": "^5.0.1", - "inquirer": "^6.2.1", + "inquirer": "^9.0.0", "jest": "^27.5.1", "jest-image-snapshot": "^4.5.1", "jest-snapshot": "^23.6.0", @@ -69,8 +72,9 @@ "rollup-plugin-rename-node-modules": "^1.3.1", "rollup-plugin-typescript2": "^0.31.2", "rollup-plugin-web-worker-loader": "^1.6.1", + "simple-peer-light": "^9.10.0", "ts-jest": "^27.1.3", - "ts-node": "^10.7.0", + "ts-node": "^10.9.1", "tslib": "^2.3.1", "typescript": "^4.7.3" }, diff --git a/packages/rrweb/rollup.config.js b/packages/rrweb/rollup.config.js index 00876723..79f17e03 100644 --- a/packages/rrweb/rollup.config.js +++ b/packages/rrweb/rollup.config.js @@ -89,6 +89,16 @@ const baseConfigs = [ name: 'rrwebConsoleRecord', pathFn: toPluginPath('console', 'record'), }, + { + input: './src/plugins/canvas-webrtc/record/index.ts', + name: 'rrwebCanvasWebRTCRecord', + pathFn: toPluginPath('canvas-webrtc', 'record'), + }, + { + input: './src/plugins/canvas-webrtc/replay/index.ts', + name: 'rrwebCanvasWebRTCReplay', + pathFn: toPluginPath('canvas-webrtc', 'replay'), + }, { input: './src/plugins/console/replay/index.ts', name: 'rrwebConsoleReplay', @@ -121,7 +131,7 @@ function getPlugins(options = {}) { minify, }), postcss({ - extract: false, + extract: true, inject: false, minimize: minify, sourceMap, @@ -209,33 +219,29 @@ if (process.env.BROWSER_ONLY) { name: 'rrwebConsoleRecord', pathFn: toPluginPath('console', 'record'), }, + { + input: './src/plugins/canvas-webrtc/record/index.ts', + name: 'rrwebCanvasWebRTCRecord', + pathFn: toPluginPath('canvas-webrtc', 'record'), + }, + { + input: './src/plugins/canvas-webrtc/replay/index.ts', + name: 'rrwebCanvasWebRTCReplay', + pathFn: toPluginPath('canvas-webrtc', 'replay'), + }, ]; configs = []; - // browser record + replay, unminified (for profiling and performance testing) - configs.push({ - input: './src/index.ts', - plugins: getPlugins(), - output: [ - { - name: 'rrweb', - format: 'iife', - file: pkg.unpkg, - }, - ], - }); - for (const c of browserOnlyBaseConfigs) { configs.push({ input: c.input, - plugins: getPlugins({ sourceMap: true, minify: true }), + plugins: getPlugins(), output: [ { name: c.name, format: 'iife', - file: toMinPath(c.pathFn(pkg.unpkg)), - sourcemap: true, + file: c.pathFn(pkg.unpkg), }, ], }); diff --git a/packages/rrweb/scripts/repl.js b/packages/rrweb/scripts/repl.js index 9afdb709..5319bcf9 100644 --- a/packages/rrweb/scripts/repl.js +++ b/packages/rrweb/scripts/repl.js @@ -1,15 +1,19 @@ /* eslint:disable: no-console */ -const fs = require('fs'); -const path = require('path'); -const EventEmitter = require('events'); -const inquirer = require('inquirer'); -const puppeteer = require('puppeteer'); +import * as path from 'path'; +import * as fs from 'fs'; +import { EventEmitter } from 'node:events'; +import inquirer from 'inquirer'; +import puppeteer from 'puppeteer'; +import { fileURLToPath } from 'url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); const emitter = new EventEmitter(); function getCode() { - const bundlePath = path.resolve(__dirname, '../dist/rrweb.min.js'); + const bundlePath = path.resolve(__dirname, '../dist/rrweb.js'); return fs.readFileSync(bundlePath, 'utf8'); } @@ -165,7 +169,7 @@ void (async () => { } await page.addStyleTag({ - path: path.resolve(__dirname, '../dist/rrweb.min.css'), + path: path.resolve(__dirname, '../dist/rrweb.css'), }); await page.evaluate(`${code} const events = ${JSON.stringify(events)}; @@ -196,10 +200,10 @@ void (async () => { Record @${time} - + - +