The desktop pointer cursor is not representative of mobile (#662)

* Simplify css for click animation

* Refactor to transfer responsibility for casting multiple synchronous to index.ts from machine.ts (so they can be dealt with in bulk)

* During synchronous application of a batch of events, move the mouse to the last position so that it's in the correct place when the timer starts

 - previous `needCastInSyncMode` added in 4bf533a675 meant that the isSync versions of MouseMove/TouchMove were being accidentally ignored
 - each synchronous MouseMove would have resulted in a separate mouse position update
 - the Click/TouchStart/TouchEnd events didn't have an async version

* The desktop pointer cursor is not representative of what is happening on a mobile device.

Instead, check a recording for any presence of a Touch event, and switch to a touch visualisation mode for the entire recording.
(for now, we use this mode even for mixed touch/mouse devices - this could be improved upon in future)

Show a round circle representing the users' finger which is visible only between TouchStart and TouchEnd events
Again this can be evolved upon, but this change should be a good start in the right direction.

* It's more correct to not have a transition for repositioning of touch as user can lift finger off screen and place elsewhere; however we can now have much smoother touch movement during the .touch-active phase as we know the finger is on the screen. This has a .25s delaying effect on the touch position which IMO is acceptable; e.g. scroll position can lag behind a touch movement and this seems to bring them more in sync

* Ensure we end up with the correct touch-active state after a series of synchronous events

* Important to discontinue tail animations and position transitions when user has lifted their finger and placed it into a new position. This is apparent in a replay session where the user is scrolling the page using repeated TouchMove bottom-to-top movements

* Simplify by unwrapping `mouseState.touchActive` and `mouseState.pos` into their own global vars

packages/rrweb/src/replay/index.ts

@@ -37,6 +37,7 @@ import {
   ElementState,
   styleAttributeValue,
   styleValueWithPriority,
+  mouseMovePos,
 } from '../types';
 import {
   createMirror,
@@ -77,6 +78,13 @@ const defaultMouseTailConfig = {
   strokeStyle: 'red',
 } as const;
 
+function indicatesTouchDevice(e: eventWithTime) {
+  return e.type == EventType.IncrementalSnapshot &&
+    (e.data.source == IncrementalSource.TouchMove ||
+     (e.data.source == IncrementalSource.MouseInteraction &&
+      e.data.type == MouseInteractions.TouchStart));
+}
+
 export class Replayer {
   public wrapper: HTMLDivElement;
   public iframe: HTMLIFrameElement;
@@ -117,6 +125,9 @@ export class Replayer {
 
   private newDocumentQueue: addedNodeMutation[] = [];
 
+  private mousePos: mouseMovePos | null = null;
+  private touchActive: boolean | null = null;
+
   constructor(
     events: Array<eventWithTime | string>,
     config?: Partial<playerConfig>,
@@ -144,6 +155,7 @@ export class Replayer {
 
     this.handleResize = this.handleResize.bind(this);
     this.getCastFn = this.getCastFn.bind(this);
+    this.applyEventsSynchronously = this.applyEventsSynchronously.bind(this);
     this.emitter.on(ReplayerEvents.Resize, this.handleResize as Handler);
 
     this.setupDom();
@@ -197,6 +209,7 @@ export class Replayer {
       },
       {
         getCastFn: this.getCastFn,
+        applyEventsSynchronously: this.applyEventsSynchronously,
         emitter: this.emitter,
       },
     );
@@ -250,6 +263,10 @@ export class Replayer {
         );
       }, 1);
     }
+    if (this.service.state.context.events.find(indicatesTouchDevice)) {
+      this.mouse.classList.add('touch-device');
+    }
+
   }
 
   public on(event: string, handler: Handler) {
@@ -373,6 +390,9 @@ export class Replayer {
     const event = this.config.unpackFn
       ? this.config.unpackFn(rawEvent as string)
       : (rawEvent as eventWithTime);
+    if (indicatesTouchDevice(event)) {
+      this.mouse.classList.add('touch-device');
+    }
     Promise.resolve().then(() =>
       this.service.send({ type: 'ADD_EVENT', payload: { event } }),
     );
@@ -443,6 +463,43 @@ export class Replayer {
     }
   }
 
+  private applyEventsSynchronously(events: Array<eventWithTime>) {
+    for (const event of events) {
+      switch (event.type) {
+        case EventType.DomContentLoaded:
+        case EventType.Load:
+        case EventType.Custom:
+          continue;
+        case EventType.FullSnapshot:
+        case EventType.Meta:
+        case EventType.Plugin:
+          break;
+        case EventType.IncrementalSnapshot:
+          switch (event.data.source) {
+            case IncrementalSource.MediaInteraction:
+              continue;
+            default:
+              break;
+          }
+          break;
+        default:
+          break;
+      }
+      const castFn = this.getCastFn(event, true);
+      castFn();
+    }
+    if (this.mousePos) {
+      this.moveAndHover(this.mousePos.x, this.mousePos.y, this.mousePos.id, true, this.mousePos.debugData);
+    }
+    this.mousePos = null;
+    if (this.touchActive === true) {
+      this.mouse.classList.add('touch-active');
+    } else if (this.touchActive === false) {
+      this.mouse.classList.remove('touch-active');
+    }
+    this.touchActive = null;
+  }
+
   private getCastFn(event: eventWithTime, isSync = false) {
     let castFn: undefined | (() => void);
     switch (event.type) {
@@ -803,12 +860,17 @@ export class Replayer {
       case IncrementalSource.MouseMove:
         if (isSync) {
           const lastPosition = d.positions[d.positions.length - 1];
-          this.moveAndHover(d, lastPosition.x, lastPosition.y, lastPosition.id);
+          this.mousePos = {
+            x: lastPosition.x,
+            y: lastPosition.y,
+            id: lastPosition.id,
+            debugData: d,
+          };
         } else {
           d.positions.forEach((p) => {
             const action = {
               doAction: () => {
-                this.moveAndHover(d, p.x, p.y, p.id);
+                this.moveAndHover(p.x, p.y, p.id, isSync, d);
               },
               delay:
                 p.timeOffset +
@@ -857,19 +919,51 @@ export class Replayer {
           case MouseInteractions.Click:
           case MouseInteractions.TouchStart:
           case MouseInteractions.TouchEnd:
-            /**
+            if (isSync) {
-             * Click has no visual impact when replaying and may
+              let touchActive: boolean | undefined;
-             * trigger navigation when apply to an <a> link.
+              if (d.type === MouseInteractions.TouchStart) {
-             * So we will not call click(), instead we add an
+                this.touchActive = true;
-             * animation to the mouse element which indicate user
+              } else if (d.type === MouseInteractions.TouchEnd) {
-             * clicked at this moment.
+                this.touchActive = false;
-             */
+              }
-            if (!isSync) {
+              this.mousePos = {
-              this.moveAndHover(d, d.x, d.y, d.id);
+                x: d.x,
-              this.mouse.classList.remove('active');
+                y: d.y,
-              // tslint:disable-next-line
+                id: d.id,
-              void this.mouse.offsetWidth;
+                debugData: d,
-              this.mouse.classList.add('active');
+              };
+            } else {
+              if (d.type === MouseInteractions.TouchStart) {
+                // don't draw a trail as user has lifted finger and is placing at a new point
+                this.tailPositions.length = 0;
+              }
+              this.moveAndHover(d.x, d.y, d.id, isSync, d);
+              if (d.type === MouseInteractions.Click) {
+                /*
+                 * don't want target.click() here as could trigger an iframe navigation
+                 * instead any effects of the click should already be covered by mutations
+                 */
+                /*
+                 * removal and addition of .active class (along with void line to trigger repaint)
+                 * triggers the 'click' css animation in styles/style.css
+                 */
+                this.mouse.classList.remove('active');
+                // tslint:disable-next-line
+                void this.mouse.offsetWidth;
+                this.mouse.classList.add('active');
+              } else if (d.type === MouseInteractions.TouchStart) {
+                void this.mouse.offsetWidth;  // needed for the position update of moveAndHover to apply without the .touch-active transition
+                this.mouse.classList.add('touch-active');
+              } else if (d.type === MouseInteractions.TouchEnd) {
+                this.mouse.classList.remove('touch-active');
+              }
+            }
+            break;
+        case MouseInteractions.TouchCancel:
+            if (isSync) {
+              this.touchActive = false;
+            } else {
+              this.mouse.classList.remove('touch-active');
             }
             break;
           default:
@@ -1487,10 +1581,10 @@ export class Replayer {
     }
   }
 
-  private moveAndHover(d: incrementalData, x: number, y: number, id: number) {
+  private moveAndHover(x: number, y: number, id: number, isSync: boolean, debugData: incrementalData) {
     const target = this.mirror.getNode(id);
     if (!target) {
-      return this.debugNodeNotFound(d, id);
+      return this.debugNodeNotFound(debugData, id);
     }
 
     const base = getBaseDimension(target, this.iframe);
@@ -1499,7 +1593,9 @@ export class Replayer {
 
     this.mouse.style.left = `${_x}px`;
     this.mouse.style.top = `${_y}px`;
-    this.drawMouseTail({ x: _x, y: _y });
+    if (!isSync) {
+      this.drawMouseTail({ x: _x, y: _y });
+    }
     this.hoverElements((target as Node) as Element);
   }
 

packages/rrweb/src/replay/machine.ts

@@ -9,7 +9,6 @@ import {
   IncrementalSource,
 } from '../types';
 import { Timer, addDelay } from './timer';
-import { needCastInSyncMode } from '../utils';
 
 export type PlayerContext = {
   events: eventWithTime[];
@@ -77,11 +76,12 @@ export function discardPriorSnapshots(
 
 type PlayerAssets = {
   emitter: Emitter;
+  applyEventsSynchronously(events: Array<eventWithTime>): void;
   getCastFn(event: eventWithTime, isSync: boolean): () => void;
 };
 export function createPlayerService(
   context: PlayerContext,
-  { getCastFn, emitter }: PlayerAssets,
+  { getCastFn, applyEventsSynchronously, emitter }: PlayerAssets,
 ) {
   const playerMachine = createMachine<PlayerContext, PlayerEvent, PlayerState>(
     {
@@ -186,6 +186,7 @@ export function createPlayerService(
             emitter.emit(ReplayerEvents.PlayBack);
           }
 
+          const syncEvents = new Array<eventWithTime>();
           const actions = new Array<actionWithDelay>();
           for (const event of neededEvents) {
             if (
@@ -196,14 +197,10 @@ export function createPlayerService(
             ) {
               continue;
             }
-            const isSync = event.timestamp < baselineTime;
+            if (event.timestamp < baselineTime) {
-            if (isSync && !needCastInSyncMode(event)) {
+              syncEvents.push(event);
-              continue;
-            }
-            const castFn = getCastFn(event, isSync);
-            if (isSync) {
-              castFn();
             } else {
+              const castFn = getCastFn(event, false);
               actions.push({
                 doAction: () => {
                   castFn();
@@ -213,6 +210,7 @@ export function createPlayerService(
               });
             }
           }
+          applyEventsSynchronously(syncEvents);
           emitter.emit(ReplayerEvents.Flush);
           timer.addActions(actions);
           timer.start();

packages/rrweb/src/replay/styles/style.css

@@ -5,25 +5,48 @@
   position: absolute;
   width: 20px;
   height: 20px;
-  transition: 0.05s linear;
+  transition: left 0.05s linear, top 0.05s linear;
   background-size: contain;
   background-position: center center;
   background-repeat: no-repeat;
   background-image: url('data:image/svg+xml;base64,PHN2ZyBoZWlnaHQ9JzMwMHB4JyB3aWR0aD0nMzAwcHgnICBmaWxsPSIjMDAwMDAwIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGRhdGEtbmFtZT0iTGF5ZXIgMSIgdmlld0JveD0iMCAwIDUwIDUwIiB4PSIwcHgiIHk9IjBweCI+PHRpdGxlPkRlc2lnbl90bnA8L3RpdGxlPjxwYXRoIGQ9Ik00OC43MSw0Mi45MUwzNC4wOCwyOC4yOSw0NC4zMywxOEExLDEsMCwwLDAsNDQsMTYuMzlMMi4zNSwxLjA2QTEsMSwwLDAsMCwxLjA2LDIuMzVMMTYuMzksNDRhMSwxLDAsMCwwLDEuNjUuMzZMMjguMjksMzQuMDgsNDIuOTEsNDguNzFhMSwxLDAsMCwwLDEuNDEsMGw0LjM4LTQuMzhBMSwxLDAsMCwwLDQ4LjcxLDQyLjkxWm0tNS4wOSwzLjY3TDI5LDMyYTEsMSwwLDAsMC0xLjQxLDBsLTkuODUsOS44NUwzLjY5LDMuNjlsMzguMTIsMTRMMzIsMjcuNThBMSwxLDAsMCwwLDMyLDI5TDQ2LjU5LDQzLjYyWiI+PC9wYXRoPjwvc3ZnPg==');
+  border-color: transparent;  /* otherwise we transition from black when .touch-device class is added */
 }
 .replayer-mouse::after {
   content: '';
   display: inline-block;
   width: 20px;
   height: 20px;
-  border-radius: 10px;
   background: rgb(73, 80, 246);
-  transform: translate(-10px, -10px);
+  border-radius: 100%;
+  transform: translate(-50%, -50%);
   opacity: 0.3;
 }
 .replayer-mouse.active::after {
   animation: click 0.2s ease-in-out 1;
 }
+.replayer-mouse.touch-device {
+  background-image: none;  /* there's no passive cursor on touch-only screens */
+  width: 70px;
+  height: 70px;
+  border-width: 4px;
+  border-style: solid;
+  border-radius: 100%;
+  margin-left: -37px;
+  margin-top: -37px;
+  border-color: rgba(73, 80, 246, 0);
+  transition: left 0s linear, top 0s linear, border-color 0.2s ease-in-out;
+}
+.replayer-mouse.touch-device.touch-active {
+  border-color: rgba(73, 80, 246, 1);
+  transition: left 0.25s linear, top 0.25s linear, border-color 0.2s ease-in-out;
+}
+.replayer-mouse.touch-device::after {
+  opacity: 0;  /* there's no passive cursor on touch-only screens */
+}
+.replayer-mouse.touch-device.active::after {
+  animation: touch-click 0.2s ease-in-out 1;
+}
 .replayer-mouse-tail {
   position: absolute;
   pointer-events: none;
@@ -34,14 +57,23 @@
     opacity: 0.3;
     width: 20px;
     height: 20px;
-    border-radius: 10px;
-    transform: translate(-10px, -10px);
   }
   50% {
     opacity: 0.5;
     width: 10px;
     height: 10px;
-    border-radius: 5px;
+  }
-    transform: translate(-5px, -5px);
+}
+
+@keyframes touch-click {
+  0% {
+    opacity: 0;
+    width: 20px;
+    height: 20px;
+  }
+  50% {
+    opacity: 0.5;
+    width: 10px;
+    height: 10px;
   }
 }

packages/rrweb/src/types.ts

@@ -352,6 +352,13 @@ export type mousePosition = {
   timeOffset: number;
 };
 
+export type mouseMovePos = {
+  x: number;
+  y: number;
+  id: number;
+  debugData: incrementalData;
+};
+
 export enum MouseInteractions {
   MouseUp,
   MouseDown,
@@ -363,6 +370,7 @@ export enum MouseInteractions {
   TouchStart,
   TouchMove_Departed, // we will start a separate observer for touch move event
   TouchEnd,
+  TouchCancel,
 }
 
 type mouseInteractionParam = {

packages/rrweb/src/utils.ts

@@ -314,38 +314,6 @@ export function polyfill(win = window) {
   }
 }
 
-export function needCastInSyncMode(event: eventWithTime): boolean {
-  switch (event.type) {
-    case EventType.DomContentLoaded:
-    case EventType.Load:
-    case EventType.Custom:
-      return false;
-    case EventType.FullSnapshot:
-    case EventType.Meta:
-    case EventType.Plugin:
-      return true;
-    default:
-      break;
-  }
-
-  switch (event.data.source) {
-    case IncrementalSource.MouseMove:
-    case IncrementalSource.MouseInteraction:
-    case IncrementalSource.TouchMove:
-    case IncrementalSource.MediaInteraction:
-      return false;
-    case IncrementalSource.ViewportResize:
-    case IncrementalSource.StyleSheetRule:
-    case IncrementalSource.Scroll:
-    case IncrementalSource.Input:
-      return true;
-    default:
-      break;
-  }
-
-  return true;
-}
-
 export type TreeNode = {
   id: number;
   mutation: addedNodeMutation;

packages/rrweb/typings/utils.d.ts

@@ -15,7 +15,6 @@ export declare function isIgnored(n: Node | INode): boolean;
 export declare function isAncestorRemoved(target: INode, mirror: Mirror): boolean;
 export declare function isTouchEvent(event: MouseEvent | TouchEvent): event is TouchEvent;
 export declare function polyfill(win?: Window & typeof globalThis): void;
-export declare function needCastInSyncMode(event: eventWithTime): boolean;
 export declare type TreeNode = {
     id: number;
     mutation: addedNodeMutation;