Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ref(rrweb): allow multi touch gestures to be shown for mobile replays #190

Merged
merged 28 commits into from
May 31, 2024
Merged

ref(rrweb): allow multi touch gestures to be shown for mobile replays #190

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
5f4dfd3
ref(rrweb): allow multi touch gestures to be shown for mobile replays
michellewzhang May 16, 2024
94e7595
some type fixes
michellewzhang May 16, 2024
d87bbb9
rm files
michellewzhang May 16, 2024
8e44b94
more fixes
michellewzhang May 16, 2024
1d0cce1
touchId -> pointerId
michellewzhang May 16, 2024
673d62a
name changes
michellewzhang May 16, 2024
c92bc1b
types
michellewzhang May 16, 2024
52e4399
some fixes
michellewzhang May 28, 2024
07d1d67
forgot a spot
michellewzhang May 28, 2024
b2b12c8
rm export
michellewzhang May 28, 2024
c7039dd
rm changes to package.json and yarn.lock
michellewzhang May 28, 2024
1b453b7
ts
michellewzhang May 28, 2024
ad7d1ce
fix delete
michellewzhang May 28, 2024
583d808
turn "run for branch" on for size limit GHA
billyvg May 29, 2024
a41c292
Update packages/rrweb/src/replay/index.ts
billyvg May 29, 2024
74a19cd
add size limit entry for Replayer
billyvg May 29, 2024
fd3e3ed
optimizations
michellewzhang May 29, 2024
fbb92f6
Merge remote-tracking branch 'origin/sentry-v2' into mz/rrweb-ref-mul…
michellewzhang May 29, 2024
c424b56
rm -1
michellewzhang May 29, 2024
3830e04
update snapshots for failing tests
michellewzhang May 29, 2024
3044563
add tests
michellewzhang May 29, 2024
a2aac2a
:white-check-mark: add test
michellewzhang May 29, 2024
830d8f3
move func out of the class
michellewzhang May 30, 2024
d9d3075
TESTS.
michellewzhang May 30, 2024
56ed9a0
tests that pass
michellewzhang May 30, 2024
0a7aeee
rm code for creating pointers upfront
michellewzhang May 31, 2024
5e465eb
add touch-device class inside createPointer
michellewzhang May 31, 2024
8fee9ca
rm play() from tests
michellewzhang May 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@
"prettier": "2.8.4",
"size-limit": "~8.2.6",
"turbo": "^1.2.4",
"typescript": "^4.9.5"
"typescript": "^4.9.5",
"yalc": "^1.0.0-pre.53"
michellewzhang marked this conversation as resolved.
Show resolved Hide resolved
},
"scripts": {
"build:all": "NODE_OPTIONS='--max-old-space-size=4096' yarn run concurrently --success=all -r -m=1 'yarn workspaces-to-typescript-project-references' 'yarn turbo run prepare'",
Expand Down
176 changes: 123 additions & 53 deletions packages/rrweb/src/replay/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,14 @@ const defaultMouseTailConfig = {
strokeStyle: 'red',
} as const;

function indicatesTouchDevice(e: eventWithTime) {
export type incrementalSnapshotEventWithTime = incrementalSnapshotEvent & {
michellewzhang marked this conversation as resolved.
Show resolved Hide resolved
timestamp: number;
delay?: number;
};

function indicatesTouchDevice(
e: eventWithTime,
): e is incrementalSnapshotEventWithTime {
return (
e.type == EventType.IncrementalSnapshot &&
(e.data.source == IncrementalSource.TouchMove ||
Expand All @@ -123,9 +130,7 @@ export class Replayer {
public usingVirtualDom = false;
public virtualDom: RRDocument = new RRDocument();

private mouse: HTMLDivElement;
private mouseTail: HTMLCanvasElement | null = null;
private tailPositions: Array<{ x: number; y: number }> = [];

private emitter: Emitter = mitt();

Expand All @@ -148,8 +153,16 @@ export class Replayer {

private newDocumentQueue: addedNodeMutation[] = [];

private mousePos: mouseMovePos | null = null;
private touchActive: boolean | null = null;
// Map of pointer ID to the unique vars used to show gestures
michellewzhang marked this conversation as resolved.
Show resolved Hide resolved
private pointers: Record<
number,
{
touchActive: boolean | null;
pointerEl: HTMLDivElement;
tailPositions: Array<{ x: number; y: number }>;
pointerPosition: mouseMovePos | null;
}
> = {};
private lastMouseDownEvent: [Node, Event] | null = null;

// Keep the rootNode of the last hovered element. So when hovering a new element, we can remove the last hovered element's :hover style.
Expand Down Expand Up @@ -293,23 +306,30 @@ export class Replayer {
this.adoptedStyleSheets = [];
}

if (this.mousePos) {
this.moveAndHover(
this.mousePos.x,
this.mousePos.y,
this.mousePos.id,
true,
this.mousePos.debugData,
);
this.mousePos = null;
}
for (const [
pointerId,
{ pointerPosition, touchActive },
] of Object.entries(this.pointers)) {
const id = parseInt(pointerId);
if (pointerPosition) {
this.moveAndHover(
pointerPosition.x,
pointerPosition.y,
pointerPosition.id,
true,
pointerPosition.debugData,
id,
);
this.pointers[id].pointerPosition = null;
michellewzhang marked this conversation as resolved.
Show resolved Hide resolved
}

if (this.touchActive === true) {
this.mouse.classList.add('touch-active');
} else if (this.touchActive === false) {
this.mouse.classList.remove('touch-active');
if (touchActive === true) {
this.pointers[id].pointerEl.classList.add('touch-active');
} else if (touchActive === false) {
this.pointers[id].pointerEl.classList.remove('touch-active');
}
this.pointers[id].touchActive = null;
}
this.touchActive = null;

if (this.lastMouseDownEvent) {
const [target, event] = this.lastMouseDownEvent;
Expand Down Expand Up @@ -402,9 +422,30 @@ export class Replayer {
);
}, 1);
}
if (this.service.state.context.events.find(indicatesTouchDevice)) {
this.mouse.classList.add('touch-device');
}
this.service.state.context.events.forEach((e: eventWithTime) => {
if (indicatesTouchDevice(e)) {
const d = e.data;
const pointerId =
'pointerId' in d && d.pointerId !== undefined ? d.pointerId : -1;

if (!this.pointers[pointerId]) {
this.createPointer(pointerId);
}
this.pointers[pointerId].pointerEl.classList.add('touch-device');
}
});
}

private createPointer(pointerId: number) {
const newMouse = document.createElement('div');
newMouse.classList.add('replayer-mouse');
this.pointers[pointerId] = {
touchActive: null,
pointerEl: newMouse,
tailPositions: [],
pointerPosition: null,
};
this.wrapper.appendChild(newMouse);
}

public on(event: string, handler: Handler) {
Expand Down Expand Up @@ -542,7 +583,13 @@ export class Replayer {
? this.config.unpackFn(rawEvent as string)
: (rawEvent as eventWithTime);
if (indicatesTouchDevice(event)) {
this.mouse.classList.add('touch-device');
const d = event.data;
const pointerId =
'pointerId' in d && d.pointerId !== undefined ? d.pointerId : -1;
if (!this.pointers[pointerId]) {
this.createPointer(pointerId);
}
this.pointers[pointerId].pointerEl.classList.add('touch-device');
}
void Promise.resolve().then(() =>
this.service.send({ type: 'ADD_EVENT', payload: { event } }),
Expand Down Expand Up @@ -571,10 +618,7 @@ export class Replayer {
this.wrapper = document.createElement('div');
this.wrapper.classList.add('replayer-wrapper');
this.config.root.appendChild(this.wrapper);

this.mouse = document.createElement('div');
this.mouse.classList.add('replayer-mouse');
this.wrapper.appendChild(this.mouse);
// this.createPointer(-1);
billyvg marked this conversation as resolved.
Show resolved Hide resolved

if (this.config.mouseTail !== false) {
this.mouseTail = document.createElement('canvas');
Expand Down Expand Up @@ -1085,10 +1129,15 @@ export class Replayer {
}
case IncrementalSource.Drag:
case IncrementalSource.TouchMove:
case IncrementalSource.MouseMove:
case IncrementalSource.MouseMove: {
const pointerId =
Copy link
Member Author

@michellewzhang michellewzhang May 16, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is assuming we adjust the data type to

// mousemoveData
{
    "type": 3,
    "timestamp": 1713999610580,
    "data": {
        "source":  IncrementalSource.TouchMove,
        "positions": [{
          "id": 0,
          "x": 149.436,
          "y": 433.929,
        },
       {
          "id": 1,
          "x": 243.436,
          "y": 155.929,
        }],
        "pointerType": 2
        "pointerId": 2,
    }
}

'pointerId' in d && d.pointerId !== undefined ? d.pointerId : -1;
if (!this.pointers[pointerId]) {
this.createPointer(pointerId);
}
if (isSync) {
const lastPosition = d.positions[d.positions.length - 1];
this.mousePos = {
this.pointers[pointerId].pointerPosition = {
x: lastPosition.x,
y: lastPosition.y,
id: lastPosition.id,
Expand All @@ -1098,7 +1147,7 @@ export class Replayer {
d.positions.forEach((p) => {
const action = {
doAction: () => {
this.moveAndHover(p.x, p.y, p.id, isSync, d);
this.moveAndHover(p.x, p.y, p.id, isSync, d, pointerId);
},
delay:
p.timeOffset +
Expand All @@ -1117,7 +1166,13 @@ export class Replayer {
});
}
break;
}
case IncrementalSource.MouseInteraction: {
const pointerId = d.pointerId ?? Object.keys(this.pointers).length;
if (!this.pointers[pointerId]) {
this.createPointer(pointerId);
}

/**
* Same as the situation of missing input target.
*/
Expand Down Expand Up @@ -1154,16 +1209,16 @@ export class Replayer {
case MouseInteractions.MouseUp:
if (isSync) {
if (d.type === MouseInteractions.TouchStart) {
this.touchActive = true;
this.pointers[pointerId].touchActive = true;
} else if (d.type === MouseInteractions.TouchEnd) {
this.touchActive = false;
this.pointers[pointerId].touchActive = false;
}
if (d.type === MouseInteractions.MouseDown) {
this.lastMouseDownEvent = [target, event];
} else if (d.type === MouseInteractions.MouseUp) {
this.lastMouseDownEvent = null;
}
this.mousePos = {
this.pointers[pointerId].pointerPosition = {
x: d.x || 0,
y: d.y || 0,
id: d.id,
Expand All @@ -1172,9 +1227,9 @@ export class Replayer {
} else {
if (d.type === MouseInteractions.TouchStart) {
// don't draw a trail as user has lifted finger and is placing at a new point
this.tailPositions.length = 0;
this.pointers[pointerId].tailPositions.length = 0;
}
this.moveAndHover(d.x || 0, d.y || 0, d.id, isSync, d);
this.moveAndHover(d.x || 0, d.y || 0, d.id, isSync, d, pointerId);
if (d.type === MouseInteractions.Click) {
/*
* don't want target.click() here as could trigger an iframe navigation
Expand All @@ -1184,14 +1239,19 @@ export class Replayer {
* removal and addition of .active class (along with void line to trigger repaint)
* triggers the 'click' css animation in styles/style.css
*/
this.mouse.classList.remove('active');
void this.mouse.offsetWidth;
this.mouse.classList.add('active');
this.pointers[pointerId].pointerEl.classList.remove('active');
void this.pointers[pointerId].pointerEl.offsetWidth;
this.pointers[pointerId].pointerEl.classList.add('active');
} else if (d.type === MouseInteractions.TouchStart) {
void this.mouse.offsetWidth; // needed for the position update of moveAndHover to apply without the .touch-active transition
this.mouse.classList.add('touch-active');
void this.pointers[pointerId].pointerEl.offsetWidth; // needed for the position update of moveAndHover to apply without the .touch-active transition
this.pointers[pointerId].pointerEl.classList.add(
'touch-active',
);
} else if (d.type === MouseInteractions.TouchEnd) {
this.mouse.classList.remove('touch-active');
this.pointers[pointerId].pointerEl.classList.remove(
'touch-active',
);
// delete this.pointers[pointerId];
michellewzhang marked this conversation as resolved.
Show resolved Hide resolved
} else {
// for MouseDown & MouseUp also invoke default behavior
target.dispatchEvent(event);
Expand All @@ -1200,9 +1260,11 @@ export class Replayer {
break;
case MouseInteractions.TouchCancel:
if (isSync) {
this.touchActive = false;
this.pointers[pointerId].touchActive = false;
} else {
this.mouse.classList.remove('touch-active');
this.pointers[pointerId].pointerEl.classList.remove(
'touch-active',
);
}
break;
default:
Expand Down Expand Up @@ -2080,6 +2142,7 @@ export class Replayer {
id: number,
isSync: boolean,
debugData: incrementalData,
pointerId: number,
) {
const target = this.mirror.getNode(id);
if (!target) {
Expand All @@ -2090,15 +2153,15 @@ export class Replayer {
const _x = x * base.absoluteScale + base.x;
const _y = y * base.absoluteScale + base.y;

this.mouse.style.left = `${_x}px`;
this.mouse.style.top = `${_y}px`;
this.pointers[pointerId].pointerEl.style.left = `${_x}px`;
this.pointers[pointerId].pointerEl.style.top = `${_y}px`;
if (!isSync) {
this.drawMouseTail({ x: _x, y: _y });
this.drawMouseTail({ x: _x, y: _y }, pointerId);
}
this.hoverElements(target as Element);
}

private drawMouseTail(position: { x: number; y: number }) {
private drawMouseTail(position: { x: number; y: number }, pointerId: number) {
if (!this.mouseTail) {
return;
}
Expand All @@ -2113,23 +2176,30 @@ export class Replayer {
return;
}
const ctx = this.mouseTail.getContext('2d');
if (!ctx || !this.tailPositions.length) {
if (!ctx || !this.pointers[pointerId].tailPositions.length) {
return;
}
ctx.clearRect(0, 0, this.mouseTail.width, this.mouseTail.height);
ctx.beginPath();
ctx.lineWidth = lineWidth;
ctx.lineCap = lineCap;
ctx.strokeStyle = strokeStyle;
ctx.moveTo(this.tailPositions[0].x, this.tailPositions[0].y);
this.tailPositions.forEach((p) => ctx.lineTo(p.x, p.y));
ctx.moveTo(
this.pointers[pointerId].tailPositions[0].x,
this.pointers[pointerId].tailPositions[0].y,
);
this.pointers[pointerId].tailPositions.forEach((p) =>
ctx.lineTo(p.x, p.y),
);
ctx.stroke();
};

this.tailPositions.push(position);
this.pointers[pointerId].tailPositions.push(position);
draw();
setTimeout(() => {
this.tailPositions = this.tailPositions.filter((p) => p !== position);
this.pointers[pointerId].tailPositions = this.pointers[
pointerId
].tailPositions.filter((p) => p !== position);
draw();
}, duration / this.speedService.state.context.timer.speed);
}
Expand Down
Loading
Loading