From 3bf3efebb6ae3b72ef4903bca388ea26005836d8 Mon Sep 17 00:00:00 2001 From: Marcin Cichocki Date: Sun, 22 Oct 2023 10:58:56 +0200 Subject: [PATCH] fix(common): move cursor to the bottom right corner via nircmd fixes #374 --- src/common/node/robot/nircmd.ts | 28 +++++++++---- src/common/node/robot/robot.ts | 5 +-- src/core/ocr/bouding-box.test.ts | 64 ++++++++++++++++++++++++++++++ src/core/ocr/bounding-box.ts | 34 ++++++++++++++++ src/core/ocr/fragments/base.ts | 4 +- src/core/ocr/image-container.ts | 29 -------------- src/core/ocr/index.ts | 1 + src/core/ocr/ocr.test.ts | 67 -------------------------------- src/electron/worker/worker.ts | 9 +++-- 9 files changed, 129 insertions(+), 112 deletions(-) create mode 100644 src/core/ocr/bouding-box.test.ts create mode 100644 src/core/ocr/bounding-box.ts diff --git a/src/common/node/robot/nircmd.ts b/src/common/node/robot/nircmd.ts index d664b512..f8c65a49 100644 --- a/src/common/node/robot/nircmd.ts +++ b/src/common/node/robot/nircmd.ts @@ -1,5 +1,5 @@ import { sleep } from '@/common'; -import { BreachProtocolRobotKeys } from './robot'; +import { BreachProtocolRobotKeys, RobotSettings } from './robot'; import { WindowsRobot } from './win32'; export class NirCmdRobot extends WindowsRobot { @@ -8,6 +8,15 @@ export class NirCmdRobot extends WindowsRobot { protected readonly binPath = './resources/win32/nircmd/nircmd.exe'; + constructor( + settings: RobotSettings, + private readonly scaling: number, + private readonly width: number, + private readonly height: number + ) { + super(settings); + } + async activateGameWindow() { await this.bin(`win activate stitle ${this.gameWindowTitle}`); // Wait extra time as nircmd will not wait for window to be actually @@ -25,9 +34,8 @@ export class NirCmdRobot extends WindowsRobot { await this.moveAway(); } - const scaling = this.settings.useScaling ? this.scaling : 1; - const sX = (x - this.x) / scaling; - const sY = (y - this.y) / scaling; + const sX = x - this.x; + const sY = y - this.y; const r = await this.moveRelative(sX, sY); this.x = x; @@ -37,10 +45,10 @@ export class NirCmdRobot extends WindowsRobot { } moveAway() { - this.x = 0; - this.y = 0; + this.x = this.width; + this.y = this.height; - return this.moveRelative(-9999, -9999); + return this.moveRelative(this.width, this.height); } pressKey(key: BreachProtocolRobotKeys) { @@ -48,6 +56,10 @@ export class NirCmdRobot extends WindowsRobot { } private moveRelative(x: number, y: number) { - return this.bin(`sendmouse move ${x} ${y}`); + return this.bin(`sendmouse move ${this.scale(x)} ${this.scale(y)}`); + } + + private scale(value: number) { + return Math.round(value / this.scaling); } } diff --git a/src/common/node/robot/robot.ts b/src/common/node/robot/robot.ts index 4c028429..eb180563 100644 --- a/src/common/node/robot/robot.ts +++ b/src/common/node/robot/robot.ts @@ -43,10 +43,7 @@ export abstract class BreachProtocolRobot { protected gameWindowTitle = 'Cyberpunk 2077'; - constructor( - protected readonly settings: RobotSettings, - protected readonly scaling: number = 1 - ) {} + constructor(protected readonly settings: RobotSettings) {} protected abstract getMappedKey(key: BreachProtocolRobotKeys): string; diff --git a/src/core/ocr/bouding-box.test.ts b/src/core/ocr/bouding-box.test.ts new file mode 100644 index 00000000..e0254c63 --- /dev/null +++ b/src/core/ocr/bouding-box.test.ts @@ -0,0 +1,64 @@ +import { BoudingBox } from './bounding-box'; + +describe('BoudingBox', () => { + const horizontal = [ + [1024, 768], + [1024, 1280], + [1152, 864], + [1280, 768], + [1280, 800], + [1280, 960], + [1280, 1024], + [1600, 1024], + [1600, 1200], + [1680, 1050], + [1920, 1200], + [1920, 1440], + ]; + + const vertical = [ + [2560, 1080], + [3440, 1440], + [3840, 1080], + ]; + + const regular = [ + [1280, 720], + [1360, 768], + [1366, 768], + [1600, 900], + [1920, 1080], + [2560, 1440], + [3840, 2160], + ]; + + it.each(horizontal)('should crop horizontal black bars(%ix%i)', (x, y) => { + const box = new BoudingBox(x, y); + + expect(box.aspectRatio).toBe(BoudingBox.ASPECT_RATIO); + expect(box.width).toBe(x); + expect(box.height).toBe(y - 2 * box.top); + expect(box.left).toBe(0); + expect(box.top).toBe((y - box.height) / 2); + }); + + it.each(vertical)('should crop vertical black bars(%ix%i)', (x, y) => { + const box = new BoudingBox(x, y); + + expect(box.aspectRatio).toBe(BoudingBox.ASPECT_RATIO); + expect(box.width).toBe(x - 2 * box.left); + expect(box.height).toBe(y); + expect(box.left).toBe((x - box.width) / 2); + expect(box.top).toBe(0); + }); + + it.each(regular)('should not crop 16:9 resolutions(%ix%i)', (x, y) => { + const box = new BoudingBox(x, y); + + expect(box.aspectRatio).toBe(BoudingBox.ASPECT_RATIO); + expect(box.width).toBe(x); + expect(box.height).toBe(y); + expect(box.left).toBe(0); + expect(box.top).toBe(0); + }); +}); diff --git a/src/core/ocr/bounding-box.ts b/src/core/ocr/bounding-box.ts new file mode 100644 index 00000000..899a7333 --- /dev/null +++ b/src/core/ocr/bounding-box.ts @@ -0,0 +1,34 @@ +export class BoudingBox { + /** Expected aspect ratio of breach protocol. */ + static readonly ASPECT_RATIO = 16 / 9; + + private readonly ratio = + this.getAspectRatio(this.x, this.y) / BoudingBox.ASPECT_RATIO; + + /** Width of the breach protocol. */ + readonly width = this.ratio > 1 ? this.y * BoudingBox.ASPECT_RATIO : this.x; + + /** Height of the breach protocol. */ + readonly height = this.ratio < 1 ? this.x / BoudingBox.ASPECT_RATIO : this.y; + + /** Distance in pixels from left edge to breach protocol. */ + readonly left = (this.x - this.width) / 2; + + /** Distance in pixels from top edge to breach protocol. */ + readonly top = (this.y - this.height) / 2; + + /** Aspect ratio of breach protocol. */ + readonly aspectRatio = this.getAspectRatio(this.width, this.height); + + constructor(public readonly x: number, public readonly y: number) {} + + private getAspectRatio(x: number, y: number) { + // WXGA, very close to 16:9 + // https://en.wikipedia.org/wiki/Graphics_display_resolution#WXGA + if (y === 768 && (x === 1366 || x === 1360)) { + return BoudingBox.ASPECT_RATIO; + } + + return x / y; + } +} diff --git a/src/core/ocr/fragments/base.ts b/src/core/ocr/fragments/base.ts index 5ef55bbd..0f114119 100644 --- a/src/core/ocr/fragments/base.ts +++ b/src/core/ocr/fragments/base.ts @@ -1,5 +1,6 @@ import { chunk, getClosest, Point, unique } from '@/common'; import { HexCode, HEX_CODES } from '../../common'; +import { BoudingBox } from '../bounding-box'; import { FragmentContainer, ImageContainer } from '../image-container'; import { BreachProtocolRecognizer, @@ -102,7 +103,8 @@ export abstract class BreachProtocolFragment< protected getFragmentBoundingBox() { const { p1, p2 } = this; - const { width, height, left, top } = this.container.getCroppedBoundingBox(); + const { width: x, height: y } = this.container.dimensions; + const { width, height, left, top } = new BoudingBox(x, y); return { left: Math.round(p1.x * width + left), diff --git a/src/core/ocr/image-container.ts b/src/core/ocr/image-container.ts index cbf9914c..67a24c90 100644 --- a/src/core/ocr/image-container.ts +++ b/src/core/ocr/image-container.ts @@ -38,9 +38,6 @@ export interface FragmentContainer { } export abstract class ImageContainer { - /** Aspect ratio of breach protocol. */ - static readonly ASPECT_RATIO = 16 / 9; - abstract readonly instance: T; abstract readonly dimensions: Dimensions; @@ -48,30 +45,4 @@ export abstract class ImageContainer { abstract toFragmentContainer( config: FragmentContainerConfig ): FragmentContainer; - - /** Return aspect ratio for given resolution and handle edge cases. */ - getAspectRatio(x: number, y: number) { - // WXGA, very close to 16:9 - // https://en.wikipedia.org/wiki/Graphics_display_resolution#WXGA - if (y === 768 && (x === 1366 || x === 1360)) { - return ImageContainer.ASPECT_RATIO; - } - - return x / y; - } - - getCroppedBoundingBox() { - const { width: x, height: y } = this.dimensions; - // Resolution with ratio less than one have horizontal black - // bars, and ratio greater than one have vertical. - // Resolutions with ratio equal to 1 are in 16:9 aspect ratio - // and do not require cropping. - const ratio = this.getAspectRatio(x, y) / ImageContainer.ASPECT_RATIO; - const width = ratio > 1 ? y * ImageContainer.ASPECT_RATIO : x; - const height = ratio < 1 ? x / ImageContainer.ASPECT_RATIO : y; - const left = (x - width) / 2; - const top = (y - height) / 2; - - return { width, height, left, top }; - } } diff --git a/src/core/ocr/index.ts b/src/core/ocr/index.ts index 3364bf2c..cd615504 100644 --- a/src/core/ocr/index.ts +++ b/src/core/ocr/index.ts @@ -3,3 +3,4 @@ export * from './image-container'; export * from './ocr'; export * from './recognizer'; export * from './result'; +export { BoudingBox } from './bounding-box'; diff --git a/src/core/ocr/ocr.test.ts b/src/core/ocr/ocr.test.ts index dfd13bcd..3b6a78fb 100644 --- a/src/core/ocr/ocr.test.ts +++ b/src/core/ocr/ocr.test.ts @@ -42,73 +42,6 @@ type Resolution = | '3440x1440' | '3840x2160'; -describe('image container', () => { - const aspectRatio = ImageContainer.ASPECT_RATIO; - const horizontal = [ - [1024, 768], - [1024, 1280], - [1152, 864], - [1280, 768], - [1280, 800], - [1280, 960], - [1280, 1024], - [1600, 1024], - [1600, 1200], - [1680, 1050], - [1920, 1200], - [1920, 1440], - ]; - - const vertical = [ - [2560, 1080], - [3440, 1440], - [3840, 1080], - ]; - - const regular = [ - [1280, 720], - [1360, 768], - [1366, 768], - [1600, 900], - [1920, 1080], - [2560, 1440], - [3840, 2160], - ]; - - it.each(horizontal)('should crop horizontal black bars(%ix%i)', (x, y) => { - const container = new NoopImageContainer({ width: x, height: y }); - const { width, height, left, top } = container.getCroppedBoundingBox(); - - expect(container.getAspectRatio(width, height)).toBe(aspectRatio); - expect(width).toBe(x); - expect(height).toBe(y - 2 * top); - expect(left).toBe(0); - expect(top).toBe((y - height) / 2); - }); - - it.each(vertical)('should crop vertical black bars(%ix%i)', (x, y) => { - const container = new NoopImageContainer({ width: x, height: y }); - const { width, height, left, top } = container.getCroppedBoundingBox(); - - expect(container.getAspectRatio(width, height)).toBe(aspectRatio); - expect(width).toBe(x - 2 * left); - expect(height).toBe(y); - expect(left).toBe((x - width) / 2); - expect(top).toBe(0); - }); - - it.each(regular)('should not crop 16:9 resolutions(%ix%i)', (x, y) => { - const container = new NoopImageContainer({ width: x, height: y }); - const { width, height, left, top } = container.getCroppedBoundingBox(); - - expect(container.getAspectRatio(width, height)).toBe(aspectRatio); - expect(width).toBe(x); - expect(height).toBe(y); - expect(left).toBe(0); - expect(top).toBe(0); - }); -}); - describe('raw data validation', () => { let recognizer: TestBreachProtocolRecognizer; let container: NoopImageContainer; diff --git a/src/electron/worker/worker.ts b/src/electron/worker/worker.ts index 0a1b94e1..33be4334 100644 --- a/src/electron/worker/worker.ts +++ b/src/electron/worker/worker.ts @@ -7,6 +7,7 @@ import { XDoToolRobot, } from '@/common/node'; import { + BoudingBox, BreachProtocolBufferSizeFragment, BreachProtocolDaemonsFragment, BreachProtocolFragmentOptions, @@ -310,12 +311,14 @@ export class BreachProtocolWorker { case 'ahk': return new AutoHotkeyRobot(this.settings); case 'nircmd': - const { activeDisplayId } = this.settings; - const { dpiScale } = this.displays.find( + const { activeDisplayId, useScaling } = this.settings; + const { dpiScale, width, height } = this.displays.find( (d) => d.id === activeDisplayId ); + const scaling = useScaling ? dpiScale : 1; + const box = new BoudingBox(width, height); - return new NirCmdRobot(this.settings, dpiScale); + return new NirCmdRobot(this.settings, scaling, box.width, box.height); default: throw new Error(`Invalid engine "${this.settings.engine}" selected!`); }