From de0497a2dc8b4166f19ba702b2f0735cd0652c8d Mon Sep 17 00:00:00 2001 From: Louis Pontoise Date: Sat, 28 May 2022 15:30:19 +0900 Subject: [PATCH] feat: improve windows detection --- src/api-wrappers/AXUIElement.swift | 4 +- src/api-wrappers/CGWindow.swift | 2 + src/api-wrappers/PrivateApis.swift | 28 ++++++- src/logic/Application.swift | 94 +++++++++++----------- src/logic/Applications.swift | 16 ++-- src/logic/Spaces.swift | 7 +- src/logic/events/AccessibilityEvents.swift | 2 +- 7 files changed, 88 insertions(+), 65 deletions(-) diff --git a/src/api-wrappers/AXUIElement.swift b/src/api-wrappers/AXUIElement.swift index aff4e2da3..a54174b76 100644 --- a/src/api-wrappers/AXUIElement.swift +++ b/src/api-wrappers/AXUIElement.swift @@ -40,8 +40,6 @@ extension AXUIElement { AXUIElementSetMessagingTimeout(AXUIElementCreateSystemWide(), globalTimeoutInSeconds + 5) } - static let normalLevel = CGWindowLevelForKey(.normalWindow) - func axCallWhichCanThrow(_ result: AXError, _ successValue: inout T) throws -> T? { switch result { case .success: return successValue @@ -87,7 +85,7 @@ extension AXUIElement { size != nil && size!.width > 100 && size!.height > 100 && (books(runningApp) || keynote(runningApp) || iina(runningApp) || ( // CGWindowLevel == .normalWindow helps filter out iStats Pro and other top-level pop-overs, and floating windows - level == AXUIElement.normalLevel && + level == CGWindow.normalLevel && jetbrainApp(runningApp, title, subrole) && ([kAXStandardWindowSubrole, kAXDialogSubrole].contains(subrole) || openBoard(runningApp) || diff --git a/src/api-wrappers/CGWindow.swift b/src/api-wrappers/CGWindow.swift index 202443218..ff71eae51 100644 --- a/src/api-wrappers/CGWindow.swift +++ b/src/api-wrappers/CGWindow.swift @@ -3,6 +3,8 @@ import Cocoa typealias CGWindow = [CFString: Any] extension CGWindow { + static let normalLevel = CGWindowLevelForKey(.normalWindow) + static func windows(_ option: CGWindowListOption) -> [CGWindow] { return CGWindowListCopyWindowInfo([.excludeDesktopElements, option], kCGNullWindowID) as! [CGWindow] } diff --git a/src/api-wrappers/PrivateApis.swift b/src/api-wrappers/PrivateApis.swift index ae230e2a9..0dd509c34 100644 --- a/src/api-wrappers/PrivateApis.swift +++ b/src/api-wrappers/PrivateApis.swift @@ -110,11 +110,30 @@ func CGSGetConnectionPSN(_ cid: CGSConnectionID, _ psn: inout ProcessSerialNumbe @_silgen_name("CGSCopyManagedDisplaySpaces") func CGSCopyManagedDisplaySpaces(_ cid: CGSConnectionID) -> CFArray +struct CGSCopyWindowsOptions: OptionSet { + let rawValue: Int + static let minimized = CGSCopyWindowsOptions(rawValue: 1 << 0) + static let screenSaverLevel1000 = CGSCopyWindowsOptions(rawValue: 1 << 1) + static let minimized2 = CGSCopyWindowsOptions(rawValue: 1 << 2) + static let unknown1 = CGSCopyWindowsOptions(rawValue: 1 << 3) + static let unknown2 = CGSCopyWindowsOptions(rawValue: 1 << 4) + static let desktopIconWindowLevel2147483603 = CGSCopyWindowsOptions(rawValue: 1 << 5) +} + +struct CGSCopyWindowsTags: OptionSet { + let rawValue: Int + static let level0 = CGSCopyWindowsTags(rawValue: 1 << 0) + static let noTitleMaybePopups = CGSCopyWindowsTags(rawValue: 1 << 1) + static let unknown1 = CGSCopyWindowsTags(rawValue: 1 << 2) + static let mainMenuWindowAndDesktopIconWindow = CGSCopyWindowsTags(rawValue: 1 << 3) + static let unknown2 = CGSCopyWindowsTags(rawValue: 1 << 4) +} + // returns an array of window IDs (as UInt32) for the space(s) provided as `spaces` // the elements of the array are ordered by the z-index order of the windows in each space, with some exceptions where spaces mix // * macOS 10.10+ @_silgen_name("CGSCopyWindowsWithOptionsAndTags") -func CGSCopyWindowsWithOptionsAndTags(_ cid: CGSConnectionID, _ owner: UInt32, _ spaces: CFArray, _ options: UInt32, _ setTags: inout UInt64, _ clearTags: inout UInt64) -> CFArray +func CGSCopyWindowsWithOptionsAndTags(_ cid: CGSConnectionID, _ owner: Int, _ spaces: CFArray, _ options: Int, _ setTags: inout Int, _ clearTags: inout Int) -> CFArray // returns the current space ID on the provided display UUID // * macOS 10.10+ @@ -217,6 +236,13 @@ enum CGSSpaceType: Int { func CGSSpaceGetType(_ cid: CGSConnectionID, _ sid: CGSSpaceID) -> CGSSpaceType +// move a window to a Space; works with fullscreen windows +// with fullscreen window, sending it back to its original state later seems to mess with macOS internals. The Space appears fully black +// this API seems unreliable to use +// the last param seem to work with 0x80007; not sure what it means +// * macOS 10.10-12.2 +@_silgen_name("CGSSpaceAddWindowsAndRemoveFromSpaces") +func CGSSpaceAddWindowsAndRemoveFromSpaces(_ cid: CGSConnectionID, _ sid: CGSSpaceID, _ wid: NSArray, _ notSure: Int) -> Void // ------------------------------------------------------------ // below are some notes on some private APIs I experimented with diff --git a/src/logic/Application.swift b/src/logic/Application.swift index 8ab986d92..6d531943c 100644 --- a/src/logic/Application.swift +++ b/src/logic/Application.swift @@ -13,29 +13,27 @@ class Application: NSObject { var icon: NSImage? var dockLabel: String? var pid: pid_t! - var wasLaunchedBeforeAltTab = false var focusedWindow: Window? = nil - init(_ runningApplication: NSRunningApplication, _ wasLaunchedBeforeAltTab: Bool = false) { + init(_ runningApplication: NSRunningApplication) { self.runningApplication = runningApplication - self.wasLaunchedBeforeAltTab = wasLaunchedBeforeAltTab pid = runningApplication.processIdentifier super.init() isHidden = runningApplication.isHidden hasBeenActiveOnce = runningApplication.isActive icon = runningApplication.icon - addAndObserveWindows() + observeEventsIfEligible() kvObservers = [ runningApplication.observe(\.isFinishedLaunching, options: [.new]) { [weak self] _, _ in guard let self = self else { return } - self.addAndObserveWindows() + self.observeEventsIfEligible() }, runningApplication.observe(\.activationPolicy, options: [.new]) { [weak self] _, _ in guard let self = self else { return } if self.runningApplication.activationPolicy != .regular { self.removeWindowslessAppWindow() } - self.addAndObserveWindows() + self.observeEventsIfEligible() }, ] } @@ -51,7 +49,7 @@ class Application: NSObject { } } - func addAndObserveWindows() { + func observeEventsIfEligible() { if runningApplication.activationPolicy != .prohibited && axUiElement == nil { axUiElement = AXUIElementCreateApplication(pid) AXObserverCreate(pid, axObserverCallback, &axObserver) @@ -60,49 +58,46 @@ class Application: NSObject { } } - func observeNewWindows(_ group: DispatchGroup? = nil) { - if runningApplication.isFinishedLaunching && runningApplication.activationPolicy != .prohibited { - retryAxCallUntilTimeout(group, 5) { [weak self] in - guard let self = self else { return } - if let axWindows_ = try self.axUiElement!.windows(), axWindows_.count > 0 { - // bug in macOS: sometimes the OS returns multiple duplicate windows (e.g. Mail.app starting at login) - let axWindows = try Array(Set(axWindows_)).compactMap { - if let wid = try $0.cgWindowId() { - let title = try $0.title() - let subrole = try $0.subrole() - let role = try $0.role() - let size = try $0.size() - let level = try wid.level() - if AXUIElement.isActualWindow(self.runningApplication, wid, level, title, subrole, role, size) { - return ($0, wid, title, try $0.isFullscreen(), try $0.isMinimized(), try $0.position(), size) - } + func manuallyUpdateWindows(_ group: DispatchGroup? = nil) { + // TODO: this method manually checks windows, but will not find windows on other Spaces + retryAxCallUntilTimeout(group, 5) { [weak self] in + guard let self = self else { return } + if let axWindows_ = try self.axUiElement!.windows(), axWindows_.count > 0 { + // bug in macOS: sometimes the OS returns multiple duplicate windows (e.g. Mail.app starting at login) + let axWindows = try Array(Set(axWindows_)).compactMap { + if let wid = try $0.cgWindowId() { + let title = try $0.title() + let subrole = try $0.subrole() + let role = try $0.role() + let size = try $0.size() + let level = try wid.level() + if AXUIElement.isActualWindow(self.runningApplication, wid, level, title, subrole, role, size) { + return ($0, wid, title, try $0.isFullscreen(), try $0.isMinimized(), try $0.position(), size) } - return nil - } as [(AXUIElement, CGWindowID, String?, Bool, Bool, CGPoint?, CGSize?)] - DispatchQueue.main.async { [weak self] in - guard let self = self else { return } - var windows = self.addWindows(axWindows) - if let window = self.addWindowslessAppsIfNeeded() { - windows.append(contentsOf: window) - } - App.app.refreshOpenUi(windows) } - } else { - DispatchQueue.main.async { [weak self] in - guard let self = self else { return } - let window = self.addWindowslessAppsIfNeeded() - App.app.refreshOpenUi(window) - } - if group == nil && !self.wasLaunchedBeforeAltTab && ( - // workaround: opening an app while the active app is fullscreen; we wait out the space transition animation - CGSSpaceGetType(cgsMainConnectionId, Spaces.currentSpaceId) == .fullscreen || - // workaround: some apps launch but have no window ready instantly. It's very unlikely an app would launch with no window - // so we retry until timeout, in those rare cases (e.g. Bear.app) - // we only do this for active app, to avoid wasting CPU, with the trade-off of maybe missing some windows - self.runningApplication.isActive - ) { - throw AxError.runtimeError + return nil + } as [(AXUIElement, CGWindowID, String?, Bool, Bool, CGPoint?, CGSize?)] + DispatchQueue.main.async { [weak self] in + guard let self = self else { return } + var windows = self.addWindows(axWindows) + if let window = self.addWindowslessAppsIfNeeded() { + windows.append(contentsOf: window) } + App.app.refreshOpenUi(windows) + } + } else { + DispatchQueue.main.async { [weak self] in + guard let self = self else { return } + let window = self.addWindowslessAppsIfNeeded() + App.app.refreshOpenUi(window) + } + // workaround: some apps launch but take a while to create their window(s) + // initial windows don't trigger a windowCreated notification, so we won't get notified + // it's very unlikely an app would launch with no initial window + // so we retry until timeout, in those rare cases (e.g. Bear.app) + // we only do this for active app, to avoid wasting CPU, with the trade-off of maybe missing some windows + if self.runningApplication.isActive { + throw AxError.runtimeError } } } @@ -151,10 +146,11 @@ class Application: NSObject { DispatchQueue.main.async { [weak self] in guard let self = self else { return } // some apps have `isFinishedLaunching == true` but are actually not finished, and will return .cannotComplete - // we consider them ready when the first subscription succeeds, and list their windows again at that point + // we consider them ready when the first subscription succeeds + // windows opened before that point won't send a notification, so check those windows manually here if !self.isReallyFinishedLaunching { self.isReallyFinishedLaunching = true - self.observeNewWindows() + self.manuallyUpdateWindows() } } }, self.runningApplication) diff --git a/src/logic/Applications.swift b/src/logic/Applications.swift index 0901ce853..f5b70a1bb 100644 --- a/src/logic/Applications.swift +++ b/src/logic/Applications.swift @@ -4,12 +4,12 @@ import ApplicationServices class Applications { static var list = [Application]() - static func observeNewWindowsBlocking() { + static func manuallyUpdateWindowsFor2s() { let group = DispatchGroup() for app in list { - app.wasLaunchedBeforeAltTab = true - guard app.runningApplication.isFinishedLaunching else { continue } - app.observeNewWindows(group) + if app.runningApplication.isFinishedLaunching && app.runningApplication.activationPolicy != .prohibited { + app.manuallyUpdateWindows(group) + } } _ = group.wait(wallTimeout: .now() + .seconds(2)) } @@ -21,7 +21,7 @@ class Applications { } static func addInitialRunningApplications() { - addRunningApplications(NSWorkspace.shared.runningApplications, true) + addRunningApplications(NSWorkspace.shared.runningApplications) } static func addInitialRunningApplicationsWindows() { @@ -33,16 +33,16 @@ class Applications { if windowsOnlyOnOtherSpaces.count > 0 { // on initial launch, we use private APIs to bring windows from other spaces into the current space, observe them, then remove them from the current space CGSAddWindowsToSpaces(cgsMainConnectionId, windowsOnlyOnOtherSpaces as NSArray, [Spaces.currentSpaceId]) - Applications.observeNewWindowsBlocking() + Applications.manuallyUpdateWindowsFor2s() CGSRemoveWindowsFromSpaces(cgsMainConnectionId, windowsOnlyOnOtherSpaces as NSArray, [Spaces.currentSpaceId]) } } } - static func addRunningApplications(_ runningApps: [NSRunningApplication], _ wasLaunchedBeforeAltTab: Bool = false) { + static func addRunningApplications(_ runningApps: [NSRunningApplication]) { runningApps.forEach { if isActualApplication($0) { - Applications.list.append(Application($0, wasLaunchedBeforeAltTab)) + Applications.list.append(Application($0)) } } } diff --git a/src/logic/Spaces.swift b/src/logic/Spaces.swift index 9b616d852..bb1c8aae3 100644 --- a/src/logic/Spaces.swift +++ b/src/logic/Spaces.swift @@ -68,9 +68,10 @@ class Spaces { } static func windowsInSpaces(_ spaceIds: [CGSSpaceID]) -> [CGWindowID] { - var set_tags = UInt64(0) - var clear_tags = UInt64(0) - return CGSCopyWindowsWithOptionsAndTags(cgsMainConnectionId, 0, spaceIds as CFArray, 2, &set_tags, &clear_tags) as! [CGWindowID] + let options = ([.minimized] as CGSCopyWindowsOptions).rawValue + var set_tags = ([] as CGSCopyWindowsTags).rawValue + var clear_tags = ([] as CGSCopyWindowsTags).rawValue + return CGSCopyWindowsWithOptionsAndTags(cgsMainConnectionId, 0, spaceIds as CFArray, options, &set_tags, &clear_tags) as! [CGWindowID] } static func isSingleSpace() -> Bool { diff --git a/src/logic/events/AccessibilityEvents.swift b/src/logic/events/AccessibilityEvents.swift index dc09a70bb..0e796a3de 100644 --- a/src/logic/events/AccessibilityEvents.swift +++ b/src/logic/events/AccessibilityEvents.swift @@ -114,7 +114,7 @@ fileprivate func focusedWindowChanged(_ element: AXUIElement, _ pid: pid_t) thro // these apps report isHidden=false, don't generate windowCreated events initially, and have a delay before their windows are created // our only recourse is to manually check their windows once they emit if (!app.hasBeenActiveOnce) { - app.observeNewWindows() + app.manuallyUpdateWindows() } } }