Window decorations: improved caption hit testing to better support Ta…

…bbedPane, SplitPane and ToolBar in title bar area (e.g. for fullWindowContent mode)

flatlaf-core/src/main/java/com/formdev/flatlaf/FlatClientProperties.java

@@ -257,18 +257,39 @@ public interface FlatClientProperties
 	String COMPONENT_FOCUS_OWNER = "JComponent.focusOwner";
 
 	/**
-	 * Specifies whether a component in an embedded menu bar should behave as caption
+	 * Specifies whether a component shown in a window title bar area should behave as caption
 	 * (left-click allows moving window, right-click shows window system menu).
-	 * The component does not receive mouse pressed/released/clicked/dragged events,
+	 * The caption component does not receive mouse pressed/released/clicked/dragged events,
 	 * but it gets mouse entered/exited/moved events.
 	 * <p>
+	 * Since 3.4, this client property also supports using a function that can check
+	 * whether a given location in the component should behave as caption.
+	 * Useful for components that do not use mouse input on whole component bounds.
+	 *
+	 * <pre>{@code
+	 * myComponent.putClientProperty( "JComponent.titleBarCaption",
+	 *     (Function<Point, Boolean>) pt -> {
+	 *         // parameter pt contains mouse location (in myComponent coordinates)
+	 *         // return true if the component is not interested in mouse input at the given location
+	 *         // return false if the component wants process mouse input at the given location
+	 *         // return null if the component children should be checked
+	 *         return ...; // check here
+	 *     } );
+	 * }</pre>
+	 * <b>Warning</b>:
+	 * <ul>
+	 *   <li>This function is invoked often when mouse is moved over window title bar area
+	 *       and should therefore return quickly.
+	 *   <li>This function is invoked on 'AWT-Windows' thread (not 'AWT-EventQueue' thread)
+	 *       while processing Windows messages.
+	 *       It <b>must not</b> change any component property or layout because this could cause a dead lock.
+	 * </ul>
+	 * <p>
 	 * <strong>Component</strong> {@link javax.swing.JComponent}<br>
-	 * <strong>Value type</strong> {@link java.lang.Boolean}
+	 * <strong>Value type</strong> {@link java.lang.Boolean} or {@link java.util.function.Function}&lt;Point, Boolean&gt;
 	 *
 	 * @since 2.5
-	 * @deprecated No longer used since FlatLaf 3.4. Retained for API compatibility.
 	 */
-	@Deprecated
 	String COMPONENT_TITLE_BAR_CAPTION = "JComponent.titleBarCaption";
 
 

flatlaf-core/src/main/java/com/formdev/flatlaf/ui/FlatNativeWindowBorder.java

@@ -219,13 +219,13 @@ public static void setHasCustomDecoration( Window window, boolean hasCustomDecor
 	}
 
 	static void setTitleBarHeightAndHitTestSpots( Window window, int titleBarHeight,
-		Predicate<Point> hitTestCallback, Rectangle appIconBounds, Rectangle minimizeButtonBounds,
+		Predicate<Point> captionHitTestCallback, Rectangle appIconBounds, Rectangle minimizeButtonBounds,
 		Rectangle maximizeButtonBounds, Rectangle closeButtonBounds )
 	{
 		if( !isSupported() )
 			return;
 
-		nativeProvider.updateTitleBarInfo( window, titleBarHeight, hitTestCallback,
+		nativeProvider.updateTitleBarInfo( window, titleBarHeight, captionHitTestCallback,
 			appIconBounds, minimizeButtonBounds, maximizeButtonBounds, closeButtonBounds );
 	}
 
@@ -271,7 +271,7 @@ public interface Provider
 	{
 		boolean hasCustomDecoration( Window window );
 		void setHasCustomDecoration( Window window, boolean hasCustomDecoration );
-		void updateTitleBarInfo( Window window, int titleBarHeight, Predicate<Point> hitTestCallback,
+		void updateTitleBarInfo( Window window, int titleBarHeight, Predicate<Point> captionHitTestCallback,
 			Rectangle appIconBounds, Rectangle minimizeButtonBounds, Rectangle maximizeButtonBounds,
 			Rectangle closeButtonBounds );
 

flatlaf-core/src/main/java/com/formdev/flatlaf/ui/FlatSplitPaneUI.java

@@ -84,7 +84,7 @@
  */
 public class FlatSplitPaneUI
 	extends BasicSplitPaneUI
-	implements StyleableUI
+	implements StyleableUI, FlatTitlePane.TitleBarCaptionHitTest
 {
 	@Styleable protected String arrowType;
 	/** @since 3.3 */ @Styleable protected Color draggingColor;
@@ -227,6 +227,15 @@ private void paintDragDivider( Graphics g, int dividerLocation ) {
 			((FlatSplitPaneDivider)divider).paintStyle( g, x, y, width, height );
 	}
 
+	//---- interface FlatTitlePane.TitleBarCaptionHitTest ----
+
+	/** @since 3.4 */
+	@Override
+	public Boolean isTitleBarCaptionAt( int x, int y ) {
+		// necessary because BasicSplitPaneDivider adds some mouse listeners for dragging divider
+		return null; // check children
+	}
+
 	//---- class FlatSplitPaneDivider -----------------------------------------
 
 	protected class FlatSplitPaneDivider

flatlaf-core/src/main/java/com/formdev/flatlaf/ui/FlatTabbedPaneUI.java

@@ -182,7 +182,7 @@
  */
 public class FlatTabbedPaneUI
 	extends BasicTabbedPaneUI
-	implements StyleableUI
+	implements StyleableUI, FlatTitlePane.TitleBarCaptionHitTest
 {
 	// tab type
 	/** @since 2 */ protected static final int TAB_TYPE_UNDERLINED = 0;
@@ -2300,6 +2300,17 @@ private int rectsTotalHeight() {
 		return (rects[last].y + rects[last].height) - rects[0].y;
 	}
 
+	//---- interface FlatTitlePane.TitleBarCaptionHitTest ----
+
+	/** @since 3.4 */
+	@Override
+	public Boolean isTitleBarCaptionAt( int x, int y ) {
+		if( tabForCoordinate( tabPane, x, y ) >= 0 )
+			return false;
+
+		return null; // check children
+	}
+
 	//---- class TabCloseButton -----------------------------------------------
 
 	private static class TabCloseButton

flatlaf-core/src/main/java/com/formdev/flatlaf/ui/FlatTitlePane.java

@@ -49,6 +49,7 @@
 import java.beans.PropertyChangeListener;
 import java.util.List;
 import java.util.Objects;
+import java.util.function.Function;
 import javax.accessibility.AccessibleContext;
 import javax.swing.BorderFactory;
 import javax.swing.Box;
@@ -65,6 +66,7 @@
 import javax.swing.UIManager;
 import javax.swing.border.AbstractBorder;
 import javax.swing.border.Border;
+import javax.swing.plaf.ComponentUI;
 import com.formdev.flatlaf.FlatClientProperties;
 import com.formdev.flatlaf.FlatSystemProperties;
 import com.formdev.flatlaf.ui.FlatNativeWindowBorder.WindowTopBorder;
@@ -314,7 +316,7 @@ public void layoutContainer( Container target ) {
 				}
 
 				// clear hit-test cache
-				lastHitTestTime = 0;
+				lastCaptionHitTestTime = 0;
 			}
 		} );
 
@@ -1004,10 +1006,10 @@ protected void updateNativeTitleBarHeightAndHitTestSpots() {
 		Rectangle closeButtonBounds = boundsInWindow( closeButton );
 
 		// clear hit-test cache
-		lastHitTestTime = 0;
+		lastCaptionHitTestTime = 0;
 
 		FlatNativeWindowBorder.setTitleBarHeightAndHitTestSpots( window, titleBarHeight,
-			this::hitTest, appIconBounds, minimizeButtonBounds, maximizeButtonBounds, closeButtonBounds );
+			this::captionHitTest, appIconBounds, minimizeButtonBounds, maximizeButtonBounds, closeButtonBounds );
 
 		debugTitleBarHeight = titleBarHeight;
 		debugAppIconBounds = appIconBounds;
@@ -1024,18 +1026,8 @@ private Rectangle boundsInWindow( JComponent c ) {
 			: null;
 	}
 
-	protected Rectangle getNativeHitTestSpot( JComponent c ) {
-		Dimension size = c.getSize();
-		if( size.width <= 0 || size.height <= 0 )
-			return null;
-
-		Point location = SwingUtilities.convertPoint( c, 0, 0, window );
-		Rectangle r = new Rectangle( location, size );
-		return r;
-	}
-
 	/**
-	 * Returns wheter there is a component at the given location, that processes
+	 * Returns whether there is a component at the given location, that processes
 	 * mouse events. E.g. buttons, menus, etc.
 	 * <p>
 	 * Note:
@@ -1046,12 +1038,12 @@ protected Rectangle getNativeHitTestSpot( JComponent c ) {
 	 *       while processing Windows messages.
 	 * </ul>
 	 */
-	private boolean hitTest( Point pt ) {
+	private boolean captionHitTest( Point pt ) {
 		// Windows invokes this method every ~200ms, even if the mouse has not moved
 		long time = System.currentTimeMillis();
-		if( pt.x == lastHitTestX && pt.y == lastHitTestY && time < lastHitTestTime + 300 ) {
-			lastHitTestTime = time;
-			return lastHitTestResult;
+		if( pt.x == lastCaptionHitTestX && pt.y == lastCaptionHitTestY && time < lastCaptionHitTestTime + 300 ) {
+			lastCaptionHitTestTime = time;
+			return lastCaptionHitTestResult;
 		}
 
 		// convert pt from window coordinates to layeredPane coordinates
@@ -1063,35 +1055,70 @@ private boolean hitTest( Point pt ) {
 			y -= c.getY();
 		}
 
-		lastHitTestX = pt.x;
-		lastHitTestY = pt.y;
-		lastHitTestTime = time;
-		lastHitTestResult = isComponentWithMouseListenerAt( layeredPane, x, y );
-		return lastHitTestResult;
+		lastCaptionHitTestX = pt.x;
+		lastCaptionHitTestY = pt.y;
+		lastCaptionHitTestTime = time;
+		lastCaptionHitTestResult = isTitleBarCaptionAt( layeredPane, x, y );
+		return lastCaptionHitTestResult;
 	}
 
-	private boolean isComponentWithMouseListenerAt( Component c, int x, int y ) {
+	private boolean isTitleBarCaptionAt( Component c, int x, int y ) {
 		if( !c.isDisplayable() || !c.isVisible() || !c.contains( x, y ) || c == mouseLayer )
-			return false;
+			return true; // continue checking with next component
 
-		if( c.getMouseListeners().length > 0 ||
-			c.getMouseMotionListeners().length > 0 ||
-			c.getMouseWheelListeners().length > 0 )
-		  return true;
+		if( c.isEnabled() &&
+			(c.getMouseListeners().length > 0 ||
+			 c.getMouseMotionListeners().length > 0) )
+		{
+			if( !(c instanceof JComponent) )
+				return false; // assume that this is not a caption because the component has mouse listeners
+
+			// check client property boolean value
+			Object caption = ((JComponent)c).getClientProperty( COMPONENT_TITLE_BAR_CAPTION );
+			if( caption instanceof Boolean )
+				return (boolean) caption;
+
+			// if component is not fully layouted, do not invoke function
+			// because it is too dangerous that the function tries to layout the component,
+			// which could cause a dead lock
+			if( !c.isValid() )
+				return false; // assume that this is not a caption because the component has mouse listeners
+
+			if( caption instanceof Function ) {
+				// check client property function value
+				@SuppressWarnings( "unchecked" )
+				Function<Point, Boolean> hitTest = (Function<Point, Boolean>) caption;
+				Boolean result = hitTest.apply( new Point( x, y ) );
+				if( result != null )
+					return result;
+			} else {
+				// check component UI
+				ComponentUI ui = JavaCompatibility2.getUI( (JComponent) c );
+				if( !(ui instanceof TitleBarCaptionHitTest) )
+					return false; // assume that this is not a caption because the component has mouse listeners
+
+				Boolean result = ((TitleBarCaptionHitTest)ui).isTitleBarCaptionAt( x, y );
+				if( result != null )
+					return result;
+			}
 
+			// else continue checking children
+		}
+
+		// check children
 		if( c instanceof Container ) {
 			for( Component child : ((Container)c).getComponents() ) {
-				if( isComponentWithMouseListenerAt( child, x - child.getX(), y - child.getY() ) )
-					return true;
+				if( !isTitleBarCaptionAt( child, x - child.getX(), y - child.getY() ) )
+					return false;
 			}
 		}
-		return false;
+		return true;
 	}
 
-	private int lastHitTestX;
-	private int lastHitTestY;
-	private long lastHitTestTime;
-	private boolean lastHitTestResult;
+	private int lastCaptionHitTestX;
+	private int lastCaptionHitTestY;
+	private long lastCaptionHitTestTime;
+	private boolean lastCaptionHitTestResult;
 
 	private int debugTitleBarHeight;
 	private Rectangle debugAppIconBounds;
@@ -1490,4 +1517,27 @@ public void componentShown( ComponentEvent e ) {
 		@Override public void componentMoved( ComponentEvent e ) {}
 		@Override public void componentHidden( ComponentEvent e ) {}
 	}
+
+	//---- interface TitleBarCaptionHitTest -----------------------------------
+
+	/**
+	 * For custom components use {@link FlatClientProperties#COMPONENT_TITLE_BAR_CAPTION}
+	 * instead of this interface.
+	 *
+	 * @since 3.4
+	 */
+	public interface TitleBarCaptionHitTest {
+		/**
+		 * Invoked for a component that is enabled and has mouse listeners,
+		 * to check whether it processes mouse input at the given x/y location.
+		 * Useful for components that do not use mouse input on whole component bounds.
+		 * E.g. a tabbed pane with a few tabs has some empty space beside the tabs
+		 * that can be used to move the window.
+		 *
+		 * @return {@code true} if the component is not interested in mouse input at the given location
+		 *         {@code false} if the component wants process mouse input at the given location
+		 *         {@code null} if the component children should be checked
+		 */
+		Boolean isTitleBarCaptionAt( int x, int y );
+	}
 }

flatlaf-core/src/main/java/com/formdev/flatlaf/ui/FlatToolBarUI.java

@@ -82,7 +82,7 @@
  */
 public class FlatToolBarUI
 	extends BasicToolBarUI
-	implements StyleableUI
+	implements StyleableUI, FlatTitlePane.TitleBarCaptionHitTest
 {
 	/** @since 1.4 */ @Styleable protected boolean focusableButtons;
 	/** @since 2 */ @Styleable protected boolean arrowKeysOnlyNavigation;
@@ -453,6 +453,15 @@ private ButtonGroup getButtonGroup( AbstractButton b ) {
 			: null;
 	}
 
+	//---- interface FlatTitlePane.TitleBarCaptionHitTest ----
+
+	/** @since 3.4 */
+	@Override
+	public Boolean isTitleBarCaptionAt( int x, int y ) {
+		// necessary because BasicToolBarUI adds some mouse listeners for dragging when toolbar is floatable
+		return null; // check children
+	}
+
 	//---- class FlatToolBarFocusTraversalPolicy ------------------------------
 
 	/**

flatlaf-core/src/main/java/com/formdev/flatlaf/ui/FlatWindowsNativeWindowBorder.java

@@ -159,7 +159,7 @@ private void uninstall( Window window ) {
 	}
 
 	@Override
-	public void updateTitleBarInfo( Window window, int titleBarHeight, Predicate<Point> hitTestCallback,
+	public void updateTitleBarInfo( Window window, int titleBarHeight, Predicate<Point> captionHitTestCallback,
 		Rectangle appIconBounds, Rectangle minimizeButtonBounds, Rectangle maximizeButtonBounds,
 		Rectangle closeButtonBounds )
 	{
@@ -168,7 +168,7 @@ public void updateTitleBarInfo( Window window, int titleBarHeight, Predicate<Poi
 			return;
 
 		wndProc.titleBarHeight = titleBarHeight;
-		wndProc.hitTestCallback = hitTestCallback;
+		wndProc.captionHitTestCallback = captionHitTestCallback;
 		wndProc.appIconBounds = cloneRectange( appIconBounds );
 		wndProc.minimizeButtonBounds = cloneRectange( minimizeButtonBounds );
 		wndProc.maximizeButtonBounds = cloneRectange( maximizeButtonBounds );
@@ -289,7 +289,7 @@ private class WndProc
 
 		// Swing coordinates/values may be scaled on a HiDPI screen
 		private int titleBarHeight; // measured from window top edge, which may be out-of-screen if maximized
-		private Predicate<Point> hitTestCallback;
+		private Predicate<Point> captionHitTestCallback;
 		private Rectangle appIconBounds;
 		private Rectangle minimizeButtonBounds;
 		private Rectangle maximizeButtonBounds;
@@ -376,7 +376,7 @@ private int onNcHitTest( int x, int y, boolean isOnResizeBorder ) {
 				// that processes mouse events (e.g. buttons, menus, etc)
 				//   - Windows ignores mouse events in this area
 				try {
-					if( hitTestCallback != null && hitTestCallback.test( pt ) )
+					if( captionHitTestCallback != null && !captionHitTestCallback.test( pt ) )
 						return HTCLIENT;
 				} catch( Throwable ex ) {
 					// ignore