diff --git a/src/affinder/_tests/test_affinder.py b/src/affinder/_tests/test_affinder.py
index ef5f0c7..3e042b0 100644
--- a/src/affinder/_tests/test_affinder.py
+++ b/src/affinder/_tests/test_affinder.py
@@ -7,6 +7,7 @@
 import napari
 import pytest
 from scipy import ndimage as ndi
+from pathlib import Path
 
 
 layer0_pts = np.array([[140.38371886,
@@ -21,8 +22,9 @@
 # get reference and moving layer types
 im0 = data.camera()
 im1 = transform.rotate(im0[100:, 32:496], 60)
-labels0 = zarr.open('./src/affinder/_tests/labels0.zarr', mode='r')
-labels1 = zarr.open('./src/affinder/_tests/labels1.zarr', mode='r')
+this_dir = Path(__file__).parent.absolute()
+labels0 = zarr.open(this_dir / 'labels0.zarr', mode='r')
+labels1 = zarr.open(this_dir / 'labels1.zarr', mode='r')
 
 
 def make_vector_border(layer_pts):
@@ -126,6 +128,20 @@ def test_apply_affine():
     np.testing.assert_allclose(res_layer[0], ref_im)
 
 
+def test_apply_affine_with_scale():
+    ref_im = np.random.random((5, 5))
+    mov_im = ndi.zoom(ref_im, 2, order=0)
+
+    ref_layer = napari.layers.Image(ref_im, scale=(0.2, 0.2))
+    mov_layer = napari.layers.Image(mov_im, scale=(0.4, 0.4))
+    mov_layer.affine = np.array([[0.25, 0, 0], [0, 0.25, 0], [0, 0, 1]])
+
+    widget = apply_affine()
+    res_layer = widget(ref_layer, mov_layer)
+
+    np.testing.assert_allclose(res_layer[0], ref_im)
+
+
 def test_apply_affine_nonimage():
     ref_im = np.random.random((5, 5))
     mov_pts = np.random.random((5, 2))
diff --git a/src/affinder/apply_tf.py b/src/affinder/apply_tf.py
index 9029e3f..3d4f3ad 100644
--- a/src/affinder/apply_tf.py
+++ b/src/affinder/apply_tf.py
@@ -1,5 +1,6 @@
 from magicgui import magic_factory
 from skimage import transform
+from napari.utils.transforms import CompositeAffine
 import numpy as np
 
 
@@ -62,8 +63,24 @@ def apply_affine(
                 'Only image transforms supported at this point.'
                 )
 
+    reference_meta = CompositeAffine(
+            scale=reference_layer.scale,
+            translate=reference_layer.translate,
+            rotate=reference_layer.rotate,
+            shear=reference_layer.shear,
+            )
+    moving_meta = CompositeAffine(
+            scale=moving_layer.scale,
+            translate=moving_layer.translate,
+            rotate=moving_layer.rotate,
+            shear=moving_layer.shear,
+            )
     # Find the transformation relative to the reference image
-    affine = np.linalg.inv(reference_layer.affine) @ moving_layer.affine
+    affine = (
+            np.linalg.inv(reference_meta)
+            @ np.linalg.inv(reference_layer.affine) @ moving_layer.affine
+            @ moving_meta
+            )
 
     # Apply the transformation
     transformed = _apply_affine_image(