Attempt to fix CT intermittent failures

Continuous testing tests can fail sometimes as an
additional unexpected run is triggered. This is because
a write is actually two seperate operations, a truncate
followed by a write, so the update processor can see this
as two seperate changes and trigger two seperate test
runs (depending on the timing). This fixes it by re-checking
the timestamps after compilation, and also attempting to
re-compile if they don't match.

core/deployment/src/main/java/io/quarkus/deployment/dev/RuntimeUpdatesProcessor.java

@@ -576,17 +576,63 @@ && sourceFileWasRecentModified(p, ignoreFirstScanChanges))
                     classScanResult.compilationHappened = true;
                     log.info("Changed source files detected, recompiling "
                             + changedSourceFiles.stream().map(File::getName).collect(Collectors.joining(", ")));
-                    try {
-                        final Set<Path> changedPaths = changedSourceFiles.stream()
-                                .map(File::toPath)
-                                .collect(Collectors.toSet());
-                        moduleChangedSourceFilePaths.addAll(changedPaths);
-                        compiler.compile(sourcePath.toString(), changedSourceFiles.stream()
-                                .collect(groupingBy(this::getFileExtension, Collectors.toSet())));
-                        compileProblem = null;
-                    } catch (Exception e) {
-                        compileProblem = e;
-                        return new ClassScanResult();
+                    //so this is pretty yuck, but on a lot of systems a write is actually a truncate + write
+                    //its possible we see the truncated file timestamp, then the write updates the timestamp
+                    //which will then re-trigger continuous testing/live reload
+                    //the empty fine does not normally cause issues as by the time we actually compile it the write
+                    //has completed (but the old timestamp is used)
+                    for (File i : changedSourceFiles) {
+                        if (i.length() == 0) {
+                            try {
+                                //give time for the write to complete
+                                //note that this is just 'best effort'
+                                //the file time may have already been updated by the time we get here
+                                Thread.sleep(200);
+                                break;
+                            } catch (InterruptedException e) {
+                                //ignore
+                            }
+                        }
+                    }
+                    Map<File, Long> compileTimestamps = new HashMap<>();
+
+                    //now we record the timestamps as they are before the compile phase
+                    for (File i : changedSourceFiles) {
+                        compileTimestamps.put(i, i.lastModified());
+                    }
+                    for (;;) {
+                        try {
+                            final Set<Path> changedPaths = changedSourceFiles.stream()
+                                    .map(File::toPath)
+                                    .collect(Collectors.toSet());
+                            moduleChangedSourceFilePaths.addAll(changedPaths);
+                            compiler.compile(sourcePath.toString(), changedSourceFiles.stream()
+                                    .collect(groupingBy(this::getFileExtension, Collectors.toSet())));
+                            compileProblem = null;
+                        } catch (Exception e) {
+                            compileProblem = e;
+                            return new ClassScanResult();
+                        }
+                        boolean timestampsChanged = false;
+                        //check to make sure no changes have occurred while the compilation was
+                        //taking place. If they have changed we update the timestamp in the compile
+                        //time set, and re-run the compilation, as we have no idea if the compiler
+                        //saw the old or new version
+                        for (Map.Entry<File, Long> entry : compileTimestamps.entrySet()) {
+                            if (entry.getKey().lastModified() != entry.getValue()) {
+                                timestampsChanged = true;
+                                entry.setValue(entry.getKey().lastModified());
+                            }
+                        }
+                        if (!timestampsChanged) {
+                            break;
+                        }
+                    }
+                    //now we re-update the underlying timestamps, to the values we just compiled
+                    //if the file has changed in the meantime it will be picked up in the next
+                    //scan
+                    for (Map.Entry<File, Long> entry : compileTimestamps.entrySet()) {
+                        sourceFileTimestamps.put(entry.getKey().toPath(), entry.getValue());
                     }
                 }
 
@@ -780,6 +826,20 @@ Set<String> checkForFileChange(Function<DevModeContext.ModuleInfo, DevModeContex
                             //as there is both normal and test resources, but only one set of watched timestampts
                             if (existing != null && value > existing) {
                                 ret.add(path);
+                                //a write can be a 'truncate' + 'write'
+                                //if the file is empty we may be seeing the middle of a write
+                                if (Files.size(file) == 0) {
+                                    try {
+                                        Thread.sleep(200);
+                                    } catch (InterruptedException e) {
+                                        //ignore
+                                    }
+                                }
+                                //re-read, as we may have read the original TS if the middle of
+                                //a truncate+write, even if the write had completed by the time
+                                //we read the size
+                                value = Files.getLastModifiedTime(file).toMillis();
+
                                 log.infof("File change detected: %s", file);
                                 if (doCopy && !Files.isDirectory(file)) {
                                     Path target = outputDir.resolve(path);