logging requirement

build.gradle

@@ -1,5 +1,5 @@
 plugins {
-    id 'org.springframework.boot' version '3.3.4'
+    id 'org.springframework.boot' version '3.3.5'
     id 'io.spring.dependency-management' version '1.1.6'
     id 'java'
     id 'checkstyle'

src/main/java/it/gov/innovazione/ndc/alerter/AlerterService.java

@@ -3,6 +3,7 @@
 import it.gov.innovazione.ndc.alerter.data.EventService;
 import it.gov.innovazione.ndc.alerter.dto.EventDto;
 import it.gov.innovazione.ndc.alerter.event.AlertableEvent;
+import it.gov.innovazione.ndc.service.logging.LoggingContext;
 import lombok.RequiredArgsConstructor;
 import org.springframework.security.core.context.SecurityContext;
 import org.springframework.security.core.context.SecurityContextHolder;
@@ -12,6 +13,7 @@
 import java.time.Instant;
 import java.util.Optional;
 
+import static it.gov.innovazione.ndc.service.logging.NDCHarvesterLogger.logApplicationInfo;
 import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
 
 @Service
@@ -30,6 +32,12 @@ public void alert(AlertableEvent alertableEvent) {
                 .createdBy(getUser())
                 .occurredAt(defaultIfNull(alertableEvent.getOccurredAt(), Instant.now()))
                 .build());
+        logApplicationInfo(LoggingContext.builder()
+                .component("alerter")
+                .message("Alerted event")
+                .details(alertableEvent.getDescription())
+                .eventCategory(alertableEvent.getCategory())
+                .build());
     }
 
     public static String getUser() {

src/main/java/it/gov/innovazione/ndc/alerter/controller/AbstractCrudController.java

@@ -5,6 +5,8 @@
 import it.gov.innovazione.ndc.alerter.data.EntityService;
 import it.gov.innovazione.ndc.alerter.dto.SlimPager;
 import it.gov.innovazione.ndc.alerter.entities.Nameable;
+import it.gov.innovazione.ndc.service.logging.LoggingContext;
+import it.gov.innovazione.ndc.service.logging.NDCHarvesterLogger;
 import jakarta.validation.Valid;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -82,6 +84,14 @@ protected void handlePreCreate(D dto) {
 
     protected void handlePostCreate(D createdEntity) {
         log.info("Created entity: {}", createdEntity);
+        NDCHarvesterLogger.logApplicationInfo(
+                LoggingContext.builder()
+                        .component("EntityCreation")
+                        .message("Created entity")
+                        .additionalInfo("entityName", createdEntity.getName())
+                        .additionalInfo("entityId", createdEntity.getId())
+                        .additionalInfo("entityType", createdEntity.getClass().getSimpleName())
+                        .build());
     }
 
     @PatchMapping
@@ -103,6 +113,14 @@ protected void handlePreUpdate(D dto) {
 
     protected void handlePostUpdate(D updatedDto) {
         log.info("Updated entity: {}", updatedDto);
+        NDCHarvesterLogger.logApplicationInfo(
+                LoggingContext.builder()
+                        .component("EntityUpdate")
+                        .message("Updated entity")
+                        .additionalInfo("entityName", updatedDto.getName())
+                        .additionalInfo("entityId", updatedDto.getId())
+                        .additionalInfo("entityType", updatedDto.getClass().getSimpleName())
+                        .build());
     }
 
     @DeleteMapping("{id}")
@@ -123,5 +141,13 @@ protected void handlePreDelete(String id) {
 
     protected void handlePostDelete(D deletedDto) {
         log.info("Deleted entity: {}", deletedDto);
+        NDCHarvesterLogger.logApplicationInfo(
+                LoggingContext.builder()
+                        .component("EntityDeletion")
+                        .message("Deleted entity")
+                        .additionalInfo("entityName", deletedDto.getName())
+                        .additionalInfo("entityId", deletedDto.getId())
+                        .additionalInfo("entityType", deletedDto.getClass().getSimpleName())
+                        .build());
     }
 }

src/main/java/it/gov/innovazione/ndc/alerter/data/ProfileInitializer.java

@@ -31,7 +31,7 @@ public class ProfileInitializer implements Initializer {
     public void init() {
         List<String> existingProfileNames = repository.findAll().stream()
                 .map(Profile::getName)
-                .collect(Collectors.toList());
+                .toList();
         DEFAULT_PROFILES.stream()
                 .filter(not(p -> existingProfileNames.contains(p.getLeft())))
                 .map(pair -> Profile.builder()

src/main/java/it/gov/innovazione/ndc/config/GitHubConfig.java

@@ -5,6 +5,8 @@
 import it.gov.innovazione.ndc.alerter.event.DefaultAlertableEvent;
 import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher;
 import it.gov.innovazione.ndc.service.NdcGitHubClient;
+import it.gov.innovazione.ndc.service.logging.LoggingContext;
+import it.gov.innovazione.ndc.service.logging.NDCHarvesterLogger;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -16,6 +18,8 @@
 
 import java.time.Instant;
 
+import static it.gov.innovazione.ndc.service.logging.NDCHarvesterLogger.logApplicationInfo;
+
 @Slf4j
 @Configuration
 @RequiredArgsConstructor
@@ -39,11 +43,24 @@ NdcGitHubClient gitHub(@Value("${github.personal-access-token}") String token) {
                             .severity(Severity.WARNING)
                             .build());
 
+            NDCHarvesterLogger.logApplicationWarn(LoggingContext.builder()
+                    .component("GitHubConfig")
+                    .message("GitHubConfig not provided")
+                    .details("GitHubConfig personal access token not provided. The GitHub issuer capability will be disabled")
+                    .eventCategory(EventCategory.APPLICATION)
+                    .build());
+
             return NdcGitHubClient.builder()
                     .enabled(false)
                     .build();
         }
         log.info("GitHub personal access token provided. The GitHub issuer capability will be enabled");
+        logApplicationInfo(LoggingContext.builder()
+                .component("GitHubConfig")
+                .message("GitHubConfig provided")
+                .details("GitHub personal access token provided. The GitHub issuer capability will be enabled")
+                .eventCategory(EventCategory.APPLICATION)
+                .build());
         return NdcGitHubClient.builder()
                 .gitHub(new GitHubBuilder().withOAuthToken(token).build())
                 .enabled(true)

src/main/java/it/gov/innovazione/ndc/config/SimpleHarvestRepositoryProcessor.java

@@ -18,6 +18,7 @@
 import it.gov.innovazione.ndc.service.InstanceManager;
 import it.gov.innovazione.ndc.service.logging.HarvesterStage;
 import it.gov.innovazione.ndc.service.logging.LoggingContext;
+import it.gov.innovazione.ndc.service.logging.NDCHarvesterLoggerUtils;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.ThreadUtils;
@@ -86,13 +87,24 @@ public void execute(String runId, Repository repository, String correlationId, S
                             .instance(instanceToHarvest)
                             .build());
 
+            NDCHarvesterLoggerUtils.setInitialContext(
+                    LoggingContext.builder()
+                            .jobId(runId)
+                            .repoUrl(repository.getUrl())
+                            .stage(HarvesterStage.START)
+                            .harvesterStatus(RUNNING)
+                            .component("HARVESTER")
+                            .additionalInfo("revision", revision)
+                            .additionalInfo("correlationId", correlationId)
+                            .additionalInfo("currentUserLogin", currentUserLogin)
+                            .additionalInfo("instance", instanceToHarvest)
+                            .additionalInfo("force", force)
+                            .build());
+
             publishHarvesterStartedEvent(repository, correlationId, revision, runId, currentUserLogin, instanceToHarvest);
 
             logSemanticInfo(LoggingContext.builder()
-                    .stage(HarvesterStage.START)
-                    .harvesterStatus(RUNNING)
                     .message("Starting harvester on repo " + repository.getUrl())
-                    .additionalInfo("force", force)
                     .build());
 
             synchronized (locks) {
@@ -110,11 +122,9 @@ public void execute(String runId, Repository repository, String correlationId, S
                             currentUserLogin);
 
                     logSemanticError(LoggingContext.builder()
-                            .stage(HarvesterStage.START)
                             .harvesterStatus(ALREADY_RUNNING)
                             .message("Failed harvester on repo " + repository.getUrl())
                             .details("Harvester on repo " + repository.getUrl() + " is already running")
-                            .additionalInfo("force", force)
                             .build());
 
                     setThreadName(runId, repository.getId(), revision, "IDLE");
@@ -148,7 +158,9 @@ public void execute(String runId, Repository repository, String correlationId, S
         } finally {
             log.info("Cleaning up after processing {}", repository.getUrl());
             harvesterService.clearTempGraphIfExists(repository.getUrl());
+            NDCHarvesterLoggerUtils.clearContext();
             log.info("Cleaned up after processing {}", repository.getUrl());
+
         }
 
         // cleanup
@@ -165,7 +177,6 @@ private void verifyNoNdcIssuesInRepoIfNecessary(Repository repository) {
         if (hasIssues) {
             URL issueUrl = ndcIssue.get().getUrl();
             logSemanticError(LoggingContext.builder()
-                    .stage(HarvesterStage.START)
                     .harvesterStatus(FAILURE)
                     .repoUrl(repository.getUrl())
                     .message("Repository has at least one open NDC issues")
@@ -187,12 +198,9 @@ private synchronized void verifySameRunWasNotExecuted(Repository repository, Str
         Optional<HarvesterRun> harvesterRun = harvesterRunService.isHarvestingAlreadyExecuted(repository.getId(), revision);
         if (harvesterRun.isPresent()) {
             logSemanticWarn(LoggingContext.builder()
-                    .stage(HarvesterStage.START)
                     .harvesterStatus(HarvesterRun.Status.UNCHANGED)
-                    .repoUrl(repository.getUrl())
                     .message("Harvesting for repo '" + repository.getUrl() + "' was already executed")
                     .additionalInfo("otherJobId", harvesterRun.get().getId())
-                    .additionalInfo("revision", revision)
                     .build());
             throw new HarvesterAlreadyExecutedException(format("Harvesting for repo '%s' with revision '%s' was already executed and no force param was passed",
                     repository.getUrl(), revision));
@@ -203,10 +211,8 @@ private synchronized void verifyHarvestingIsNotInProgress(String runId, Reposito
         Optional<HarvesterRun> harvestingInProgress = harvesterRunService.isHarvestingInProgress(runId, repository);
         if (harvestingInProgress.isPresent()) {
             logSemanticError(LoggingContext.builder()
-                    .jobId(runId)
                     .message("Harvesting for repo '" + repository.getUrl() + "' is already in progress")
                     .harvesterStatus(ALREADY_RUNNING)
-                    .repoUrl(repository.getUrl())
                     .additionalInfo("otherJobId", harvestingInProgress.get().getId())
                     .build());
             throw new HarvesterAlreadyInProgressException(format("Harvesting for repo '%s' is already in progress", repository.getUrl()));

src/main/java/it/gov/innovazione/ndc/controller/ConfigurationController.java

@@ -1,7 +1,6 @@
 package it.gov.innovazione.ndc.controller;
 
 import io.swagger.v3.oas.annotations.Operation;
-import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher;
 import it.gov.innovazione.ndc.eventhandler.event.ConfigService;
 import it.gov.innovazione.ndc.harvester.service.ActualConfigService;
 import lombok.RequiredArgsConstructor;
@@ -30,7 +29,6 @@
 public class ConfigurationController {
 
     private final ActualConfigService configService;
-    private final NdcEventPublisher eventPublisher;
 
     @GetMapping
     @Operation(

src/main/java/it/gov/innovazione/ndc/harvester/AgencyRepositoryService.java

@@ -11,6 +11,8 @@
 import it.gov.innovazione.ndc.harvester.util.GitUtils;
 import it.gov.innovazione.ndc.harvester.util.PropertiesUtils;
 import it.gov.innovazione.ndc.harvester.util.Version;
+import it.gov.innovazione.ndc.service.logging.HarvesterStage;
+import it.gov.innovazione.ndc.service.logging.LoggingContext;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
@@ -22,8 +24,9 @@
 import java.util.List;
 import java.util.Optional;
 import java.util.function.Predicate;
-import java.util.stream.Collectors;
 
+import static it.gov.innovazione.ndc.service.logging.NDCHarvesterLogger.logSemanticInfo;
+import static it.gov.innovazione.ndc.service.logging.NDCHarvesterLogger.logSemanticWarn;
 import static java.util.function.Predicate.not;
 
 @Component
@@ -84,8 +87,21 @@ private <P extends SemanticAssetPath> List<P> findPaths(Path clonedRepo, Semanti
         if (!fileUtils.folderExists(assetRootPath)) {
             log.warn("No {} folder found in {}", type.getDescription(), clonedRepo);
 
+            logSemanticWarn(LoggingContext.builder()
+                    .stage(HarvesterStage.PATH_SCANNING)
+                    .message("No " + type.getFolderName() + " folder found in " + clonedRepo)
+                    .additionalInfo("path", clonedRepo.toString())
+                    .build());
+
             assetRootPath = Path.of(clonedRepo.toString(), type.getLegacyFolderName()); 
             if (!fileUtils.folderExists(assetRootPath)) {
+                logSemanticWarn(LoggingContext.builder()
+                        .stage(HarvesterStage.PATH_SCANNING)
+                        .message("No " + type.getLegacyFolderName() + " folder found in " + clonedRepo)
+                        .additionalInfo("path", clonedRepo.toString())
+                        .build());
+                log.warn("No {} or {} folder found in {}", type.getDescription(), type.getLegacyFolderName(), clonedRepo);
+
                 return List.of();
             }
         }
@@ -95,14 +111,23 @@ private <P extends SemanticAssetPath> List<P> findPaths(Path clonedRepo, Semanti
 
     private boolean isDirectoryToBeSkipped(Path path) {
         String directoryName = fileUtils.getLowerCaseFileName(path);
-        return fileUtils.isDirectory(path) && this.lowerSkipWords.stream().anyMatch(directoryName::contains);
+        boolean skip = fileUtils.isDirectory(path) && this.lowerSkipWords.stream().anyMatch(directoryName::contains);
+        if (skip) {
+            logSemanticWarn(LoggingContext.builder()
+                    .stage(HarvesterStage.PATH_SCANNING)
+                    .message("Skipping directory " + path)
+                    .additionalInfo("directory", path.toString())
+                    .additionalInfo("skipWords", lowerSkipWords)
+                    .build());
+        }
+        return skip;
     }
 
     @SneakyThrows
     private <P extends SemanticAssetPath> List<P> createSemanticAssetPaths(Path path, FolderScanner<P> scanner, boolean ignoreObsoleteVersions) {
         List<Path> dirContents = fileUtils.listContents(path).stream()
                 .filter(c -> !isDirectoryToBeSkipped(c))
-                .collect(Collectors.toList());
+                .toList();
         boolean hasSubDir = dirContents.stream().anyMatch(fileUtils::isDirectory);
         if (!hasSubDir) {
             return tryScanDir(path, scanner);
@@ -117,21 +142,36 @@ private <P extends SemanticAssetPath> List<P> createSemanticAssetPaths(Path path
             }
         }
 
-        return dirContents.stream()
+        List<P> assets = dirContents.stream()
                 // consider folders for recursion
                 .filter(fileUtils::isDirectory)
                 // only consider folders which are not obsolete
                 .filter(not(isObsoleteVersion))
                 // recurse and flatten
                 .flatMap(subDir -> createSemanticAssetPaths(subDir, scanner, ignoreObsoleteVersions).stream())
                 // then collect
-                .collect(Collectors.toList());
+                .toList();
+
+        logSemanticInfo(LoggingContext.builder()
+                .stage(HarvesterStage.PATH_SCANNING)
+                .message("Scanned folder for " + scanner.getClass().getSimpleName())
+                .additionalInfo("folder", path.toString())
+                .additionalInfo("assets", assets.size())
+                .build());
+
+        return assets;
     }
 
     private <P extends SemanticAssetPath> List<P> tryScanDir(Path dir, FolderScanner<P> scanner) throws IOException {
         try {
             return scanner.scanFolder(dir);
         } catch (InvalidAssetFolderException e) {
+            logSemanticWarn(LoggingContext.builder()
+                    .stage(HarvesterStage.PATH_SCANNING)
+                    .message("Invalid folder " + dir + "; skipping")
+                    .details(e.getRealErrorMessage())
+                    .additionalInfo("folder", dir.toString())
+                    .build());
             log.warn("Invalid folder {}; skipping", dir, e);
             return Collections.emptyList();
         }
@@ -148,7 +188,14 @@ private Predicate<Path> isObsoleteVersionPredicate(String latestVersionString) {
         return path -> {
             String fileName = path.getFileName().toString();
             boolean hasValidVersion = Version.of(fileName).isPresent();
-            return hasValidVersion && !latestVersionString.equals(fileName);
+            boolean isObsolete = hasValidVersion && !latestVersionString.equals(fileName);
+            logSemanticWarn(LoggingContext.builder()
+                    .stage(HarvesterStage.PATH_SCANNING)
+                    .message("Skipping obsolete version " + path)
+                    .additionalInfo("path", path.toString())
+                    .additionalInfo("latestVersion", latestVersionString)
+                    .build());
+            return isObsolete;
         };
     }
 }