Skip to content

Commit

Permalink
Use background job to calculate admin statistics
Browse files Browse the repository at this point in the history 
Add db migration
Fix tests
  • Loading branch information
amvanbaren committed May 8, 2023
1 parent 2252936 commit a122c43
Show file tree
Hide file tree
Showing 14 changed files with 375 additions and 302 deletions.
42 changes: 41 additions & 1 deletion server/src/main/java/org/eclipse/openvsx/admin/AdminAPI.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import java.util.stream.Collectors;
import java.net.URI;

import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.base.Strings;

import org.eclipse.openvsx.LocalRegistryService;
Expand Down Expand Up @@ -42,6 +43,7 @@

@RestController
public class AdminAPI {

@Autowired
RepositoryService repositories;

Expand All @@ -57,6 +59,41 @@ public class AdminAPI {
@Autowired
SearchUtilService search;

@GetMapping(
path = "/admin/reports",
produces = MediaType.APPLICATION_JSON_VALUE
)
public ResponseEntity<Map<String, List<String>>> getReports(
@RequestParam("token") String tokenValue
) {
try {
validateToken(tokenValue);
return ResponseEntity.ok(admins.getReports());
} catch (ErrorResultException exc) {
return ResponseEntity.status(exc.getStatus()).build();
}
}

@PostMapping(
path = "/admin/report/schedule",
consumes = MediaType.APPLICATION_JSON_VALUE,
produces = MediaType.APPLICATION_JSON_VALUE
)
public ResponseEntity<ResultJson> scheduleReport(
@RequestParam String token,
@RequestBody JsonNode json
) {
try {
validateToken(token);
var year = json.get("year").asInt();
var month = json.get("month").asInt();
admins.scheduleReport(year, month);
return ResponseEntity.accepted().build();
} catch (ErrorResultException exc) {
return exc.toResponseEntity(ResultJson.class);
}
}

@GetMapping(
path = "/admin/report",
produces = MediaType.APPLICATION_JSON_VALUE
Expand Down Expand Up @@ -91,12 +128,15 @@ public ResponseEntity<String> getReportCsv(
}
}

private AdminStatistics getReport(String tokenValue, int year, int month) {
private void validateToken(String tokenValue) {
var accessToken = repositories.findAccessToken(tokenValue);
if(accessToken == null || !accessToken.isActive() || accessToken.getUser() == null || !ROLE_ADMIN.equals(accessToken.getUser().getRole())) {
throw new ErrorResultException("Invalid access token", HttpStatus.FORBIDDEN);
}
}

private AdminStatistics getReport(String tokenValue, int year, int month) {
validateToken(tokenValue);
return admins.getAdminStatistics(year, month);
}

Expand Down
162 changes: 48 additions & 114 deletions server/src/main/java/org/eclipse/openvsx/admin/AdminService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,33 +17,33 @@
import org.eclipse.openvsx.eclipse.EclipseService;
import org.eclipse.openvsx.entities.*;
import org.eclipse.openvsx.json.*;
import org.eclipse.openvsx.migration.HandlerJobRequest;
import org.eclipse.openvsx.migration.MigrationRunner;
import org.eclipse.openvsx.repositories.RepositoryService;
import org.eclipse.openvsx.search.SearchUtilService;
import org.eclipse.openvsx.storage.StorageUtilService;
import org.eclipse.openvsx.util.*;
import org.jobrunr.scheduling.JobRequestScheduler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.jobrunr.scheduling.cron.Cron;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.event.ApplicationStartedEvent;
import org.springframework.context.event.EventListener;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Component;
import org.springframework.util.StopWatch;

import javax.persistence.EntityManager;
import javax.transaction.Transactional;
import java.time.DateTimeException;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.Comparator;
import java.util.LinkedHashSet;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.time.ZoneId;
import java.util.*;
import java.util.stream.Collectors;

import static org.eclipse.openvsx.entities.FileResource.*;

@Component
public class AdminService {

private static final Logger LOGGER = LoggerFactory.getLogger(AdminService.class);
@Autowired
RepositoryService repositories;

Expand Down Expand Up @@ -77,6 +77,12 @@ public class AdminService {
@Autowired
JobRequestScheduler scheduler;

@EventListener
public void applicationStarted(ApplicationStartedEvent event) {
var jobRequest = new HandlerJobRequest<>(MonthlyAdminStatisticsJobRequestHandler.class);
scheduler.scheduleRecurrently("MonthlyAdminStatistics", Cron.monthly(1, 0, 3), ZoneId.of("UTC"), jobRequest);
}

@Transactional(rollbackOn = ErrorResultException.class)
public ResultJson deleteExtension(String namespaceName, String extensionName, UserData admin)
throws ErrorResultException {
Expand Down Expand Up @@ -347,121 +353,49 @@ public void logAdminAction(UserData admin, ResultJson result) {
}
}

@Transactional
public AdminStatistics getAdminStatistics(int year, int month) throws ErrorResultException {
validateYearAndMonth(year, month);
var statistics = repositories.findAdminStatisticsByYearAndMonth(year, month);
if(statistics == null) {
throw new NotFoundException();
}

return statistics;
}

public void scheduleReport(int year, int month) {
validateYearAndMonth(year, month);
if(repositories.findAdminStatisticsByYearAndMonth(year, month) != null) {
throw new ErrorResultException("Report for " + year + "/" + month + " already exists");
}

var jobIdText = "AdminStatistics::year=" + year + ",month=" + month;
var jobId = UUID.nameUUIDFromBytes(jobIdText.getBytes(StandardCharsets.UTF_8));
scheduler.enqueue(jobId, new AdminStatisticsJobRequest(year, month));
}

private void validateYearAndMonth(int year, int month) {
if(year < 0) {
throw new ErrorResultException("Year can't be negative", HttpStatus.BAD_REQUEST);
}
if(month < 1 || month > 12) {
throw new ErrorResultException("Month must be a value between 1 and 12", HttpStatus.BAD_REQUEST);
}

var now = LocalDateTime.now();
if(year > now.getYear() || (year == now.getYear() && month > now.getMonthValue())) {
var now = TimeUtil.getCurrentUTC();
if(year > now.getYear() || (year == now.getYear() && month >= now.getMonthValue())) {
throw new ErrorResultException("Combination of year and month lies in the future", HttpStatus.BAD_REQUEST);
}
}

var statistics = repositories.findAdminStatisticsByYearAndMonth(year, month);
if(statistics == null) {
LocalDateTime startInclusive;
try {
startInclusive = LocalDateTime.of(year, month, 1, 0, 0);
} catch(DateTimeException e) {
throw new ErrorResultException("Invalid month or year", HttpStatus.BAD_REQUEST);
}

var currentYearAndMonth = now.getYear() == year && now.getMonthValue() == month;
var endExclusive = currentYearAndMonth
? now.truncatedTo(ChronoUnit.MINUTES)
: startInclusive.plusMonths(1);

LOGGER.info(">> ADMIN REPORT STATS");
var stopwatch = new StopWatch();
stopwatch.start("repositories.countActiveExtensions");
var extensions = repositories.countActiveExtensions(endExclusive);
stopwatch.stop();
LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis());

stopwatch.start("repositories.downloadsBetween");
var downloads = repositories.downloadsBetween(startInclusive, endExclusive);
stopwatch.stop();
LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis());

stopwatch.start("repositories.downloadsUntil");
var downloadsTotal = repositories.downloadsUntil(endExclusive);
stopwatch.stop();
LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis());

stopwatch.start("repositories.countActiveExtensionPublishers");
var publishers = repositories.countActiveExtensionPublishers(endExclusive);
stopwatch.stop();
LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis());

stopwatch.start("repositories.averageNumberOfActiveReviewsPerActiveExtension");
var averageReviewsPerExtension = repositories.averageNumberOfActiveReviewsPerActiveExtension(endExclusive);
stopwatch.stop();
LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis());

stopwatch.start("repositories.countPublishersThatClaimedNamespaceOwnership");
var namespaceOwners = repositories.countPublishersThatClaimedNamespaceOwnership(endExclusive);
stopwatch.stop();
LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis());

stopwatch.start("repositories.countActiveExtensionsGroupedByExtensionReviewRating");
var extensionsByRating = repositories.countActiveExtensionsGroupedByExtensionReviewRating(endExclusive);
stopwatch.stop();
LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis());

stopwatch.start("repositories.countActiveExtensionPublishersGroupedByExtensionsPublished");
var publishersByExtensionsPublished = repositories.countActiveExtensionPublishersGroupedByExtensionsPublished(endExclusive);
stopwatch.stop();
LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis());

var limit = 10;

stopwatch.start("repositories.topMostActivePublishingUsers");
var topMostActivePublishingUsers = repositories.topMostActivePublishingUsers(endExclusive, limit);
stopwatch.stop();
LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis());

stopwatch.start("repositories.topNamespaceExtensions");
var topNamespaceExtensions = repositories.topNamespaceExtensions(endExclusive, limit);
stopwatch.stop();
LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis());

stopwatch.start("repositories.topNamespaceExtensionVersions");
var topNamespaceExtensionVersions = repositories.topNamespaceExtensionVersions(endExclusive, limit);
stopwatch.stop();
LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis());

stopwatch.start("repositories.topMostDownloadedExtensions");
var topMostDownloadedExtensions = repositories.topMostDownloadedExtensions(endExclusive, limit);
stopwatch.stop();
LOGGER.info("{} took {} ms", stopwatch.getLastTaskName(), stopwatch.getLastTaskTimeMillis());
LOGGER.info("<< ADMIN REPORT STATS");

statistics = new AdminStatistics();
statistics.setYear(year);
statistics.setMonth(month);
statistics.setExtensions(extensions);
statistics.setDownloads(downloads);
statistics.setDownloadsTotal(downloadsTotal);
statistics.setPublishers(publishers);
statistics.setAverageReviewsPerExtension(averageReviewsPerExtension);
statistics.setNamespaceOwners(namespaceOwners);
statistics.setExtensionsByRating(extensionsByRating);
statistics.setPublishersByExtensionsPublished(publishersByExtensionsPublished);
statistics.setTopMostActivePublishingUsers(topMostActivePublishingUsers);
statistics.setTopNamespaceExtensions(topNamespaceExtensions);
statistics.setTopNamespaceExtensionVersions(topNamespaceExtensionVersions);
statistics.setTopMostDownloadedExtensions(topMostDownloadedExtensions);

if(!currentYearAndMonth) {
// archive statistics for quicker lookup next time
entityManager.persist(statistics);
}
}

return statistics;
public Map<String, List<String>> getReports() {
return repositories.findAllAdminStatistics().stream()
.sorted(Comparator.comparingInt(AdminStatistics::getYear).thenComparing(AdminStatistics::getMonth))
.map(stat -> {
var yearText = String.valueOf(stat.getYear());
var monthText = String.valueOf(stat.getMonth());
return new AbstractMap.SimpleEntry<>(yearText, monthText);
})
.collect(Collectors.groupingBy(Map.Entry::getKey, () -> new LinkedHashMap<>(), Collectors.mapping(Map.Entry::getValue, Collectors.toList())));
}
}
47 changes: 47 additions & 0 deletions server/src/main/java/org/eclipse/openvsx/admin/AdminStatisticsJobRequest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/** ******************************************************************************
* Copyright (c) 2023 Precies. Software Ltd and others
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License v. 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0.
*
* SPDX-License-Identifier: EPL-2.0
* ****************************************************************************** */
package org.eclipse.openvsx.admin;

import org.jobrunr.jobs.lambdas.JobRequest;
import org.jobrunr.jobs.lambdas.JobRequestHandler;

public class AdminStatisticsJobRequest implements JobRequest {

private int year;
private int month;

public AdminStatisticsJobRequest() {}

public AdminStatisticsJobRequest(int year, int month) {
this.year = year;
this.month = month;
}

public int getYear() {
return year;
}

public void setYear(int year) {
this.year = year;
}

public int getMonth() {
return month;
}

public void setMonth(int month) {
this.month = month;
}

@Override
public Class<? extends JobRequestHandler> getJobRequestHandler() {
return AdminStatisticsJobRequestHandler.class;
}
}
Loading

0 comments on commit a122c43

Please sign in to comment.