From 75365372067140d6407aeb4d9fb301722c3d1e72 Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Fri, 29 Dec 2023 05:55:48 +0100 Subject: [PATCH 1/2] 90830/104654: issue PR#8267 - Only first 10 predb records resynced (default size) - IT proving issue --- .../SolrDatabaseResyncIT.java | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/dspace-api/src/test/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncIT.java b/dspace-api/src/test/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncIT.java index 4fa881257e0f..fd7c3c4934f0 100644 --- a/dspace-api/src/test/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncIT.java +++ b/dspace-api/src/test/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncIT.java @@ -35,6 +35,9 @@ import org.junit.Before; import org.junit.Test; +/** + * IT for {@link org.dspace.app.solrdatabaseresync.SolrDatabaseResyncIT} + */ public class SolrDatabaseResyncIT extends AbstractIntegrationTestWithDatabase { private final ConfigurationService configurationService = @@ -48,6 +51,15 @@ public class SolrDatabaseResyncIT extends AbstractIntegrationTestWithDatabase { private Collection col; private Item item1; private Item item2; + private Item item3; + private Item item4; + private Item item5; + private Item item6; + private Item item7; + private Item item8; + private Item item9; + private Item item10; + private Item item11; @Before public void setUp() throws Exception { @@ -75,6 +87,16 @@ public void setUp() throws Exception { .withSubject("TestingForMore") .build(); + item3 = ItemBuilder.createItem(context, col).withTitle("Public item 3").build(); + item4 = ItemBuilder.createItem(context, col).withTitle("Public item 4").build(); + item5 = ItemBuilder.createItem(context, col).withTitle("Public item 5").build(); + item6 = ItemBuilder.createItem(context, col).withTitle("Public item 6").build(); + item7 = ItemBuilder.createItem(context, col).withTitle("Public item 7").build(); + item8 = ItemBuilder.createItem(context, col).withTitle("Public item 8").build(); + item9 = ItemBuilder.createItem(context, col).withTitle("Public item 9").build(); + item10 = ItemBuilder.createItem(context, col).withTitle("Public item 10").build(); + item11 = ItemBuilder.createItem(context, col).withTitle("Public item 11").build(); + context.setDispatcher("noindex"); } @@ -83,12 +105,30 @@ public void solrPreDBStatusExistingItemTest() throws Exception { // Items were created, they should contain a predb status in solr assertHasPreDBStatus(item1); assertHasPreDBStatus(item2); + assertHasPreDBStatus(item3); + assertHasPreDBStatus(item4); + assertHasPreDBStatus(item5); + assertHasPreDBStatus(item6); + assertHasPreDBStatus(item7); + assertHasPreDBStatus(item8); + assertHasPreDBStatus(item9); + assertHasPreDBStatus(item10); + assertHasPreDBStatus(item11); performSolrDatabaseResyncScript(); // Database status script was performed, their predb status should be removed assertHasNoPreDBStatus(item1); assertHasNoPreDBStatus(item2); + assertHasNoPreDBStatus(item3); + assertHasNoPreDBStatus(item4); + assertHasNoPreDBStatus(item5); + assertHasNoPreDBStatus(item6); + assertHasNoPreDBStatus(item7); + assertHasNoPreDBStatus(item8); + assertHasNoPreDBStatus(item9); + assertHasNoPreDBStatus(item10); + assertHasNoPreDBStatus(item11); context.restoreAuthSystemState(); } @@ -98,22 +138,50 @@ public void solrPreDBStatusRemovedItemTest() throws Exception { // Items were created, they should contain a predb status in solr assertHasPreDBStatus(item1); assertHasPreDBStatus(item2); + assertHasPreDBStatus(item3); + assertHasPreDBStatus(item4); + assertHasPreDBStatus(item5); + assertHasPreDBStatus(item6); + assertHasPreDBStatus(item7); + assertHasPreDBStatus(item8); + assertHasPreDBStatus(item9); + assertHasPreDBStatus(item10); + assertHasPreDBStatus(item11); collectionService.delete(context, col); // Items were deleted, they should still contain a predb status in solr for now assertHasPreDBStatus(item1); assertHasPreDBStatus(item2); + assertHasPreDBStatus(item3); + assertHasPreDBStatus(item4); + assertHasPreDBStatus(item5); + assertHasPreDBStatus(item6); + assertHasPreDBStatus(item7); + assertHasPreDBStatus(item8); + assertHasPreDBStatus(item9); + assertHasPreDBStatus(item10); + assertHasPreDBStatus(item11); performSolrDatabaseResyncScript(); // Database status script was performed, their solr document should have been removed assertNoSolrDocument(item1); assertNoSolrDocument(item2); + assertNoSolrDocument(item3); + assertNoSolrDocument(item4); + assertNoSolrDocument(item5); + assertNoSolrDocument(item6); + assertNoSolrDocument(item7); + assertNoSolrDocument(item8); + assertNoSolrDocument(item9); + assertNoSolrDocument(item10); + assertNoSolrDocument(item11); context.restoreAuthSystemState(); } + public void assertHasNoPreDBStatus(Item item) throws Exception { assertNotEquals(STATUS_FIELD_PREDB, getStatus(item)); } From 91f53c22870bcb28719ded08bf642d269d13f4a2 Mon Sep 17 00:00:00 2001 From: Marie Verdonck Date: Fri, 29 Dec 2023 05:58:25 +0100 Subject: [PATCH 2/2] 90830/104654: issue PR#8267 - Only first 10 predb records resynced (default size) - Fix: - Query with 0 rows to get total number of records to process - Process them in batches of 100 --- .../SolrDatabaseResyncCli.java | 70 +++++++++++-------- .../SolrDatabaseResyncIT.java | 1 + 2 files changed, 43 insertions(+), 28 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncCli.java b/dspace-api/src/main/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncCli.java index f901c9ca569e..aac42ce1acf9 100644 --- a/dspace-api/src/main/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncCli.java +++ b/dspace-api/src/main/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncCli.java @@ -105,35 +105,17 @@ private void performStatusUpdate(Context context) throws SearchServiceException, solrQuery.addFilterQuery(dateRangeFilter); solrQuery.addField(SearchUtils.RESOURCE_ID_FIELD); solrQuery.addField(SearchUtils.RESOURCE_UNIQUE_ID); + solrQuery.setRows(0); QueryResponse response = solrSearchCore.getSolr().query(solrQuery, solrSearchCore.REQUEST_METHOD); - - if (response != null) { - logInfoAndOut(response.getResults().size() + " items found to process"); - - for (SolrDocument doc : response.getResults()) { - String uuid = (String) doc.getFirstValue(SearchUtils.RESOURCE_ID_FIELD); - String uniqueId = (String) doc.getFirstValue(SearchUtils.RESOURCE_UNIQUE_ID); - logDebugAndOut("Processing item with UUID: " + uuid); - - Optional indexableObject = Optional.empty(); - try { - indexableObject = indexObjectServiceFactory - .getIndexableObjectFactory(uniqueId).findIndexableObject(context, uuid); - } catch (SQLException e) { - log.warn("An exception occurred when attempting to retrieve item with UUID \"" + uuid + - "\" from the database, removing related solr document", e); - } - - try { - if (indexableObject.isPresent()) { - logDebugAndOut("Item exists in DB, updating solr document"); - updateItem(context, indexableObject.get()); - } else { - logDebugAndOut("Item doesn't exist in DB, removing solr document"); - removeItem(context, uniqueId); - } - } catch (SQLException | IOException e) { - log.error(e.getMessage(), e); + if (response != null && response.getResults() != null) { + long nrOfPreDBResults = response.getResults().getNumFound(); + if (nrOfPreDBResults > 0) { + logInfoAndOut(nrOfPreDBResults + " items found to process"); + int batchSize = configurationService.getIntProperty("script.solr-database-resync.batch-size", 100); + for (int start = 0; start < nrOfPreDBResults; start += batchSize) { + solrQuery.setStart(start); + solrQuery.setRows(batchSize); + performStatusUpdateOnNextBatch(context, solrQuery); } } } @@ -141,6 +123,38 @@ private void performStatusUpdate(Context context) throws SearchServiceException, indexingService.commit(); } + private void performStatusUpdateOnNextBatch(Context context, SolrQuery solrQuery) + throws SolrServerException, IOException { + QueryResponse response = solrSearchCore.getSolr().query(solrQuery, solrSearchCore.REQUEST_METHOD); + + for (SolrDocument doc : response.getResults()) { + String uuid = (String) doc.getFirstValue(SearchUtils.RESOURCE_ID_FIELD); + String uniqueId = (String) doc.getFirstValue(SearchUtils.RESOURCE_UNIQUE_ID); + logDebugAndOut("Processing item with UUID: " + uuid); + + Optional indexableObject = Optional.empty(); + try { + indexableObject = indexObjectServiceFactory + .getIndexableObjectFactory(uniqueId).findIndexableObject(context, uuid); + } catch (SQLException e) { + log.warn("An exception occurred when attempting to retrieve item with UUID \"" + uuid + + "\" from the database, removing related solr document", e); + } + + try { + if (indexableObject.isPresent()) { + logDebugAndOut("Item exists in DB, updating solr document"); + updateItem(context, indexableObject.get()); + } else { + logDebugAndOut("Item doesn't exist in DB, removing solr document"); + removeItem(context, uniqueId); + } + } catch (SQLException | IOException e) { + log.error(e.getMessage(), e); + } + } + } + private void updateItem(Context context, IndexableObject indexableObject) throws SolrServerException, IOException { Map fieldModifier = new HashMap<>(1); fieldModifier.put("remove", STATUS_FIELD_PREDB); diff --git a/dspace-api/src/test/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncIT.java b/dspace-api/src/test/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncIT.java index fd7c3c4934f0..d1fa0db9089b 100644 --- a/dspace-api/src/test/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncIT.java +++ b/dspace-api/src/test/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncIT.java @@ -65,6 +65,7 @@ public class SolrDatabaseResyncIT extends AbstractIntegrationTestWithDatabase { public void setUp() throws Exception { super.setUp(); configurationService.setProperty("solr-database-resync.time-until-reindex", 1); + configurationService.setProperty("script.solr-database-resync.batch-size", 5); ServiceManager serviceManager = DSpaceServicesFactory.getInstance().getServiceManager(); searchService = serviceManager.getServiceByName(null, MockSolrSearchCore.class);