Skip to content

Commit

Permalink
on subtemplate's update, trigger indexation for every md referencing it
Browse files Browse the repository at this point in the history 
rely on metadataIndexer.batchIndexInThreadPool for async indexing
test threadPoolIndexation starts for expected md
throw exception when updating a subtemplate too resource hungry (too many md to index, i.e. more than 10000)
  • Loading branch information
@cmangeat
cmangeat committed Feb 26, 2024
1 parent f5a0710 commit 3718541
Show file tree
Hide file tree
Showing 4 changed files with 156 additions and 25 deletions.
34 changes: 34 additions & 0 deletions common/src/main/java/org/fao/geonet/exceptions/GNException.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
//=============================================================================
//=== Copyright (C) 2001-2023 Food and Agriculture Organization of the
//=== United Nations (FAO-UN), United Nations World Food Programme (WFP)
//=== and United Nations Environment Programme (UNEP)
//===
//=== This program is free software; you can redistribute it and/or modify
//=== it under the terms of the GNU General Public License as published by
//=== the Free Software Foundation; either version 2 of the License, or (at
//=== your option) any later version.
//===
//=== This program is distributed in the hope that it will be useful, but
//=== WITHOUT ANY WARRANTY; without even the implied warranty of
//=== MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
//=== General Public License for more details.
//===
//=== You should have received a copy of the GNU General Public License
//=== along with this program; if not, write to the Free Software
//=== Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
//===
//=== Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2,
//=== Rome - Italy. email: [email protected]
//==============================================================================
package org.fao.geonet.exceptions;

public class GNException extends RuntimeException {

public GNException(String msg) {
super(msg);
}

public GNException(String msg, Throwable cause) {
super(msg, cause);
}
}
35 changes: 32 additions & 3 deletions core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

package org.fao.geonet.kernel.datamanager.base;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Optional;
import com.google.common.base.Predicate;
Expand All @@ -33,19 +34,20 @@
import jeeves.transaction.TransactionTask;
import jeeves.xlink.Processor;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.action.search.SearchResponse;
import org.fao.geonet.ApplicationContextHolder;
import org.fao.geonet.constants.Edit;
import org.fao.geonet.constants.Geonet;
import org.fao.geonet.constants.Params;
import org.fao.geonet.domain.*;
import org.fao.geonet.exceptions.GNException;
import org.fao.geonet.exceptions.UnAuthorizedException;
import org.fao.geonet.kernel.*;
import org.fao.geonet.kernel.datamanager.*;
import org.fao.geonet.kernel.schema.MetadataSchema;
import org.fao.geonet.kernel.schema.SchemaPlugin;
import org.fao.geonet.kernel.search.EsSearchManager;
import org.fao.geonet.kernel.search.IndexingMode;
import org.fao.geonet.kernel.search.MetaSearcher;
import org.fao.geonet.kernel.search.index.BatchOpsMetadataReindexer;
import org.fao.geonet.kernel.setting.SettingManager;
import org.fao.geonet.kernel.setting.Settings;
Expand Down Expand Up @@ -104,8 +106,6 @@ public class BaseMetadataManager implements IMetadataManager {
@Autowired
private GroupRepository groupRepository;
@Autowired
private MetadataStatusRepository metadataStatusRepository;
@Autowired
private MetadataValidationRepository metadataValidationRepository;
@Autowired
private MetadataRepository metadataRepository;
Expand Down Expand Up @@ -141,6 +141,10 @@ public class BaseMetadataManager implements IMetadataManager {

@Autowired
private ApplicationContext _applicationContext;

@VisibleForTesting
public int maxMdsReferencingSubTemplate = 10000;

@PersistenceContext
private EntityManager _entityManager;

Expand Down Expand Up @@ -747,6 +751,9 @@ public synchronized AbstractMetadata updateMetadata(final ServiceContext context
getSearchManager().delete(String.format("+uuid:\"%s\"", uuidBeforeUfo));
}
metadataIndexer.indexMetadata(metadataId, true, indexingMode);
if (metadata.getDataInfo().getType() == MetadataType.SUB_TEMPLATE) {
indexMdsReferencingSubTemplate(context, metadata);
}
}
}

Expand Down Expand Up @@ -1314,4 +1321,26 @@ boolean hasReferencingMetadata(ServiceContext context, AbstractMetadata metadata
return this.searchManager.query(query.toString(), null, 0, 0).getHits().getTotalHits().value > 0;
}

private void indexMdsReferencingSubTemplate(ServiceContext context, AbstractMetadata subTemplate) throws Exception {
StringBuilder query = new StringBuilder(String.format("xlink:*%s*", subTemplate.getUuid()));
SearchResponse response = this.searchManager.query(query.toString(), null, 0, maxMdsReferencingSubTemplate);
if (response.getHits().getTotalHits().value > maxMdsReferencingSubTemplate) {
throw new GNException("Not implemented");
}
ArrayList<String> toIndex = new ArrayList<>();
response.getHits().forEach(consumer -> {
String consumerUuid = consumer.getId();
try {
String consumerId = this.metadataUtils.getMetadataId(consumerUuid);
if (consumerId != null) {
toIndex.add(consumerId);
}
} catch (RuntimeException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException(e);
}
});
metadataIndexer.batchIndexInThreadPool(context, toIndex);
}
}
107 changes: 85 additions & 22 deletions core/src/test/java/org/fao/geonet/kernel/LocalXLinksUpdateDeleteTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,15 @@
package org.fao.geonet.kernel;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import jeeves.server.context.ServiceContext;
import org.elasticsearch.action.search.SearchResponse;
import org.fao.geonet.AbstractCoreIntegrationTest;
import org.fao.geonet.domain.AbstractMetadata;
import org.fao.geonet.kernel.datamanager.IMetadataManager;
import org.fao.geonet.exceptions.GNException;
import org.fao.geonet.kernel.datamanager.base.BaseMetadataIndexer;
import org.fao.geonet.kernel.datamanager.base.BaseMetadataManager;
import org.fao.geonet.kernel.search.EsSearchManager;
import org.fao.geonet.kernel.search.IndexingMode;
import org.fao.geonet.kernel.setting.SettingManager;
import org.fao.geonet.kernel.setting.Settings;
Expand All @@ -12,30 +18,43 @@
import org.jdom.Element;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;

import java.net.URL;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import static org.fao.geonet.domain.MetadataType.SUB_TEMPLATE;
import static org.fao.geonet.domain.MetadataType.TEMPLATE;
import static org.fao.geonet.schema.iso19139.ISO19139Namespaces.GCO;
import static org.fao.geonet.schema.iso19139.ISO19139Namespaces.GMD;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.when;


public class LocalXLinksUpdateDeleteTest extends AbstractIntegrationTestWithMockedSingletons {
@Autowired
private IMetadataManager metadataManager;
private BaseMetadataManager metadataManager;

@Autowired
private SettingManager settingManager;

@Autowired
private EsSearchManager searchManager;

private ServiceContext serviceContext;

@Autowired
private BaseMetadataIndexer metadataIndexer;

@Before
public void setUp() throws Exception {
serviceContext = createServiceContext();
Expand All @@ -44,28 +63,42 @@ public void setUp() throws Exception {

@Test
public void updateHasToRegisterReferrersForIndexation() throws Exception {
URL contactResource = AbstractCoreIntegrationTest.class.getResource("kernel/babarContact.xml");
Element contactElement = Xml.loadStream(contactResource.openStream());
AbstractMetadata contactMetadata = insertContact(contactElement);
AbstractMetadata vicinityMapMetadata = insertVicinityMap(contactMetadata);
Mockito.reset(metadataIndexer);

// TODOES
// Object document = searchForMetadataTagged("babar");
// assertFalse(context.getBean(IndexingList.class).getIdentifiers().contains(vicinityMapMetadata.getId()));
AbstractMetadata vicinityMapMetadata = updateASubtemplateUsedByOneMetadata();

Xml.selectElement(contactElement, "gmd:individualName/gco:CharacterString", Arrays.asList(GMD, GCO)).setText("momo");
metadataManager.updateMetadata(serviceContext,
Integer.toString(contactMetadata.getId()),
contactElement,
false,
false,
null,
null,
false,
IndexingMode.full);
ArgumentCaptor<List> toIndexCaptor = ArgumentCaptor.forClass(List.class);
Mockito.verify(metadataIndexer).batchIndexInThreadPool(Mockito.anyObject(), toIndexCaptor.capture());
assertTrue(toIndexCaptor.getValue().contains(Integer.toString(vicinityMapMetadata.getId())));
}

@Test
public void exceedMaxMdsReferencingSubTemplateThrowsException() {
Mockito.reset(metadataIndexer);
try {
metadataManager.maxMdsReferencingSubTemplate = 0;

assertThrows(GNException.class, () -> updateASubtemplateUsedByOneMetadata());

Mockito.verify(metadataIndexer, never()).batchIndexInThreadPool(Mockito.anyObject(), Mockito.anyList());
} finally {
metadataManager.maxMdsReferencingSubTemplate = 10000;
}
}

private SearchResponse countMetadataUsing(String mdUuid, String contactForResourceIndividual) throws Exception {
ObjectMapper objectMapper = new ObjectMapper();

JsonNode query = objectMapper.createObjectNode()
.set("nested", objectMapper.createObjectNode()
.put("path", "contactForResource")
.set("query", objectMapper.createObjectNode()
.set("bool", objectMapper.createObjectNode().set("must", objectMapper.createArrayNode()
.add(objectMapper.createObjectNode().set("match", objectMapper.createObjectNode().put("_id", mdUuid)))
.add(objectMapper.createObjectNode().set("match", objectMapper.createObjectNode().put("contactForResource.individual", contactForResourceIndividual))))))
);

// TODOES assertEquals(vicinityMapMetadata.getUuid(), document.getField("_uuid").stringValue());
// assertTrue(context.getBean(IndexingList.class).getIdentifiers().contains(vicinityMapMetadata.getId()));
return this.searchManager.query(query, Collections.emptySet(), 0, 10, null);
}

@Test
Expand Down Expand Up @@ -124,4 +157,34 @@ private AbstractMetadata insertContact(Element contactElement) throws Exception
when(springLocalServiceInvoker.invoke(any(String.class))).thenReturn(contactElement);
return contactMetadata;
}

private AbstractMetadata updateASubtemplateUsedByOneMetadata() throws Exception {
URL contactResource = AbstractCoreIntegrationTest.class.getResource("kernel/babarContact.xml");
Element contactElement = Xml.loadStream(contactResource.openStream());
AbstractMetadata contactMetadata = insertContact(contactElement);
AbstractMetadata vicinityMapMetadata = insertVicinityMap(contactMetadata);
assertEquals(1, countMetadataUsing(vicinityMapMetadata.getUuid(), "babar").getHits().getHits().length);
assertEquals(0, countMetadataUsing(vicinityMapMetadata.getUuid(), "momo").getHits().getHits().length);

Xml.selectElement(contactElement, "gmd:individualName/gco:CharacterString", Arrays.asList(GMD, GCO)).setText("momo");
metadataManager.updateMetadata(serviceContext,
Integer.toString(contactMetadata.getId()),
contactElement,
false,
false,
null,
null,
false,
IndexingMode.full);
return vicinityMapMetadata;
}

static public class SpyFactory {
@Autowired
BaseMetadataIndexer baseMetadataIndexer;

BaseMetadataIndexer build() {
return Mockito.spy(baseMetadataIndexer);
}
}
}
5 changes: 5 additions & 0 deletions core/src/test/resources/mocked-core-repository-test-context.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,9 @@
<bean id="localInvokerMock" class="org.mockito.Mockito" factory-method="mock" primary="true">
<constructor-arg value="org.fao.geonet.kernel.SpringLocalServiceInvoker" />
</bean>

<bean id="spyFactory" class="org.fao.geonet.kernel.LocalXLinksUpdateDeleteTest.SpyFactory" autowire = "byType"/>

<bean id="baseMetadataIndexerSpy" factory-bean="spyFactory" factory-method="build" primary="true" />

</beans>

0 comments on commit 3718541

Please sign in to comment.