Skip to content

Commit

Permalink
feat: sql federated catalog cache (#218)
Browse files Browse the repository at this point in the history
  • Loading branch information
wolf4ood authored Jul 10, 2024
1 parent 4fbfdaa commit 42b2607
Show file tree
Hide file tree
Showing 19 changed files with 916 additions and 131 deletions.
17 changes: 13 additions & 4 deletions DEPENDENCIES
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,16 @@ maven/mavencentral/com.apicatalog/titanium-json-ld/1.4.0, Apache-2.0, approved,
maven/mavencentral/com.fasterxml.jackson.core/jackson-annotations/2.10.3, Apache-2.0, approved, CQ21280
maven/mavencentral/com.fasterxml.jackson.core/jackson-annotations/2.16.2, Apache-2.0, approved, #11606
maven/mavencentral/com.fasterxml.jackson.core/jackson-annotations/2.17.0, Apache-2.0, approved, #13672
maven/mavencentral/com.fasterxml.jackson.core/jackson-annotations/2.17.1, Apache-2.0, approved, #13672
maven/mavencentral/com.fasterxml.jackson.core/jackson-annotations/2.17.2, Apache-2.0, approved, #13672
maven/mavencentral/com.fasterxml.jackson.core/jackson-core/2.16.2, Apache-2.0 AND MIT, approved, #11602
maven/mavencentral/com.fasterxml.jackson.core/jackson-core/2.17.2, , approved, #13665
maven/mavencentral/com.fasterxml.jackson.core/jackson-databind/2.16.2, Apache-2.0, approved, #11605
maven/mavencentral/com.fasterxml.jackson.core/jackson-databind/2.17.0, Apache-2.0, approved, #13671
maven/mavencentral/com.fasterxml.jackson.core/jackson-databind/2.17.1, Apache-2.0, approved, #13671
maven/mavencentral/com.fasterxml.jackson.core/jackson-databind/2.17.2, Apache-2.0, approved, #13671
maven/mavencentral/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/2.16.2, Apache-2.0, approved, #11855
maven/mavencentral/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/2.17.2, Apache-2.0, approved, #13669
maven/mavencentral/com.fasterxml.jackson.datatype/jackson-datatype-jakarta-jsonp/2.17.1, Apache-2.0, approved, #14161
maven/mavencentral/com.fasterxml.jackson.datatype/jackson-datatype-jakarta-jsonp/2.17.2, Apache-2.0, approved, #14161
maven/mavencentral/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.16.2, Apache-2.0, approved, #11853
maven/mavencentral/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.17.1, Apache-2.0, approved, #14160
maven/mavencentral/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.17.2, Apache-2.0, approved, #14160
maven/mavencentral/com.fasterxml.jackson.jakarta.rs/jackson-jakarta-rs-base/2.17.2, Apache-2.0, approved, #14194
maven/mavencentral/com.fasterxml.jackson.jakarta.rs/jackson-jakarta-rs-json-provider/2.16.2, Apache-2.0, approved, #11858
Expand Down Expand Up @@ -93,9 +89,12 @@ maven/mavencentral/jakarta.xml.bind/jakarta.xml.bind-api/4.0.2, BSD-3-Clause, ap
maven/mavencentral/javax.servlet/javax.servlet-api/3.1.0, (CDDL-1.1 OR GPL-2.0-only WITH Classpath-exception-2.0) AND Apache-2.0, approved, CQ7248
maven/mavencentral/javax.ws.rs/javax.ws.rs-api/2.1, (CDDL-1.1 OR GPL-2.0 WITH Classpath-exception-2.0) AND Apache-2.0, approved, CQ18121
maven/mavencentral/junit/junit/4.13.2, EPL-2.0, approved, CQ23636
maven/mavencentral/net.bytebuddy/byte-buddy-agent/1.14.1, Apache-2.0, approved, #7164
maven/mavencentral/net.bytebuddy/byte-buddy-agent/1.14.15, Apache-2.0, approved, #7164
maven/mavencentral/net.bytebuddy/byte-buddy/1.14.1, Apache-2.0 AND BSD-3-Clause, approved, #7163
maven/mavencentral/net.bytebuddy/byte-buddy/1.14.15, Apache-2.0 AND BSD-3-Clause, approved, #7163
maven/mavencentral/net.bytebuddy/byte-buddy/1.14.16, Apache-2.0 AND BSD-3-Clause, approved, #7163
maven/mavencentral/net.bytebuddy/byte-buddy/1.14.18, Apache-2.0 AND BSD-3-Clause, approved, #7163
maven/mavencentral/net.java.dev.jna/jna/5.13.0, Apache-2.0 AND LGPL-2.1-or-later, approved, #15196
maven/mavencentral/net.sf.saxon/Saxon-HE/12.4, MPL-2.0 AND (MPL-2.0 AND Apache-2.0) AND (MPL-2.0 AND LicenseRef-X11-style) AND MPL-1.0 AND W3C, approved, #12716
maven/mavencentral/org.antlr/antlr4-runtime/4.13.1, BSD-3-Clause, approved, #10767
Expand Down Expand Up @@ -124,11 +123,13 @@ maven/mavencentral/org.apache.maven.doxia/doxia-sink-api/1.12.0, Apache-2.0, app
maven/mavencentral/org.apache.xbean/xbean-reflect/3.7, Apache-2.0, approved, clearlydefined
maven/mavencentral/org.apiguardian/apiguardian-api/1.1.2, Apache-2.0, approved, clearlydefined
maven/mavencentral/org.assertj/assertj-core/3.26.0, Apache-2.0, approved, #14886
maven/mavencentral/org.assertj/assertj-core/3.26.3, Apache-2.0, approved, #14886
maven/mavencentral/org.awaitility/awaitility/4.2.0, Apache-2.0, approved, #14178
maven/mavencentral/org.bouncycastle/bcpkix-jdk18on/1.78.1, MIT, approved, #14434
maven/mavencentral/org.bouncycastle/bcprov-jdk18on/1.78.1, MIT AND CC0-1.0, approved, #14433
maven/mavencentral/org.bouncycastle/bcutil-jdk18on/1.78.1, MIT, approved, #14435
maven/mavencentral/org.ccil.cowan.tagsoup/tagsoup/1.2.1, Apache-2.0, approved, clearlydefined
maven/mavencentral/org.checkerframework/checker-qual/3.42.0, MIT, approved, clearlydefined
maven/mavencentral/org.checkerframework/checker-qual/3.43.0, MIT, approved, clearlydefined
maven/mavencentral/org.codehaus.plexus/plexus-classworlds/2.6.0, Apache-2.0 AND Plexus, approved, CQ22821
maven/mavencentral/org.codehaus.plexus/plexus-component-annotations/2.1.0, Apache-2.0, approved, #809
Expand Down Expand Up @@ -237,6 +238,7 @@ maven/mavencentral/org.eclipse.edc/policy-spi/0.8.1-SNAPSHOT, Apache-2.0, approv
maven/mavencentral/org.eclipse.edc/query-lib/0.8.1-SNAPSHOT, Apache-2.0, approved, technology.edc
maven/mavencentral/org.eclipse.edc/runtime-metamodel/0.8.1-SNAPSHOT, Apache-2.0, approved, technology.edc
maven/mavencentral/org.eclipse.edc/secrets-spi/0.8.1-SNAPSHOT, Apache-2.0, approved, technology.edc
maven/mavencentral/org.eclipse.edc/sql-core/0.8.1-SNAPSHOT, Apache-2.0, approved, technology.edc
maven/mavencentral/org.eclipse.edc/state-machine-lib/0.8.1-SNAPSHOT, Apache-2.0, approved, technology.edc
maven/mavencentral/org.eclipse.edc/store-lib/0.8.1-SNAPSHOT, Apache-2.0, approved, technology.edc
maven/mavencentral/org.eclipse.edc/token-core/0.8.1-SNAPSHOT, Apache-2.0, approved, technology.edc
Expand Down Expand Up @@ -311,12 +313,15 @@ maven/mavencentral/org.jetbrains/annotations/13.0, Apache-2.0, approved, clearly
maven/mavencentral/org.jetbrains/annotations/17.0.0, Apache-2.0, approved, clearlydefined
maven/mavencentral/org.jetbrains/annotations/24.1.0, Apache-2.0, approved, clearlydefined
maven/mavencentral/org.junit-pioneer/junit-pioneer/2.2.0, EPL-2.0, approved, #11857
maven/mavencentral/org.junit.jupiter/junit-jupiter-api/5.10.2, EPL-2.0, approved, #9714
maven/mavencentral/org.junit.jupiter/junit-jupiter-api/5.10.3, EPL-2.0, approved, #9714
maven/mavencentral/org.junit.jupiter/junit-jupiter-engine/5.10.3, EPL-2.0, approved, #9711
maven/mavencentral/org.junit.jupiter/junit-jupiter-params/5.10.3, EPL-2.0, approved, #15250
maven/mavencentral/org.junit.platform/junit-platform-commons/1.10.2, EPL-2.0, approved, #9715
maven/mavencentral/org.junit.platform/junit-platform-commons/1.10.3, EPL-2.0, approved, #9715
maven/mavencentral/org.junit.platform/junit-platform-engine/1.10.3, EPL-2.0, approved, #9709
maven/mavencentral/org.junit.platform/junit-platform-launcher/1.10.3, EPL-2.0, approved, #15216
maven/mavencentral/org.junit/junit-bom/5.10.2, EPL-2.0, approved, #9844
maven/mavencentral/org.junit/junit-bom/5.10.3, EPL-2.0, approved, #9844
maven/mavencentral/org.junit/junit-bom/5.9.2, EPL-2.0, approved, #4711
maven/mavencentral/org.jvnet.mimepull/mimepull/1.9.15, CDDL-1.1 OR GPL-2.0-only WITH Classpath-exception-2.0, approved, CQ21484
Expand All @@ -330,13 +335,17 @@ maven/mavencentral/org.ow2.asm/asm-tree/9.5, BSD-3-Clause, approved, #7555
maven/mavencentral/org.ow2.asm/asm-tree/9.7, BSD-3-Clause, approved, #14073
maven/mavencentral/org.ow2.asm/asm/9.5, BSD-3-Clause, approved, #7554
maven/mavencentral/org.ow2.asm/asm/9.7, BSD-3-Clause, approved, #14076
maven/mavencentral/org.postgresql/postgresql/42.7.3, BSD-2-Clause AND Apache-2.0, approved, #11681
maven/mavencentral/org.reflections/reflections/0.10.2, Apache-2.0 AND WTFPL, approved, clearlydefined
maven/mavencentral/org.rnorth.duct-tape/duct-tape/1.0.8, MIT, approved, clearlydefined
maven/mavencentral/org.slf4j/slf4j-api/1.7.25, MIT, approved, CQ13368
maven/mavencentral/org.slf4j/slf4j-api/1.7.30, MIT, approved, CQ13368
maven/mavencentral/org.slf4j/slf4j-api/1.7.36, MIT, approved, CQ13368
maven/mavencentral/org.slf4j/slf4j-api/2.0.9, MIT, approved, #5915
maven/mavencentral/org.testcontainers/database-commons/1.19.8, Apache-2.0, approved, #10345
maven/mavencentral/org.testcontainers/jdbc/1.19.8, Apache-2.0, approved, #10348
maven/mavencentral/org.testcontainers/junit-jupiter/1.19.8, MIT, approved, #10344
maven/mavencentral/org.testcontainers/postgresql/1.19.8, MIT, approved, #10350
maven/mavencentral/org.testcontainers/testcontainers/1.19.8, MIT, approved, #15203
maven/mavencentral/org.xmlresolver/xmlresolver/5.2.2, Apache-2.0, approved, clearlydefined
maven/mavencentral/org.yaml/snakeyaml/2.2, Apache-2.0 AND (Apache-2.0 OR BSD-3-Clause OR EPL-1.0 OR GPL-2.0-or-later OR LGPL-2.1-or-later), approved, #10232
2 changes: 2 additions & 0 deletions core/federated-catalog-core/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ dependencies {
testImplementation(libs.edc.junit)
testImplementation(libs.edc.ext.http)
testImplementation(libs.awaitility)

testImplementation(testFixtures(project(":spi:federated-catalog-spi")))
}

edcBuild {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,138 +15,20 @@
package org.eclipse.edc.catalog.defaults.store;


import org.eclipse.edc.catalog.spi.CatalogConstants;
import org.eclipse.edc.catalog.spi.FederatedCatalogCache;
import org.eclipse.edc.catalog.spi.testfixtures.FederatedCatalogCacheTestBase;
import org.eclipse.edc.catalog.store.InMemoryFederatedCatalogCache;
import org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset;
import org.eclipse.edc.connector.controlplane.catalog.spi.Catalog;
import org.eclipse.edc.connector.controlplane.catalog.spi.DataService;
import org.eclipse.edc.connector.controlplane.catalog.spi.Dataset;
import org.eclipse.edc.connector.controlplane.catalog.spi.Distribution;
import org.eclipse.edc.query.CriterionOperatorRegistryImpl;
import org.eclipse.edc.spi.query.QuerySpec;
import org.eclipse.edc.util.concurrency.LockManager;
import org.junit.jupiter.api.Test;

import java.util.List;
import java.util.UUID;
import java.util.concurrent.locks.ReentrantReadWriteLock;

import static org.assertj.core.api.Assertions.assertThat;

class InMemoryFederatedCatalogCacheTest {
class InMemoryFederatedCatalogCacheTest extends FederatedCatalogCacheTestBase {

private final InMemoryFederatedCatalogCache store = new InMemoryFederatedCatalogCache(new LockManager(new ReentrantReadWriteLock()), CriterionOperatorRegistryImpl.ofDefaults());

@Test
void queryCacheContainingOneElementWithNoCriterion_shouldReturnUniqueElement() {
var contractOfferId = UUID.randomUUID().toString();
var assetId = UUID.randomUUID().toString();
var catalogEntry = createCatalog(contractOfferId, createAsset(assetId));

store.save(catalogEntry);

var result = store.query(QuerySpec.none());

assertThat(result)
.hasSize(1)
.allSatisfy(co -> assertThat(co.getDatasets().get(0).getId()).isEqualTo(assetId));
}

@Test
void queryCacheAfterInsertingSameAssetTwice_shouldReturnLastInsertedContractOfferOnly() {
var contractOfferId1 = UUID.randomUUID().toString();
var assetId = UUID.randomUUID().toString();
var entry1 = createCatalog(contractOfferId1, createAsset(assetId));
var entry2 = createCatalog(contractOfferId1, createAsset(assetId));

store.save(entry1);
store.save(entry2);

var result = store.query(QuerySpec.none());

assertThat(result)
.hasSize(1)
.allSatisfy(co -> {
assertThat(co.getId()).isEqualTo(contractOfferId1);
assertThat(co.getDatasets().get(0).getId()).isEqualTo(assetId);
});

}

@Test
void queryCacheContainingTwoDistinctAssets_shouldReturnBothContractOffers() {
var contractOfferId1 = UUID.randomUUID().toString();
var contractOfferId2 = UUID.randomUUID().toString();
var assetId1 = UUID.randomUUID().toString();
var assetId2 = UUID.randomUUID().toString();
var entry1 = createCatalog(contractOfferId1, createAsset(assetId1));
var entry2 = createCatalog(contractOfferId2, createAsset(assetId2));

store.save(entry1);
store.save(entry2);

var result = store.query(QuerySpec.none());

assertThat(result)
.hasSize(2)
.anySatisfy(co -> assertThat(co.getDatasets().get(0).getId()).isEqualTo(assetId1))
.anySatisfy(co -> assertThat(co.getDatasets().get(0).getId()).isEqualTo(assetId2));
}

@Test
void removedMarked_noneMarked() {
var contractOfferId1 = UUID.randomUUID().toString();
var contractOfferId2 = UUID.randomUUID().toString();
var assetId1 = UUID.randomUUID().toString();
var assetId2 = UUID.randomUUID().toString();
var entry1 = createCatalog(contractOfferId1, createAsset(assetId1));
var entry2 = createCatalog(contractOfferId2, createAsset(assetId2));

store.save(entry1);
store.save(entry2);

assertThat(store.query(QuerySpec.none())).hasSize(2);

store.deleteExpired(); // none of them is marked, d
assertThat(store.query(QuerySpec.none())).containsExactlyInAnyOrder(entry1, entry2);

}

@Test
void removedMarked_shouldDeleteMarked() {
var contractOfferId1 = UUID.randomUUID().toString();
var contractOfferId2 = UUID.randomUUID().toString();
var assetId1 = UUID.randomUUID().toString();
var assetId2 = UUID.randomUUID().toString();
var entry1 = createCatalog(contractOfferId1, createAsset(assetId1));
var entry2 = createCatalog(contractOfferId2, createAsset(assetId2));

store.save(entry1);
store.save(entry2);

assertThat(store.query(QuerySpec.none())).hasSize(2);

store.expireAll(); // two items marked
store.save(createCatalog(UUID.randomUUID().toString(), createAsset(UUID.randomUUID().toString())));
store.deleteExpired(); // should delete only marked items
assertThat(store.query(QuerySpec.none())).hasSize(1)
.doesNotContain(entry1, entry2);

}

private Catalog createCatalog(String id, Asset asset) {
var dataService = DataService.Builder.newInstance().build();
return Catalog.Builder.newInstance()
.id(id)
.dataServices(List.of(dataService))
.datasets(List.of(Dataset.Builder.newInstance().id(asset.getId()).distributions(List.of(Distribution.Builder.newInstance().dataService(dataService).format("test-format").build())).build()))
.property(CatalogConstants.PROPERTY_ORIGINATOR, "https://test.source/" + id)
.build();
}

private Asset createAsset(String id) {
return Asset.Builder.newInstance()
.id(id)
.build();

@Override
protected FederatedCatalogCache getStore() {
return store;
}
}
29 changes: 29 additions & 0 deletions extensions/store/sql/federated-catalog-cache-sql/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0
*
* SPDX-License-Identifier: Apache-2.0
*
* Contributors:
* Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
*
*/

plugins {
`java-library`
}

dependencies {
api(project(":spi:federated-catalog-spi"))
testImplementation(testFixtures(libs.edc.core.sql))
implementation(libs.edc.core.sql) // for the SqlStatements
implementation(libs.edc.spi.transaction.datasource)
implementation(libs.edc.lib.util)

testImplementation(libs.edc.junit)
testImplementation(testFixtures(libs.edc.core.sql))
testImplementation(testFixtures(project(":spi:federated-catalog-spi")))
}
21 changes: 21 additions & 0 deletions extensions/store/sql/federated-catalog-cache-sql/docs/schema.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
/*
* Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0
*
* SPDX-License-Identifier: Apache-2.0
*
* Contributors:
* Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
*
*/

-- only intended for and tested with Postgres!
CREATE TABLE IF NOT EXISTS edc_federated_catalog
(
id VARCHAR PRIMARY KEY NOT NULL,
catalog JSON,
marked BOOLEAN DEFAULT FALSE
);
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
/*
* Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0
*
* SPDX-License-Identifier: Apache-2.0
*
* Contributors:
* Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
*
*/

package org.eclipse.edc.catalog.store.sql;

import org.eclipse.edc.catalog.store.sql.schema.postgres.FederatedCatalogMapping;
import org.eclipse.edc.spi.query.QuerySpec;
import org.eclipse.edc.sql.translation.SqlOperatorTranslator;
import org.eclipse.edc.sql.translation.SqlQueryStatement;

import static java.lang.String.format;

public abstract class BaseSqlDialectStatements implements FederatedCatalogCacheStatements {

protected final SqlOperatorTranslator operatorTranslator;

protected BaseSqlDialectStatements(SqlOperatorTranslator operatorTranslator) {
this.operatorTranslator = operatorTranslator;
}

@Override
public String getFindByIdTemplate() {
return format("SELECT * FROM %s WHERE %s = ?", getFederatedCatalogTable(), getIdColumn());
}

@Override
public String getUpdateAsMarkedTemplate() {
return format("UPDATE %s SET %s = ?", getFederatedCatalogTable(), getMarkedColumn());
}

@Override
public String getDeleteByMarkedTemplate() {
return executeStatement()
.delete(getFederatedCatalogTable(), getMarkedColumn());
}

@Override
public String getInsertTemplate() {
return executeStatement()
.column(getIdColumn())
.jsonColumn(getCatalogColumn())
.column(getMarkedColumn())
.insertInto(getFederatedCatalogTable());
}

@Override
public String getUpdateTemplate() {
return executeStatement()
.jsonColumn(getCatalogColumn())
.column(getMarkedColumn())
.update(getFederatedCatalogTable(), getIdColumn());
}

@Override
public SqlQueryStatement createQuery(QuerySpec querySpec) {
var select = getSelectStatement();
return new SqlQueryStatement(select, querySpec, new FederatedCatalogMapping(this), operatorTranslator);
}

@Override
public String getSelectStatement() {
return format("SELECT * FROM %s", getFederatedCatalogTable());
}
}
Loading

0 comments on commit 42b2607

Please sign in to comment.