Skip to content

Commit

Permalink
fix(openapi): fix openapi v2 endpoints & v3 documentation update
Browse files Browse the repository at this point in the history
  • Loading branch information
@david-leifker
david-leifker committed Aug 9, 2024
1 parent 080f2a2 commit 573c1cb
Show file tree
Hide file tree
Showing 4 changed files with 169 additions and 110 deletions.
204 changes: 115 additions & 89 deletions docs/api/tutorials/structured-properties.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,29 +158,37 @@ curl -X 'POST' -v \
-H 'accept: application/json' \
-H 'Content-Type: application/json' \
-d '{
"value": {
"qualifiedName": "io.acryl.privacy.retentionTime",
"valueType": "urn:li:dataType:datahub.number",
"description": "Retention Time is used to figure out how long to retain records in a dataset",
"displayName": "Retention Time",
"cardinality": "MULTIPLE",
"entityTypes": [
"urn:li:entityType:datahub.dataset",
"urn:li:entityType:datahub.dataFlow"
],
"allowedValues": [
{
"value": {"double": 30},
"description": "30 days, usually reserved for datasets that are ephemeral and contain pii"
},
{
"value": {"double": 60},
"description": "Use this for datasets that drive monthly reporting but contain pii"
},
{
"value": {"double": 365},
"description": "Use this for non-sensitive data that can be retained for longer"
}
]
"valueType": "urn:li:dataType:datahub.number",
"description": "Retention Time is used to figure out how long to retain records in a dataset",
"displayName": "Retention Time",
"cardinality": "MULTIPLE",
"entityTypes": [
"urn:li:entityType:datahub.dataset",
"urn:li:entityType:datahub.dataFlow"
],
"allowedValues": [
{
"value": {
"double": 30
},
"description": "30 days, usually reserved for datasets that are ephemeral and contain pii"
},
{
"value": {
"double": 60
},
"description": "Use this for datasets that drive monthly reporting but contain pii"
},
{
"value": {
"double": 365
},
"description": "Use this for non-sensitive data that can be retained for longer"
}
]
}
}' | jq
```

Expand Down Expand Up @@ -474,14 +482,16 @@ curl -X 'POST' -v \
-H 'accept: application/json' \
-H 'Content-Type: application/json' \
-d '{
"properties": [
{
"propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime",
"values": [
{"double": 60.0}
]
}
]
"value": {
"properties": [
{
"propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime",
"values": [
{"double": 60.0}
]
}
]
}
}' | jq
```
Example Response:
Expand Down Expand Up @@ -627,23 +637,25 @@ curl -X 'POST' -v \
-H 'accept: application/json' \
-H 'Content-Type: application/json' \
-d '{
"qualifiedName": "io.acryl.privacy.retentionTime02",
"displayName": "Retention Time 02",
"valueType": "urn:li:dataType:datahub.string",
"allowedValues": [
{
"value": {"string": "foo2"},
"description": "test foo2 value"
},
{
"value": {"string": "bar2"},
"description": "test bar2 value"
}
],
"cardinality": "SINGLE",
"entityTypes": [
"urn:li:entityType:datahub.dataset"
]
"value": {
"qualifiedName": "io.acryl.privacy.retentionTime02",
"displayName": "Retention Time 02",
"valueType": "urn:li:dataType:datahub.string",
"allowedValues": [
{
"value": {"string": "foo2"},
"description": "test foo2 value"
},
{
"value": {"string": "bar2"},
"description": "test bar2 value"
}
],
"cardinality": "SINGLE",
"entityTypes": [
"urn:li:entityType:datahub.dataset"
]
}
}' | jq
```

Expand Down Expand Up @@ -686,24 +698,26 @@ Specically, this will set `io.acryl.privacy.retentionTime` as `60.0` and `io.acr

```shell
curl -X 'POST' -v \
'http://localhost:8080/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \
'http://localhost:8080/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties?createIfNotExists=false' \
-H 'accept: application/json' \
-H 'Content-Type: application/json' \
-d '{
"properties": [
{
"propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime",
"values": [
{"double": 60.0}
]
},
{
"propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02",
"values": [
{"string": "bar2"}
]
}
]
"value": {
"properties": [
{
"propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime",
"values": [
{"double": 60.0}
]
},
{
"propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02",
"values": [
{"string": "bar2"}
]
}
]
}
}' | jq
```

Expand Down Expand Up @@ -1111,7 +1125,9 @@ curl -X 'POST' \
-H 'accept: application/json' \
-H 'Content-Type: application/json' \
-d '{
"removed": true
"value": {
"removed": true
}
}' | jq
```

Expand All @@ -1132,11 +1148,13 @@ If you want to **remove the soft delete**, you can do so by either hard deleting

```shell
curl -X 'POST' \
'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/status?systemMetadata=false' \
'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/status?systemMetadata=false&createIfNotExists=false' \
-H 'accept: application/json' \
-H 'Content-Type: application/json' \
-d '{
"removed": false
"value": {
"removed": true
}
}' | jq
```

Expand Down Expand Up @@ -1271,34 +1289,42 @@ Change the cardinality to `SINGLE` and add a `version`.

```shell
curl -X 'POST' -v \
'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/propertyDefinition' \
'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/propertyDefinition?createIfNotExists=false' \
-H 'accept: application/json' \
-H 'Content-Type: application/json' \
-d '{
"value": {
"qualifiedName": "io.acryl.privacy.retentionTime",
"valueType": "urn:li:dataType:datahub.number",
"description": "Retention Time is used to figure out how long to retain records in a dataset",
"displayName": "Retention Time",
"cardinality": "SINGLE",
"version": "20240614080000",
"entityTypes": [
"urn:li:entityType:datahub.dataset",
"urn:li:entityType:datahub.dataFlow"
],
"allowedValues": [
{
"value": {"double": 30},
"description": "30 days, usually reserved for datasets that are ephemeral and contain pii"
},
{
"value": {"double": 60},
"description": "Use this for datasets that drive monthly reporting but contain pii"
},
{
"value": {"double": 365},
"description": "Use this for non-sensitive data that can be retained for longer"
}
]
"valueType": "urn:li:dataType:datahub.number",
"description": "Retention Time is used to figure out how long to retain records in a dataset",
"displayName": "Retention Time",
"cardinality": "SINGLE",
"version": "20240614080000",
"entityTypes": [
"urn:li:entityType:datahub.dataset",
"urn:li:entityType:datahub.dataFlow"
],
"allowedValues": [
{
"value": {
"double": 30
},
"description": "30 days, usually reserved for datasets that are ephemeral and contain pii"
},
{
"value": {
"double": 60
},
"description": "Use this for datasets that drive monthly reporting but contain pii"
},
{
"value": {
"double": 365
},
"description": "Use this for non-sensitive data that can be retained for longer"
}
]
}
}' | jq
```

Expand Down
23 changes: 2 additions & 21 deletions ...servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,11 @@
import com.datahub.authorization.AuthorizerChain;
import com.datahub.util.RecordUtils;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableSet;
import com.linkedin.common.urn.Urn;
import com.linkedin.data.ByteString;
import com.linkedin.data.template.RecordTemplate;
import com.linkedin.entity.EnvelopedAspect;
import com.linkedin.events.metadata.ChangeType;
import com.linkedin.metadata.aspect.AspectRetriever;
import com.linkedin.metadata.aspect.batch.AspectsBatch;
import com.linkedin.metadata.aspect.batch.ChangeMCP;
Expand All @@ -41,7 +38,6 @@
import com.linkedin.metadata.search.SearchEntityArray;
import com.linkedin.metadata.search.SearchService;
import com.linkedin.metadata.utils.AuditStampUtils;
import com.linkedin.metadata.utils.GenericRecordUtils;
import com.linkedin.metadata.utils.SearchUtil;
import com.linkedin.mxe.SystemMetadata;
import com.linkedin.util.Pair;
Expand All @@ -57,7 +53,6 @@
import jakarta.servlet.http.HttpServletRequest;
import java.lang.reflect.InvocationTargetException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
Expand Down Expand Up @@ -726,28 +721,14 @@ protected RecordTemplate toRecordTemplate(
aspectSpec.getDataTemplateClass(), envelopedAspect.getValue().data());
}

protected ChangeMCP toUpsertItem(
protected abstract ChangeMCP toUpsertItem(
@Nonnull AspectRetriever aspectRetriever,
Urn entityUrn,
AspectSpec aspectSpec,
Boolean createIfNotExists,
String jsonAspect,
Actor actor)
throws JsonProcessingException {
JsonNode jsonNode = objectMapper.readTree(jsonAspect);
String aspectJson = jsonNode.get("value").toString();
return ChangeItemImpl.builder()
.urn(entityUrn)
.aspectName(aspectSpec.getName())
.changeType(Boolean.TRUE.equals(createIfNotExists) ? ChangeType.CREATE : ChangeType.UPSERT)
.auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr()))
.recordTemplate(
GenericRecordUtils.deserializeAspect(
ByteString.copyString(aspectJson, StandardCharsets.UTF_8),
GenericRecordUtils.JSON,
aspectSpec))
.build(aspectRetriever);
}
throws URISyntaxException, JsonProcessingException;

protected ChangeMCP toUpsertItem(
@Nonnull AspectRetriever aspectRetriever,
Expand Down
25 changes: 25 additions & 0 deletions ...enapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,11 @@
import com.linkedin.data.ByteString;
import com.linkedin.data.template.RecordTemplate;
import com.linkedin.entity.EnvelopedAspect;
import com.linkedin.events.metadata.ChangeType;
import com.linkedin.metadata.aspect.AspectRetriever;
import com.linkedin.metadata.aspect.batch.AspectsBatch;
import com.linkedin.metadata.aspect.batch.BatchItem;
import com.linkedin.metadata.aspect.batch.ChangeMCP;
import com.linkedin.metadata.entity.EntityApiUtils;
import com.linkedin.metadata.entity.IngestResult;
import com.linkedin.metadata.entity.UpdateAspectResult;
Expand Down Expand Up @@ -260,4 +263,26 @@ protected List<GenericEntityV2> buildEntityList(
}
return responseList;
}

@Override
protected ChangeMCP toUpsertItem(
@Nonnull AspectRetriever aspectRetriever,
Urn entityUrn,
AspectSpec aspectSpec,
Boolean createIfNotExists,
String jsonAspect,
Actor actor)
throws URISyntaxException {
return ChangeItemImpl.builder()
.urn(entityUrn)
.aspectName(aspectSpec.getName())
.changeType(Boolean.TRUE.equals(createIfNotExists) ? ChangeType.CREATE : ChangeType.UPSERT)
.auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr()))
.recordTemplate(
GenericRecordUtils.deserializeAspect(
ByteString.copyString(jsonAspect, StandardCharsets.UTF_8),
GenericRecordUtils.JSON,
aspectSpec))
.build(aspectRetriever);
}
}
Loading

0 comments on commit 573c1cb

Please sign in to comment.