From 7949498de58ad9cf74d28644e9b166f44ba38e1c Mon Sep 17 00:00:00 2001 From: Jisha Abubaker Date: Wed, 9 Aug 2017 11:55:17 -0700 Subject: [PATCH] adding DLP Quickstart + redact image sample (#797) * adding DLP Quickstart + redact image sample * adding comments --- dlp/README.md | 8 +- .../main/java/com/example/dlp/Inspect.java | 232 +++++++++--------- .../main/java/com/example/dlp/Metadata.java | 13 +- .../main/java/com/example/dlp/QuickStart.java | 100 ++++++++ dlp/src/main/java/com/example/dlp/Redact.java | 182 ++++++++++---- .../test/java/com/example/dlp/InspectIT.java | 26 +- .../test/java/com/example/dlp/MetadataIT.java | 24 +- .../java/com/example/dlp/QuickStartIT.java | 55 +++++ .../test/java/com/example/dlp/RedactIT.java | 49 +++- 9 files changed, 480 insertions(+), 209 deletions(-) create mode 100644 dlp/src/main/java/com/example/dlp/QuickStart.java create mode 100644 dlp/src/test/java/com/example/dlp/QuickStartIT.java diff --git a/dlp/README.md b/dlp/README.md index 8715bb5770c..2c976b5a73f 100644 --- a/dlp/README.md +++ b/dlp/README.md @@ -31,6 +31,13 @@ info types for a given category. eg. HEALTH or GOVERNMENT. java -cp target/dlp-samples-1.0-jar-with-dependencies.jar com.example.dlp.Metadata ``` +## Run the quickstart + +The Quickstart demonstrates using the DLP API to identify an InfoType in a given string. +``` + java -cp target/dlp-samples-1.0-jar-with-dependencies.jar com.example.dlp.QuickStart +``` + ## Inspect data for sensitive elements Inspect strings, files locally and on Google Cloud Storage and Cloud Datastore kinds with the DLP API. @@ -112,4 +119,3 @@ Run all tests: ``` mvn clean verify ``` - diff --git a/dlp/src/main/java/com/example/dlp/Inspect.java b/dlp/src/main/java/com/example/dlp/Inspect.java index dc010493dac..03e3b616b09 100644 --- a/dlp/src/main/java/com/example/dlp/Inspect.java +++ b/dlp/src/main/java/com/example/dlp/Inspect.java @@ -1,10 +1,11 @@ /** - * Copyright 2017, Google, Inc. + * Copyright 2017 Google Inc. + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -38,15 +39,6 @@ import com.google.privacy.dlp.v2beta1.ResultName; import com.google.privacy.dlp.v2beta1.StorageConfig; import com.google.protobuf.ByteString; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionGroup; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; - import java.net.URLConnection; import java.nio.file.Files; import java.nio.file.Path; @@ -55,11 +47,23 @@ import java.util.Collections; import java.util.List; import javax.activation.MimetypesFileTypeMap; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; public class Inspect { - private static void inspectString(String string, Likelihood minLikelihood, int maxFindings, - List infoTypes, boolean includeQuote) { + private static void inspectString( + String string, + Likelihood minLikelihood, + int maxFindings, + List infoTypes, + boolean includeQuote) { // [START dlp_inspect_string] // instantiate a client try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { @@ -75,24 +79,24 @@ private static void inspectString(String string, Likelihood minLikelihood, int m // Whether to include the matching string // includeQuote = true; - InspectConfig inspectConfig = InspectConfig.newBuilder() - .addAllInfoTypes(infoTypes) - .setMinLikelihood(minLikelihood) - .setMaxFindings(maxFindings) - .setIncludeQuote(includeQuote) - .build(); + InspectConfig inspectConfig = + InspectConfig.newBuilder() + .addAllInfoTypes(infoTypes) + .setMinLikelihood(minLikelihood) + .setMaxFindings(maxFindings) + .setIncludeQuote(includeQuote) + .build(); // The string to inspect // string = 'My name is Gary and my email is gary@example.com'; - ContentItem contentItem = ContentItem.newBuilder() - .setType("text/plain") - .setValue(string) - .build(); - - InspectContentRequest request = InspectContentRequest.newBuilder() - .setInspectConfig(inspectConfig) - .addItems(contentItem) - .build(); + ContentItem contentItem = + ContentItem.newBuilder().setType("text/plain").setValue(string).build(); + + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setInspectConfig(inspectConfig) + .addItems(contentItem) + .build(); InspectContentResponse response = dlpServiceClient.inspectContent(request); for (InspectResult result : response.getResultsList()) { @@ -115,8 +119,12 @@ private static void inspectString(String string, Likelihood minLikelihood, int m // [END dlp_inspect_string] } - private static void inspectFile(String filePath, Likelihood minLikelihood, int maxFindings, - List infoTypes, boolean includeQuote) { + private static void inspectFile( + String filePath, + Likelihood minLikelihood, + int maxFindings, + List infoTypes, + boolean includeQuote) { // [START dlp_inspect_file] // Instantiates a client try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { @@ -146,22 +154,22 @@ private static void inspectFile(String filePath, Likelihood minLikelihood, int m } byte[] data = Files.readAllBytes(path); - ContentItem contentItem = ContentItem.newBuilder() - .setType(mimeType) - .setData(ByteString.copyFrom(data)) - .build(); - - InspectConfig inspectConfig = InspectConfig.newBuilder() - .addAllInfoTypes(infoTypes) - .setMinLikelihood(minLikelihood) - .setMaxFindings(maxFindings) - .setIncludeQuote(includeQuote) - .build(); - - InspectContentRequest request = InspectContentRequest.newBuilder() - .setInspectConfig(inspectConfig) - .addItems(contentItem) - .build(); + ContentItem contentItem = + ContentItem.newBuilder().setType(mimeType).setData(ByteString.copyFrom(data)).build(); + + InspectConfig inspectConfig = + InspectConfig.newBuilder() + .addAllInfoTypes(infoTypes) + .setMinLikelihood(minLikelihood) + .setMaxFindings(maxFindings) + .setIncludeQuote(includeQuote) + .build(); + + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setInspectConfig(inspectConfig) + .addItems(contentItem) + .build(); InspectContentResponse response = dlpServiceClient.inspectContent(request); for (InspectResult result : response.getResultsList()) { @@ -185,13 +193,13 @@ private static void inspectFile(String filePath, Likelihood minLikelihood, int m // [END dlp_inspect_file] } - private static void inspectGcsFile(String bucketName, String fileName, - Likelihood minLikelihood, List infoTypes) + private static void inspectGcsFile( + String bucketName, String fileName, Likelihood minLikelihood, List infoTypes) throws Exception { // [START dlp_inspect_gcs] // Instantiates a client try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { -// The name of the bucket where the file resides. + // The name of the bucket where the file resides. // bucketName = 'YOUR-BUCKET'; // The path to the file within the bucket to inspect. @@ -207,21 +215,19 @@ private static void inspectGcsFile(String bucketName, String fileName, // The infoTypes of information to match // infoTypes = ['US_MALE_NAME', 'US_FEMALE_NAME']; - CloudStorageOptions cloudStorageOptions = CloudStorageOptions - .newBuilder() - .setFileSet(FileSet.newBuilder().setUrl( - "gs://" + bucketName + "/" + fileName - )) - .build(); + CloudStorageOptions cloudStorageOptions = + CloudStorageOptions.newBuilder() + .setFileSet(FileSet.newBuilder().setUrl("gs://" + bucketName + "/" + fileName)) + .build(); - StorageConfig storageConfig = StorageConfig.newBuilder() - .setCloudStorageOptions(cloudStorageOptions) - .build(); + StorageConfig storageConfig = + StorageConfig.newBuilder().setCloudStorageOptions(cloudStorageOptions).build(); - InspectConfig inspectConfig = InspectConfig.newBuilder() - .addAllInfoTypes(infoTypes) - .setMinLikelihood(minLikelihood) - .build(); + InspectConfig inspectConfig = + InspectConfig.newBuilder() + .addAllInfoTypes(infoTypes) + .setMinLikelihood(minLikelihood) + .build(); // optionally provide an output configuration to store results, default : none OutputStorageConfig outputConfig = OutputStorageConfig.getDefaultInstance(); @@ -252,8 +258,12 @@ private static void inspectGcsFile(String bucketName, String fileName, // [END dlp_inspect_gcs] } - private static void inspectDatastore(String projectId, String namespaceId, String kind, - Likelihood minLikelihood, List infoTypes) { + private static void inspectDatastore( + String projectId, + String namespaceId, + String kind, + Likelihood minLikelihood, + List infoTypes) { // [START dlp_inspect_datastore] // Instantiates a client try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { @@ -274,19 +284,24 @@ private static void inspectDatastore(String projectId, String namespaceId, Strin // The infoTypes of information to match // infoTypes = ['US_MALE_NAME', 'US_FEMALE_NAME']; - // Get reference to the file to be inspected - PartitionId partitionId = PartitionId.newBuilder().setProjectId(projectId) - .setNamespaceId(namespaceId).build(); + // Reference to the Datastore namespace + PartitionId partitionId = + PartitionId.newBuilder().setProjectId(projectId).setNamespaceId(namespaceId).build(); + + // Reference to the Datastore kind KindExpression kindExpression = KindExpression.newBuilder().setName(kind).build(); - DatastoreOptions datastoreOptions = DatastoreOptions.newBuilder() - .setKind(kindExpression).setPartitionId(partitionId).build(); - StorageConfig storageConfig = StorageConfig.newBuilder() - .setDatastoreOptions(datastoreOptions).build(); + DatastoreOptions datastoreOptions = + DatastoreOptions.newBuilder().setKind(kindExpression).setPartitionId(partitionId).build(); + + // Construct Datastore configuration to be inspected + StorageConfig storageConfig = + StorageConfig.newBuilder().setDatastoreOptions(datastoreOptions).build(); - InspectConfig inspectConfig = InspectConfig.newBuilder() - .addAllInfoTypes(infoTypes) - .setMinLikelihood(minLikelihood) - .build(); + InspectConfig inspectConfig = + InspectConfig.newBuilder() + .addAllInfoTypes(infoTypes) + .setMinLikelihood(minLikelihood) + .build(); // optionally provide an output configuration to store results, default : none OutputStorageConfig outputConfig = OutputStorageConfig.getDefaultInstance(); @@ -317,6 +332,10 @@ private static void inspectDatastore(String projectId, String namespaceId, Strin // [END dlp_inspect_datastore] } + /** + * Command line application to inspect data using the Data Loss Prevention API. + * Supported data formats : string, file, text files on GCS and Datastore entities + */ public static void main(String[] args) throws Exception { OptionGroup optionsGroup = new OptionGroup(); @@ -336,61 +355,37 @@ public static void main(String[] args) throws Exception { Options commandLineOptions = new Options(); commandLineOptions.addOptionGroup(optionsGroup); - Option minLikelihoodOption = Option.builder("minLikelihood") - .hasArg(true) - .required(false) - .build(); + Option minLikelihoodOption = + Option.builder("minLikelihood").hasArg(true).required(false).build(); commandLineOptions.addOption(minLikelihoodOption); - Option maxFindingsOption = Option.builder("maxFindings") - .hasArg(true) - .required(false) - .build(); + Option maxFindingsOption = Option.builder("maxFindings").hasArg(true).required(false).build(); commandLineOptions.addOption(maxFindingsOption); - Option infoTypesOption = Option.builder("infoTypes") - .hasArg(true) - .required(false) - .build(); + Option infoTypesOption = Option.builder("infoTypes").hasArg(true).required(false).build(); infoTypesOption.setArgs(Option.UNLIMITED_VALUES); commandLineOptions.addOption(infoTypesOption); - Option includeQuoteOption = Option.builder("includeQuote") - .hasArg(true) - .required(false) - .build(); + Option includeQuoteOption = Option.builder("includeQuote").hasArg(true).required(false).build(); commandLineOptions.addOption(includeQuoteOption); - Option bucketNameOption = Option.builder("bucketName") - .hasArg(true) - .required(false) - .build(); + Option bucketNameOption = Option.builder("bucketName").hasArg(true).required(false).build(); commandLineOptions.addOption(bucketNameOption); - Option gcsFileNameOption = Option.builder("fileName") - .hasArg(true) - .required(false) - .build(); + Option gcsFileNameOption = Option.builder("fileName").hasArg(true).required(false).build(); commandLineOptions.addOption(gcsFileNameOption); - Option datastoreProjectIdOption = Option.builder("projectId") - .hasArg(true) - .required(false) - .build(); + Option datastoreProjectIdOption = + Option.builder("projectId").hasArg(true).required(false).build(); commandLineOptions.addOption(datastoreProjectIdOption); - Option datastoreNamespaceOption = Option.builder("namespace") - .hasArg(true) - .required(false) - .build(); + Option datastoreNamespaceOption = + Option.builder("namespace").hasArg(true).required(false).build(); commandLineOptions.addOption(datastoreNamespaceOption); - Option datastoreKindOption = Option.builder("kind") - .hasArg(true) - .required(false) - .build(); + Option datastoreKindOption = Option.builder("kind").hasArg(true).required(false).build(); commandLineOptions.addOption(datastoreKindOption); CommandLineParser parser = new DefaultParser(); @@ -406,11 +401,13 @@ public static void main(String[] args) throws Exception { return; } - Likelihood minLikelihood = Likelihood.valueOf(cmd.getOptionValue(minLikelihoodOption.getOpt(), - Likelihood.LIKELIHOOD_UNSPECIFIED.name())); + Likelihood minLikelihood = + Likelihood.valueOf( + cmd.getOptionValue( + minLikelihoodOption.getOpt(), Likelihood.LIKELIHOOD_UNSPECIFIED.name())); int maxFindings = Integer.parseInt(cmd.getOptionValue(maxFindingsOption.getOpt(), "0")); - boolean includeQuote = Boolean - .parseBoolean(cmd.getOptionValue(includeQuoteOption.getOpt(), "true")); + boolean includeQuote = + Boolean.parseBoolean(cmd.getOptionValue(includeQuoteOption.getOpt(), "true")); List infoTypesList = Collections.emptyList(); if (cmd.hasOption(infoTypesOption.getOpt())) { @@ -437,8 +434,9 @@ public static void main(String[] args) throws Exception { String namespaceId = cmd.getOptionValue(datastoreNamespaceOption.getOpt(), ""); String kind = cmd.getOptionValue(datastoreKindOption.getOpt()); // use default project id when project id is not specified - String projectId = cmd.getOptionValue(datastoreProjectIdOption.getOpt(), - ServiceOptions.getDefaultProjectId()); + String projectId = + cmd.getOptionValue( + datastoreProjectIdOption.getOpt(), ServiceOptions.getDefaultProjectId()); inspectDatastore(projectId, namespaceId, kind, minLikelihood, infoTypesList); } } diff --git a/dlp/src/main/java/com/example/dlp/Metadata.java b/dlp/src/main/java/com/example/dlp/Metadata.java index 8045a22ef35..c6ffda065f6 100644 --- a/dlp/src/main/java/com/example/dlp/Metadata.java +++ b/dlp/src/main/java/com/example/dlp/Metadata.java @@ -1,10 +1,11 @@ /** - * Copyright 2017, Google, Inc. + * Copyright 2017 Google Inc. + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,6 +21,7 @@ import com.google.privacy.dlp.v2beta1.InfoTypeDescription; import com.google.privacy.dlp.v2beta1.ListInfoTypesResponse; import com.google.privacy.dlp.v2beta1.ListRootCategoriesResponse; +import java.util.List; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; @@ -28,8 +30,6 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import java.util.List; - public class Metadata { private static void listInfoTypes(String category, String languageCode) throws Exception { @@ -54,8 +54,8 @@ private static void listRootCategories(String languageCode) throws Exception { try (DlpServiceClient dlpClient = DlpServiceClient.create()) { // The BCP-47 language code to use, e.g. 'en-US' // languageCode = 'en-US' - ListRootCategoriesResponse rootCategoriesResponse = dlpClient - .listRootCategories(languageCode); + ListRootCategoriesResponse rootCategoriesResponse = + dlpClient.listRootCategories(languageCode); for (CategoryDescription categoryDescription : rootCategoriesResponse.getCategoriesList()) { System.out.println("Name : " + categoryDescription.getName()); System.out.println("Display name : " + categoryDescription.getDisplayName()); @@ -64,6 +64,7 @@ private static void listRootCategories(String languageCode) throws Exception { // [END dlp_list_root_categories] } + /** Retrieve infoTypes. */ public static void main(String[] args) throws Exception { Options options = new Options(); Option languageCodeOption = new Option("language", null, true, "BCP-47 language code"); diff --git a/dlp/src/main/java/com/example/dlp/QuickStart.java b/dlp/src/main/java/com/example/dlp/QuickStart.java new file mode 100644 index 00000000000..ff5f932ef8c --- /dev/null +++ b/dlp/src/main/java/com/example/dlp/QuickStart.java @@ -0,0 +1,100 @@ +/** + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.dlp; + +import com.google.cloud.dlp.v2beta1.DlpServiceClient; +import com.google.privacy.dlp.v2beta1.ContentItem; +import com.google.privacy.dlp.v2beta1.Finding; +import com.google.privacy.dlp.v2beta1.InfoType; +import com.google.privacy.dlp.v2beta1.InspectConfig; +import com.google.privacy.dlp.v2beta1.InspectContentRequest; +import com.google.privacy.dlp.v2beta1.InspectContentResponse; +import com.google.privacy.dlp.v2beta1.InspectResult; +import com.google.privacy.dlp.v2beta1.Likelihood; +import java.util.Arrays; +import java.util.List; + +// [START dlp_quickstart] +public class QuickStart { + + /** Quick start to DLP API : inspects a given string for an InfoType. */ + public static void main(String[] args) throws Exception { + + // string to inspect + String text = "Robert Frost"; + + // The minimum likelihood required before returning a match: + // LIKELIHOOD_UNSPECIFIED, VERY_UNLIKELY, UNLIKELY, POSSIBLE, LIKELY, VERY_LIKELY, UNRECOGNIZED + Likelihood minLikelihood = Likelihood.VERY_LIKELY; + + // The maximum number of findings to report (0 = server maximum) + int maxFindings = 0; + + // The infoTypes of information to match + List infoTypes = + Arrays.asList( + InfoType.newBuilder().setName("US_MALE_NAME").build(), + InfoType.newBuilder().setName("US_FEMALE_NAME").build()); + + // Whether to include the matching string + boolean includeQuote = true; + + // instantiate a client + try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) { + + InspectConfig inspectConfig = + InspectConfig.newBuilder() + .addAllInfoTypes(infoTypes) + .setMinLikelihood(minLikelihood) + .setMaxFindings(maxFindings) + .setIncludeQuote(includeQuote) + .build(); + + ContentItem contentItem = + ContentItem.newBuilder().setType("text/plain").setValue(text).build(); + + InspectContentRequest request = + InspectContentRequest.newBuilder() + .setInspectConfig(inspectConfig) + .addItems(contentItem) + .build(); + + // Inspect the text for info types + InspectContentResponse response = dlpServiceClient.inspectContent(request); + + // Print the response + for (InspectResult result : response.getResultsList()) { + if (result.getFindingsCount() > 0) { + System.out.println("Findings: "); + for (Finding finding : result.getFindingsList()) { + if (includeQuote) { + System.out.print("Quote: " + finding.getQuote()); + } + System.out.print("\tInfo type: " + finding.getInfoType().getName()); + System.out.println("\tLikelihood: " + finding.getLikelihood()); + } + } else { + System.out.println("No findings."); + } + } + } catch (Exception e) { + System.out.println("Error in inspectString: " + e.getMessage()); + } + } +} + +// [END dlp_quickstart] diff --git a/dlp/src/main/java/com/example/dlp/Redact.java b/dlp/src/main/java/com/example/dlp/Redact.java index 780c34dd3ff..8a332ca063e 100644 --- a/dlp/src/main/java/com/example/dlp/Redact.java +++ b/dlp/src/main/java/com/example/dlp/Redact.java @@ -1,5 +1,6 @@ /** - * Copyright 2017, Google, Inc. + * Copyright 2017 Google Inc. + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -20,61 +21,66 @@ import com.google.privacy.dlp.v2beta1.InfoType; import com.google.privacy.dlp.v2beta1.InspectConfig; import com.google.privacy.dlp.v2beta1.Likelihood; +import com.google.privacy.dlp.v2beta1.RedactContentRequest; +import com.google.privacy.dlp.v2beta1.RedactContentRequest.ImageRedactionConfig; import com.google.privacy.dlp.v2beta1.RedactContentRequest.ReplaceConfig; import com.google.privacy.dlp.v2beta1.RedactContentResponse; import com.google.protobuf.ByteString; +import java.io.FileOutputStream; +import java.net.URLConnection; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import javax.activation.MimetypesFileTypeMap; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - public class Redact { - private static void redactString(String string, String replacement, Likelihood minLikelihood, - List infoTypes) throws Exception { + private static void redactString( + String string, String replacement, Likelihood minLikelihood, List infoTypes) + throws Exception { // [START dlp_redact_string] // Instantiate the DLP client try (DlpServiceClient dlpClient = DlpServiceClient.create()) { // The minimum likelihood required before returning a match // eg.minLikelihood = LIKELIHOOD_VERY_LIKELY; - InspectConfig inspectConfig = InspectConfig.newBuilder() - .addAllInfoTypes(infoTypes) - .setMinLikelihood(minLikelihood) - .build(); - - ContentItem contentItem = ContentItem.newBuilder() - .setType("text/plain") - .setData(ByteString.copyFrom(string.getBytes())) - .build(); + InspectConfig inspectConfig = + InspectConfig.newBuilder() + .addAllInfoTypes(infoTypes) + .setMinLikelihood(minLikelihood) + .build(); + + ContentItem contentItem = + ContentItem.newBuilder() + .setType("text/plain") + .setData(ByteString.copyFrom(string.getBytes())) + .build(); List replaceConfigs = new ArrayList<>(); if (infoTypes.isEmpty()) { // replace all detected sensitive elements with replacement string - replaceConfigs.add( - ReplaceConfig.newBuilder() - .setReplaceWith(replacement) - .build()); + replaceConfigs.add(ReplaceConfig.newBuilder().setReplaceWith(replacement).build()); } else { // Replace select info types with chosen replacement string for (InfoType infoType : infoTypes) { replaceConfigs.add( - ReplaceConfig.newBuilder() - .setInfoType(infoType) - .setReplaceWith(replacement) - .build()); + ReplaceConfig.newBuilder().setInfoType(infoType).setReplaceWith(replacement).build()); } } - RedactContentResponse contentResponse = dlpClient.redactContent( - inspectConfig, Collections.singletonList(contentItem), replaceConfigs); + RedactContentResponse contentResponse = + dlpClient.redactContent( + inspectConfig, Collections.singletonList(contentItem), replaceConfigs); for (ContentItem responseItem : contentResponse.getItemsList()) { // print out string with redacted content System.out.println(responseItem.getData().toStringUtf8()); @@ -83,37 +89,99 @@ private static void redactString(String string, String replacement, Likelihood m // [END dlp_redact_string] } - // Command line application to redact strings using the Data Loss Prevention API + private static void redactImage( + String filePath, Likelihood minLikelihood, List infoTypes, String outputPath) + throws Exception { + // [START dlp_redact_image] + // Instantiate the DLP client + try (DlpServiceClient dlpClient = DlpServiceClient.create()) { + // The path to a local file to inspect. Can be a JPG or PNG image file. + // filePath = 'path/to/image.png' + // detect file mime type, default to application/octet-stream + String mimeType = URLConnection.guessContentTypeFromName(filePath); + if (mimeType == null) { + mimeType = MimetypesFileTypeMap.getDefaultFileTypeMap().getContentType(filePath); + } + if (mimeType == null) { + mimeType = "application/octet-stream"; + } + + byte[] data = Files.readAllBytes(Paths.get(filePath)); + + // The minimum likelihood required before redacting a match + // minLikelihood = 'LIKELIHOOD_UNSPECIFIED' + + // The infoTypes of information to redact + // infoTypes = [{ name: 'EMAIL_ADDRESS' }, { name: 'PHONE_NUMBER' }] + + // The local path to save the resulting image to. + // outputPath = 'result.png' + + InspectConfig inspectConfig = + InspectConfig.newBuilder() + .addAllInfoTypes(infoTypes) + .setMinLikelihood(minLikelihood) + .build(); + ContentItem contentItem = + ContentItem.newBuilder().setType(mimeType).setData(ByteString.copyFrom(data)).build(); + + List imageRedactionConfigs = new ArrayList<>(); + for (InfoType infoType : infoTypes) { + // clear the specific info type if detected in the image + // use .setRedactionColor to color detected info type without clearing + ImageRedactionConfig imageRedactionConfig = + ImageRedactionConfig.newBuilder().setInfoType(infoType).clearTarget().build(); + imageRedactionConfigs.add(imageRedactionConfig); + } + RedactContentRequest redactContentRequest = + RedactContentRequest.newBuilder() + .setInspectConfig(inspectConfig) + .addAllImageRedactionConfigs(imageRedactionConfigs) + .addItems(contentItem) + .build(); + + RedactContentResponse contentResponse = dlpClient.redactContent(redactContentRequest); + for (ContentItem responseItem : contentResponse.getItemsList()) { + // redacted image data + ByteString redactedImageData = responseItem.getData(); + FileOutputStream outputStream = new FileOutputStream(outputPath); + outputStream.write(redactedImageData.toByteArray()); + outputStream.close(); + } + // [END dlp_redact_image] + } + } + + /** Command line application to redact strings, images using the Data Loss Prevention API. */ public static void main(String[] args) throws Exception { + OptionGroup optionsGroup = new OptionGroup(); + optionsGroup.setRequired(true); + Option stringOption = new Option("s", "string", true, "redact string"); + optionsGroup.addOption(stringOption); + + Option fileOption = new Option("f", "file path", true, "redact input file path"); + optionsGroup.addOption(fileOption); + Options commandLineOptions = new Options(); + commandLineOptions.addOptionGroup(optionsGroup); - Option stringOption = Option.builder("s") - .longOpt("source string") - .hasArg(true) - .required(true) - .build(); - commandLineOptions.addOption(stringOption); - - Option replaceOption = Option.builder("r") - .longOpt("replace string") - .hasArg(true) - .required(true) - .build(); - commandLineOptions.addOption(replaceOption); + Option minLikelihoodOption = + Option.builder("minLikelihood").hasArg(true).required(false).build(); - Option minLikelihoodOption = Option.builder("minLikelihood") - .hasArg(true) - .required(false) - .build(); commandLineOptions.addOption(minLikelihoodOption); - Option infoTypesOption = Option.builder("infoTypes") - .hasArg(true) - .required(false) - .build(); + Option replaceOption = + Option.builder("r").longOpt("replace string").hasArg(true).required(false).build(); + commandLineOptions.addOption(replaceOption); + + Option infoTypesOption = Option.builder("infoTypes").hasArg(true).required(false).build(); infoTypesOption.setArgs(Option.UNLIMITED_VALUES); commandLineOptions.addOption(infoTypesOption); + Option outputFilePathOption = + Option.builder("o").hasArg(true).longOpt("outputFilePath").required(false).build(); + commandLineOptions.addOption(outputFilePathOption); + CommandLineParser parser = new DefaultParser(); HelpFormatter formatter = new HelpFormatter(); CommandLine cmd; @@ -127,8 +195,7 @@ public static void main(String[] args) throws Exception { return; } - String source = cmd.getOptionValue(stringOption.getOpt()); - String replacement = cmd.getOptionValue(replaceOption.getOpt()); + String replacement = cmd.getOptionValue(replaceOption.getOpt(), "_REDACTED_"); List infoTypesList = new ArrayList<>(); String[] infoTypes = cmd.getOptionValues(infoTypesOption.getOpt()); @@ -137,6 +204,19 @@ public static void main(String[] args) throws Exception { infoTypesList.add(InfoType.newBuilder().setName(infoType).build()); } } - redactString(source, replacement, Likelihood.LIKELIHOOD_UNSPECIFIED, infoTypesList); + Likelihood minLikelihood = + Likelihood.valueOf( + cmd.getOptionValue( + minLikelihoodOption.getOpt(), Likelihood.LIKELIHOOD_UNSPECIFIED.name())); + + // string inspection + if (cmd.hasOption("s")) { + String source = cmd.getOptionValue(stringOption.getOpt()); + redactString(source, replacement, minLikelihood, infoTypesList); + } else if (cmd.hasOption("f")) { + String filePath = cmd.getOptionValue(fileOption.getOpt()); + String outputFilePath = cmd.getOptionValue(outputFilePathOption.getOpt()); + redactImage(filePath, minLikelihood, infoTypesList, outputFilePath); + } } } diff --git a/dlp/src/test/java/com/example/dlp/InspectIT.java b/dlp/src/test/java/com/example/dlp/InspectIT.java index fff56b734b0..fbeb536bd5f 100644 --- a/dlp/src/test/java/com/example/dlp/InspectIT.java +++ b/dlp/src/test/java/com/example/dlp/InspectIT.java @@ -1,32 +1,34 @@ /** - * Copyright 2017, Google, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You may obtain a copy of the License - * at + * Copyright 2017 Google Inc. * - *

http://www.apache.org/licenses/LICENSE-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ + package com.example.dlp; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.PrintStream; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.PrintStream; - @RunWith(JUnit4.class) -@SuppressWarnings("checkstyle:abbreviationaswordinname") public class InspectIT { private ByteArrayOutputStream bout; private PrintStream out; diff --git a/dlp/src/test/java/com/example/dlp/MetadataIT.java b/dlp/src/test/java/com/example/dlp/MetadataIT.java index ebd0d1a2e9e..044d534dbaa 100644 --- a/dlp/src/test/java/com/example/dlp/MetadataIT.java +++ b/dlp/src/test/java/com/example/dlp/MetadataIT.java @@ -1,31 +1,33 @@ /** - * Copyright 2017, Google, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You may obtain a copy of the License - * at + * Copyright 2017 Google Inc. * - *

http://www.apache.org/licenses/LICENSE-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ + package com.example.dlp; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; - @RunWith(JUnit4.class) -@SuppressWarnings("checkstyle:abbreviationaswordinname") public class MetadataIT { private ByteArrayOutputStream bout; diff --git a/dlp/src/test/java/com/example/dlp/QuickStartIT.java b/dlp/src/test/java/com/example/dlp/QuickStartIT.java new file mode 100644 index 00000000000..9b156f1f87a --- /dev/null +++ b/dlp/src/test/java/com/example/dlp/QuickStartIT.java @@ -0,0 +1,55 @@ +/** + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.dlp; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class QuickStartIT { + private ByteArrayOutputStream bout; + private PrintStream out; + + @Before + public void setUp() { + bout = new ByteArrayOutputStream(); + out = new PrintStream(bout); + System.setOut(out); + assertNotNull(System.getenv("GOOGLE_APPLICATION_CREDENTIALS")); + } + + @Test + public void testQuickStart() throws Exception { + QuickStart.main(new String[] {}); + String output = bout.toString(); + assertTrue(output.contains("US_MALE_NAME")); + } + + @After + public void tearDown() { + System.setOut(null); + bout.reset(); + } +} diff --git a/dlp/src/test/java/com/example/dlp/RedactIT.java b/dlp/src/test/java/com/example/dlp/RedactIT.java index 6e768a1cedf..d93c097ead3 100644 --- a/dlp/src/test/java/com/example/dlp/RedactIT.java +++ b/dlp/src/test/java/com/example/dlp/RedactIT.java @@ -1,31 +1,35 @@ /** - * Copyright 2017, Google, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You may obtain a copy of the License - * at + * Copyright 2017 Google Inc. * - *

http://www.apache.org/licenses/LICENSE-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ + package com.example.dlp; +import static junit.framework.TestCase.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.PrintStream; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; - @RunWith(JUnit4.class) -@SuppressWarnings("checkstyle:abbreviationaswordinname") public class RedactIT { private ByteArrayOutputStream bout; private PrintStream out; @@ -47,6 +51,29 @@ public void testInfoTypesInStringAreReplaced() throws Exception { assertTrue(output.contains("My phone number is _REDACTED_ and my email address is _REDACTED_")); } + @Test + public void testInfoTypesInImageAreReplaced() throws Exception { + ClassLoader classLoader = getClass().getClassLoader(); + // confirm that current data contains info types + File file = new File(classLoader.getResource("test.png").getFile()); + Inspect.main(new String[] {"-f", file.getAbsolutePath()}); + String output = bout.toString(); + assertTrue(output.contains("PHONE_NUMBER")); + assertTrue(output.contains("EMAIL_ADDRESS")); + bout.reset(); + + String outputFilePath = "output.png"; + + Redact.main( + new String[] { + "-f", file.getAbsolutePath(), "-infoTypes", "PHONE_NUMBER", "-o", outputFilePath + }); + Inspect.main(new String[] {"-f", outputFilePath}); + output = bout.toString(); + assertFalse(output.contains("PHONE_NUMBER")); + assertTrue(output.contains("EMAIL_ADDRESS")); + } + @After public void tearDown() { System.setOut(null);