Merge branch 'main' into aking-1077-classify-data-category-dropdown

* main:
  Restrict startsWith comparisons in CheckboxTree more (#1126)
  [1076] ui/plus: Approve dataset classification button (#1129)
  Do not rely on order for checking intercept results (#1131)
  Prepare 1.9.1 release (#1137)
  Bump fideslang to 1.3.1 (#1136)
  Prepare changelog for 1.9.0 release (#1134)
  Update CHANGELOG.md (#1132)
  Fix bug causing missing datamap rows (#1124)
  cls migration (#1060)

.fides/dataset.yml

@@ -144,6 +144,106 @@ dataset:
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
       retention: null
       fields: null
+  - name: cls_classification_detail
+    description: 'Fides Generated Description for Table: cls_classification_detail'
+    data_categories: null
+    data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    fields:
+    - name: collection
+      description: 'Fides Generated Description for Column: collection'
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: created_at
+      description: 'Fides Generated Description for Column: created_at'
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: dataset
+      description: 'Fides Generated Description for Column: dataset'
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: field
+      description: 'Fides Generated Description for Column: field'
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: id
+      description: 'Fides Generated Description for Column: id'
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: instance_id
+      description: 'Fides Generated Description for Column: instance_id'
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: labels
+      description: 'Fides Generated Description for Column: labels'
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: status
+      description: 'Fides Generated Description for Column: status'
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: updated_at
+      description: 'Fides Generated Description for Column: updated_at'
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+  - name: cls_classification_instance
+    description: 'Fides Generated Description for Table: cls_classification_instance'
+    data_categories: null
+    data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    fields:
+    - name: created_at
+      description: 'Fides Generated Description for Column: created_at'
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: id
+      description: 'Fides Generated Description for Column: id'
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: organization_key
+      description: 'Fides Generated Description for Column: organization_key'
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: dataset_key
+      description: 'Fides Generated Description for Column: dataset_key'
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: dataset_name
+      description: 'Fides Generated Description for Column: dataset_name'
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: status
+      description: 'Fides Generated Description for Column: status'
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: target
+      description: 'Fides Generated Description for Column: target'
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: type
+      description: 'Fides Generated Description for Column: type'
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: updated_at
+      description: 'Fides Generated Description for Column: updated_at'
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
   - name: ctl_data_categories
     description: null
     data_categories: null

CHANGELOG.md

@@ -15,7 +15,19 @@ The types of changes are:
 * `Fixed` for any bug fixes.
 * `Security` in case of vulnerabilities.
 
-## [Unreleased](https://github.com/ethyca/fides/compare/1.8.5...main)
+## [Unreleased](https://github.com/ethyca/fides/compare/1.9.1...main)
+
+## [1.9.1](https://github.com/ethyca/fides/compare/1.9.0...1.9.1)
+
+### Changed
+
+* Update fideslang to v1.3.1 [#1136](https://github.com/ethyca/fides/pull/1136)
+
+### Changed
+
+* Update fideslang to v1.3.1 [#1136](https://github.com/ethyca/fides/pull/1136)
+
+## [1.9.0](https://github.com/ethyca/fides/compare/1.8.6...1.9.0) - 2022-09-29
 
 ### Added
 
@@ -25,6 +37,7 @@ The types of changes are:
   * Initial Classification & Review status for generated datasets. [#1074](https://github.com/ethyca/fides/pull/1074)
   * Component for choosing data categories based on classification results. [#1110](https://github.com/ethyca/fides/pull/1110)
   * The dataset fields table shows data categories from the classifier (if available). [#1088](https://github.com/ethyca/fides/pull/1088)
+  * The "Approve" button can be used to update the dataset with the classifier's suggestions. [#1129](https://github.com/ethyca/fides/pull/1129)
 * System management UI:
   * New page to add a system via yaml [#1062](https://github.com/ethyca/fides/pull/1062)
   * Skeleton of page to add a system manually [#1068](https://github.com/ethyca/fides/pull/1068)
@@ -33,6 +46,7 @@ The types of changes are:
   * Delete a system through the UI [#1085](https://github.com/ethyca/fides/pull/1085)
   * Edit a system through the UI [#1096](https://github.com/ethyca/fides/pull/1096)
 * Cypress component testing [#1106](https://github.com/ethyca/fides/pull/1106)
+
 ### Changed
 
 * Changed behavior of `load_default_taxonomy` to append instead of upsert [#1040](https://github.com/ethyca/fides/pull/1040)
@@ -43,12 +57,23 @@ The types of changes are:
 ### Fixed
 
 * Fixed the "help" link in the UI header [#1078](https://github.com/ethyca/fides/pull/1078)
+* Fixed a bug in Data Category Dropdowns where checking i.e. `user.biometric` would also check `user.biometric_health` [#1126](https://github.com/ethyca/fides/pull/1126)
 
 ### Security
 
 * Upgraded pymysql to version `1.0.2` [#1094](https://github.com/ethyca/fides/pull/1094)
 
-## [1.8.5](https://github.com/ethyca/fides/compare/1.8.4...1.8.5)
+## [1.8.6](https://github.com/ethyca/fides/compare/1.8.5...1.8.6) - 2022-09-28
+
+### Added
+
+* Added classification tables for Plus users [#1060](https://github.com/ethyca/fides/pull/1060)
+
+### Fixed
+
+* Fixed a bug where rows were being excluded from a data map [#1124](https://github.com/ethyca/fides/pull/1124)
+
+## [1.8.5](https://github.com/ethyca/fides/compare/1.8.4...1.8.5) - 2022-09-21
 
 ### Changed
 

clients/ctl/admin-ui/__tests__/checkbox-tree.test.tsx

@@ -28,6 +28,11 @@ const MOCK_NODES = [
         value: "grandparent.aunt",
         children: [],
       },
+      {
+        label: "aunt_second",
+        value: "grandparent.aunt_second",
+        children: [],
+      },
     ],
   },
   {
@@ -236,6 +241,7 @@ describe("Checkbox tree", () => {
           "grandparent.parent.child",
           "grandparent.parent.sibling",
           "grandparent.aunt",
+          "grandparent.aunt_second",
         ].sort()
       );
       expect(
@@ -254,6 +260,13 @@ describe("Checkbox tree", () => {
           (d) => d.value
         )
       ).toEqual(["grandparent.parent.child"]);
+
+      // make sure aunt_second does not sneak in
+      expect(
+        getDescendantsAndCurrent(MOCK_NODES, "grandparent.aunt").map(
+          (d) => d.value
+        )
+      ).toEqual(["grandparent.aunt"]);
     });
 
     it("can determine if an ancestor is selected", () => {

clients/ctl/admin-ui/__tests__/features/dataset-helpers.test.tsx

@@ -1,11 +1,16 @@
 import {
   getUpdatedCollectionFromField,
+  getUpdatedDatasetFromClassifyDataset,
   getUpdatedDatasetFromCollection,
   getUpdatedDatasetFromField,
   removeCollectionFromDataset,
   removeFieldFromDataset,
 } from "~/features/dataset/helpers";
 import {
+  mockClassification,
+  mockClassifyCollection,
+  mockClassifyDataset,
+  mockClassifyField,
   mockDataset,
   mockDatasetCollection,
   mockDatasetField,
@@ -27,6 +32,7 @@ describe("dataset helpers", () => {
       );
       expect(updatedCollection.fields[0].name).toEqual(newName);
     });
+
     it("should update a dataset from a collection", () => {
       const newDescription = "updated description";
       const originalCollection = mockDatasetCollection();
@@ -42,6 +48,7 @@ describe("dataset helpers", () => {
       );
       expect(updatedDataset.collections[0].description).toEqual(newDescription);
     });
+
     it("should update a dataset from a field", () => {
       const newName = "a fancy email";
       const originalField = mockDatasetField({ name: "a regular email" });
@@ -69,7 +76,85 @@ describe("dataset helpers", () => {
         updatedDataset.collections[collectionIndex].fields[fieldIndex].name
       ).toEqual(newName);
     });
+
+    it("should update a Dataset from a ClassifyInstance", () => {
+      const originalDataset = mockDataset({
+        collections: [
+          mockDatasetCollection({
+            name: "users",
+            fields: [
+              // Field without data categories:
+              mockDatasetField({
+                name: "email",
+                data_categories: [],
+              }),
+              // Field with data categories:
+              mockDatasetField({
+                name: "state",
+                data_categories: ["system.operations"],
+              }),
+              // Field without a corresponding classify field:
+              mockDatasetField({
+                name: "shoe_size",
+                data_categories: [],
+              }),
+            ],
+          }),
+        ],
+      });
+      const classifyDataset = mockClassifyDataset({
+        collections: [
+          mockClassifyCollection({
+            name: "users",
+            fields: [
+              mockClassifyField({
+                name: "email",
+                classifications: [
+                  mockClassification({
+                    label: "user.contact",
+                    score: 0.75,
+                  }),
+                  mockClassification({
+                    label: "user.email",
+                    score: 0.95,
+                  }),
+                ],
+              }),
+              mockClassifyField({
+                name: "state",
+                classifications: [
+                  mockClassification({
+                    label: "user.address",
+                  }),
+                ],
+              }),
+            ],
+          }),
+        ],
+      });
+
+      const updatedDataset = getUpdatedDatasetFromClassifyDataset(
+        originalDataset,
+        classifyDataset
+      );
+
+      // It should return a new object.
+      expect(updatedDataset).not.toBe(originalDataset);
+      // A field without any categories should be filled in with the high score suggestion.
+      expect(updatedDataset.collections[0].fields[0].data_categories).toEqual([
+        "user.email",
+      ]);
+      // A field that already has a category should be unchanged.
+      expect(updatedDataset.collections[0].fields[1].data_categories).toEqual([
+        "system.operations",
+      ]);
+      // A field that had no classification match should be unchanged.
+      expect(updatedDataset.collections[0].fields[2].data_categories).toEqual(
+        []
+      );
+    });
   });
+
   describe("removing from datasets", () => {
     it("should be able to remove a dataset field", () => {
       const deleteName = "remove me";
@@ -101,6 +186,7 @@ describe("dataset helpers", () => {
         )
       ).toHaveLength(0);
     });
+
     it("should be able to remove a dataset collection", () => {
       const deleteName = "remove me";
       const collection = mockDatasetCollection({ name: deleteName });

clients/ctl/admin-ui/cypress/components/DataCategoryInput.cy.tsx

@@ -1,11 +1,10 @@
+import { stubPlus } from "cypress/support/stubs";
 import * as React from "react";
 
 import DataCategoryInput from "~/features/dataset/DataCategoryInput";
 import { MOCK_DATA_CATEGORIES } from "~/mocks/data";
 import { DataCategory } from "~/types/api";
 
-import { stubPlusHealth } from "../support/stubs";
-
 describe("DataCategoryInput", () => {
   it("can check a category", () => {
     const onCheckedSpy = cy.spy().as("onCheckedSpy");
@@ -31,7 +30,7 @@ describe("DataCategoryInput", () => {
   });
 
   it("can render the classified version", () => {
-    stubPlusHealth();
+    stubPlus(true);
 
     const onCheckedSpy = cy.spy().as("onCheckedSpy");
     cy.mount(