Data quality assistant (#2657)

backend/src/api/dataQuality/dataQualityMember.ts

@@ -0,0 +1,44 @@
+import { FeatureFlag } from '@crowd/types'
+
+import DataQualityService from '@/services/dataQualityService'
+
+import isFeatureEnabled from '../../feature-flags/isFeatureEnabled'
+import Permissions from '../../security/permissions'
+import PermissionChecker from '../../services/user/permissionChecker'
+
+/**
+ * GET /tenant/{tenantId}/data-quality/member
+ * @summary Find a member data issues
+ * @tag Data Quality
+ * @security Bearer
+ * @description Find a data quality issues for members
+ * @pathParam {string} tenantId - Your workspace/tenant ID
+ * @response 200 - Ok
+ * @responseContent {DataQualityResponse} 200.application/json
+ * @response 401 - Unauthorized
+ * @response 404 - Not found
+ * @response 429 - Too many requests
+ */
+export default async (req, res) => {
+  new PermissionChecker(req).validateHas(Permissions.values.memberRead)
+
+  const segmentId = req.query.segments?.length > 0 ? req.query.segments[0] : null
+  if (!segmentId) {
+    const segmentsEnabled = await isFeatureEnabled(FeatureFlag.SEGMENTS, req)
+    if (segmentsEnabled) {
+      await req.responseHandler.error(req, res, {
+        code: 400,
+        message: 'Segment ID is required',
+      })
+      return
+    }
+  }
+
+  const payload = await new DataQualityService(req).findMemberIssues(
+    req.params.tenantId,
+    req.query,
+    segmentId,
+  )
+
+  await req.responseHandler.success(req, res, payload)
+}

backend/src/api/dataQuality/dataQualityOrganization.ts

@@ -0,0 +1,40 @@
+import { FeatureFlag } from '@crowd/types'
+
+import DataQualityService from '@/services/dataQualityService'
+
+import isFeatureEnabled from '../../feature-flags/isFeatureEnabled'
+import Permissions from '../../security/permissions'
+import PermissionChecker from '../../services/user/permissionChecker'
+
+/**
+ * GET /tenant/{tenantId}/data-quality/organization
+ * @summary Find a organization data issues
+ * @tag Data Quality
+ * @security Bearer
+ * @description Find a data quality issues for organizations
+ * @pathParam {string} tenantId - Your workspace/tenant ID
+ * @response 200 - Ok
+ * @responseContent {DataQualityResponse} 200.application/json
+ * @response 401 - Unauthorized
+ * @response 404 - Not found
+ * @response 429 - Too many requests
+ */
+export default async (req, res) => {
+  new PermissionChecker(req).validateHas(Permissions.values.organizationRead)
+
+  const segmentId = req.query.segments?.length > 0 ? req.query.segments[0] : null
+  if (!segmentId) {
+    const segmentsEnabled = await isFeatureEnabled(FeatureFlag.SEGMENTS, req)
+    if (segmentsEnabled) {
+      await req.responseHandler.error(req, res, {
+        code: 400,
+        message: 'Segment ID is required',
+      })
+      return
+    }
+  }
+
+  const payload = await new DataQualityService(req).findOrganizationIssues()
+
+  await req.responseHandler.success(req, res, payload)
+}

backend/src/api/dataQuality/index.ts

@@ -0,0 +1,9 @@
+import { safeWrap } from '../../middlewares/errorMiddleware'
+
+export default (app) => {
+  app.get(`/tenant/:tenantId/data-quality/member`, safeWrap(require('./dataQualityMember').default))
+  app.get(
+    `/tenant/:tenantId/data-quality/organization`,
+    safeWrap(require('./dataQualityOrganization').default),
+  )
+}

backend/src/api/index.ts

@@ -243,6 +243,7 @@ setImmediate(async () => {
   require('./customViews').default(routes)
   require('./dashboard').default(routes)
   require('./mergeAction').default(routes)
+  require('./dataQuality').default(routes)
   // Loads the Tenant if the :tenantId param is passed
   routes.param('tenantId', tenantMiddleware)
   routes.param('tenantId', segmentMiddleware)

backend/src/database/repositories/dataQualityRepository.ts

@@ -0,0 +1,166 @@
+import {
+  fetchMembersWithConflictingWorkExperiences,
+  fetchMembersWithMissingInfoOnWorkExperience,
+  fetchMembersWithMissingPeriodOnWorkExperience,
+  fetchMembersWithTooManyEmails,
+  fetchMembersWithTooManyIdentities,
+  fetchMembersWithTooManyIdentitiesPerPlatform,
+  fetchMembersWithoutWorkExperience,
+} from '@crowd/data-access-layer/src/data-quality'
+
+import SequelizeRepository from '@/database/repositories/sequelizeRepository'
+
+import { IRepositoryOptions } from './IRepositoryOptions'
+
+class DataQualityRepository {
+  /**
+   * Finds members with no work experience.
+   *
+   * @param {IRepositoryOptions} options - The repository options for executing the query.
+   * @param {string} tenantId - The ID of the tenant.
+   * @param {number} limit - The maximum number of results to return.
+   * @param {number} offset - The offset from which to begin returning results.
+   * @param {string} segmentId - The ID of the segment.
+   * @return {Promise<Members[]>} - A promise that resolves with an array of members having no work experience.
+   */
+  static async findMembersWithNoWorkExperience(
+    options: IRepositoryOptions,
+    tenantId: string,
+    limit: number,
+    offset: number,
+    segmentId: string,
+  ) {
+    const qx = SequelizeRepository.getQueryExecutor(options)
+    return fetchMembersWithoutWorkExperience(qx, tenantId, limit, offset, segmentId)
+  }
+
+  /**
+   * Finds and returns members with too many identities.
+   * Executes a query to fetch members who exceed a certain number of identities.
+   *
+   * @param {IRepositoryOptions} options - Repository options for querying the database.
+   * @param {string} tenantId - Identifier of the tenant whose members are being queried.
+   * @param {number} limit - The maximum number of members to retrieve.
+   * @param {number} offset - The number of members to skip before starting to collect the result set.
+   * @param {string} segmentId - Identifier of the segment to filter members.
+   * @return {Promise<Array>} A promise that resolves to an array of members with too many identities.
+   */
+  static async findMembersWithTooManyIdentities(
+    options: IRepositoryOptions,
+    tenantId: string,
+    limit: number,
+    offset: number,
+    segmentId: string,
+  ) {
+    const qx = SequelizeRepository.getQueryExecutor(options)
+    return fetchMembersWithTooManyIdentities(qx, 20, tenantId, limit, offset, segmentId)
+  }
+
+  /**
+   * Finds members with too many identities per platform.
+   *
+   * @param {IRepositoryOptions} options - The repository options for database connection and other configurations.
+   * @param {string} tenantId - The ID of the tenant to filter members by.
+   * @param {number} limit - The maximum number of members to return.
+   * @param {number} offset - The number of members to skip before starting to collect the result set.
+   * @param {string} segmentId - The ID of the segment to filter members by.
+   *
+   * @return {Promise<Array>} A promise that resolves to an array of members with too many identities per platform.
+   */
+  static async findMembersWithTooManyIdentitiesPerPlatform(
+    options: IRepositoryOptions,
+    tenantId: string,
+    limit: number,
+    offset: number,
+    segmentId: string,
+  ) {
+    const qx = SequelizeRepository.getQueryExecutor(options)
+    return fetchMembersWithTooManyIdentitiesPerPlatform(qx, 1, tenantId, limit, offset, segmentId)
+  }
+
+  /**
+   * Finds members who have too many emails within a given tenant and segment.
+   *
+   * @param {IRepositoryOptions} options - The repository options containing configuration and context for the query.
+   * @param {string} tenantId - The identifier for the tenant where members are queried.
+   * @param {number} limit - The maximum number of members to return.
+   * @param {number} offset - The starting point for pagination.
+   * @param {string} segmentId - The identifier for the segment to filter members.
+   * @return {Promise<Array>} - A promise that resolves to an array of members who have too many emails.
+   */
+  static async findMembersWithTooManyEmails(
+    options: IRepositoryOptions,
+    tenantId: string,
+    limit: number,
+    offset: number,
+    segmentId: string,
+  ) {
+    const qx = SequelizeRepository.getQueryExecutor(options)
+    return fetchMembersWithTooManyEmails(qx, 5, tenantId, limit, offset, segmentId)
+  }
+
+  /**
+   * Finds members with missing information on work experience.
+   *
+   * @param {IRepositoryOptions} options - The repository options to be used.
+   * @param {string} tenantId - The unique identifier of the tenant.
+   * @param {number} limit - The maximum number of records to fetch.
+   * @param {number} offset - The number of records to skip.
+   * @param {string} segmentId - The segment identifier to be used for filtering.
+   * @return {Promise<Array>} A promise that resolves to an array of members with missing work experience information.
+   */
+  static async findMembersWithMissingInfoOnWorkExperience(
+    options: IRepositoryOptions,
+    tenantId: string,
+    limit: number,
+    offset: number,
+    segmentId: string,
+  ) {
+    const qx = SequelizeRepository.getQueryExecutor(options)
+    return fetchMembersWithMissingInfoOnWorkExperience(qx, tenantId, limit, offset, segmentId)
+  }
+
+  /**
+   * Fetches members whose work experience period is missing.
+   *
+   * @param {IRepositoryOptions} options - The repository options for database access.
+   * @param {string} tenantId - The ID of the tenant to find members for.
+   * @param {number} limit - The maximum number of members to return.
+   * @param {number} offset - The offset to start fetching members from.
+   * @param {string} segmentId - The ID of the segment to filter members.
+   * @return {Promise<Member[]>} A promise that resolves to an array of members with missing work experience periods.
+   */
+  static async findMembersWithMissingPeriodOnWorkExperience(
+    options: IRepositoryOptions,
+    tenantId: string,
+    limit: number,
+    offset: number,
+    segmentId: string,
+  ) {
+    const qx = SequelizeRepository.getQueryExecutor(options)
+    return fetchMembersWithMissingPeriodOnWorkExperience(qx, tenantId, limit, offset, segmentId)
+  }
+
+  /**
+   * Finds members with conflicting work experience based on specified options.
+   *
+   * @param {IRepositoryOptions} options - The repository options for database query execution.
+   * @param {string} tenantId - The ID of the tenant to filter members.
+   * @param {number} limit - The maximum number of records to fetch.
+   * @param {number} offset - The number of records to skip for pagination.
+   * @param {string} segmentId - The ID of the segment to filter members.
+   * @return {Promise<Array>} - A promise that resolves to an array of members with conflicting work experiences.
+   */
+  static async findMembersWithConflictingWorkExperience(
+    options: IRepositoryOptions,
+    tenantId: string,
+    limit: number,
+    offset: number,
+    segmentId: string,
+  ) {
+    const qx = SequelizeRepository.getQueryExecutor(options)
+    return fetchMembersWithConflictingWorkExperiences(qx, tenantId, limit, offset, segmentId)
+  }
+}
+
+export default DataQualityRepository

backend/src/database/repositories/memberRepository.ts

@@ -534,11 +534,13 @@ class MemberRepository {
             {
               id: i.id,
               displayName: i.primaryDisplayName,
+              activityCount: i.primaryActivityCount,
               avatarUrl: i.primaryAvatarUrl,
             },
             {
               id: i.toMergeId,
               displayName: i.toMergeDisplayName,
+              activityCount: i.toActivityCount,
               avatarUrl: i.toMergeAvatarUrl,
             },
           ],

backend/src/services/dataQualityService.ts

@@ -0,0 +1,54 @@
+/* eslint-disable no-continue */
+import { LoggerBase } from '@crowd/logging'
+
+import DataQualityRepository from '@/database/repositories/dataQualityRepository'
+import { IDataQualityParams, IDataQualityType } from '@/types/data-quality/data-quality-filters'
+
+import { IServiceOptions } from './IServiceOptions'
+
+export default class DataQualityService extends LoggerBase {
+  options: IServiceOptions
+
+  constructor(options: IServiceOptions) {
+    super(options.log)
+    this.options = options
+  }
+
+  /**
+   * Finds issues related to member data quality based on the specified type.
+   *
+   * @param {string} tenantId - The ID of the tenant for whom to find member issues.
+   * @param {IDataQualityParams} params - The parameters for finding member issues, including the type of issue, limit, and offset.
+   * @param {string} segmentId - The ID of the segment where the members belong.
+   * @return {Promise<Array>} A promise that resolves to an array of members with the specified data quality issues.
+   */
+  async findMemberIssues(tenantId: string, params: IDataQualityParams, segmentId: string) {
+    const methodMap = {
+      [IDataQualityType.NO_WORK_EXPERIENCE]: DataQualityRepository.findMembersWithNoWorkExperience,
+      [IDataQualityType.TOO_MANY_IDENTITIES]:
+        DataQualityRepository.findMembersWithTooManyIdentities,
+      [IDataQualityType.TOO_MANY_IDENTITIES_PER_PLATFORM]:
+        DataQualityRepository.findMembersWithTooManyIdentitiesPerPlatform,
+      [IDataQualityType.TOO_MANY_EMAILS]: DataQualityRepository.findMembersWithTooManyEmails,
+      [IDataQualityType.WORK_EXPERIENCE_MISSING_INFO]:
+        DataQualityRepository.findMembersWithMissingInfoOnWorkExperience,
+      [IDataQualityType.WORK_EXPERIENCE_MISSING_PERIOD]:
+        DataQualityRepository.findMembersWithMissingPeriodOnWorkExperience,
+      [IDataQualityType.CONFLICTING_WORK_EXPERIENCE]:
+        DataQualityRepository.findMembersWithConflictingWorkExperience,
+    }
+
+    const method = methodMap[params.type]
+
+    if (method) {
+      return method(this.options, tenantId, params.limit || 10, params.offset || 0, segmentId)
+    }
+    return []
+  }
+
+  // TODO: Implement this method when there are checks available
+  // eslint-disable-next-line class-methods-use-this
+  async findOrganizationIssues() {
+    return Promise.resolve([])
+  }
+}

backend/src/types/data-quality/data-quality-filters.ts

@@ -0,0 +1,15 @@
+export enum IDataQualityType {
+  TOO_MANY_IDENTITIES = 'too-many-identities',
+  TOO_MANY_IDENTITIES_PER_PLATFORM = 'too-many-identities-per-platform',
+  TOO_MANY_EMAILS = 'too-many-emails',
+  NO_WORK_EXPERIENCE = 'no-work-experience',
+  WORK_EXPERIENCE_MISSING_INFO = 'work-experience-missing-info',
+  WORK_EXPERIENCE_MISSING_PERIOD = 'work-experience-missing-period',
+  CONFLICTING_WORK_EXPERIENCE = 'conflicting-work-experience',
+}
+
+export interface IDataQualityParams {
+  type: IDataQualityType
+  limit?: number
+  offset?: number
+}

frontend/config/styles/components/badge.scss

@@ -34,4 +34,14 @@
   --lf-badge-tertiary-background: var(--lf-color-gray-100);
   --lf-badge-tertiary-border: var(--lf-color-gray-200);
   --lf-badge-tertiary-text: var(--lf-color-gray-900);
+
+  /* Danger */
+  --lf-badge-danger-background: var(--lf-color-red-100);
+  --lf-badge-danger-border: var(--lf-color-red-200);
+  --lf-badge-danger-text: var(--lf-color-red-800);
+
+  /* Warning */
+  --lf-badge-warning-background: var(--lf-color-yellow-100);
+  --lf-badge-warning-border: var(--lf-color-yellow-200);
+  --lf-badge-warning-text: var(--lf-color-yellow-800);
 }

frontend/src/config/permissions/admin.ts

@@ -75,6 +75,8 @@ const admin: Record<LfPermission, boolean> = {
   [LfPermission.mergeOrganizations]: true,
   [LfPermission.customViewsCreate]: true,
   [LfPermission.customViewsTenantManage]: true,
+  [LfPermission.dataQualityRead]: true,
+  [LfPermission.dataQualityEdit]: true,
 };
 
 export default admin;

frontend/src/config/permissions/projectAdmin.ts

@@ -75,6 +75,8 @@ const projectAdmin: Record<LfPermission, boolean> = {
   [LfPermission.mergeOrganizations]: true,
   [LfPermission.customViewsCreate]: true,
   [LfPermission.customViewsTenantManage]: true,
+  [LfPermission.dataQualityRead]: true,
+  [LfPermission.dataQualityEdit]: true,
 };
 
 export default projectAdmin;

frontend/src/config/permissions/readonly.ts

@@ -75,6 +75,8 @@ const readonly: Record<LfPermission, boolean> = {
   [LfPermission.mergeOrganizations]: false,
   [LfPermission.customViewsCreate]: true,
   [LfPermission.customViewsTenantManage]: false,
+  [LfPermission.dataQualityRead]: true,
+  [LfPermission.dataQualityEdit]: false,
 };
 
 export default readonly;