From 38c96782bea394d4916fbc9bb850f6b08085befd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C5=A1per=20Grom?= Date: Wed, 30 Oct 2024 15:27:41 +0100 Subject: [PATCH] Data quality assistant (#2657) --- .../src/api/dataQuality/dataQualityMember.ts | 44 +++ .../dataQuality/dataQualityOrganization.ts | 40 +++ backend/src/api/dataQuality/index.ts | 9 + backend/src/api/index.ts | 1 + .../repositories/dataQualityRepository.ts | 166 +++++++++ .../database/repositories/memberRepository.ts | 2 + backend/src/services/dataQualityService.ts | 54 +++ .../data-quality/data-quality-filters.ts | 15 + frontend/config/styles/components/badge.scss | 10 + frontend/src/config/permissions/admin.ts | 2 + .../src/config/permissions/projectAdmin.ts | 2 + frontend/src/config/permissions/readonly.ts | 2 + .../components/data-quality-member.vue | 51 +++ .../components/data-quality-organization.vue | 44 +++ .../data-quality-member-issues-item.vue | 54 +++ .../member/data-quality-member-issues.vue | 124 +++++++ ...-quality-member-merge-suggestions-item.vue | 50 +++ .../data-quality-member-merge-suggestions.vue | 143 ++++++++ ...ty-organization-merge-suggestions-item.vue | 51 +++ ...quality-organization-merge-suggestions.vue | 126 +++++++ .../shared/data-quality-project-dropdown.vue | 88 +++++ .../shared/data-quality-type-dropdown.vue | 80 +++++ .../data-quality/config/data-issue-types.ts | 52 +++ .../types/conflicting-work-experience.ts | 10 + .../config/types/no-work-experience.ts | 10 + .../config/types/too-many-emails.ts | 10 + .../types/too-many-identities-per-platform.ts | 10 + .../config/types/too-many-identities.ts | 10 + .../types/work-experience-missing-info.ts | 10 + .../types/work-experience-missing-period.ts | 10 + .../data-quality/data-quality.module.ts | 5 + .../data-quality/data-quality.routes.ts | 35 ++ .../data-quality/pages/data-quality.page.vue | 51 +++ .../services/data-quality.api.service.ts | 38 ++ .../data-quality/types/DataIssueType.ts | 9 + frontend/src/modules/index.ts | 2 + .../modules/layout/components/menu/menu.vue | 13 +- .../suggestions/member-merge-similarity.vue | 19 +- .../member-merge-suggestion-dropdown.vue | 114 ++++++ .../pages/member-merge-suggestions-page.vue | 92 +---- .../shared/modules/monitoring/types/event.ts | 1 + .../modules/permissions/types/Permissions.ts | 4 + frontend/src/ui-kit/badge/badge.scss | 12 + frontend/src/ui-kit/badge/types/BadgeType.ts | 2 + frontend/src/ui-kit/tabs/Tab.vue | 6 +- scripts/cli | 14 + .../src/data-quality/index.ts | 2 + .../src/data-quality/members.ts | 338 ++++++++++++++++++ .../src/data-quality/organizations.ts | 144 ++++++++ 49 files changed, 2081 insertions(+), 100 deletions(-) create mode 100644 backend/src/api/dataQuality/dataQualityMember.ts create mode 100644 backend/src/api/dataQuality/dataQualityOrganization.ts create mode 100644 backend/src/api/dataQuality/index.ts create mode 100644 backend/src/database/repositories/dataQualityRepository.ts create mode 100644 backend/src/services/dataQualityService.ts create mode 100644 backend/src/types/data-quality/data-quality-filters.ts create mode 100644 frontend/src/modules/data-quality/components/data-quality-member.vue create mode 100644 frontend/src/modules/data-quality/components/data-quality-organization.vue create mode 100644 frontend/src/modules/data-quality/components/member/data-quality-member-issues-item.vue create mode 100644 frontend/src/modules/data-quality/components/member/data-quality-member-issues.vue create mode 100644 frontend/src/modules/data-quality/components/member/data-quality-member-merge-suggestions-item.vue create mode 100644 frontend/src/modules/data-quality/components/member/data-quality-member-merge-suggestions.vue create mode 100644 frontend/src/modules/data-quality/components/organization/data-quality-organization-merge-suggestions-item.vue create mode 100644 frontend/src/modules/data-quality/components/organization/data-quality-organization-merge-suggestions.vue create mode 100644 frontend/src/modules/data-quality/components/shared/data-quality-project-dropdown.vue create mode 100644 frontend/src/modules/data-quality/components/shared/data-quality-type-dropdown.vue create mode 100644 frontend/src/modules/data-quality/config/data-issue-types.ts create mode 100644 frontend/src/modules/data-quality/config/types/conflicting-work-experience.ts create mode 100644 frontend/src/modules/data-quality/config/types/no-work-experience.ts create mode 100644 frontend/src/modules/data-quality/config/types/too-many-emails.ts create mode 100644 frontend/src/modules/data-quality/config/types/too-many-identities-per-platform.ts create mode 100644 frontend/src/modules/data-quality/config/types/too-many-identities.ts create mode 100644 frontend/src/modules/data-quality/config/types/work-experience-missing-info.ts create mode 100644 frontend/src/modules/data-quality/config/types/work-experience-missing-period.ts create mode 100644 frontend/src/modules/data-quality/data-quality.module.ts create mode 100644 frontend/src/modules/data-quality/data-quality.routes.ts create mode 100644 frontend/src/modules/data-quality/pages/data-quality.page.vue create mode 100644 frontend/src/modules/data-quality/services/data-quality.api.service.ts create mode 100644 frontend/src/modules/data-quality/types/DataIssueType.ts create mode 100644 frontend/src/modules/member/components/suggestions/member-merge-suggestion-dropdown.vue create mode 100644 services/libs/data-access-layer/src/data-quality/index.ts create mode 100644 services/libs/data-access-layer/src/data-quality/members.ts create mode 100644 services/libs/data-access-layer/src/data-quality/organizations.ts diff --git a/backend/src/api/dataQuality/dataQualityMember.ts b/backend/src/api/dataQuality/dataQualityMember.ts new file mode 100644 index 0000000000..7973c7dace --- /dev/null +++ b/backend/src/api/dataQuality/dataQualityMember.ts @@ -0,0 +1,44 @@ +import { FeatureFlag } from '@crowd/types' + +import DataQualityService from '@/services/dataQualityService' + +import isFeatureEnabled from '../../feature-flags/isFeatureEnabled' +import Permissions from '../../security/permissions' +import PermissionChecker from '../../services/user/permissionChecker' + +/** + * GET /tenant/{tenantId}/data-quality/member + * @summary Find a member data issues + * @tag Data Quality + * @security Bearer + * @description Find a data quality issues for members + * @pathParam {string} tenantId - Your workspace/tenant ID + * @response 200 - Ok + * @responseContent {DataQualityResponse} 200.application/json + * @response 401 - Unauthorized + * @response 404 - Not found + * @response 429 - Too many requests + */ +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.memberRead) + + const segmentId = req.query.segments?.length > 0 ? req.query.segments[0] : null + if (!segmentId) { + const segmentsEnabled = await isFeatureEnabled(FeatureFlag.SEGMENTS, req) + if (segmentsEnabled) { + await req.responseHandler.error(req, res, { + code: 400, + message: 'Segment ID is required', + }) + return + } + } + + const payload = await new DataQualityService(req).findMemberIssues( + req.params.tenantId, + req.query, + segmentId, + ) + + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/api/dataQuality/dataQualityOrganization.ts b/backend/src/api/dataQuality/dataQualityOrganization.ts new file mode 100644 index 0000000000..fe251bc9a4 --- /dev/null +++ b/backend/src/api/dataQuality/dataQualityOrganization.ts @@ -0,0 +1,40 @@ +import { FeatureFlag } from '@crowd/types' + +import DataQualityService from '@/services/dataQualityService' + +import isFeatureEnabled from '../../feature-flags/isFeatureEnabled' +import Permissions from '../../security/permissions' +import PermissionChecker from '../../services/user/permissionChecker' + +/** + * GET /tenant/{tenantId}/data-quality/organization + * @summary Find a organization data issues + * @tag Data Quality + * @security Bearer + * @description Find a data quality issues for organizations + * @pathParam {string} tenantId - Your workspace/tenant ID + * @response 200 - Ok + * @responseContent {DataQualityResponse} 200.application/json + * @response 401 - Unauthorized + * @response 404 - Not found + * @response 429 - Too many requests + */ +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.organizationRead) + + const segmentId = req.query.segments?.length > 0 ? req.query.segments[0] : null + if (!segmentId) { + const segmentsEnabled = await isFeatureEnabled(FeatureFlag.SEGMENTS, req) + if (segmentsEnabled) { + await req.responseHandler.error(req, res, { + code: 400, + message: 'Segment ID is required', + }) + return + } + } + + const payload = await new DataQualityService(req).findOrganizationIssues() + + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/api/dataQuality/index.ts b/backend/src/api/dataQuality/index.ts new file mode 100644 index 0000000000..e00c3acd8d --- /dev/null +++ b/backend/src/api/dataQuality/index.ts @@ -0,0 +1,9 @@ +import { safeWrap } from '../../middlewares/errorMiddleware' + +export default (app) => { + app.get(`/tenant/:tenantId/data-quality/member`, safeWrap(require('./dataQualityMember').default)) + app.get( + `/tenant/:tenantId/data-quality/organization`, + safeWrap(require('./dataQualityOrganization').default), + ) +} diff --git a/backend/src/api/index.ts b/backend/src/api/index.ts index 0f92c94997..c40f142da6 100644 --- a/backend/src/api/index.ts +++ b/backend/src/api/index.ts @@ -243,6 +243,7 @@ setImmediate(async () => { require('./customViews').default(routes) require('./dashboard').default(routes) require('./mergeAction').default(routes) + require('./dataQuality').default(routes) // Loads the Tenant if the :tenantId param is passed routes.param('tenantId', tenantMiddleware) routes.param('tenantId', segmentMiddleware) diff --git a/backend/src/database/repositories/dataQualityRepository.ts b/backend/src/database/repositories/dataQualityRepository.ts new file mode 100644 index 0000000000..b39dad0e82 --- /dev/null +++ b/backend/src/database/repositories/dataQualityRepository.ts @@ -0,0 +1,166 @@ +import { + fetchMembersWithConflictingWorkExperiences, + fetchMembersWithMissingInfoOnWorkExperience, + fetchMembersWithMissingPeriodOnWorkExperience, + fetchMembersWithTooManyEmails, + fetchMembersWithTooManyIdentities, + fetchMembersWithTooManyIdentitiesPerPlatform, + fetchMembersWithoutWorkExperience, +} from '@crowd/data-access-layer/src/data-quality' + +import SequelizeRepository from '@/database/repositories/sequelizeRepository' + +import { IRepositoryOptions } from './IRepositoryOptions' + +class DataQualityRepository { + /** + * Finds members with no work experience. + * + * @param {IRepositoryOptions} options - The repository options for executing the query. + * @param {string} tenantId - The ID of the tenant. + * @param {number} limit - The maximum number of results to return. + * @param {number} offset - The offset from which to begin returning results. + * @param {string} segmentId - The ID of the segment. + * @return {Promise} - A promise that resolves with an array of members having no work experience. + */ + static async findMembersWithNoWorkExperience( + options: IRepositoryOptions, + tenantId: string, + limit: number, + offset: number, + segmentId: string, + ) { + const qx = SequelizeRepository.getQueryExecutor(options) + return fetchMembersWithoutWorkExperience(qx, tenantId, limit, offset, segmentId) + } + + /** + * Finds and returns members with too many identities. + * Executes a query to fetch members who exceed a certain number of identities. + * + * @param {IRepositoryOptions} options - Repository options for querying the database. + * @param {string} tenantId - Identifier of the tenant whose members are being queried. + * @param {number} limit - The maximum number of members to retrieve. + * @param {number} offset - The number of members to skip before starting to collect the result set. + * @param {string} segmentId - Identifier of the segment to filter members. + * @return {Promise} A promise that resolves to an array of members with too many identities. + */ + static async findMembersWithTooManyIdentities( + options: IRepositoryOptions, + tenantId: string, + limit: number, + offset: number, + segmentId: string, + ) { + const qx = SequelizeRepository.getQueryExecutor(options) + return fetchMembersWithTooManyIdentities(qx, 20, tenantId, limit, offset, segmentId) + } + + /** + * Finds members with too many identities per platform. + * + * @param {IRepositoryOptions} options - The repository options for database connection and other configurations. + * @param {string} tenantId - The ID of the tenant to filter members by. + * @param {number} limit - The maximum number of members to return. + * @param {number} offset - The number of members to skip before starting to collect the result set. + * @param {string} segmentId - The ID of the segment to filter members by. + * + * @return {Promise} A promise that resolves to an array of members with too many identities per platform. + */ + static async findMembersWithTooManyIdentitiesPerPlatform( + options: IRepositoryOptions, + tenantId: string, + limit: number, + offset: number, + segmentId: string, + ) { + const qx = SequelizeRepository.getQueryExecutor(options) + return fetchMembersWithTooManyIdentitiesPerPlatform(qx, 1, tenantId, limit, offset, segmentId) + } + + /** + * Finds members who have too many emails within a given tenant and segment. + * + * @param {IRepositoryOptions} options - The repository options containing configuration and context for the query. + * @param {string} tenantId - The identifier for the tenant where members are queried. + * @param {number} limit - The maximum number of members to return. + * @param {number} offset - The starting point for pagination. + * @param {string} segmentId - The identifier for the segment to filter members. + * @return {Promise} - A promise that resolves to an array of members who have too many emails. + */ + static async findMembersWithTooManyEmails( + options: IRepositoryOptions, + tenantId: string, + limit: number, + offset: number, + segmentId: string, + ) { + const qx = SequelizeRepository.getQueryExecutor(options) + return fetchMembersWithTooManyEmails(qx, 5, tenantId, limit, offset, segmentId) + } + + /** + * Finds members with missing information on work experience. + * + * @param {IRepositoryOptions} options - The repository options to be used. + * @param {string} tenantId - The unique identifier of the tenant. + * @param {number} limit - The maximum number of records to fetch. + * @param {number} offset - The number of records to skip. + * @param {string} segmentId - The segment identifier to be used for filtering. + * @return {Promise} A promise that resolves to an array of members with missing work experience information. + */ + static async findMembersWithMissingInfoOnWorkExperience( + options: IRepositoryOptions, + tenantId: string, + limit: number, + offset: number, + segmentId: string, + ) { + const qx = SequelizeRepository.getQueryExecutor(options) + return fetchMembersWithMissingInfoOnWorkExperience(qx, tenantId, limit, offset, segmentId) + } + + /** + * Fetches members whose work experience period is missing. + * + * @param {IRepositoryOptions} options - The repository options for database access. + * @param {string} tenantId - The ID of the tenant to find members for. + * @param {number} limit - The maximum number of members to return. + * @param {number} offset - The offset to start fetching members from. + * @param {string} segmentId - The ID of the segment to filter members. + * @return {Promise} A promise that resolves to an array of members with missing work experience periods. + */ + static async findMembersWithMissingPeriodOnWorkExperience( + options: IRepositoryOptions, + tenantId: string, + limit: number, + offset: number, + segmentId: string, + ) { + const qx = SequelizeRepository.getQueryExecutor(options) + return fetchMembersWithMissingPeriodOnWorkExperience(qx, tenantId, limit, offset, segmentId) + } + + /** + * Finds members with conflicting work experience based on specified options. + * + * @param {IRepositoryOptions} options - The repository options for database query execution. + * @param {string} tenantId - The ID of the tenant to filter members. + * @param {number} limit - The maximum number of records to fetch. + * @param {number} offset - The number of records to skip for pagination. + * @param {string} segmentId - The ID of the segment to filter members. + * @return {Promise} - A promise that resolves to an array of members with conflicting work experiences. + */ + static async findMembersWithConflictingWorkExperience( + options: IRepositoryOptions, + tenantId: string, + limit: number, + offset: number, + segmentId: string, + ) { + const qx = SequelizeRepository.getQueryExecutor(options) + return fetchMembersWithConflictingWorkExperiences(qx, tenantId, limit, offset, segmentId) + } +} + +export default DataQualityRepository diff --git a/backend/src/database/repositories/memberRepository.ts b/backend/src/database/repositories/memberRepository.ts index 8b77f286c6..b017fc4b99 100644 --- a/backend/src/database/repositories/memberRepository.ts +++ b/backend/src/database/repositories/memberRepository.ts @@ -534,11 +534,13 @@ class MemberRepository { { id: i.id, displayName: i.primaryDisplayName, + activityCount: i.primaryActivityCount, avatarUrl: i.primaryAvatarUrl, }, { id: i.toMergeId, displayName: i.toMergeDisplayName, + activityCount: i.toActivityCount, avatarUrl: i.toMergeAvatarUrl, }, ], diff --git a/backend/src/services/dataQualityService.ts b/backend/src/services/dataQualityService.ts new file mode 100644 index 0000000000..e8b081fba5 --- /dev/null +++ b/backend/src/services/dataQualityService.ts @@ -0,0 +1,54 @@ +/* eslint-disable no-continue */ +import { LoggerBase } from '@crowd/logging' + +import DataQualityRepository from '@/database/repositories/dataQualityRepository' +import { IDataQualityParams, IDataQualityType } from '@/types/data-quality/data-quality-filters' + +import { IServiceOptions } from './IServiceOptions' + +export default class DataQualityService extends LoggerBase { + options: IServiceOptions + + constructor(options: IServiceOptions) { + super(options.log) + this.options = options + } + + /** + * Finds issues related to member data quality based on the specified type. + * + * @param {string} tenantId - The ID of the tenant for whom to find member issues. + * @param {IDataQualityParams} params - The parameters for finding member issues, including the type of issue, limit, and offset. + * @param {string} segmentId - The ID of the segment where the members belong. + * @return {Promise} A promise that resolves to an array of members with the specified data quality issues. + */ + async findMemberIssues(tenantId: string, params: IDataQualityParams, segmentId: string) { + const methodMap = { + [IDataQualityType.NO_WORK_EXPERIENCE]: DataQualityRepository.findMembersWithNoWorkExperience, + [IDataQualityType.TOO_MANY_IDENTITIES]: + DataQualityRepository.findMembersWithTooManyIdentities, + [IDataQualityType.TOO_MANY_IDENTITIES_PER_PLATFORM]: + DataQualityRepository.findMembersWithTooManyIdentitiesPerPlatform, + [IDataQualityType.TOO_MANY_EMAILS]: DataQualityRepository.findMembersWithTooManyEmails, + [IDataQualityType.WORK_EXPERIENCE_MISSING_INFO]: + DataQualityRepository.findMembersWithMissingInfoOnWorkExperience, + [IDataQualityType.WORK_EXPERIENCE_MISSING_PERIOD]: + DataQualityRepository.findMembersWithMissingPeriodOnWorkExperience, + [IDataQualityType.CONFLICTING_WORK_EXPERIENCE]: + DataQualityRepository.findMembersWithConflictingWorkExperience, + } + + const method = methodMap[params.type] + + if (method) { + return method(this.options, tenantId, params.limit || 10, params.offset || 0, segmentId) + } + return [] + } + + // TODO: Implement this method when there are checks available + // eslint-disable-next-line class-methods-use-this + async findOrganizationIssues() { + return Promise.resolve([]) + } +} diff --git a/backend/src/types/data-quality/data-quality-filters.ts b/backend/src/types/data-quality/data-quality-filters.ts new file mode 100644 index 0000000000..9f6445044b --- /dev/null +++ b/backend/src/types/data-quality/data-quality-filters.ts @@ -0,0 +1,15 @@ +export enum IDataQualityType { + TOO_MANY_IDENTITIES = 'too-many-identities', + TOO_MANY_IDENTITIES_PER_PLATFORM = 'too-many-identities-per-platform', + TOO_MANY_EMAILS = 'too-many-emails', + NO_WORK_EXPERIENCE = 'no-work-experience', + WORK_EXPERIENCE_MISSING_INFO = 'work-experience-missing-info', + WORK_EXPERIENCE_MISSING_PERIOD = 'work-experience-missing-period', + CONFLICTING_WORK_EXPERIENCE = 'conflicting-work-experience', +} + +export interface IDataQualityParams { + type: IDataQualityType + limit?: number + offset?: number +} diff --git a/frontend/config/styles/components/badge.scss b/frontend/config/styles/components/badge.scss index 8b1226f9a8..4f0d0f1177 100644 --- a/frontend/config/styles/components/badge.scss +++ b/frontend/config/styles/components/badge.scss @@ -34,4 +34,14 @@ --lf-badge-tertiary-background: var(--lf-color-gray-100); --lf-badge-tertiary-border: var(--lf-color-gray-200); --lf-badge-tertiary-text: var(--lf-color-gray-900); + + /* Danger */ + --lf-badge-danger-background: var(--lf-color-red-100); + --lf-badge-danger-border: var(--lf-color-red-200); + --lf-badge-danger-text: var(--lf-color-red-800); + + /* Warning */ + --lf-badge-warning-background: var(--lf-color-yellow-100); + --lf-badge-warning-border: var(--lf-color-yellow-200); + --lf-badge-warning-text: var(--lf-color-yellow-800); } diff --git a/frontend/src/config/permissions/admin.ts b/frontend/src/config/permissions/admin.ts index c85677276d..e75dfcc1e8 100644 --- a/frontend/src/config/permissions/admin.ts +++ b/frontend/src/config/permissions/admin.ts @@ -75,6 +75,8 @@ const admin: Record = { [LfPermission.mergeOrganizations]: true, [LfPermission.customViewsCreate]: true, [LfPermission.customViewsTenantManage]: true, + [LfPermission.dataQualityRead]: true, + [LfPermission.dataQualityEdit]: true, }; export default admin; diff --git a/frontend/src/config/permissions/projectAdmin.ts b/frontend/src/config/permissions/projectAdmin.ts index f0c85c4c49..e6edcec4bf 100644 --- a/frontend/src/config/permissions/projectAdmin.ts +++ b/frontend/src/config/permissions/projectAdmin.ts @@ -75,6 +75,8 @@ const projectAdmin: Record = { [LfPermission.mergeOrganizations]: true, [LfPermission.customViewsCreate]: true, [LfPermission.customViewsTenantManage]: true, + [LfPermission.dataQualityRead]: true, + [LfPermission.dataQualityEdit]: true, }; export default projectAdmin; diff --git a/frontend/src/config/permissions/readonly.ts b/frontend/src/config/permissions/readonly.ts index dba0b88ce9..0ea20d2349 100644 --- a/frontend/src/config/permissions/readonly.ts +++ b/frontend/src/config/permissions/readonly.ts @@ -75,6 +75,8 @@ const readonly: Record = { [LfPermission.mergeOrganizations]: false, [LfPermission.customViewsCreate]: true, [LfPermission.customViewsTenantManage]: false, + [LfPermission.dataQualityRead]: true, + [LfPermission.dataQualityEdit]: false, }; export default readonly; diff --git a/frontend/src/modules/data-quality/components/data-quality-member.vue b/frontend/src/modules/data-quality/components/data-quality-member.vue new file mode 100644 index 0000000000..0dad57d446 --- /dev/null +++ b/frontend/src/modules/data-quality/components/data-quality-member.vue @@ -0,0 +1,51 @@ + + + + + diff --git a/frontend/src/modules/data-quality/components/data-quality-organization.vue b/frontend/src/modules/data-quality/components/data-quality-organization.vue new file mode 100644 index 0000000000..84a3ab9402 --- /dev/null +++ b/frontend/src/modules/data-quality/components/data-quality-organization.vue @@ -0,0 +1,44 @@ + + + + + diff --git a/frontend/src/modules/data-quality/components/member/data-quality-member-issues-item.vue b/frontend/src/modules/data-quality/components/member/data-quality-member-issues-item.vue new file mode 100644 index 0000000000..c88f804cd5 --- /dev/null +++ b/frontend/src/modules/data-quality/components/member/data-quality-member-issues-item.vue @@ -0,0 +1,54 @@ + + + + + diff --git a/frontend/src/modules/data-quality/components/member/data-quality-member-issues.vue b/frontend/src/modules/data-quality/components/member/data-quality-member-issues.vue new file mode 100644 index 0000000000..efeffece0e --- /dev/null +++ b/frontend/src/modules/data-quality/components/member/data-quality-member-issues.vue @@ -0,0 +1,124 @@ + + + + + diff --git a/frontend/src/modules/data-quality/components/member/data-quality-member-merge-suggestions-item.vue b/frontend/src/modules/data-quality/components/member/data-quality-member-merge-suggestions-item.vue new file mode 100644 index 0000000000..9bfd29385f --- /dev/null +++ b/frontend/src/modules/data-quality/components/member/data-quality-member-merge-suggestions-item.vue @@ -0,0 +1,50 @@ + + + + + diff --git a/frontend/src/modules/data-quality/components/member/data-quality-member-merge-suggestions.vue b/frontend/src/modules/data-quality/components/member/data-quality-member-merge-suggestions.vue new file mode 100644 index 0000000000..1e3d003a6f --- /dev/null +++ b/frontend/src/modules/data-quality/components/member/data-quality-member-merge-suggestions.vue @@ -0,0 +1,143 @@ + + + + + diff --git a/frontend/src/modules/data-quality/components/organization/data-quality-organization-merge-suggestions-item.vue b/frontend/src/modules/data-quality/components/organization/data-quality-organization-merge-suggestions-item.vue new file mode 100644 index 0000000000..75c441c785 --- /dev/null +++ b/frontend/src/modules/data-quality/components/organization/data-quality-organization-merge-suggestions-item.vue @@ -0,0 +1,51 @@ + + + + + diff --git a/frontend/src/modules/data-quality/components/organization/data-quality-organization-merge-suggestions.vue b/frontend/src/modules/data-quality/components/organization/data-quality-organization-merge-suggestions.vue new file mode 100644 index 0000000000..2f0b6ea83b --- /dev/null +++ b/frontend/src/modules/data-quality/components/organization/data-quality-organization-merge-suggestions.vue @@ -0,0 +1,126 @@ + + + + + diff --git a/frontend/src/modules/data-quality/components/shared/data-quality-project-dropdown.vue b/frontend/src/modules/data-quality/components/shared/data-quality-project-dropdown.vue new file mode 100644 index 0000000000..56632983cd --- /dev/null +++ b/frontend/src/modules/data-quality/components/shared/data-quality-project-dropdown.vue @@ -0,0 +1,88 @@ + + + + + diff --git a/frontend/src/modules/data-quality/components/shared/data-quality-type-dropdown.vue b/frontend/src/modules/data-quality/components/shared/data-quality-type-dropdown.vue new file mode 100644 index 0000000000..4d88147919 --- /dev/null +++ b/frontend/src/modules/data-quality/components/shared/data-quality-type-dropdown.vue @@ -0,0 +1,80 @@ + + + + + diff --git a/frontend/src/modules/data-quality/config/data-issue-types.ts b/frontend/src/modules/data-quality/config/data-issue-types.ts new file mode 100644 index 0000000000..28f85d6f0d --- /dev/null +++ b/frontend/src/modules/data-quality/config/data-issue-types.ts @@ -0,0 +1,52 @@ +import { DataIssueType } from '@/modules/data-quality/types/DataIssueType'; +import noWorkExperience from '@/modules/data-quality/config/types/no-work-experience'; +import workExperienceMissingInfo from '@/modules/data-quality/config/types/work-experience-missing-info'; +import workExperienceMissingPeriod from '@/modules/data-quality/config/types/work-experience-missing-period'; +import tooManyIdentitiesPerPlatform from '@/modules/data-quality/config/types/too-many-identities-per-platform'; +import tooManyEmails from '@/modules/data-quality/config/types/too-many-emails'; +import tooManyIdentities from '@/modules/data-quality/config/types/too-many-identities'; +import { Contributor } from '@/modules/contributor/types/Contributor'; +import { Organization } from '@/modules/organization/types/Organization'; +import conflictingWorkExperience from '@/modules/data-quality/config/types/conflicting-work-experience'; + +export interface DataIssueTypeConfig{ + label: string; + badgeType: string; + badgeText: (entity: Contributor | Organization) => string; + description: (entity: Contributor | Organization) => string; +} + +export interface DataIssueTypeMenu{ + label?: string; + types: DataIssueType[] +} + +export const dataIssueTypes: Record = { + [DataIssueType.TOO_MANY_IDENTITIES]: tooManyIdentities, + [DataIssueType.TOO_MANY_IDENTITIES_PER_PLATFORM]: tooManyIdentitiesPerPlatform, + [DataIssueType.TOO_MANY_EMAILS]: tooManyEmails, + [DataIssueType.NO_WORK_EXPERIENCE]: noWorkExperience, + [DataIssueType.WORK_EXPERIENCE_MISSING_INFO]: workExperienceMissingInfo, + [DataIssueType.WORK_EXPERIENCE_MISSING_PERIOD]: workExperienceMissingPeriod, + [DataIssueType.CONFLICTING_WORK_EXPERIENCE]: conflictingWorkExperience, +}; + +export const memberDataIssueTypeMenu: DataIssueTypeMenu[] = [ + { + label: 'Identities', + types: [ + DataIssueType.TOO_MANY_IDENTITIES, + DataIssueType.TOO_MANY_IDENTITIES_PER_PLATFORM, + DataIssueType.TOO_MANY_EMAILS, + ], + }, + { + label: 'Work history', + types: [ + DataIssueType.WORK_EXPERIENCE_MISSING_INFO, + DataIssueType.NO_WORK_EXPERIENCE, + DataIssueType.WORK_EXPERIENCE_MISSING_PERIOD, + DataIssueType.CONFLICTING_WORK_EXPERIENCE, + ], + }, +]; diff --git a/frontend/src/modules/data-quality/config/types/conflicting-work-experience.ts b/frontend/src/modules/data-quality/config/types/conflicting-work-experience.ts new file mode 100644 index 0000000000..38adf9f0c5 --- /dev/null +++ b/frontend/src/modules/data-quality/config/types/conflicting-work-experience.ts @@ -0,0 +1,10 @@ +import { DataIssueTypeConfig } from '@/modules/data-quality/config/data-issue-types'; + +const conflictingWorkExperience: DataIssueTypeConfig = { + label: 'Work experiences with overlapping periods', + badgeType: 'warning', + badgeText: () => 'Work experiences with overlapping periods', + description: () => null, +}; + +export default conflictingWorkExperience; diff --git a/frontend/src/modules/data-quality/config/types/no-work-experience.ts b/frontend/src/modules/data-quality/config/types/no-work-experience.ts new file mode 100644 index 0000000000..29fd0e8358 --- /dev/null +++ b/frontend/src/modules/data-quality/config/types/no-work-experience.ts @@ -0,0 +1,10 @@ +import { DataIssueTypeConfig } from '@/modules/data-quality/config/data-issue-types'; + +const noWorkExperience: DataIssueTypeConfig = { + label: 'Missing work experiences', + badgeType: 'danger', + badgeText: () => 'Missing work experiences', + description: () => null, +}; + +export default noWorkExperience; diff --git a/frontend/src/modules/data-quality/config/types/too-many-emails.ts b/frontend/src/modules/data-quality/config/types/too-many-emails.ts new file mode 100644 index 0000000000..7454dbc4ef --- /dev/null +++ b/frontend/src/modules/data-quality/config/types/too-many-emails.ts @@ -0,0 +1,10 @@ +import { DataIssueTypeConfig } from '@/modules/data-quality/config/data-issue-types'; + +const tooManyEmails: DataIssueTypeConfig = { + label: 'More than 5 verified emails', + badgeType: 'warning', + badgeText: () => 'More than 5 verified emails', + description: () => null, +}; + +export default tooManyEmails; diff --git a/frontend/src/modules/data-quality/config/types/too-many-identities-per-platform.ts b/frontend/src/modules/data-quality/config/types/too-many-identities-per-platform.ts new file mode 100644 index 0000000000..24ced6fbd4 --- /dev/null +++ b/frontend/src/modules/data-quality/config/types/too-many-identities-per-platform.ts @@ -0,0 +1,10 @@ +import { DataIssueTypeConfig } from '@/modules/data-quality/config/data-issue-types'; + +const tooManyIdentitiesPerPlatform: DataIssueTypeConfig = { + label: 'More than 1 verified identity per platform', + badgeType: 'warning', + badgeText: () => 'More than 1 verified identity per platform', + description: () => null, +}; + +export default tooManyIdentitiesPerPlatform; diff --git a/frontend/src/modules/data-quality/config/types/too-many-identities.ts b/frontend/src/modules/data-quality/config/types/too-many-identities.ts new file mode 100644 index 0000000000..59c2e7f6fd --- /dev/null +++ b/frontend/src/modules/data-quality/config/types/too-many-identities.ts @@ -0,0 +1,10 @@ +import { DataIssueTypeConfig } from '@/modules/data-quality/config/data-issue-types'; + +const tooManyIdentities: DataIssueTypeConfig = { + label: 'More than 20 identities', + badgeType: 'danger', + badgeText: () => 'More than 20 identities', + description: () => null, +}; + +export default tooManyIdentities; diff --git a/frontend/src/modules/data-quality/config/types/work-experience-missing-info.ts b/frontend/src/modules/data-quality/config/types/work-experience-missing-info.ts new file mode 100644 index 0000000000..d6f2af6788 --- /dev/null +++ b/frontend/src/modules/data-quality/config/types/work-experience-missing-info.ts @@ -0,0 +1,10 @@ +import { DataIssueTypeConfig } from '@/modules/data-quality/config/data-issue-types'; + +const workExperienceMissingInfo: DataIssueTypeConfig = { + label: 'Work experience(s) with missing information', + badgeType: 'warning', + badgeText: () => 'Work experience(s) with missing information', + description: () => null, +}; + +export default workExperienceMissingInfo; diff --git a/frontend/src/modules/data-quality/config/types/work-experience-missing-period.ts b/frontend/src/modules/data-quality/config/types/work-experience-missing-period.ts new file mode 100644 index 0000000000..cc57957936 --- /dev/null +++ b/frontend/src/modules/data-quality/config/types/work-experience-missing-period.ts @@ -0,0 +1,10 @@ +import { DataIssueTypeConfig } from '@/modules/data-quality/config/data-issue-types'; + +const workExperienceMissingPeriod: DataIssueTypeConfig = { + label: 'Work experience(s) without period', + badgeType: 'danger', + badgeText: () => 'Work experience(s) without period', + description: () => null, +}; + +export default workExperienceMissingPeriod; diff --git a/frontend/src/modules/data-quality/data-quality.module.ts b/frontend/src/modules/data-quality/data-quality.module.ts new file mode 100644 index 0000000000..e89f06a2bb --- /dev/null +++ b/frontend/src/modules/data-quality/data-quality.module.ts @@ -0,0 +1,5 @@ +import routes from '@/modules/data-quality/data-quality.routes'; + +export default { + routes, +}; diff --git a/frontend/src/modules/data-quality/data-quality.routes.ts b/frontend/src/modules/data-quality/data-quality.routes.ts new file mode 100644 index 0000000000..3e57e3b61e --- /dev/null +++ b/frontend/src/modules/data-quality/data-quality.routes.ts @@ -0,0 +1,35 @@ +import Layout from '@/modules/layout/components/layout.vue'; +import { PageEventKey } from '@/shared/modules/monitoring/types/event'; +import { PermissionGuard } from '@/shared/modules/permissions/router/PermissionGuard'; +import { LfPermission } from '@/shared/modules/permissions/types/Permissions'; + +const DataQualityPage = () => import('@/modules/data-quality/pages/data-quality.page.vue'); + +export default [ + { + name: '', + path: '', + component: Layout, + meta: { + auth: true, + title: 'Data Quality Assistant', + segments: { + requireSelectedProjectGroup: true, + }, + }, + children: [ + { + name: 'data-quality-assistant', + path: '/data-quality-assistant', + component: DataQualityPage, + meta: { + auth: true, + eventKey: PageEventKey.DATA_QUALITY_ASSISTANT, + }, + beforeEnter: [ + PermissionGuard(LfPermission.dataQualityRead), + ], + }, + ], + }, +]; diff --git a/frontend/src/modules/data-quality/pages/data-quality.page.vue b/frontend/src/modules/data-quality/pages/data-quality.page.vue new file mode 100644 index 0000000000..36f2ae2069 --- /dev/null +++ b/frontend/src/modules/data-quality/pages/data-quality.page.vue @@ -0,0 +1,51 @@ + + + + + diff --git a/frontend/src/modules/data-quality/services/data-quality.api.service.ts b/frontend/src/modules/data-quality/services/data-quality.api.service.ts new file mode 100644 index 0000000000..e0a4bcd0ea --- /dev/null +++ b/frontend/src/modules/data-quality/services/data-quality.api.service.ts @@ -0,0 +1,38 @@ +import authAxios from '@/shared/axios/auth-axios'; +import { AuthService } from '@/modules/auth/services/auth.service'; +import { Contributor } from '@/modules/contributor/types/Contributor'; +import { Organization } from '@/modules/organization/types/Organization'; + +export class DataQualityApiService { + static async findMemberIssues(params: any, segments: string[]): Promise { + const tenantId = AuthService.getTenantId(); + + const response = await authAxios.get( + `/tenant/${tenantId}/data-quality/member`, + { + params: { + ...params, + segments, + }, + }, + ); + + return response.data; + } + + static async findOrganizationIssues(params: any, segments: string[]): Promise { + const tenantId = AuthService.getTenantId(); + + const response = await authAxios.get( + `/tenant/${tenantId}/data-quality/organization`, + { + params: { + ...params, + segments, + }, + }, + ); + + return response.data; + } +} diff --git a/frontend/src/modules/data-quality/types/DataIssueType.ts b/frontend/src/modules/data-quality/types/DataIssueType.ts new file mode 100644 index 0000000000..6d6c798530 --- /dev/null +++ b/frontend/src/modules/data-quality/types/DataIssueType.ts @@ -0,0 +1,9 @@ +export enum DataIssueType { + TOO_MANY_IDENTITIES = 'too-many-identities', + TOO_MANY_IDENTITIES_PER_PLATFORM = 'too-many-identities-per-platform', + TOO_MANY_EMAILS = 'too-many-emails', + NO_WORK_EXPERIENCE = 'no-work-experience', + WORK_EXPERIENCE_MISSING_INFO = 'work-experience-missing-info', + WORK_EXPERIENCE_MISSING_PERIOD = 'work-experience-missing-period', + CONFLICTING_WORK_EXPERIENCE = 'conflicting-work-experience', +} diff --git a/frontend/src/modules/index.ts b/frontend/src/modules/index.ts index d1bbc2c588..aed0d56b4c 100644 --- a/frontend/src/modules/index.ts +++ b/frontend/src/modules/index.ts @@ -11,6 +11,7 @@ import automation from '@/modules/automation/automation-module'; import organization from '@/modules/organization/organization-module'; import lf from '@/modules/lf/lf-modules'; +import dataQuality from '@/modules/data-quality/data-quality.module'; import eagleEye from '@/modules/eagle-eye/eagle-eye-module'; const modules: Record = { @@ -27,6 +28,7 @@ const modules: Record = { eagleEye, organization, lf, + dataQuality, }; export default modules; diff --git a/frontend/src/modules/layout/components/menu/menu.vue b/frontend/src/modules/layout/components/menu/menu.vue index 041237f782..f618e27673 100644 --- a/frontend/src/modules/layout/components/menu/menu.vue +++ b/frontend/src/modules/layout/components/menu/menu.vue @@ -45,6 +45,17 @@ :to="{ path: '/activities', query: { projectGroup: selectedProjectGroup?.id } }" :disabled="!selectedProjectGroup" /> +
+
+
+
-
+
- -

- {{ percentage }}% confidence + +

+ {{ percentage }}% confidence

diff --git a/frontend/src/modules/member/components/suggestions/member-merge-suggestion-dropdown.vue b/frontend/src/modules/member/components/suggestions/member-merge-suggestion-dropdown.vue new file mode 100644 index 0000000000..62dcb1894b --- /dev/null +++ b/frontend/src/modules/member/components/suggestions/member-merge-suggestion-dropdown.vue @@ -0,0 +1,114 @@ + + + + + diff --git a/frontend/src/modules/member/pages/member-merge-suggestions-page.vue b/frontend/src/modules/member/pages/member-merge-suggestions-page.vue index 75fe7b1205..1e5e796814 100644 --- a/frontend/src/modules/member/pages/member-merge-suggestions-page.vue +++ b/frontend/src/modules/member/pages/member-merge-suggestions-page.vue @@ -98,26 +98,7 @@ View suggestion - - - - - Merge suggestion - - - - Ignore suggestion - - +
@@ -163,16 +144,14 @@ import LfButton from '@/ui-kit/button/Button.vue'; import AppMemberMergeSimilarity from '@/modules/member/components/suggestions/member-merge-similarity.vue'; import { storeToRefs } from 'pinia'; import { useLfSegmentsStore } from '@/modules/lf/segments/store'; -import LfDropdown from '@/ui-kit/dropdown/Dropdown.vue'; -import LfDropdownItem from '@/ui-kit/dropdown/DropdownItem.vue'; import AppMemberMergeSuggestionsDialog from '@/modules/member/components/member-merge-suggestions-dialog.vue'; -import useMemberMergeMessage from '@/shared/modules/merge/config/useMemberMergeMessage'; -import Message from '@/shared/message/message'; import LfSpinner from '@/ui-kit/spinner/Spinner.vue'; import AppMergeSuggestionsFilters from '@/modules/member/components/suggestions/merge-suggestions-filters.vue'; import LfTableHead from '@/ui-kit/table/TableHead.vue'; import useProductTracking from '@/shared/modules/monitoring/useProductTracking'; import { EventType, FeatureEventKey } from '@/shared/modules/monitoring/types/event'; +import LfMemberMergeSuggestionDropdown + from '@/modules/member/components/suggestions/member-merge-suggestion-dropdown.vue'; const { selectedProjectGroup } = storeToRefs(useLfSegmentsStore()); @@ -263,71 +242,6 @@ const loadMore = () => { loadMergeSuggestions(); }; -const sending = ref(''); - -const merge = (suggestion: any) => { - if (sending.value.length) { - return; - } - - trackEvent({ - key: FeatureEventKey.MERGE_MEMBER_MERGE_SUGGESTION, - type: EventType.FEATURE, - properties: { - similarity: suggestion.similarity, - }, - }); - - const primaryMember = suggestion.members[0]; - const secondaryMember = suggestion.members[1]; - sending.value = `${primaryMember.id}:${secondaryMember.id}`; - - const { loadingMessage } = useMemberMergeMessage; - - loadingMessage(); - - MemberService.merge(primaryMember, secondaryMember) - .then(() => { - Message.closeAll(); - Message.info( - "We're finalizing profiles merging. We will let you know once the process is completed.", - { - title: 'Profiles merging in progress', - }, - ); - }) - .finally(() => { - reload(); - sending.value = ''; - }); -}; - -const ignore = (suggestion: any) => { - if (sending.value.length) { - return; - } - - trackEvent({ - key: FeatureEventKey.IGNORE_MEMBER_MERGE_SUGGESTION, - type: EventType.FEATURE, - properties: { - similarity: suggestion.similarity, - }, - }); - - const primaryMember = suggestion.members[0]; - const secondaryMember = suggestion.members[1]; - sending.value = `${primaryMember.id}:${secondaryMember.id}`; - MemberService.addToNoMerge(...suggestion.members) - .then(() => { - Message.success('Merging suggestion ignored successfully'); - reload(); - }) - .finally(() => { - sending.value = ''; - }); -}; - onMounted(() => { getTotalCount(); }); diff --git a/frontend/src/shared/modules/monitoring/types/event.ts b/frontend/src/shared/modules/monitoring/types/event.ts index 955b84ed32..173afa4aa1 100644 --- a/frontend/src/shared/modules/monitoring/types/event.ts +++ b/frontend/src/shared/modules/monitoring/types/event.ts @@ -35,6 +35,7 @@ export enum PageEventKey { MEMBERS_MERGE_SUGGESTIONS = 'Contributors merge suggestions', MANAGE_PROJECTS = 'Manage projects', INTEGRATIONS = 'Integrations', + DATA_QUALITY_ASSISTANT = 'Data Quality Assistant', } export enum FeatureEventKey { diff --git a/frontend/src/shared/modules/permissions/types/Permissions.ts b/frontend/src/shared/modules/permissions/types/Permissions.ts index 86c58f021f..cbe8de042e 100644 --- a/frontend/src/shared/modules/permissions/types/Permissions.ts +++ b/frontend/src/shared/modules/permissions/types/Permissions.ts @@ -104,4 +104,8 @@ export enum LfPermission { // Custom views customViewsCreate = 'customViewsCreate', customViewsTenantManage = 'customViewsTenantManage', + + // Custom views + dataQualityRead = 'dataQualityRead', + dataQualityEdit = 'dataQualityEdit', } diff --git a/frontend/src/ui-kit/badge/badge.scss b/frontend/src/ui-kit/badge/badge.scss index a91777c90e..01e5875976 100644 --- a/frontend/src/ui-kit/badge/badge.scss +++ b/frontend/src/ui-kit/badge/badge.scss @@ -29,6 +29,18 @@ --lf-badge-text: var(--lf-badge-tertiary-text); } + &--danger{ + --lf-badge-background: var(--lf-badge-danger-background); + --lf-badge-border: var(--lf-badge-danger-border); + --lf-badge-text: var(--lf-badge-danger-text); + } + + &--warning{ + --lf-badge-background: var(--lf-badge-warning-background); + --lf-badge-border: var(--lf-badge-warning-border); + --lf-badge-text: var(--lf-badge-warning-text); + } + /* Badge sizes */ &--small{ --lf-badge-font-size: var(--lf-badge-small-font-size); diff --git a/frontend/src/ui-kit/badge/types/BadgeType.ts b/frontend/src/ui-kit/badge/types/BadgeType.ts index 17e1c79077..ca6c1db114 100644 --- a/frontend/src/ui-kit/badge/types/BadgeType.ts +++ b/frontend/src/ui-kit/badge/types/BadgeType.ts @@ -2,6 +2,8 @@ export const badgeType = [ 'primary', 'secondary', 'tertiary', + 'danger', + 'warning', ] as const; export type BadgeType = typeof badgeType[number]; diff --git a/frontend/src/ui-kit/tabs/Tab.vue b/frontend/src/ui-kit/tabs/Tab.vue index 53b752149e..aea115d93e 100644 --- a/frontend/src/ui-kit/tabs/Tab.vue +++ b/frontend/src/ui-kit/tabs/Tab.vue @@ -11,6 +11,7 @@ import { useRoute, useRouter } from 'vue-router'; const props = defineProps<{ name: string, modelValue: string; + preserveQuery?: boolean, }>(); const router = useRouter(); @@ -19,7 +20,10 @@ const route = useRoute(); const isActive = computed(() => (route?.hash.substring(1) || props.modelValue) === props.name); const selectTab = () => { - router?.push({ + router?.push(props.preserveQuery ? { + ...route, + hash: `#${props.name}`, + } : { hash: `#${props.name}`, }); }; diff --git a/scripts/cli b/scripts/cli index 1fcedd996c..43dc86b267 100755 --- a/scripts/cli +++ b/scripts/cli @@ -862,6 +862,20 @@ while test $# -gt 0; do service_start exit ;; + service-restart-fe-dev) + IGNORED_SERVICES=("frontend" "python-worker" "job-generator" "discord-ws" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "emails-worker" "entity-merging-worker" "automations-worker") + DEV=1 + kill_all_containers + service_start + exit + ;; + clean-start-fe-dev) + INGORED_SERVICES=("frontend" "python-worker" "job-generator" "discord-ws" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "emails-worker" "entity-merging-worker" "automations-worker") + CLEAN_START=1 + DEV=1 + start + exit + ;; deploy-staging) select_services deploy_staging diff --git a/services/libs/data-access-layer/src/data-quality/index.ts b/services/libs/data-access-layer/src/data-quality/index.ts new file mode 100644 index 0000000000..240d10ffa2 --- /dev/null +++ b/services/libs/data-access-layer/src/data-quality/index.ts @@ -0,0 +1,2 @@ +export * from './members' +export * from './organizations' diff --git a/services/libs/data-access-layer/src/data-quality/members.ts b/services/libs/data-access-layer/src/data-quality/members.ts new file mode 100644 index 0000000000..cb860f156d --- /dev/null +++ b/services/libs/data-access-layer/src/data-quality/members.ts @@ -0,0 +1,338 @@ +import { IMember } from '@crowd/types' + +import { QueryExecutor } from '../queryExecutor' + +/** + * Fetches members who do not have any work experience. + * + * @param {QueryExecutor} qx - The query executor for running SQL queries. + * @param {string} tenantId - The ID of the tenant to fetch members for. + * @param {number} limit - The maximum number of members to return. + * @param {number} offset - The number of members to skip before starting to collect the result set. + * @param {string} segmentId - The segment ID to filter members by. + * @return {Promise} A promise that resolves to an array of members without work experience. + */ +export async function fetchMembersWithoutWorkExperience( + qx: QueryExecutor, + tenantId: string, + limit: number, + offset: number, + segmentId: string, +): Promise { + return qx.select( + ` + SELECT m.id, m."displayName", m.attributes, msa."activityCount" + FROM members m + LEFT JOIN "memberOrganizations" mo ON m.id = mo."memberId" AND mo."deletedAt" IS NULL + INNER JOIN "memberSegmentsAgg" msa ON m.id = msa."memberId" AND msa."segmentId" = '${segmentId}' + WHERE mo."memberId" IS NULL + AND m."tenantId" = '${tenantId}' + AND m."deletedAt" IS NULL + AND COALESCE((m.attributes -> 'isBot' ->> 'default')::BOOLEAN, FALSE) = FALSE + ORDER BY msa."activityCount" DESC + LIMIT ${limit} OFFSET ${offset}; + `, + { + segmentId, + tenantId, + limit, + offset, + }, + ) +} + +/** + * Fetches members with a number of identities that exceed a specified threshold. + * + * @param {QueryExecutor} qx - The query executor to perform database operations. + * @param {number} [threshold=15] - The threshold for the number of identities a member must exceed to be included in the results. + * @param {string} tenantId - The ID of the tenant whose members are being queried. + * @param {number} limit - The maximum number of members to return. + * @param {number} offset - The number of members to skip before starting to collect the result set. + * @param {string} segmentId - The ID of the segment within which the activity count is considered. + * @return {Promise} A promise that resolves to an array of members who have more identities than the specified threshold. + */ +export async function fetchMembersWithTooManyIdentities( + qx: QueryExecutor, + threshold = 15, + tenantId: string, + limit: number, + offset: number, + segmentId: string, +): Promise { + return qx.select( + ` + SELECT + mi."memberId", + m."displayName", + m."attributes", + m.id, + COUNT(DISTINCT CASE WHEN mi."type" = 'email' THEN mi."value"::text ELSE mi.id::text END) AS "identityCount", + msa."activityCount" + FROM "memberIdentities" mi + JOIN "members" m ON mi."memberId" = m.id + INNER JOIN "memberSegmentsAgg" msa ON m.id = msa."memberId" AND msa."segmentId" = '${segmentId}' + WHERE m."tenantId" = '${tenantId}' + AND COALESCE((m.attributes -> 'isBot' ->> 'default')::BOOLEAN, FALSE) = FALSE + GROUP BY mi."memberId", m."displayName", m."attributes", m.id, msa."activityCount" + HAVING COUNT(DISTINCT CASE WHEN mi."type" = 'email' THEN mi."value"::text ELSE mi.id::text END) > ${threshold} + ORDER BY msa."activityCount" DESC + LIMIT ${limit} OFFSET ${offset}; + `, + { + threshold, + tenantId, + limit, + offset, + segmentId, + }, + ) +} + +/** + * Fetches members with a number of verified identities per platform exceeding a specified threshold for a given tenant. + * + * @param {QueryExecutor} qx - The query executor to run the database queries. + * @param {number} [threshold=1] - The minimum number of verified identities per platform to filter members by. Defaults to 1. + * @param {string} tenantId - The ID of the tenant to filter members. + * @param {number} limit - The maximum number of records to return. + * @param {number} offset - The number of records to skip. + * @param {string} segmentId - The segment ID to fetch the member activity count. + * @return {Promise} A promise that resolves to an array of members matching the criteria. + */ +export async function fetchMembersWithTooManyIdentitiesPerPlatform( + qx: QueryExecutor, + threshold = 1, + tenantId: string, + limit: number, + offset: number, + segmentId: string, +): Promise { + return qx.select( + ` + WITH platform_identities AS ( + SELECT + mi."memberId", + mi.platform, + COUNT(*) AS "identityCount" + FROM "memberIdentities" mi + WHERE mi."tenantId" = '${tenantId}' + AND mi.type = 'username' + AND mi.verified = true + AND mi.platform IN ('linkedin', 'github', 'twitter', 'lfx', 'slack', 'cvent', 'tnc', 'groupsio') + GROUP BY mi."memberId", mi.platform + HAVING COUNT(*) > ${threshold} + ), + aggregated_platforms AS ( + SELECT + p."memberId", + STRING_AGG(p.platform, ',') AS platforms + FROM platform_identities p + GROUP BY p."memberId" + ) + SELECT + p."memberId", + m."displayName", + m."attributes", + m.id, + platforms, + msa."activityCount" + FROM aggregated_platforms p + JOIN "members" m ON p."memberId" = m.id + INNER JOIN "memberSegmentsAgg" msa ON m.id = msa."memberId" + WHERE m."tenantId" = '${tenantId}' + AND msa."segmentId" = '${segmentId}' + AND COALESCE((m.attributes -> 'isBot' ->> 'default')::BOOLEAN, FALSE) = FALSE + ORDER BY msa."activityCount" DESC + LIMIT ${limit} OFFSET ${offset}; + `, + { + threshold, + tenantId, + limit, + offset, + segmentId, + }, + ) +} + +/** + * Fetches members who have more than a specified number of verified email addresses. + * + * @param {QueryExecutor} qx - The query executor to run database queries. + * @param {number} [threshold=3] - The threshold number of email addresses a member must exceed to be included. + * @param {string} tenantId - The ID of the tenant to which the members belong. + * @param {number} limit - The maximum number of members to retrieve. + * @param {number} offset - The number of members to skip before starting to collect the result set. + * @param {string} segmentId - The ID of the segment to which the members belong. + * @return {Promise} - A promise that resolves to an array of members who have more than the specified number of verified email addresses. + */ +export async function fetchMembersWithTooManyEmails( + qx: QueryExecutor, + threshold = 3, + tenantId: string, + limit: number, + offset: number, + segmentId: string, +): Promise { + return qx.select( + ` + SELECT + mi."memberId", + m."displayName", + m."attributes", + m.id, + COUNT(DISTINCT mi.value) AS "identityCount", + msa."activityCount" + FROM "memberIdentities" mi + JOIN "members" m ON mi."memberId" = m.id + INNER JOIN "memberSegmentsAgg" msa ON m.id = msa."memberId" AND msa."segmentId" = '${segmentId}' + WHERE m."tenantId" = '${tenantId}' + AND mi.verified = true + AND mi.type = 'email' + AND COALESCE((m.attributes -> 'isBot' ->> 'default')::BOOLEAN, FALSE) = FALSE + GROUP BY mi."memberId", m."displayName", m."attributes", m.id, msa."activityCount" + HAVING COUNT(DISTINCT mi.value) > ${threshold} + ORDER BY msa."activityCount" DESC + LIMIT ${limit} OFFSET ${offset}; + `, + { + threshold, + tenantId, + limit, + offset, + segmentId, + }, + ) +} + +/** + * Fetches members with missing information on work experience. + * + * @param {QueryExecutor} qx - The query executor instance used to perform database operations. + * @param {string} tenantId - The ID of the tenant to filter members by. + * @param {number} limit - The maximum number of members to retrieve. + * @param {number} offset - The starting point in the list of members to retrieve. + * @param {string} segmentId - The ID of the segment to filter members by. + * @return {Promise} A promise that resolves to an array of members + * with missing period information on their work experience. + */ +export async function fetchMembersWithMissingInfoOnWorkExperience( + qx: QueryExecutor, + tenantId: string, + limit: number, + offset: number, + segmentId: string, +): Promise { + return qx.select( + ` + SELECT + m.id, + m."displayName", + m."attributes", + msa."activityCount", + COUNT(mo.id) AS "organizationsCount" + FROM "members" m + JOIN "memberOrganizations" mo ON m.id = mo."memberId" AND mo."deletedAt" IS NULL + INNER JOIN "memberSegmentsAgg" msa ON m.id = msa."memberId" AND msa."segmentId" = '${segmentId}' + WHERE m."tenantId" = '${tenantId}' + AND (mo."title" IS NULL OR mo."title"='') + AND COALESCE((m.attributes -> 'isBot' ->> 'default')::BOOLEAN, FALSE) = FALSE + GROUP BY m.id, msa."activityCount" + ORDER BY msa."activityCount" DESC + LIMIT ${limit} OFFSET ${offset}; + `, + { + tenantId, + limit, + offset, + segmentId, + }, + ) +} + +/** + * Fetches members with missing information on work experience. + * + * @param {QueryExecutor} qx - The query executor instance used to perform database operations. + * @param {string} tenantId - The ID of the tenant to filter members by. + * @param {number} limit - The maximum number of members to retrieve. + * @param {number} offset - The starting point in the list of members to retrieve. + * @param {string} segmentId - The ID of the segment to filter members by. + * @return {Promise} A promise that resolves to an array of members + * with missing period information on their work experience. + */ +export async function fetchMembersWithMissingPeriodOnWorkExperience( + qx: QueryExecutor, + tenantId: string, + limit: number, + offset: number, + segmentId: string, +): Promise { + return qx.select( + ` + SELECT + m.id, + m."displayName", + m."attributes", + msa."activityCount", + COUNT(mo.id) AS "organizationsCount" + FROM "members" m + JOIN "memberOrganizations" mo ON m.id = mo."memberId" AND mo."deletedAt" IS NULL + INNER JOIN "memberSegmentsAgg" msa ON m.id = msa."memberId" AND msa."segmentId" = '${segmentId}' + WHERE m."tenantId" = '${tenantId}' + AND (mo."dateStart" IS NULL) + AND COALESCE((m.attributes -> 'isBot' ->> 'default')::BOOLEAN, FALSE) = FALSE + GROUP BY m.id, msa."activityCount" + ORDER BY msa."activityCount" DESC + LIMIT ${limit} OFFSET ${offset}; + `, + { + tenantId, + limit, + offset, + segmentId, + }, + ) +} + +export async function fetchMembersWithConflictingWorkExperiences( + qx: QueryExecutor, + tenantId: string, + limit: number, + offset: number, + segmentId: string, +): Promise { + return qx.select( + ` + SELECT + m.id, + m."displayName", + m."attributes", + msa."activityCount", + COUNT(DISTINCT mo1.id) AS "organizationsCount" + FROM "members" m + JOIN "memberOrganizations" mo1 ON m.id = mo1."memberId" AND mo1."deletedAt" IS NULL + INNER JOIN "memberSegmentsAgg" msa ON m.id = msa."memberId" AND msa."segmentId" = '${segmentId}' + INNER JOIN "memberOrganizations" mo2 ON mo1."memberId" = mo2."memberId" AND mo2."deletedAt" IS NULL + AND mo1.id != mo2.id + AND ( + (mo1."dateStart" < COALESCE(mo2."dateEnd", 'infinity'::timestamp) + AND COALESCE(mo1."dateEnd", 'infinity'::timestamp) > mo2."dateStart") OR + (mo2."dateStart" < COALESCE(mo1."dateEnd", 'infinity'::timestamp) + AND COALESCE(mo2."dateEnd", 'infinity'::timestamp) > mo1."dateStart") + ) + WHERE m."tenantId" = '${tenantId}' + AND COALESCE((m.attributes -> 'isBot' ->> 'default')::BOOLEAN, FALSE) = FALSE + GROUP BY m.id, msa."activityCount" + ORDER BY msa."activityCount" DESC + LIMIT ${limit} OFFSET ${offset}; + `, + { + tenantId, + limit, + offset, + segmentId, + }, + ) +} diff --git a/services/libs/data-access-layer/src/data-quality/organizations.ts b/services/libs/data-access-layer/src/data-quality/organizations.ts new file mode 100644 index 0000000000..eed30be1d6 --- /dev/null +++ b/services/libs/data-access-layer/src/data-quality/organizations.ts @@ -0,0 +1,144 @@ +import { IMemberOrganization } from '@crowd/types' + +import { QueryExecutor } from '../queryExecutor' + +export async function fetchMemberOrganizations( + qx: QueryExecutor, + memberId: string, +): Promise { + return qx.select( + ` + SELECT "id", "organizationId", "dateStart", "dateEnd", "title", "memberId", "source" + FROM "memberOrganizations" + WHERE "memberId" = $(memberId) + AND "deletedAt" IS NULL + ORDER BY + CASE + WHEN "dateEnd" IS NULL AND "dateStart" IS NOT NULL THEN 1 + WHEN "dateEnd" IS NOT NULL AND "dateStart" IS NOT NULL THEN 2 + WHEN "dateEnd" IS NULL AND "dateStart" IS NULL THEN 3 + ELSE 4 + END ASC, + "dateEnd" DESC, + "dateStart" DESC + `, + { + memberId, + }, + ) +} + +export async function fetchManyMemberOrgs( + qx: QueryExecutor, + memberIds: string[], +): Promise<{ memberId: string; organizations: IMemberOrganization[] }[]> { + return qx.select( + ` + SELECT + mo."memberId", + JSONB_AGG(mo ORDER BY mo."createdAt") AS "organizations" + FROM "memberOrganizations" mo + WHERE mo."memberId" IN ($(memberIds:csv)) + AND mo."deletedAt" IS NULL + GROUP BY mo."memberId" + `, + { + memberIds, + }, + ) +} + +export async function createMemberOrganization( + qx: QueryExecutor, + memberId: string, + data: Partial, +): Promise { + return qx.result( + ` + INSERT INTO "memberOrganizations"("memberId", "organizationId", "dateStart", "dateEnd", "title", "source", "createdAt", "updatedAt") + VALUES($(memberId), $(organizationId), $(dateStart), $(dateEnd), $(title), $(source), $(date), $(date)) + `, + { + memberId, + organizationId: data.organizationId, + dateStart: data.dateStart, + dateEnd: data.dateEnd, + title: data.title, + source: data.source, + date: new Date().toISOString(), + }, + ) +} + +export async function updateMemberOrganization( + qx: QueryExecutor, + memberId: string, + id: string, + data: Partial, +): Promise { + return qx.result( + ` + UPDATE "memberOrganizations" + SET + "organizationId" = $(organizationId), + "dateStart" = $(dateStart), + "dateEnd" = $(dateEnd), + title = $(title), + source = $(source), + "updatedAt" = $(updatedAt) + WHERE "memberId" = $(memberId) AND "id" = $(id); + `, + { + memberId, + id, + organizationId: data.organizationId, + dateStart: data.dateStart, + dateEnd: data.dateEnd, + title: data.title, + source: data.source, + updatedAt: new Date().toISOString(), + }, + ) +} + +export async function deleteMemberOrganization( + qx: QueryExecutor, + memberId: string, + id: string, +): Promise { + return qx.result( + ` + UPDATE "memberOrganizations" + SET "deletedAt" = NOW() + WHERE "memberId" = $(memberId) AND "id" = $(id); + `, + { + memberId, + id, + }, + ) +} + +export async function cleanSoftDeletedMemberOrganization( + qx: QueryExecutor, + memberId: string, + organizationId: string, + data: Partial, +): Promise { + return qx.result( + ` + DELETE FROM "memberOrganizations" + WHERE "memberId" = $(memberId) + AND "organizationId" = $(organizationId) + AND (("dateStart" = $(dateStart)) OR ("dateStart" IS NULL AND $(dateStart) IS NULL)) + AND (("dateEnd" = $(dateEnd)) OR ("dateEnd" IS NULL AND $(dateEnd) IS NULL)) + AND "deletedAt" IS NOT NULL; + `, + { + memberId, + organizationId, + dateStart: data.dateStart ?? null, + dateEnd: data.dateEnd ?? null, + }, + ) +}