Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[#4460] Feature/Corporate contributors dups #4469

Merged
merged 1 commit into from
Oct 29, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
173 changes: 100 additions & 73 deletions cla-backend-go/signatures/repository.go
Original file line number Diff line number Diff line change
@@ -4616,6 +4616,32 @@ func (repo repository) addAdditionalICLAMetaData(f logrus.Fields, intermediateRe
return finalResults, nil
}

func deduplicateSignatures(signatures []map[string]*dynamodb.AttributeValue) []map[string]*dynamodb.AttributeValue {
f := logrus.Fields{
"functionName": "v1.signatures.repository.deduplicateSignatures",
}
// Map to keep track of the latest unique signatures by SignatureReferenceID
uniqueSignatures := make(map[string]map[string]*dynamodb.AttributeValue)
for _, sig := range signatures {
log.WithFields(f).Debugf("processing signature: %+v", sig)
refID := sig["signature_reference_id"].S
existingSig, exists := uniqueSignatures[*refID]
if !exists || (sig["date_created"] != nil && existingSig["date_created"] != nil && *sig["date_created"].S > *existingSig["date_created"].S) {
uniqueSignatures[*refID] = sig
} else if !exists || (sig["date_created"] != nil && existingSig["date_created"] == nil) {
uniqueSignatures[*refID] = sig
} else if !exists || (sig["date_created"] == nil && existingSig["date_created"] != nil) {
uniqueSignatures[*refID] = sig
}
}
// Convert the map back to a slice
var uniqueSignatureList []map[string]*dynamodb.AttributeValue
for _, sig := range uniqueSignatures {
uniqueSignatureList = append(uniqueSignatureList, sig)
}
return uniqueSignatureList
}

func (repo repository) GetClaGroupCorporateContributors(ctx context.Context, claGroupID string, companyID *string, pageSize *int64, nextKey *string, searchTerm *string) (*models.CorporateContributorList, error) {
f := logrus.Fields{
"functionName": "v1.signatures.repository.GetClaGroupCorporateContributors",
@@ -4698,10 +4724,10 @@ func (repo repository) GetClaGroupCorporateContributors(ctx context.Context, cla

out := &models.CorporateContributorList{List: make([]*models.CorporateContributor, 0)}
var lastEvaluatedKey string

var allItems []map[string]*dynamodb.AttributeValue
currentCount := int64(0)

for ok := true; ok; ok = lastEvaluatedKey != "" && currentCount < *pageSize {
for ok := true; ok; ok = lastEvaluatedKey != "" {
// Make the DynamoDB Query API call
log.WithFields(f).Debug("querying signatures...")
results, queryErr := repo.dynamoDBClient.Query(queryInput)
@@ -4710,92 +4736,90 @@ func (repo repository) GetClaGroupCorporateContributors(ctx context.Context, cla
return nil, queryErr
}

var dbSignatures []ItemSignature
// Append the results...
allItems = append(allItems, results.Items...)
currentCount += int64(len(results.Items))

err := dynamodbattribute.UnmarshalListOfMaps(results.Items, &dbSignatures)
if results.LastEvaluatedKey["signature_id"] != nil && currentCount < *pageSize {
lastEvaluatedKey = *results.LastEvaluatedKey["signature_id"].S
queryInput.ExclusiveStartKey = results.LastEvaluatedKey
} else {
lastEvaluatedKey = ""
}

}

allItems = deduplicateSignatures(allItems)
var dbSignatures []ItemSignature
err = dynamodbattribute.UnmarshalListOfMaps(allItems, &dbSignatures)
if err != nil {
log.WithFields(f).Warnf("error unmarshalling icla signatures from database for cla group: %s, error: %v",
claGroupID, err)
return nil, err
}

log.WithFields(f).Debugf("located %d signatures...", len(dbSignatures))
for _, sig := range dbSignatures {
var sigCreatedTime = sig.DateCreated
t, err := utils.ParseDateTime(sig.DateCreated)
if err != nil {
log.WithFields(f).Warnf("error unmarshalling icla signatures from database for cla group: %s, error: %v",
claGroupID, err)
return nil, err
log.WithFields(f).WithError(err).Warn("unable to parse signature date created time")
} else {
sigCreatedTime = utils.TimeToString(t)
}

log.WithFields(f).Debugf("located %d signatures...", len(dbSignatures))
for _, sig := range dbSignatures {
var sigCreatedTime = sig.DateCreated
t, err := utils.ParseDateTime(sig.DateCreated)
// Set the signed date/time
var sigSignedTime string
// Use the user docusign date signed value if it is present - older signatures do not have this
if sig.UserDocusignDateSigned != "" {
// Put the date into a standard format
t, err = utils.ParseDateTime(sig.UserDocusignDateSigned)
if err != nil {
log.WithFields(f).WithError(err).Warn("unable to parse signature date created time")
log.WithFields(f).WithError(err).Warn("unable to parse signature docusign date signed time")
} else {
sigCreatedTime = utils.TimeToString(t)
sigSignedTime = utils.TimeToString(t)
}

// Set the signed date/time
var sigSignedTime string
// Use the user docusign date signed value if it is present - older signatures do not have this
if sig.UserDocusignDateSigned != "" {
// Put the date into a standard format
t, err = utils.ParseDateTime(sig.UserDocusignDateSigned)
if err != nil {
log.WithFields(f).WithError(err).Warn("unable to parse signature docusign date signed time")
} else {
sigSignedTime = utils.TimeToString(t)
}
} else {
// Put the date into a standard format
t, err = utils.ParseDateTime(sig.DateCreated)
if err != nil {
log.WithFields(f).WithError(err).Warn("unable to parse signature date created time")
} else {
// Put the date into a standard format
t, err = utils.ParseDateTime(sig.DateCreated)
if err != nil {
log.WithFields(f).WithError(err).Warn("unable to parse signature date created time")
} else {
sigSignedTime = utils.TimeToString(t)
}
sigSignedTime = utils.TimeToString(t)
}
}

signatureVersion := fmt.Sprintf("v%s.%s", strconv.Itoa(sig.SignatureDocumentMajorVersion), strconv.Itoa(sig.SignatureDocumentMinorVersion))

sigName := sig.UserName
user, userErr := repo.usersRepo.GetUser(sig.SignatureReferenceID)
if userErr != nil {
log.WithFields(f).Warnf("unable to get user for id: %s, error: %v ", sig.SignatureReferenceID, userErr)
}
if user != nil && sigName == "" {
sigName = user.Username
}
signatureVersion := fmt.Sprintf("v%s.%s", strconv.Itoa(sig.SignatureDocumentMajorVersion), strconv.Itoa(sig.SignatureDocumentMinorVersion))

out.List = append(out.List, &models.CorporateContributor{
SignatureID: sig.SignatureID,
GithubID: sig.UserGithubUsername,
LinuxFoundationID: sig.UserLFUsername,
Name: sigName,
SignatureVersion: signatureVersion,
Email: sig.UserEmail,
Timestamp: sigCreatedTime,
UserDocusignName: sig.UserDocusignName,
UserDocusignDateSigned: sigSignedTime,
SignatureModified: sig.DateModified,
SignatureApproved: sig.SignatureApproved,
SignatureSigned: sig.SignatureSigned,
})

// Increment the current count
currentCount++
if currentCount >= *pageSize {
break
}
sigName := sig.UserName
user, userErr := repo.usersRepo.GetUser(sig.SignatureReferenceID)
if userErr != nil {
log.WithFields(f).Warnf("unable to get user for id: %s, error: %v ", sig.SignatureReferenceID, userErr)
}

if results.LastEvaluatedKey["signature_id"] != nil && currentCount < *pageSize {
lastEvaluatedKey = *results.LastEvaluatedKey["signature_id"].S
queryInput.ExclusiveStartKey = results.LastEvaluatedKey
} else {
lastEvaluatedKey = ""
if user != nil && sigName == "" {
sigName = user.Username
}

out.List = append(out.List, &models.CorporateContributor{
SignatureID: sig.SignatureID,
GithubID: sig.UserGithubUsername,
LinuxFoundationID: sig.UserLFUsername,
Name: sigName,
SignatureVersion: signatureVersion,
Email: sig.UserEmail,
Timestamp: sigCreatedTime,
UserDocusignName: sig.UserDocusignName,
UserDocusignDateSigned: sigSignedTime,
SignatureModified: sig.DateModified,
SignatureApproved: sig.SignatureApproved,
SignatureSigned: sig.SignatureSigned,
})
}
sort.Slice(out.List, func(i, j int) bool {
return out.List[i].Name < out.List[j].Name
})

out.ResultCount = currentCount
out.ResultCount = int64(len(out.List))
out.TotalCount = totalCount
out.NextKey = lastEvaluatedKey

@@ -4851,6 +4875,7 @@ func (repo repository) getTotalCorporateContributorCount(ctx context.Context, cl

var lastEvaluatedKey string
var totalCount int64
var allItems []map[string]*dynamodb.AttributeValue

// Loop until we have all the records - we'll get a nil lastEvaluatedKey when we're done
for ok := true; ok; ok = lastEvaluatedKey != "" {
@@ -4861,8 +4886,7 @@ func (repo repository) getTotalCorporateContributorCount(ctx context.Context, cl
return
}

// Add the count to the total
totalCount += *results.Count
allItems = append(allItems, results.Items...)

// Set the last evaluated key
if results.LastEvaluatedKey["signature_id"] != nil {
@@ -4873,10 +4897,13 @@ func (repo repository) getTotalCorporateContributorCount(ctx context.Context, cl
}
}

log.WithFields(f).Debugf("total signature count query took: %s", time.Since(beforeQuery))

// Deduplicate the signatures
// uniqueSignatures := deduplicateSignatures(allItems)
totalCount = int64(len(allItems))
totalCountChannel <- totalCount

log.WithFields(f).Debugf("total signature count query took: %s", time.Since(beforeQuery))

}

// EclaAutoCreate this routine updates the CCLA signature record by adjusting the auto_create_ecla column to the specified value
Loading