Skip to content

Commit

Permalink
Add missing datapoints in resourcecorrelation and api calls to moc (#236
Browse files Browse the repository at this point in the history 
)

* add correlationId when call moc to login

* adding CRD correlation log

* polify telemetry method

---------

Co-authored-by: peidlu <[email protected]>
  • Loading branch information
@lpdnju
lpdnju and lpdnju authored Oct 24, 2023
1 parent 502d741 commit 26d9955
Show file tree
Hide file tree
Showing 9 changed files with 194 additions and 41 deletions.
6 changes: 3 additions & 3 deletions cloud/scope/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,11 @@ import (
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/klog/v2/klogr"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/util/conditions"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
)

// ClusterScopeParams defines the input parameters used to create a new Scope.
Expand All @@ -58,7 +58,7 @@ func NewClusterScope(params ClusterScopeParams) (*ClusterScope, error) {
}

if params.Logger == nil {
log := klogr.New()
log := zap.New(zap.UseDevMode(true))
params.Logger = &log
}

Expand All @@ -84,7 +84,7 @@ func NewClusterScope(params ClusterScopeParams) (*ClusterScope, error) {
Context: scopeContext,
}

authorizer, err := azhciauth.ReconcileAzureStackHCIAccess(scope.Context, scope.Client, agentFqdn)
authorizer, err := azhciauth.ReconcileAzureStackHCIAccess(*params.Logger, scope.Context, scope.Client, agentFqdn)
if err != nil {
return nil, errors.Wrap(err, "error creating azurestackhci services. can not authenticate to azurestackhci")
}
Expand Down
4 changes: 2 additions & 2 deletions cloud/scope/loadbalancer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@ import (
infrav1 "github.com/microsoft/cluster-api-provider-azurestackhci/api/v1beta1"
"github.com/microsoft/moc/pkg/diagnostics"
"github.com/pkg/errors"
"k8s.io/klog/v2/klogr"
"k8s.io/utils/pointer"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
capierrors "sigs.k8s.io/cluster-api/errors"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
)

// LoadBalancerScopeParams defines the input parameters used to create a new LoadBalancerScope.
Expand All @@ -53,7 +53,7 @@ func NewLoadBalancerScope(params LoadBalancerScopeParams) (*LoadBalancerScope, e
}

if params.Logger == nil {
log := klogr.New()
log := zap.New(zap.UseDevMode(true))
params.Logger = &log
}

Expand Down
4 changes: 2 additions & 2 deletions cloud/scope/machine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ import (
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/klog/v2/klogr"
"k8s.io/utils/pointer"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/cluster-api/controllers/noderefutil"
Expand All @@ -35,6 +34,7 @@ import (
"sigs.k8s.io/cluster-api/util/conditions"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
)

// MachineScopeParams defines the input parameters used to create a new MachineScope.
Expand Down Expand Up @@ -68,7 +68,7 @@ func NewMachineScope(params MachineScopeParams) (*MachineScope, error) {
}

if params.Logger == nil {
log := klogr.New()
log := zap.New(zap.UseDevMode(true))
params.Logger = &log
}

Expand Down
8 changes: 4 additions & 4 deletions cloud/scope/virtualmachine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,13 @@ import (
"github.com/microsoft/moc/pkg/auth"
"github.com/microsoft/moc/pkg/diagnostics"
"github.com/pkg/errors"
"k8s.io/klog/v2/klogr"
"k8s.io/utils/pointer"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
capierrors "sigs.k8s.io/cluster-api/errors"
"sigs.k8s.io/cluster-api/util/conditions"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
)

// MachineScopeParams defines the input parameters used to create a new VirtualMachineScope.
Expand All @@ -57,7 +57,7 @@ func NewVirtualMachineScope(params VirtualMachineScopeParams) (*VirtualMachineSc
}

if params.Logger == nil {
log := klogr.New()
log := zap.New(zap.UseDevMode(true))
params.Logger = &log
}

Expand All @@ -67,7 +67,8 @@ func NewVirtualMachineScope(params VirtualMachineScopeParams) (*VirtualMachineSc
}
params.AzureStackHCIClients.CloudAgentFqdn = agentFqdn

authorizer, err := azhciauth.ReconcileAzureStackHCIAccess(context.Background(), params.Client, agentFqdn)
scopeContext := diagnostics.NewContextWithCorrelationId(context.Background(), params.AzureStackHCIVirtualMachine.GetAnnotations()[infrav1.AzureCorrelationIDAnnotationKey])
authorizer, err := azhciauth.ReconcileAzureStackHCIAccess(*params.Logger, scopeContext, params.Client, agentFqdn)
if err != nil {
return nil, errors.Wrap(err, "failed to create azurestackhci session")
}
Expand All @@ -77,7 +78,6 @@ func NewVirtualMachineScope(params VirtualMachineScopeParams) (*VirtualMachineSc
if err != nil {
return nil, errors.Wrap(err, "failed to init patch helper")
}
scopeContext := diagnostics.NewContextWithCorrelationId(context.Background(), params.AzureStackHCIVirtualMachine.GetAnnotations()[infrav1.AzureCorrelationIDAnnotationKey])
return &VirtualMachineScope{
client: params.Client,
AzureStackHCIVirtualMachine: params.AzureStackHCIVirtualMachine,
Expand Down
81 changes: 74 additions & 7 deletions cloud/telemetry/logutils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/microsoft/cluster-api-provider-azurestackhci/cloud/services/health"
"github.com/microsoft/cluster-api-provider-azurestackhci/cloud/services/versions"
mocerrors "github.com/microsoft/moc/pkg/errors"
ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
)

type MocResourceType string
Expand All @@ -28,12 +29,21 @@ const (
Group MocResourceType = "Group"
)

type MocOperation string
type ResourceType string

const (
CreateOrUpdate MocOperation = "CreateOrUpdate"
Delete MocOperation = "Delete"
Get MocOperation = "Get"
CRD ResourceType = "CRD"
Status ResourceType = "Status"
)

type Operation string

const (
Create Operation = "Create"
CreateOrUpdate Operation = "CreateOrUpdate"
Update Operation = "Update"
Delete Operation = "Delete"
Get Operation = "Get"
)

type OperationLog struct {
Expand All @@ -47,7 +57,7 @@ type OperationLog struct {
Message string `json:"msg"`
}

func WriteMocOperationLog(logger logr.Logger, operation MocOperation, crResourceName string, mocResourceType MocResourceType, mocResourceName string, params interface{}, err error) {
func WriteMocOperationLog(logger logr.Logger, operation Operation, crResourceName string, mocResourceType MocResourceType, mocResourceName string, params interface{}, err error) {
errcode := "0"
message := ""
if err != nil {
Expand All @@ -66,9 +76,42 @@ func WriteMocOperationLog(logger logr.Logger, operation MocOperation, crResource
Message: message,
}

jsonData, err := json.Marshal(oplog)
jsonData, serializeError := json.Marshal(oplog)
if serializeError != nil {
logger.Error(serializeError, "Unable to serialize operation log object.", "resourceName", crResourceName)
} else {
logger.Info(string(jsonData))
}
}

// RecordHybridAKSCRDChange need to be called when CRD changed.
func RecordHybridAKSCRDChange(logger logr.Logger, parentResource string, resource string, action Operation, resourceType ResourceType, params interface{}, err error) {
errMessage := ""
errCode := "0"
if err != nil {
logger.Error(err, "Unable to serialize operation log object.", "resourceName", crResourceName)
errCode = "-1"
errMessage = err.Error()
}

oplog := OperationLog{
Timestamp: time.Now().Format(time.RFC3339),
ParentResource: parentResource,
Resource: resource,
FilterKeyword: "RESOURCE_ACTION",
Action: fmt.Sprintf("%s %s", action, resourceType),
Params: params,
ErrorCode: errCode,
Message: errMessage,
}

jsonData, serializeError := json.Marshal(oplog)
if serializeError != nil {
logger.Error(serializeError, "Unable to serialize operation log object",
"timestamp", time.Now().Format(time.RFC3339),
"parent_resource", parentResource,
"resource", resource,
"filter_keyword", "RESOURCE_ACTION",
"action", action)
} else {
logger.Info(string(jsonData))
}
Expand Down Expand Up @@ -132,3 +175,27 @@ func getVersionsService(scope scope.ScopeInterface) *versions.Service {
versionsService = versions.NewService(scope)
return versionsService
}

func IsCRDUpdate(operationResult ctrlutil.OperationResult) bool {
if operationResult == ctrlutil.OperationResultCreated || operationResult == ctrlutil.OperationResultUpdatedStatus ||
operationResult == ctrlutil.OperationResultUpdatedStatusOnly {
return true
}

return false
}

func ConvertOperationResult(operationResult ctrlutil.OperationResult) (Operation, ResourceType) {
switch operationResult {
case ctrlutil.OperationResultCreated:
return Create, CRD
case ctrlutil.OperationResultUpdated:
return Update, CRD
case ctrlutil.OperationResultUpdatedStatus:
fallthrough
case ctrlutil.OperationResultUpdatedStatusOnly:
return Update, Status
default:
return "", ""
}
}
38 changes: 35 additions & 3 deletions controllers/azurestackhcicluster_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,14 @@ package controllers

import (
"context"
"fmt"
"time"

"github.com/go-logr/logr"
infrav1 "github.com/microsoft/cluster-api-provider-azurestackhci/api/v1beta1"
azurestackhci "github.com/microsoft/cluster-api-provider-azurestackhci/cloud"
"github.com/microsoft/cluster-api-provider-azurestackhci/cloud/scope"
"github.com/microsoft/cluster-api-provider-azurestackhci/cloud/telemetry"
infrav1util "github.com/microsoft/cluster-api-provider-azurestackhci/pkg/util"
mocerrors "github.com/microsoft/moc/pkg/errors"
"github.com/pkg/errors"
Expand Down Expand Up @@ -248,7 +250,16 @@ func (r *AzureStackHCIClusterReconciler) deleteOrphanedMachines(clusterScope *sc
}
}
clusterScope.Info("Deleting Orphaned Machine", "Name", azhciMachine.Name, "AzureStackHCICluster", clusterScope.AzureStackHCICluster.Name)
if err := r.Client.Delete(clusterScope.Context, azhciMachine); err != nil {
err := r.Client.Delete(clusterScope.Context, azhciMachine)
telemetry.RecordHybridAKSCRDChange(
clusterScope.GetLogger(),
clusterScope.GetCustomResourceTypeWithName(),
fmt.Sprintf("%s/%s/%s", azhciMachine.TypeMeta.Kind, azhciMachine.ObjectMeta.Namespace, azhciMachine.ObjectMeta.Name),
telemetry.Delete,
telemetry.CRD,
nil,
err)
if err != nil {
if !apierrors.IsNotFound(err) {
return errors.Wrapf(err, "Failed to delete AzureStackHCIMachine %s", azhciMachine)
}
Expand Down Expand Up @@ -291,7 +302,19 @@ func (r *AzureStackHCIClusterReconciler) reconcileAzureStackHCILoadBalancer(clus
return nil
}

if _, err := controllerutil.CreateOrUpdate(clusterScope.Context, r.Client, azureStackHCILoadBalancer, mutateFn); err != nil {
operationResult, err := controllerutil.CreateOrUpdate(clusterScope.Context, r.Client, azureStackHCILoadBalancer, mutateFn)
if telemetry.IsCRDUpdate(operationResult) {
operation, resourceType := telemetry.ConvertOperationResult(operationResult)
telemetry.RecordHybridAKSCRDChange(
clusterScope.GetLogger(),
clusterScope.GetCustomResourceTypeWithName(),
fmt.Sprintf("%s/%s/%s", azureStackHCILoadBalancer.TypeMeta.Kind, azureStackHCILoadBalancer.ObjectMeta.Namespace, azureStackHCILoadBalancer.ObjectMeta.Name),
operation,
resourceType,
nil,
err)
}
if err != nil {
if !apierrors.IsAlreadyExists(err) {
conditions.MarkFalse(clusterScope.AzureStackHCICluster, infrav1.NetworkInfrastructureReadyCondition, infrav1.LoadBalancerProvisioningReason, clusterv1.ConditionSeverityWarning, err.Error())
return false, err
Expand Down Expand Up @@ -344,7 +367,16 @@ func (r *AzureStackHCIClusterReconciler) reconcileDeleteAzureStackHCILoadBalance
return errors.Wrapf(err, "Failed to update AzureStackHCILoadBalancer %s", azureStackHCILoadBalancerName)
}
}
if err := r.Client.Delete(clusterScope.Context, azureStackHCILoadBalancer); err != nil {
err := r.Client.Delete(clusterScope.Context, azureStackHCILoadBalancer)
telemetry.RecordHybridAKSCRDChange(
clusterScope.GetLogger(),
clusterScope.GetCustomResourceTypeWithName(),
fmt.Sprintf("%s/%s/%s", azureStackHCILoadBalancer.TypeMeta.Kind, azureStackHCILoadBalancer.ObjectMeta.Namespace, azureStackHCILoadBalancer.ObjectMeta.Name),
telemetry.Delete,
telemetry.CRD,
nil,
err)
if err != nil {
if !apierrors.IsNotFound(err) {
conditions.MarkFalse(clusterScope.AzureStackHCICluster, infrav1.NetworkInfrastructureReadyCondition, clusterv1.DeletionFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
return errors.Wrapf(err, "Failed to delete AzureStackHCILoadBalancer %s", azureStackHCILoadBalancerName)
Expand Down
38 changes: 34 additions & 4 deletions controllers/azurestackhciloadbalancer_virtualmachine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
infrav1 "github.com/microsoft/cluster-api-provider-azurestackhci/api/v1beta1"
azurestackhci "github.com/microsoft/cluster-api-provider-azurestackhci/cloud"
"github.com/microsoft/cluster-api-provider-azurestackhci/cloud/scope"
"github.com/microsoft/cluster-api-provider-azurestackhci/cloud/telemetry"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
Expand Down Expand Up @@ -129,7 +130,16 @@ func (r *AzureStackHCILoadBalancerReconciler) reconcileDeleteVirtualMachines(loa

for _, vm := range vmList {
if vm.GetDeletionTimestamp().IsZero() {
if err := r.Client.Delete(clusterScope.Context, vm); err != nil {
err := r.Client.Delete(clusterScope.Context, vm)
telemetry.RecordHybridAKSCRDChange(
clusterScope.GetLogger(),
clusterScope.GetCustomResourceTypeWithName(),
fmt.Sprintf("%s/%s/%s", vm.TypeMeta.Kind, vm.ObjectMeta.Namespace, vm.ObjectMeta.Name),
telemetry.Delete,
telemetry.CRD,
nil,
err)
if err != nil {
if !apierrors.IsNotFound(err) {
return errors.Wrapf(err, "failed to delete AzureStackHCIVirtualMachine %s", vm.Name)
}
Expand Down Expand Up @@ -191,8 +201,19 @@ func (r *AzureStackHCILoadBalancerReconciler) createOrUpdateVirtualMachine(loadB

return nil
}

if _, err := controllerutil.CreateOrUpdate(clusterScope.Context, r.Client, vm, mutateFn); err != nil {
operationResult, err := controllerutil.CreateOrUpdate(clusterScope.Context, r.Client, vm, mutateFn)
if telemetry.IsCRDUpdate(operationResult) {
operation, resourceType := telemetry.ConvertOperationResult(operationResult)
telemetry.RecordHybridAKSCRDChange(
loadBalancerScope.GetLogger(),
clusterScope.GetCustomResourceTypeWithName(),
fmt.Sprintf("%s/%s/%s", vm.TypeMeta.Kind, vm.ObjectMeta.Namespace, vm.ObjectMeta.Name),
operation,
resourceType,
nil,
err)
}
if err != nil {
return nil, err
}

Expand All @@ -202,7 +223,16 @@ func (r *AzureStackHCILoadBalancerReconciler) createOrUpdateVirtualMachine(loadB
// deleteVirtualMachine deletes a virtual machine
func (r *AzureStackHCILoadBalancerReconciler) deleteVirtualMachine(clusterScope *scope.ClusterScope, vm *infrav1.AzureStackHCIVirtualMachine) error {
if vm.GetDeletionTimestamp().IsZero() {
if err := r.Client.Delete(clusterScope.Context, vm); err != nil {
err := r.Client.Delete(clusterScope.Context, vm)
telemetry.RecordHybridAKSCRDChange(
clusterScope.GetLogger(),
clusterScope.GetCustomResourceTypeWithName(),
fmt.Sprintf("%s/%s/%s", vm.TypeMeta.Kind, vm.ObjectMeta.Namespace, vm.ObjectMeta.Name),
telemetry.Delete,
telemetry.CRD,
nil,
err)
if err != nil {
if !apierrors.IsNotFound(err) {
return errors.Wrapf(err, "failed to delete AzureStackHCIVirtualMachine %s", vm.Name)
}
Expand Down
Loading

0 comments on commit 26d9955

Please sign in to comment.