diff --git a/api/v1beta1/zz_generated.conversion.go b/api/v1beta1/zz_generated.conversion.go index 16331b3613..7eed081c55 100644 --- a/api/v1beta1/zz_generated.conversion.go +++ b/api/v1beta1/zz_generated.conversion.go @@ -562,6 +562,7 @@ func autoConvert_v1beta2_IBMPowerVSClusterSpec_To_v1beta1_IBMPowerVSClusterSpec( // WARNING: in.ResourceGroup requires manual conversion: does not exist in peer-type // WARNING: in.VPC requires manual conversion: does not exist in peer-type // WARNING: in.VPCSubnets requires manual conversion: does not exist in peer-type + // WARNING: in.VPCSecurityGroups requires manual conversion: does not exist in peer-type // WARNING: in.TransitGateway requires manual conversion: does not exist in peer-type // WARNING: in.LoadBalancers requires manual conversion: does not exist in peer-type // WARNING: in.CosInstance requires manual conversion: does not exist in peer-type @@ -587,6 +588,7 @@ func autoConvert_v1beta2_IBMPowerVSClusterStatus_To_v1beta1_IBMPowerVSClusterSta // WARNING: in.DHCPServer requires manual conversion: does not exist in peer-type // WARNING: in.VPC requires manual conversion: does not exist in peer-type // WARNING: in.VPCSubnet requires manual conversion: does not exist in peer-type + // WARNING: in.VPCSecurityGroups requires manual conversion: does not exist in peer-type // WARNING: in.TransitGateway requires manual conversion: does not exist in peer-type // WARNING: in.COSInstance requires manual conversion: does not exist in peer-type // WARNING: in.LoadBalancers requires manual conversion: does not exist in peer-type diff --git a/api/v1beta2/conditions_consts.go b/api/v1beta2/conditions_consts.go index 1ded21813f..a3cda55973 100644 --- a/api/v1beta2/conditions_consts.go +++ b/api/v1beta2/conditions_consts.go @@ -85,6 +85,11 @@ const ( // NetworkReconciliationFailedReason used when an error occurs during network reconciliation. NetworkReconciliationFailedReason = "NetworkReconciliationFailed" + // VPCSecurityGroupReadyCondition reports on the successful reconciliation of a VPC. + VPCSecurityGroupReadyCondition capiv1beta1.ConditionType = "VPCSecurityGroupReady" + // VPCSecurityGroupReconciliationFailedReason used when an error occurs during VPC reconciliation. + VPCSecurityGroupReconciliationFailedReason = "VPCSecurityGroupReconciliationFailed" + // VPCReadyCondition reports on the successful reconciliation of a VPC. VPCReadyCondition capiv1beta1.ConditionType = "VPCReady" // VPCReconciliationFailedReason used when an error occurs during VPC reconciliation. diff --git a/api/v1beta2/ibmpowervscluster_types.go b/api/v1beta2/ibmpowervscluster_types.go index 65efbf5cd1..7c22e4d56d 100644 --- a/api/v1beta2/ibmpowervscluster_types.go +++ b/api/v1beta2/ibmpowervscluster_types.go @@ -103,6 +103,10 @@ type IBMPowerVSClusterSpec struct { // +optional VPCSubnets []Subnet `json:"vpcSubnets,omitempty"` + // VPCSecurityGroups to attach it to the VPC resource + // +optional + VPCSecurityGroups []SecurityGroup `json:"vpcSecurityGroups,omitempty"` + // transitGateway contains information about IBM Cloud TransitGateway // IBM Cloud TransitGateway helps in establishing network connectivity between IBM Cloud Power VS and VPC infrastructure // more information about TransitGateway can be found here https://www.ibm.com/products/transit-gateway. @@ -200,6 +204,9 @@ type IBMPowerVSClusterStatus struct { // vpcSubnet is reference to IBM Cloud VPC subnet. VPCSubnet map[string]ResourceReference `json:"vpcSubnet,omitempty"` + // vpcSecurityGroups is reference to IBM Cloud VPC security group. + VPCSecurityGroups map[string]VPCSecurityGroupStatus `json:"vpcSecurityGroups,omitempty"` + // transitGateway is reference to IBM Cloud TransitGateway. TransitGateway *ResourceReference `json:"transitGateway,omitempty"` diff --git a/api/v1beta2/ibmvpccluster_types.go b/api/v1beta2/ibmvpccluster_types.go index 945f34ea58..9d04c52e3d 100644 --- a/api/v1beta2/ibmvpccluster_types.go +++ b/api/v1beta2/ibmvpccluster_types.go @@ -93,6 +93,17 @@ type AdditionalListenerSpec struct { Port int64 `json:"port"` } +// VPCSecurityGroupStatus defines a vpc security group resource status with its id and respective rule's ids. +type VPCSecurityGroupStatus struct { + // id represents the id of the resource. + ID *string `json:"id,omitempty"` + // rules contains the id of rules created under the security group + RuleIDs []*string `json:"ruleIDs,omitempty"` + // +kubebuilder:default=false + // controllerCreated indicates whether the resource is created by the controller. + ControllerCreated *bool `json:"controllerCreated,omitempty"` +} + // VPCLoadBalancerStatus defines the status VPC load balancer. type VPCLoadBalancerStatus struct { // id of VPC load balancer. diff --git a/api/v1beta2/types.go b/api/v1beta2/types.go index b1fa3c7e53..5bdeb06a10 100644 --- a/api/v1beta2/types.go +++ b/api/v1beta2/types.go @@ -232,9 +232,9 @@ const ( // For example: // - any - Any source or destination (0.0.0.0/0) // - cidr - A CIDR representing a set of IP's (10.0.0.0/28) -// - ip - A specific IP address (192.168.0.1) +// - address - A specific address (192.168.0.1) // - sg - A Security Group. -// +kubebuilder:validation:Enum=any;cidr;ip;sg +// +kubebuilder:validation:Enum=any;cidr;address;sg type SecurityGroupRuleRemoteType string const ( @@ -242,8 +242,8 @@ const ( SecurityGroupRuleRemoteTypeAny SecurityGroupRuleRemoteType = SecurityGroupRuleRemoteType("any") // SecurityGroupRuleRemoteTypeCIDR defines the destination or source for the Rule is a CIDR block. SecurityGroupRuleRemoteTypeCIDR SecurityGroupRuleRemoteType = SecurityGroupRuleRemoteType("cidr") - // SecurityGroupRuleRemoteTypeIP defines the destination or source for the Rule is an IP address. - SecurityGroupRuleRemoteTypeIP SecurityGroupRuleRemoteType = SecurityGroupRuleRemoteType("ip") + // SecurityGroupRuleRemoteTypeAddress defines the destination or source for the Rule is an address. + SecurityGroupRuleRemoteTypeAddress SecurityGroupRuleRemoteType = SecurityGroupRuleRemoteType("address") // SecurityGroupRuleRemoteTypeSG defines the destination or source for the Rule is a VPC Security Group. SecurityGroupRuleRemoteTypeSG SecurityGroupRuleRemoteType = SecurityGroupRuleRemoteType("sg") ) @@ -328,20 +328,20 @@ type SecurityGroupRule struct { // SecurityGroupRuleRemote defines a VPC Security Group Rule's remote details. // The type of remote defines the additional remote details where are used for defining the remote. -// +kubebuilder:validation:XValidation:rule="self.remoteType == 'any' ? (!has(self.cidrSubnetName) && !has(self.ip) && !has(self.securityGroupName)) : true",message="cidrSubnetName, ip, and securityGroupName are not valid for SecurityGroupRuleRemoteTypeAny remoteType" -// +kubebuilder:validation:XValidation:rule="self.remoteType == 'cidr' ? (has(self.cidrSubnetName) && !has(self.ip) && !has(self.securityGroupName)) : true",message="only cidrSubnetName is valid for SecurityGroupRuleRemoteTypeCIDR remoteType" -// +kubebuilder:validation:XValidation:rule="self.remoteType == 'ip' ? (has(self.ip) && !has(self.cidrSubnetName) && !has(self.securityGroupName)) : true",message="only ip is valid for SecurityGroupRuleRemoteTypeIP remoteType" -// +kubebuilder:validation:XValidation:rule="self.remoteType == 'sg' ? (has(self.securityGroupName) && !has(self.cidrSubnetName) && !has(self.ip)) : true",message="only securityGroupName is valid for SecurityGroupRuleRemoteTypeSG remoteType" +// +kubebuilder:validation:XValidation:rule="self.remoteType == 'any' ? (!has(self.cidrSubnetName) && !has(self.address) && !has(self.securityGroupName)) : true",message="cidrSubnetName, addresss, and securityGroupName are not valid for SecurityGroupRuleRemoteTypeAny remoteType" +// +kubebuilder:validation:XValidation:rule="self.remoteType == 'cidr' ? (has(self.cidrSubnetName) && !has(self.address) && !has(self.securityGroupName)) : true",message="only cidrSubnetName is valid for SecurityGroupRuleRemoteTypeCIDR remoteType" +// +kubebuilder:validation:XValidation:rule="self.remoteType == 'address' ? (has(self.address) && !has(self.cidrSubnetName) && !has(self.securityGroupName)) : true",message="only address is valid for SecurityGroupRuleRemoteTypeIP remoteType" +// +kubebuilder:validation:XValidation:rule="self.remoteType == 'sg' ? (has(self.securityGroupName) && !has(self.cidrSubnetName) && !has(self.address)) : true",message="only securityGroupName is valid for SecurityGroupRuleRemoteTypeSG remoteType" type SecurityGroupRuleRemote struct { // cidrSubnetName is the name of the VPC Subnet to retrieve the CIDR from, to use for the remote's destination/source. // Only used when remoteType is SecurityGroupRuleRemoteTypeCIDR. // +optional CIDRSubnetName *string `json:"cidrSubnetName,omitempty"` - // ip is the IP to use for the remote's destination/source. + // address is the address to use for the remote's destination/source. // Only used when remoteType is SecurityGroupRuleRemoteTypeIP. // +optional - IP *string `json:"ip,omitempty"` + Address *string `json:"address,omitempty"` // remoteType defines the type of filter to define for the remote's destination/source. // +required diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 20a61a0280..39b74da7d7 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -193,6 +193,13 @@ func (in *IBMPowerVSClusterSpec) DeepCopyInto(out *IBMPowerVSClusterSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.VPCSecurityGroups != nil { + in, out := &in.VPCSecurityGroups, &out.VPCSecurityGroups + *out = make([]SecurityGroup, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.TransitGateway != nil { in, out := &in.TransitGateway, &out.TransitGateway *out = new(TransitGateway) @@ -262,6 +269,13 @@ func (in *IBMPowerVSClusterStatus) DeepCopyInto(out *IBMPowerVSClusterStatus) { (*out)[key] = *val.DeepCopy() } } + if in.VPCSecurityGroups != nil { + in, out := &in.VPCSecurityGroups, &out.VPCSecurityGroups + *out = make(map[string]VPCSecurityGroupStatus, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } if in.TransitGateway != nil { in, out := &in.TransitGateway, &out.TransitGateway *out = new(ResourceReference) @@ -1448,8 +1462,8 @@ func (in *SecurityGroupRuleRemote) DeepCopyInto(out *SecurityGroupRuleRemote) { *out = new(string) **out = **in } - if in.IP != nil { - in, out := &in.IP, &out.IP + if in.Address != nil { + in, out := &in.Address, &out.Address *out = new(string) **out = **in } @@ -1665,6 +1679,42 @@ func (in *VPCResourceReference) DeepCopy() *VPCResourceReference { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VPCSecurityGroupStatus) DeepCopyInto(out *VPCSecurityGroupStatus) { + *out = *in + if in.ID != nil { + in, out := &in.ID, &out.ID + *out = new(string) + **out = **in + } + if in.RuleIDs != nil { + in, out := &in.RuleIDs, &out.RuleIDs + *out = make([]*string, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(string) + **out = **in + } + } + } + if in.ControllerCreated != nil { + in, out := &in.ControllerCreated, &out.ControllerCreated + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VPCSecurityGroupStatus. +func (in *VPCSecurityGroupStatus) DeepCopy() *VPCSecurityGroupStatus { + if in == nil { + return nil + } + out := new(VPCSecurityGroupStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VPCVolume) DeepCopyInto(out *VPCVolume) { *out = *in diff --git a/cloud/scope/powervs_cluster.go b/cloud/scope/powervs_cluster.go index b1a1160b04..cc3270dfb1 100644 --- a/cloud/scope/powervs_cluster.go +++ b/cloud/scope/powervs_cluster.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "reflect" "strings" "github.com/go-logr/logr" @@ -494,6 +495,44 @@ func (s *PowerVSClusterScope) SetVPCSubnetID(name string, resource infrav1beta2. s.IBMPowerVSCluster.Status.VPCSubnet[name] = resource } +// GetVPCSecurityGroupByName returns the VPC security group id and its ruleIDs. +func (s *PowerVSClusterScope) GetVPCSecurityGroupByName(name string) (*string, []*string, *bool) { + if s.IBMPowerVSCluster.Status.VPCSecurityGroups == nil { + return nil, nil, nil + } + if val, ok := s.IBMPowerVSCluster.Status.VPCSecurityGroups[name]; ok { + return val.ID, val.RuleIDs, val.ControllerCreated + } + return nil, nil, nil +} + +// GetVPCSecurityGroupByID returns the VPC security group's ruleIDs. +func (s *PowerVSClusterScope) GetVPCSecurityGroupByID(securityGroupID string) (*string, []*string, *bool) { + if s.IBMPowerVSCluster.Status.VPCSecurityGroups == nil { + return nil, nil, nil + } + for _, sg := range s.IBMPowerVSCluster.Status.VPCSecurityGroups { + if *sg.ID == securityGroupID { + return sg.ID, sg.RuleIDs, sg.ControllerCreated + } + } + return nil, nil, nil +} + +// SetVPCSecurityGroup set the VPC security group id. +func (s *PowerVSClusterScope) SetVPCSecurityGroup(name string, resource infrav1beta2.VPCSecurityGroupStatus) { + s.V(3).Info("Setting VPC security group status", "name", name, "resource", resource) + if s.IBMPowerVSCluster.Status.VPCSecurityGroups == nil { + s.IBMPowerVSCluster.Status.VPCSecurityGroups = make(map[string]infrav1beta2.VPCSecurityGroupStatus) + } + if val, ok := s.IBMPowerVSCluster.Status.VPCSecurityGroups[name]; ok { + if val.ControllerCreated != nil && *val.ControllerCreated { + resource.ControllerCreated = val.ControllerCreated + } + } + s.IBMPowerVSCluster.Status.VPCSecurityGroups[name] = resource +} + // TransitGateway returns the cluster Transit Gateway information. func (s *PowerVSClusterScope) TransitGateway() *infrav1beta2.TransitGateway { return s.IBMPowerVSCluster.Spec.TransitGateway @@ -1166,6 +1205,409 @@ func (s *PowerVSClusterScope) createVPCSubnet(subnet infrav1beta2.Subnet) (*stri return subnetDetails.ID, nil } +// ReconcileVPCSecurityGroups reconciles VPC security group. +func (s *PowerVSClusterScope) ReconcileVPCSecurityGroups() error { + for _, securityGroup := range s.IBMPowerVSCluster.Spec.VPCSecurityGroups { + var securityGroupID *string + var securityGroupRuleIDs []*string + + if securityGroup.Name != nil { + securityGroupID, securityGroupRuleIDs, _ = s.GetVPCSecurityGroupByName(*securityGroup.Name) + } else { + _, securityGroupRuleIDs, _ = s.GetVPCSecurityGroupByID(*securityGroup.ID) + } + + if securityGroupID != nil && securityGroupRuleIDs != nil { + if _, _, err := s.IBMVPCClient.GetSecurityGroup(&vpcv1.GetSecurityGroupOptions{ + ID: securityGroupID, + }); err != nil { + return fmt.Errorf("failed to fetch security group: %w", err) + } + for _, rule := range securityGroupRuleIDs { + if _, _, err := s.IBMVPCClient.GetSecurityGroupRule(&vpcv1.GetSecurityGroupRuleOptions{ + SecurityGroupID: securityGroupID, + ID: rule, + }); err != nil { + return fmt.Errorf("failed to fetch security group rules: %w", err) + } + } + continue + } + + sg, ruleIDs, err := s.validateVPCSecurityGroup(securityGroup) + if err != nil { + return fmt.Errorf("failed to validate existing security group: %w", err) + } + if sg != nil { + s.V(3).Info("VPC security group already exists", "name", *sg.Name) + s.SetVPCSecurityGroup(*sg.Name, infrav1beta2.VPCSecurityGroupStatus{ + ID: sg.ID, + RuleIDs: ruleIDs, + ControllerCreated: ptr.To(false), + }) + continue + } + + securityGroupID, err = s.createVPCSecurityGroup(securityGroup) + if err != nil { + return fmt.Errorf("failed to create security group: %w", err) + } + s.V(3).Info("VPC security group created", "name", *securityGroup.Name) + s.SetVPCSecurityGroup(*securityGroup.Name, infrav1beta2.VPCSecurityGroupStatus{ + ID: securityGroupID, + ControllerCreated: ptr.To(true), + }) + + if err := s.createVPCSecurityGroupRulesAndSetStatus(securityGroup.Rules, securityGroupID, securityGroup.Name); err != nil { + return err + } + } + + return nil +} + +// createVPCSecurityGroupRule creates a specific rule for a existing security group. +func (s *PowerVSClusterScope) createVPCSecurityGroupRule(securityGroupID, direction, protocol *string, portMin, portMax *int64, remote infrav1beta2.SecurityGroupRuleRemote) (*string, error) { + setRemote := func(remote infrav1beta2.SecurityGroupRuleRemote, remoteOption *vpcv1.SecurityGroupRuleRemotePrototype) error { + switch remote.RemoteType { + case infrav1beta2.SecurityGroupRuleRemoteTypeCIDR: + cidrSubnet, err := s.IBMVPCClient.GetVPCSubnetByName(*remote.CIDRSubnetName) + if err != nil { + return fmt.Errorf("failed to find vpc subnet by name '%s' for getting cidr block: %w", *remote.CIDRSubnetName, err) + } + if cidrSubnet == nil { + return fmt.Errorf("subnet by name '%s' not exist", *remote.CIDRSubnetName) + } + s.V(3).Info("Creating VPC security group rule", "securityGroupID", *securityGroupID, "direction", *direction, "protocol", *protocol, "cidrBlockSubnet", *remote.CIDRSubnetName, "cidr", *cidrSubnet.Ipv4CIDRBlock) + remoteOption.CIDRBlock = cidrSubnet.Ipv4CIDRBlock + case infrav1beta2.SecurityGroupRuleRemoteTypeAddress: + s.V(3).Info("Creating VPC security group rule", "securityGroupID", *securityGroupID, "direction", *direction, "protocol", *protocol, "ip", *remote.Address) + remoteOption.Address = remote.Address + case infrav1beta2.SecurityGroupRuleRemoteTypeSG: + sg, err := s.IBMVPCClient.GetSecurityGroupByName(*remote.SecurityGroupName) + if err != nil { + return fmt.Errorf("failed to find security group by name '%s' details to create security group rule: %w", *remote.SecurityGroupName, err) + } + if sg.Name != nil { + return fmt.Errorf("security group by name '%s' not exist", *remote.SecurityGroupName) + } + s.V(3).Info("Creating VPC security group rule", "securityGroupID", *securityGroupID, "direction", *direction, "protocol", *protocol, "securityGroup", *remote.SecurityGroupName, "securityGroupCRN", *sg.CRN) + remoteOption.CRN = sg.CRN + default: + s.V(3).Info("Creating VPC security group rule", "securityGroupID", *securityGroupID, "direction", *direction, "protocol", *protocol, "cidr", "0.0.0.0/0") + remoteOption.CIDRBlock = ptr.To("0.0.0.0/0") + } + + return nil + } + + remoteOption := &vpcv1.SecurityGroupRuleRemotePrototype{} + if err := setRemote(remote, remoteOption); err != nil { + return nil, fmt.Errorf("failed to set remote option while creating security group rule: %w", err) + } + + options := vpcv1.CreateSecurityGroupRuleOptions{ + SecurityGroupID: securityGroupID, + } + + options.SetSecurityGroupRulePrototype(&vpcv1.SecurityGroupRulePrototype{ + Direction: direction, + Protocol: protocol, + PortMin: portMin, + PortMax: portMax, + Remote: remoteOption, + }) + + var ruleID *string + ruleIntf, _, err := s.IBMVPCClient.CreateSecurityGroupRule(&options) + if err != nil { + return nil, err + } + + switch reflect.TypeOf(ruleIntf).String() { + case "*vpcv1.SecurityGroupRuleSecurityGroupRuleProtocolAll": + rule := ruleIntf.(*vpcv1.SecurityGroupRuleSecurityGroupRuleProtocolAll) + ruleID = rule.ID + case "*vpcv1.SecurityGroupRuleSecurityGroupRuleProtocolTcpudp": + rule := ruleIntf.(*vpcv1.SecurityGroupRuleSecurityGroupRuleProtocolTcpudp) + ruleID = rule.ID + case "*vpcv1.SecurityGroupRuleSecurityGroupRuleProtocolIcmp": + rule := ruleIntf.(*vpcv1.SecurityGroupRuleSecurityGroupRuleProtocolIcmp) + ruleID = rule.ID + } + + return ruleID, nil +} + +// createVPCSecurityGroupRules creates rules for a security group. +func (s *PowerVSClusterScope) createVPCSecurityGroupRules(ogSecurityGroupRules []*infrav1beta2.SecurityGroupRule, securityGroupID *string) ([]*string, error) { + ruleIDs := []*string{} + s.V(3).Info("Creating VPC security group rules") + + for _, rule := range ogSecurityGroupRules { + var protocol *string + var portMax, portMin *int64 + + direction := ptr.To(string(rule.Direction)) + switch rule.Direction { + case infrav1beta2.SecurityGroupRuleDirectionInbound: + protocol = ptr.To(string(rule.Source.Protocol)) + if rule.Source.PortRange != nil { + portMin = ptr.To(rule.Source.PortRange.MinimumPort) + portMax = ptr.To(rule.Source.PortRange.MaximumPort) + } + + for _, remote := range rule.Source.Remotes { + id, err := s.createVPCSecurityGroupRule(securityGroupID, direction, protocol, portMin, portMax, remote) + if err != nil { + return nil, fmt.Errorf("failed to create security group rule: %v", err) + } + ruleIDs = append(ruleIDs, id) + } + case infrav1beta2.SecurityGroupRuleDirectionOutbound: + protocol = ptr.To(string(rule.Destination.Protocol)) + if rule.Destination.PortRange != nil { + portMin = ptr.To(rule.Destination.PortRange.MinimumPort) + portMax = ptr.To(rule.Destination.PortRange.MaximumPort) + } + + for _, remote := range rule.Destination.Remotes { + id, err := s.createVPCSecurityGroupRule(securityGroupID, direction, protocol, portMin, portMax, remote) + if err != nil { + return nil, fmt.Errorf("failed to create security group rule: %v", err) + } + ruleIDs = append(ruleIDs, id) + } + } + } + + return ruleIDs, nil +} + +// createVPCSecurityGroupRulesAndSetStatus creates VPC security group rules and sets its status. +func (s *PowerVSClusterScope) createVPCSecurityGroupRulesAndSetStatus(ogSecurityGroupRules []*infrav1beta2.SecurityGroupRule, securityGroupID, securityGroupName *string) error { + ruleIDs, err := s.createVPCSecurityGroupRules(ogSecurityGroupRules, securityGroupID) + if err != nil { + return fmt.Errorf("failed to create security group rules: %w", err) + } + s.V(3).Info("VPC security group rules created", "security group name", *securityGroupName) + + s.SetVPCSecurityGroup(*securityGroupName, infrav1beta2.VPCSecurityGroupStatus{ + ID: securityGroupID, + RuleIDs: ruleIDs, + ControllerCreated: ptr.To(true), + }) + + return nil +} + +// createVPCSecurityGroup creates a VPC security group. +func (s *PowerVSClusterScope) createVPCSecurityGroup(spec infrav1beta2.SecurityGroup) (*string, error) { + s.V(3).Info("Creating VPC security group", "name", *spec.Name) + + options := &vpcv1.CreateSecurityGroupOptions{ + VPC: &vpcv1.VPCIdentity{ + ID: s.GetVPCID(), + }, + Name: spec.Name, + ResourceGroup: &vpcv1.ResourceGroupIdentity{ + ID: ptr.To(s.GetResourceGroupID()), + }, + } + + securityGroup, _, err := s.IBMVPCClient.CreateSecurityGroup(options) + if err != nil { + return nil, err + } + // To-Do: Add tags to VPC security group, need to implement the client for "github.com/IBM/platform-services-go-sdk/globaltaggingv1". + + return securityGroup.ID, nil +} + +// validateVPCSecurityGroupRuleRemote compares a specific security group rule's remote with the spec and existing security group rule's remote. +func (s *PowerVSClusterScope) validateVPCSecurityGroupRuleRemote(originalSGRemote *vpcv1.SecurityGroupRuleRemote, expectedSGRemote infrav1beta2.SecurityGroupRuleRemote) (bool, error) { + var match bool + + switch expectedSGRemote.RemoteType { + case infrav1beta2.SecurityGroupRuleRemoteTypeAny: + if originalSGRemote.CIDRBlock != nil && *originalSGRemote.CIDRBlock == "0.0.0.0/0" { + match = true + } + case infrav1beta2.SecurityGroupRuleRemoteTypeAddress: + if originalSGRemote.Address != nil && *originalSGRemote.Address == *expectedSGRemote.Address { + match = true + } + case infrav1beta2.SecurityGroupRuleRemoteTypeCIDR: + cidrSubnet, err := s.IBMVPCClient.GetVPCSubnetByName(*expectedSGRemote.CIDRSubnetName) + if err != nil { + return false, fmt.Errorf("failed to find vpc subnet by name '%s' for getting cidr block: %w", *expectedSGRemote.CIDRSubnetName, err) + } + + if originalSGRemote.CIDRBlock != nil && cidrSubnet != nil && *originalSGRemote.CIDRBlock == *cidrSubnet.Ipv4CIDRBlock { + match = true + } + case infrav1beta2.SecurityGroupRuleRemoteTypeSG: + securityGroup, err := s.IBMVPCClient.GetSecurityGroupByName(*expectedSGRemote.SecurityGroupName) + if err != nil { + return false, fmt.Errorf("failed to find id for resource group '%s': %w", *expectedSGRemote.SecurityGroupName, err) + } + + if originalSGRemote.CRN != nil && securityGroup.Name != nil && *originalSGRemote.CRN == *securityGroup.CRN { + match = true + } + } + + return match, nil +} + +// validateSecurityGroupRule compares a specific security group's rule with the spec and existing security group's rule. +func (s *PowerVSClusterScope) validateSecurityGroupRule(originalSecurityGroupRules []vpcv1.SecurityGroupRuleIntf, direction infrav1beta2.SecurityGroupRuleDirection, protocol string, portMin, portMax int64, icmpCode, icmpType *int64, remote infrav1beta2.SecurityGroupRuleRemote) (ruleID *string, match bool, err error) { + updateError := func(e error) { + err = fmt.Errorf("failed to validate security group rule's remote: %w", e) + } + + for _, ogRuleIntf := range originalSecurityGroupRules { + switch reflect.TypeOf(ogRuleIntf).String() { + case "*vpcv1.SecurityGroupRuleSecurityGroupRuleProtocolAll": + ogRule := ogRuleIntf.(*vpcv1.SecurityGroupRuleSecurityGroupRuleProtocolAll) + ruleID = ogRule.ID + + if *ogRule.Direction == string(direction) && *ogRule.Protocol == protocol { + ogRemote := ogRule.Remote.(*vpcv1.SecurityGroupRuleRemote) + match, err = s.validateVPCSecurityGroupRuleRemote(ogRemote, remote) + if err != nil { + updateError(err) + return nil, false, err + } + } + case "*vpcv1.SecurityGroupRuleSecurityGroupRuleProtocolTcpudp": + ogRule := ogRuleIntf.(*vpcv1.SecurityGroupRuleSecurityGroupRuleProtocolTcpudp) + ruleID = ogRule.ID + + if *ogRule.Direction == string(direction) && *ogRule.Protocol == protocol && *ogRule.PortMax == portMax && *ogRule.PortMin == portMin { + ogRemote := ogRule.Remote.(*vpcv1.SecurityGroupRuleRemote) + match, err = s.validateVPCSecurityGroupRuleRemote(ogRemote, remote) + if err != nil { + updateError(err) + return nil, false, err + } + } + case "*vpcv1.SecurityGroupRuleSecurityGroupRuleProtocolIcmp": + ogRule := ogRuleIntf.(*vpcv1.SecurityGroupRuleSecurityGroupRuleProtocolIcmp) + ruleID = ogRule.ID + + if *ogRule.Direction == string(direction) && *ogRule.Protocol == protocol && *ogRule.Code == *icmpCode && *ogRule.Type == *icmpType { + ogRemote := ogRule.Remote.(*vpcv1.SecurityGroupRuleRemote) + match, err = s.validateVPCSecurityGroupRuleRemote(ogRemote, remote) + if err != nil { + updateError(err) + return nil, false, err + } + } + } + if match { + return ruleID, match, nil + } + } + + return nil, false, nil +} + +// validateVPCSecurityGroupRules compares a specific security group rules spec with the existing security group's rules. +func (s *PowerVSClusterScope) validateVPCSecurityGroupRules(originalSecurityGroup *vpcv1.SecurityGroup, expectedSecurityGroupRules []*infrav1beta2.SecurityGroupRule) ([]*string, bool, error) { + ruleIDs := []*string{} + for _, expectedRule := range expectedSecurityGroupRules { + direction := expectedRule.Direction + var protocol string + var portMin, portMax int64 + + switch direction { + case infrav1beta2.SecurityGroupRuleDirectionInbound: + protocol = string(expectedRule.Source.Protocol) + portMin = expectedRule.Source.PortRange.MinimumPort + portMax = expectedRule.Source.PortRange.MaximumPort + icmpCode := expectedRule.Source.ICMPCode + icmpType := expectedRule.Source.ICMPType + + for _, remote := range expectedRule.Source.Remotes { + id, match, err := s.validateSecurityGroupRule(originalSecurityGroup.Rules, direction, protocol, portMin, portMax, icmpCode, icmpType, remote) + if err != nil { + return nil, false, fmt.Errorf("failed to validate security group rule: %w", err) + } + if !match { + return nil, false, nil + } + ruleIDs = append(ruleIDs, id) + } + case infrav1beta2.SecurityGroupRuleDirectionOutbound: + protocol = string(expectedRule.Destination.Protocol) + portMin = expectedRule.Destination.PortRange.MinimumPort + portMax = expectedRule.Destination.PortRange.MaximumPort + icmpCode := expectedRule.Destination.ICMPCode + icmpType := expectedRule.Destination.ICMPType + + for _, remote := range expectedRule.Destination.Remotes { + id, match, err := s.validateSecurityGroupRule(originalSecurityGroup.Rules, direction, protocol, portMin, portMax, icmpCode, icmpType, remote) + if err != nil { + return nil, false, fmt.Errorf("failed to validate security group rule: %v", err) + } + if !match { + return nil, false, nil + } + ruleIDs = append(ruleIDs, id) + } + } + } + + return ruleIDs, true, nil +} + +// validateVPCSecurityGroup validates the security group and it's rules provided by user via spec. +func (s *PowerVSClusterScope) validateVPCSecurityGroup(securityGroup infrav1beta2.SecurityGroup) (*vpcv1.SecurityGroup, []*string, error) { + var securityGroupDet *vpcv1.SecurityGroup + var err error + + if securityGroup.Name != nil { + securityGroupDet, err = s.IBMVPCClient.GetSecurityGroupByName(*securityGroup.Name) + if err != nil && err.Error() != vpc.SecurityGroupByNameNotFound(*securityGroup.Name).Error() { + return nil, nil, err + } + if securityGroupDet == nil { + return nil, nil, nil + } + } else { + securityGroupDet, _, err = s.IBMVPCClient.GetSecurityGroup(&vpcv1.GetSecurityGroupOptions{ + ID: securityGroup.ID, + }) + if err != nil { + return nil, nil, err + } + if securityGroupDet == nil { + return nil, nil, fmt.Errorf("failed to find vpc security group with provided id '%v'", securityGroup.ID) + } + } + if securityGroupDet != nil && *securityGroupDet.VPC.ID != *s.GetVPCID() { + return nil, nil, fmt.Errorf("security group by name exist but not attached to VPC") + } + + ruleIDs, ok, err := s.validateVPCSecurityGroupRules(securityGroupDet, securityGroup.Rules) + if err != nil { + return nil, nil, fmt.Errorf("failed to validate security group rules: %v", err) + } + if !ok { + if _, _, controllerCreated := s.GetVPCSecurityGroupByName(*securityGroup.Name); *controllerCreated { + if err := s.createVPCSecurityGroupRulesAndSetStatus(securityGroup.Rules, securityGroupDet.ID, securityGroupDet.Name); err != nil { + return nil, nil, err + } + return nil, nil, nil + } else { + return nil, nil, fmt.Errorf("security group by name exist but rules are not matching") + } + } + + return securityGroupDet, ruleIDs, nil +} + // ReconcileTransitGateway reconcile transit gateway. func (s *PowerVSClusterScope) ReconcileTransitGateway() (bool, error) { if s.GetTransitGatewayID() != nil { @@ -1874,6 +2316,35 @@ func (s *PowerVSClusterScope) DeleteLoadBalancer() (bool, error) { return false, nil } +// DeleteVPCSecurityGroups deletes VPC security group. +func (s *PowerVSClusterScope) DeleteVPCSecurityGroups() error { + for _, securityGroup := range s.IBMPowerVSCluster.Status.VPCSecurityGroups { + if securityGroup.ControllerCreated == nil || !*securityGroup.ControllerCreated { + s.V(3).Info("Skipping VPC security group deletion as resource is not created by controller") + continue + } + if _, _, err := s.IBMVPCClient.GetSecurityGroup(&vpcv1.GetSecurityGroupOptions{ + ID: securityGroup.ID, + }); err != nil { + if strings.Contains(err.Error(), string(VPCSecurityGroupNotFound)) { + s.V(3).Info("VPC security group has been already deleted", "ID", securityGroup.ID) + continue + } + return fmt.Errorf("failed to fetch security group: %w", err) + } + + s.V(3).Info("Deleting VPC security group", "ID", *securityGroup.ID) + options := &vpcv1.DeleteSecurityGroupOptions{ + ID: securityGroup.ID, + } + if _, err := s.IBMVPCClient.DeleteSecurityGroup(options); err != nil { + return fmt.Errorf("failed to delete VPC security group: %w", err) + } + s.V(3).Info("VPC security group successfully deleted", "ID", securityGroup.ID) + } + return nil +} + // DeleteVPCSubnet deletes VPC subnet. func (s *PowerVSClusterScope) DeleteVPCSubnet() (bool, error) { for _, subnet := range s.IBMPowerVSCluster.Status.VPCSubnet { diff --git a/cloud/scope/types.go b/cloud/scope/types.go index 6a122004cb..04530337b1 100644 --- a/cloud/scope/types.go +++ b/cloud/scope/types.go @@ -37,4 +37,7 @@ var ( // COSInstanceNotFound is the error returned when a COS service instance is not found. COSInstanceNotFound = ResourceNotFound("COS instance unavailable") + + // VPCSecurityGroupNotFound is the error returned when a VPC security group is not found. + VPCSecurityGroupNotFound = ResourceNotFound("Security group not found") ) diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ibmpowervsclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ibmpowervsclusters.yaml index ad94420e44..8859e4ca1e 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ibmpowervsclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ibmpowervsclusters.yaml @@ -434,6 +434,356 @@ spec: it is expected to set the region, not setting will result in webhook error. type: string type: object + vpcSecurityGroups: + description: VPCSecurityGroups to attach it to the VPC resource + items: + description: SecurityGroup defines a VPC Security Group that should + exist or be created within the specified VPC, with the specified + Security Group Rules. + properties: + id: + description: id of the Security Group. + type: string + name: + description: name of the Security Group. + type: string + resourceGroup: + description: resourceGroup of the Security Group. + type: string + rules: + description: rules are the Security Group Rules for the Security + Group. + items: + description: SecurityGroupRule defines a VPC Security Group + Rule for a specified Security Group. + properties: + action: + description: action defines whether to allow or deny traffic + defined by the Security Group Rule. + enum: + - allow + - deny + type: string + destination: + description: |- + destination is a SecurityGroupRulePrototype which defines the destination of outbound traffic for the Security Group Rule. + Only used when direction is SecurityGroupRuleDirectionOutbound. + properties: + icmpCode: + description: |- + icmpCode is the ICMP code for the Rule. + Only used when Protocol is SecurityGroupProtocolICMP. + format: int64 + type: integer + icmpType: + description: |- + icmpType is the ICMP type for the Rule. + Only used when Protocol is SecurityGroupProtocolICMP. + format: int64 + type: integer + portRange: + description: portRange is a range of ports allowed + for the Rule's remote. + properties: + maximumPort: + description: maximumPort is the inclusive upper + range of ports. + format: int64 + maximum: 65535 + minimum: 1 + type: integer + minimumPort: + description: minimumPort is the inclusive lower + range of ports. + format: int64 + maximum: 65535 + minimum: 1 + type: integer + type: object + x-kubernetes-validations: + - message: maximum port must be greater than or equal + to minimum port + rule: self.maximumPort >= self.minimumPort + protocol: + description: protocol defines the traffic protocol + used for the Security Group Rule. + enum: + - all + - icmp + - tcp + - udp + type: string + remotes: + description: |- + remotes is a set of SecurityGroupRuleRemote's that define the traffic allowed by the Rule's remote. + Specifying multiple SecurityGroupRuleRemote's creates a unique Security Group Rule with the shared Protocol, PortRange, etc. + This allows for easier management of Security Group Rule's for sets of CIDR's, IP's, etc. + items: + description: |- + SecurityGroupRuleRemote defines a VPC Security Group Rule's remote details. + The type of remote defines the additional remote details where are used for defining the remote. + properties: + address: + description: |2- + address is the address to use for the remote's destination/source. + Only used when remoteType is SecurityGroupRuleRemoteTypeIP. + type: string + cidrSubnetName: + description: |- + cidrSubnetName is the name of the VPC Subnet to retrieve the CIDR from, to use for the remote's destination/source. + Only used when remoteType is SecurityGroupRuleRemoteTypeCIDR. + type: string + remoteType: + description: remoteType defines the type of + filter to define for the remote's destination/source. + enum: + - any + - cidr + - address + - sg + type: string + securityGroupName: + description: |- + securityGroupName is the name of the VPC Security Group to use for the remote's destination/source. + Only used when remoteType is SecurityGroupRuleRemoteTypeSG + type: string + required: + - remoteType + type: object + x-kubernetes-validations: + - message: cidrSubnetName, addresss, and securityGroupName + are not valid for SecurityGroupRuleRemoteTypeAny + remoteType + rule: 'self.remoteType == ''any'' ? (!has(self.cidrSubnetName) + && !has(self.address) && !has(self.securityGroupName)) + : true' + - message: only cidrSubnetName is valid for SecurityGroupRuleRemoteTypeCIDR + remoteType + rule: 'self.remoteType == ''cidr'' ? (has(self.cidrSubnetName) + && !has(self.address) && !has(self.securityGroupName)) + : true' + - message: only address is valid for SecurityGroupRuleRemoteTypeIP + remoteType + rule: 'self.remoteType == ''address'' ? (has(self.address) + && !has(self.cidrSubnetName) && !has(self.securityGroupName)) + : true' + - message: only securityGroupName is valid for SecurityGroupRuleRemoteTypeSG + remoteType + rule: 'self.remoteType == ''sg'' ? (has(self.securityGroupName) + && !has(self.cidrSubnetName) && !has(self.address)) + : true' + type: array + required: + - protocol + - remotes + type: object + x-kubernetes-validations: + - message: icmpCode and icmpType are only supported for + SecurityGroupRuleProtocolIcmp protocol + rule: 'self.protocol != ''icmp'' ? (!has(self.icmpCode) + && !has(self.icmpType)) : true' + - message: portRange is not valid for SecurityGroupRuleProtocolAll + protocol + rule: 'self.protocol == ''all'' ? !has(self.portRange) + : true' + - message: portRange is not valid for SecurityGroupRuleProtocolIcmp + protocol + rule: 'self.protocol == ''icmp'' ? !has(self.portRange) + : true' + direction: + description: direction defines whether the traffic is + inbound or outbound for the Security Group Rule. + enum: + - inbound + - outbound + type: string + securityGroupID: + description: securityGroupID is the ID of the Security + Group for the Security Group Rule. + type: string + source: + description: |- + source is a SecurityGroupRulePrototype which defines the source of inbound traffic for the Security Group Rule. + Only used when direction is SecurityGroupRuleDirectionInbound. + properties: + icmpCode: + description: |- + icmpCode is the ICMP code for the Rule. + Only used when Protocol is SecurityGroupProtocolICMP. + format: int64 + type: integer + icmpType: + description: |- + icmpType is the ICMP type for the Rule. + Only used when Protocol is SecurityGroupProtocolICMP. + format: int64 + type: integer + portRange: + description: portRange is a range of ports allowed + for the Rule's remote. + properties: + maximumPort: + description: maximumPort is the inclusive upper + range of ports. + format: int64 + maximum: 65535 + minimum: 1 + type: integer + minimumPort: + description: minimumPort is the inclusive lower + range of ports. + format: int64 + maximum: 65535 + minimum: 1 + type: integer + type: object + x-kubernetes-validations: + - message: maximum port must be greater than or equal + to minimum port + rule: self.maximumPort >= self.minimumPort + protocol: + description: protocol defines the traffic protocol + used for the Security Group Rule. + enum: + - all + - icmp + - tcp + - udp + type: string + remotes: + description: |- + remotes is a set of SecurityGroupRuleRemote's that define the traffic allowed by the Rule's remote. + Specifying multiple SecurityGroupRuleRemote's creates a unique Security Group Rule with the shared Protocol, PortRange, etc. + This allows for easier management of Security Group Rule's for sets of CIDR's, IP's, etc. + items: + description: |- + SecurityGroupRuleRemote defines a VPC Security Group Rule's remote details. + The type of remote defines the additional remote details where are used for defining the remote. + properties: + address: + description: |2- + address is the address to use for the remote's destination/source. + Only used when remoteType is SecurityGroupRuleRemoteTypeIP. + type: string + cidrSubnetName: + description: |- + cidrSubnetName is the name of the VPC Subnet to retrieve the CIDR from, to use for the remote's destination/source. + Only used when remoteType is SecurityGroupRuleRemoteTypeCIDR. + type: string + remoteType: + description: remoteType defines the type of + filter to define for the remote's destination/source. + enum: + - any + - cidr + - address + - sg + type: string + securityGroupName: + description: |- + securityGroupName is the name of the VPC Security Group to use for the remote's destination/source. + Only used when remoteType is SecurityGroupRuleRemoteTypeSG + type: string + required: + - remoteType + type: object + x-kubernetes-validations: + - message: cidrSubnetName, addresss, and securityGroupName + are not valid for SecurityGroupRuleRemoteTypeAny + remoteType + rule: 'self.remoteType == ''any'' ? (!has(self.cidrSubnetName) + && !has(self.address) && !has(self.securityGroupName)) + : true' + - message: only cidrSubnetName is valid for SecurityGroupRuleRemoteTypeCIDR + remoteType + rule: 'self.remoteType == ''cidr'' ? (has(self.cidrSubnetName) + && !has(self.address) && !has(self.securityGroupName)) + : true' + - message: only address is valid for SecurityGroupRuleRemoteTypeIP + remoteType + rule: 'self.remoteType == ''address'' ? (has(self.address) + && !has(self.cidrSubnetName) && !has(self.securityGroupName)) + : true' + - message: only securityGroupName is valid for SecurityGroupRuleRemoteTypeSG + remoteType + rule: 'self.remoteType == ''sg'' ? (has(self.securityGroupName) + && !has(self.cidrSubnetName) && !has(self.address)) + : true' + type: array + required: + - protocol + - remotes + type: object + x-kubernetes-validations: + - message: icmpCode and icmpType are only supported for + SecurityGroupRuleProtocolIcmp protocol + rule: 'self.protocol != ''icmp'' ? (!has(self.icmpCode) + && !has(self.icmpType)) : true' + - message: portRange is not valid for SecurityGroupRuleProtocolAll + protocol + rule: 'self.protocol == ''all'' ? !has(self.portRange) + : true' + - message: portRange is not valid for SecurityGroupRuleProtocolIcmp + protocol + rule: 'self.protocol == ''icmp'' ? !has(self.portRange) + : true' + required: + - action + - direction + type: object + x-kubernetes-validations: + - message: both destination and source cannot be provided + rule: (has(self.destination) && !has(self.source)) || (!has(self.destination) + && has(self.source)) + - message: source must be set for SecurityGroupRuleDirectionInbound + direction + rule: 'self.direction == ''inbound'' ? has(self.source) + : true' + - message: destination is not valid for SecurityGroupRuleDirectionInbound + direction + rule: 'self.direction == ''inbound'' ? !has(self.destination) + : true' + - message: destination must be set for SecurityGroupRuleDirectionOutbound + direction + rule: 'self.direction == ''outbound'' ? has(self.destination) + : true' + - message: source is not valid for SecurityGroupRuleDirectionOutbound + direction + rule: 'self.direction == ''outbound'' ? !has(self.source) + : true' + type: array + tags: + description: tags are tags to add to the Security Group. + items: + type: string + type: array + vpc: + description: vpc is the IBM Cloud VPC for the Security Group. + properties: + id: + description: id of resource. + maxLength: 64 + minLength: 1 + pattern: ^[-0-9a-z_]+$ + type: string + name: + description: name of resource. + maxLength: 63 + minLength: 1 + pattern: ^([a-z]|[a-z][-a-z0-9]*[a-z0-9])$ + type: string + region: + description: |- + region of IBM Cloud VPC. + when powervs.cluster.x-k8s.io/create-infra=true annotation is set on IBMPowerVSCluster resource, + it is expected to set the region, not setting will result in webhook error. + type: string + type: object + type: object + x-kubernetes-validations: + - message: either an id or name must be specified + rule: has(self.id) || has(self.name) + type: array vpcSubnets: description: |- vpcSubnets contains information about IBM Cloud VPC Subnet resources. @@ -634,6 +984,29 @@ spec: description: id represents the id of the resource. type: string type: object + vpcSecurityGroups: + additionalProperties: + description: VPCSecurityGroupStatus defines a vpc security group + resource status with its id and respective rule's ids. + properties: + controllerCreated: + default: false + description: controllerCreated indicates whether the resource + is created by the controller. + type: boolean + id: + description: id represents the id of the resource. + type: string + ruleIDs: + description: rules contains the id of rules created under the + security group + items: + type: string + type: array + type: object + description: vpcSecurityGroups is reference to IBM Cloud VPC security + group. + type: object vpcSubnet: additionalProperties: description: ResourceReference identifies a resource with id. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ibmpowervsclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ibmpowervsclustertemplates.yaml index fa43d35190..4958ee5fd4 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ibmpowervsclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ibmpowervsclustertemplates.yaml @@ -465,6 +465,362 @@ spec: it is expected to set the region, not setting will result in webhook error. type: string type: object + vpcSecurityGroups: + description: VPCSecurityGroups to attach it to the VPC resource + items: + description: SecurityGroup defines a VPC Security Group + that should exist or be created within the specified VPC, + with the specified Security Group Rules. + properties: + id: + description: id of the Security Group. + type: string + name: + description: name of the Security Group. + type: string + resourceGroup: + description: resourceGroup of the Security Group. + type: string + rules: + description: rules are the Security Group Rules for + the Security Group. + items: + description: SecurityGroupRule defines a VPC Security + Group Rule for a specified Security Group. + properties: + action: + description: action defines whether to allow or + deny traffic defined by the Security Group Rule. + enum: + - allow + - deny + type: string + destination: + description: |- + destination is a SecurityGroupRulePrototype which defines the destination of outbound traffic for the Security Group Rule. + Only used when direction is SecurityGroupRuleDirectionOutbound. + properties: + icmpCode: + description: |- + icmpCode is the ICMP code for the Rule. + Only used when Protocol is SecurityGroupProtocolICMP. + format: int64 + type: integer + icmpType: + description: |- + icmpType is the ICMP type for the Rule. + Only used when Protocol is SecurityGroupProtocolICMP. + format: int64 + type: integer + portRange: + description: portRange is a range of ports + allowed for the Rule's remote. + properties: + maximumPort: + description: maximumPort is the inclusive + upper range of ports. + format: int64 + maximum: 65535 + minimum: 1 + type: integer + minimumPort: + description: minimumPort is the inclusive + lower range of ports. + format: int64 + maximum: 65535 + minimum: 1 + type: integer + type: object + x-kubernetes-validations: + - message: maximum port must be greater than + or equal to minimum port + rule: self.maximumPort >= self.minimumPort + protocol: + description: protocol defines the traffic + protocol used for the Security Group Rule. + enum: + - all + - icmp + - tcp + - udp + type: string + remotes: + description: |- + remotes is a set of SecurityGroupRuleRemote's that define the traffic allowed by the Rule's remote. + Specifying multiple SecurityGroupRuleRemote's creates a unique Security Group Rule with the shared Protocol, PortRange, etc. + This allows for easier management of Security Group Rule's for sets of CIDR's, IP's, etc. + items: + description: |- + SecurityGroupRuleRemote defines a VPC Security Group Rule's remote details. + The type of remote defines the additional remote details where are used for defining the remote. + properties: + address: + description: |2- + address is the address to use for the remote's destination/source. + Only used when remoteType is SecurityGroupRuleRemoteTypeIP. + type: string + cidrSubnetName: + description: |- + cidrSubnetName is the name of the VPC Subnet to retrieve the CIDR from, to use for the remote's destination/source. + Only used when remoteType is SecurityGroupRuleRemoteTypeCIDR. + type: string + remoteType: + description: remoteType defines the + type of filter to define for the remote's + destination/source. + enum: + - any + - cidr + - address + - sg + type: string + securityGroupName: + description: |- + securityGroupName is the name of the VPC Security Group to use for the remote's destination/source. + Only used when remoteType is SecurityGroupRuleRemoteTypeSG + type: string + required: + - remoteType + type: object + x-kubernetes-validations: + - message: cidrSubnetName, addresss, and + securityGroupName are not valid for + SecurityGroupRuleRemoteTypeAny remoteType + rule: 'self.remoteType == ''any'' ? (!has(self.cidrSubnetName) + && !has(self.address) && !has(self.securityGroupName)) + : true' + - message: only cidrSubnetName is valid + for SecurityGroupRuleRemoteTypeCIDR + remoteType + rule: 'self.remoteType == ''cidr'' ? (has(self.cidrSubnetName) + && !has(self.address) && !has(self.securityGroupName)) + : true' + - message: only address is valid for SecurityGroupRuleRemoteTypeIP + remoteType + rule: 'self.remoteType == ''address'' + ? (has(self.address) && !has(self.cidrSubnetName) + && !has(self.securityGroupName)) : true' + - message: only securityGroupName is valid + for SecurityGroupRuleRemoteTypeSG remoteType + rule: 'self.remoteType == ''sg'' ? (has(self.securityGroupName) + && !has(self.cidrSubnetName) && !has(self.address)) + : true' + type: array + required: + - protocol + - remotes + type: object + x-kubernetes-validations: + - message: icmpCode and icmpType are only supported + for SecurityGroupRuleProtocolIcmp protocol + rule: 'self.protocol != ''icmp'' ? (!has(self.icmpCode) + && !has(self.icmpType)) : true' + - message: portRange is not valid for SecurityGroupRuleProtocolAll + protocol + rule: 'self.protocol == ''all'' ? !has(self.portRange) + : true' + - message: portRange is not valid for SecurityGroupRuleProtocolIcmp + protocol + rule: 'self.protocol == ''icmp'' ? !has(self.portRange) + : true' + direction: + description: direction defines whether the traffic + is inbound or outbound for the Security Group + Rule. + enum: + - inbound + - outbound + type: string + securityGroupID: + description: securityGroupID is the ID of the + Security Group for the Security Group Rule. + type: string + source: + description: |- + source is a SecurityGroupRulePrototype which defines the source of inbound traffic for the Security Group Rule. + Only used when direction is SecurityGroupRuleDirectionInbound. + properties: + icmpCode: + description: |- + icmpCode is the ICMP code for the Rule. + Only used when Protocol is SecurityGroupProtocolICMP. + format: int64 + type: integer + icmpType: + description: |- + icmpType is the ICMP type for the Rule. + Only used when Protocol is SecurityGroupProtocolICMP. + format: int64 + type: integer + portRange: + description: portRange is a range of ports + allowed for the Rule's remote. + properties: + maximumPort: + description: maximumPort is the inclusive + upper range of ports. + format: int64 + maximum: 65535 + minimum: 1 + type: integer + minimumPort: + description: minimumPort is the inclusive + lower range of ports. + format: int64 + maximum: 65535 + minimum: 1 + type: integer + type: object + x-kubernetes-validations: + - message: maximum port must be greater than + or equal to minimum port + rule: self.maximumPort >= self.minimumPort + protocol: + description: protocol defines the traffic + protocol used for the Security Group Rule. + enum: + - all + - icmp + - tcp + - udp + type: string + remotes: + description: |- + remotes is a set of SecurityGroupRuleRemote's that define the traffic allowed by the Rule's remote. + Specifying multiple SecurityGroupRuleRemote's creates a unique Security Group Rule with the shared Protocol, PortRange, etc. + This allows for easier management of Security Group Rule's for sets of CIDR's, IP's, etc. + items: + description: |- + SecurityGroupRuleRemote defines a VPC Security Group Rule's remote details. + The type of remote defines the additional remote details where are used for defining the remote. + properties: + address: + description: |2- + address is the address to use for the remote's destination/source. + Only used when remoteType is SecurityGroupRuleRemoteTypeIP. + type: string + cidrSubnetName: + description: |- + cidrSubnetName is the name of the VPC Subnet to retrieve the CIDR from, to use for the remote's destination/source. + Only used when remoteType is SecurityGroupRuleRemoteTypeCIDR. + type: string + remoteType: + description: remoteType defines the + type of filter to define for the remote's + destination/source. + enum: + - any + - cidr + - address + - sg + type: string + securityGroupName: + description: |- + securityGroupName is the name of the VPC Security Group to use for the remote's destination/source. + Only used when remoteType is SecurityGroupRuleRemoteTypeSG + type: string + required: + - remoteType + type: object + x-kubernetes-validations: + - message: cidrSubnetName, addresss, and + securityGroupName are not valid for + SecurityGroupRuleRemoteTypeAny remoteType + rule: 'self.remoteType == ''any'' ? (!has(self.cidrSubnetName) + && !has(self.address) && !has(self.securityGroupName)) + : true' + - message: only cidrSubnetName is valid + for SecurityGroupRuleRemoteTypeCIDR + remoteType + rule: 'self.remoteType == ''cidr'' ? (has(self.cidrSubnetName) + && !has(self.address) && !has(self.securityGroupName)) + : true' + - message: only address is valid for SecurityGroupRuleRemoteTypeIP + remoteType + rule: 'self.remoteType == ''address'' + ? (has(self.address) && !has(self.cidrSubnetName) + && !has(self.securityGroupName)) : true' + - message: only securityGroupName is valid + for SecurityGroupRuleRemoteTypeSG remoteType + rule: 'self.remoteType == ''sg'' ? (has(self.securityGroupName) + && !has(self.cidrSubnetName) && !has(self.address)) + : true' + type: array + required: + - protocol + - remotes + type: object + x-kubernetes-validations: + - message: icmpCode and icmpType are only supported + for SecurityGroupRuleProtocolIcmp protocol + rule: 'self.protocol != ''icmp'' ? (!has(self.icmpCode) + && !has(self.icmpType)) : true' + - message: portRange is not valid for SecurityGroupRuleProtocolAll + protocol + rule: 'self.protocol == ''all'' ? !has(self.portRange) + : true' + - message: portRange is not valid for SecurityGroupRuleProtocolIcmp + protocol + rule: 'self.protocol == ''icmp'' ? !has(self.portRange) + : true' + required: + - action + - direction + type: object + x-kubernetes-validations: + - message: both destination and source cannot be provided + rule: (has(self.destination) && !has(self.source)) + || (!has(self.destination) && has(self.source)) + - message: source must be set for SecurityGroupRuleDirectionInbound + direction + rule: 'self.direction == ''inbound'' ? has(self.source) + : true' + - message: destination is not valid for SecurityGroupRuleDirectionInbound + direction + rule: 'self.direction == ''inbound'' ? !has(self.destination) + : true' + - message: destination must be set for SecurityGroupRuleDirectionOutbound + direction + rule: 'self.direction == ''outbound'' ? has(self.destination) + : true' + - message: source is not valid for SecurityGroupRuleDirectionOutbound + direction + rule: 'self.direction == ''outbound'' ? !has(self.source) + : true' + type: array + tags: + description: tags are tags to add to the Security Group. + items: + type: string + type: array + vpc: + description: vpc is the IBM Cloud VPC for the Security + Group. + properties: + id: + description: id of resource. + maxLength: 64 + minLength: 1 + pattern: ^[-0-9a-z_]+$ + type: string + name: + description: name of resource. + maxLength: 63 + minLength: 1 + pattern: ^([a-z]|[a-z][-a-z0-9]*[a-z0-9])$ + type: string + region: + description: |- + region of IBM Cloud VPC. + when powervs.cluster.x-k8s.io/create-infra=true annotation is set on IBMPowerVSCluster resource, + it is expected to set the region, not setting will result in webhook error. + type: string + type: object + type: object + x-kubernetes-validations: + - message: either an id or name must be specified + rule: has(self.id) || has(self.name) + type: array vpcSubnets: description: |- vpcSubnets contains information about IBM Cloud VPC Subnet resources. diff --git a/controllers/ibmpowervscluster_controller.go b/controllers/ibmpowervscluster_controller.go index a3e785f482..84a197abe1 100644 --- a/controllers/ibmpowervscluster_controller.go +++ b/controllers/ibmpowervscluster_controller.go @@ -189,6 +189,15 @@ func (r *IBMPowerVSClusterReconciler) reconcile(clusterScope *scope.PowerVSClust } conditions.MarkTrue(powerVSCluster, infrav1beta2.VPCSubnetReadyCondition) + // reconcile VPC security group + clusterScope.Info("Reconciling VPC security group") + if err := clusterScope.ReconcileVPCSecurityGroups(); err != nil { + clusterScope.Error(err, "failed to reconcile VPC security group") + conditions.MarkFalse(powerVSCluster, infrav1beta2.VPCSecurityGroupReadyCondition, infrav1beta2.VPCSecurityGroupReconciliationFailedReason, capiv1beta1.ConditionSeverityError, err.Error()) + return reconcile.Result{}, err + } + conditions.MarkTrue(powerVSCluster, infrav1beta2.VPCSecurityGroupReadyCondition) + // reconcile Transit Gateway clusterScope.Info("Reconciling Transit Gateway") if requeue, err := clusterScope.ReconcileTransitGateway(); err != nil { @@ -279,6 +288,11 @@ func (r *IBMPowerVSClusterReconciler) reconcileDelete(ctx context.Context, clust return reconcile.Result{RequeueAfter: 1 * time.Minute}, nil } + clusterScope.Info("Deleting VPC security group") + if err := clusterScope.DeleteVPCSecurityGroups(); err != nil { + allErrs = append(allErrs, errors.Wrapf(err, "failed to delete VPC subnet")) + } + clusterScope.Info("Deleting VPC subnet") if requeue, err := clusterScope.DeleteVPCSubnet(); err != nil { allErrs = append(allErrs, errors.Wrapf(err, "failed to delete VPC subnet")) diff --git a/pkg/cloud/services/vpc/mock/vpc_generated.go b/pkg/cloud/services/vpc/mock/vpc_generated.go index 370da61389..7312a76eb2 100644 --- a/pkg/cloud/services/vpc/mock/vpc_generated.go +++ b/pkg/cloud/services/vpc/mock/vpc_generated.go @@ -119,6 +119,22 @@ func (mr *MockVpcMockRecorder) CreatePublicGateway(options any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreatePublicGateway", reflect.TypeOf((*MockVpc)(nil).CreatePublicGateway), options) } +// CreateSecurityGroup mocks base method. +func (m *MockVpc) CreateSecurityGroup(options *vpcv1.CreateSecurityGroupOptions) (*vpcv1.SecurityGroup, *core.DetailedResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateSecurityGroup", options) + ret0, _ := ret[0].(*vpcv1.SecurityGroup) + ret1, _ := ret[1].(*core.DetailedResponse) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// CreateSecurityGroup indicates an expected call of CreateSecurityGroup. +func (mr *MockVpcMockRecorder) CreateSecurityGroup(options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSecurityGroup", reflect.TypeOf((*MockVpc)(nil).CreateSecurityGroup), options) +} + // CreateSecurityGroupRule mocks base method. func (m *MockVpc) CreateSecurityGroupRule(options *vpcv1.CreateSecurityGroupRuleOptions) (vpcv1.SecurityGroupRuleIntf, *core.DetailedResponse, error) { m.ctrl.T.Helper() @@ -227,6 +243,21 @@ func (mr *MockVpcMockRecorder) DeletePublicGateway(options any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeletePublicGateway", reflect.TypeOf((*MockVpc)(nil).DeletePublicGateway), options) } +// DeleteSecurityGroup mocks base method. +func (m *MockVpc) DeleteSecurityGroup(options *vpcv1.DeleteSecurityGroupOptions) (*core.DetailedResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteSecurityGroup", options) + ret0, _ := ret[0].(*core.DetailedResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteSecurityGroup indicates an expected call of DeleteSecurityGroup. +func (mr *MockVpcMockRecorder) DeleteSecurityGroup(options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSecurityGroup", reflect.TypeOf((*MockVpc)(nil).DeleteSecurityGroup), options) +} + // DeleteSubnet mocks base method. func (m *MockVpc) DeleteSubnet(options *vpcv1.DeleteSubnetOptions) (*core.DetailedResponse, error) { m.ctrl.T.Helper() @@ -320,6 +351,53 @@ func (mr *MockVpcMockRecorder) GetLoadBalancerByName(loadBalancerName any) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLoadBalancerByName", reflect.TypeOf((*MockVpc)(nil).GetLoadBalancerByName), loadBalancerName) } +// GetSecurityGroup mocks base method. +func (m *MockVpc) GetSecurityGroup(options *vpcv1.GetSecurityGroupOptions) (*vpcv1.SecurityGroup, *core.DetailedResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSecurityGroup", options) + ret0, _ := ret[0].(*vpcv1.SecurityGroup) + ret1, _ := ret[1].(*core.DetailedResponse) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetSecurityGroup indicates an expected call of GetSecurityGroup. +func (mr *MockVpcMockRecorder) GetSecurityGroup(options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSecurityGroup", reflect.TypeOf((*MockVpc)(nil).GetSecurityGroup), options) +} + +// GetSecurityGroupByName mocks base method. +func (m *MockVpc) GetSecurityGroupByName(name string) (*vpcv1.SecurityGroup, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSecurityGroupByName", name) + ret0, _ := ret[0].(*vpcv1.SecurityGroup) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSecurityGroupByName indicates an expected call of GetSecurityGroupByName. +func (mr *MockVpcMockRecorder) GetSecurityGroupByName(name any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSecurityGroupByName", reflect.TypeOf((*MockVpc)(nil).GetSecurityGroupByName), name) +} + +// GetSecurityGroupRule mocks base method. +func (m *MockVpc) GetSecurityGroupRule(options *vpcv1.GetSecurityGroupRuleOptions) (vpcv1.SecurityGroupRuleIntf, *core.DetailedResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSecurityGroupRule", options) + ret0, _ := ret[0].(vpcv1.SecurityGroupRuleIntf) + ret1, _ := ret[1].(*core.DetailedResponse) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetSecurityGroupRule indicates an expected call of GetSecurityGroupRule. +func (mr *MockVpcMockRecorder) GetSecurityGroupRule(options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSecurityGroupRule", reflect.TypeOf((*MockVpc)(nil).GetSecurityGroupRule), options) +} + // GetSubnet mocks base method. func (m *MockVpc) GetSubnet(arg0 *vpcv1.GetSubnetOptions) (*vpcv1.Subnet, *core.DetailedResponse, error) { m.ctrl.T.Helper() @@ -493,6 +571,22 @@ func (mr *MockVpcMockRecorder) ListLoadBalancers(options any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListLoadBalancers", reflect.TypeOf((*MockVpc)(nil).ListLoadBalancers), options) } +// ListSecurityGroups mocks base method. +func (m *MockVpc) ListSecurityGroups(options *vpcv1.ListSecurityGroupsOptions) (*vpcv1.SecurityGroupCollection, *core.DetailedResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListSecurityGroups", options) + ret0, _ := ret[0].(*vpcv1.SecurityGroupCollection) + ret1, _ := ret[1].(*core.DetailedResponse) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// ListSecurityGroups indicates an expected call of ListSecurityGroups. +func (mr *MockVpcMockRecorder) ListSecurityGroups(options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListSecurityGroups", reflect.TypeOf((*MockVpc)(nil).ListSecurityGroups), options) +} + // ListSubnets mocks base method. func (m *MockVpc) ListSubnets(options *vpcv1.ListSubnetsOptions) (*vpcv1.SubnetCollection, *core.DetailedResponse, error) { m.ctrl.T.Helper() diff --git a/pkg/cloud/services/vpc/service.go b/pkg/cloud/services/vpc/service.go index 69d025b58c..dfbc9cdc1f 100644 --- a/pkg/cloud/services/vpc/service.go +++ b/pkg/cloud/services/vpc/service.go @@ -26,6 +26,9 @@ import ( "sigs.k8s.io/cluster-api-provider-ibmcloud/pkg/cloud/services/utils" ) +// SecurityGroupByNameNotFound returns an appropriate error when security group by name not found. +var SecurityGroupByNameNotFound = func(name string) error { return fmt.Errorf("failed to find security group by name '%s'", name) } + // Service holds the VPC Service specific information. type Service struct { vpcService *vpcv1.VpcV1 @@ -337,6 +340,58 @@ func (s *Service) GetSubnetAddrPrefix(vpcID, zone string) (string, error) { return "", fmt.Errorf("not found a valid CIDR for VPC %s in zone %s", vpcID, zone) } +// CreateSecurityGroup creates a new security group. +func (s *Service) CreateSecurityGroup(options *vpcv1.CreateSecurityGroupOptions) (*vpcv1.SecurityGroup, *core.DetailedResponse, error) { + return s.vpcService.CreateSecurityGroup(options) +} + +// DeleteSecurityGroup deletes the security group passed. +func (s *Service) DeleteSecurityGroup(options *vpcv1.DeleteSecurityGroupOptions) (*core.DetailedResponse, error) { + return s.vpcService.DeleteSecurityGroup(options) +} + +// ListSecurityGroups lists security group. +func (s *Service) ListSecurityGroups(options *vpcv1.ListSecurityGroupsOptions) (*vpcv1.SecurityGroupCollection, *core.DetailedResponse, error) { + return s.vpcService.ListSecurityGroups(options) +} + +// GetSecurityGroup gets a specific security group by id. +func (s *Service) GetSecurityGroup(options *vpcv1.GetSecurityGroupOptions) (*vpcv1.SecurityGroup, *core.DetailedResponse, error) { + return s.vpcService.GetSecurityGroup(options) +} + +// GetSecurityGroupByName gets a specific security group by name. +func (s *Service) GetSecurityGroupByName(name string) (*vpcv1.SecurityGroup, error) { + securityGroupPager, err := s.vpcService.NewSecurityGroupsPager(&vpcv1.ListSecurityGroupsOptions{}) + if err != nil { + return nil, fmt.Errorf("error listing security group: %v", err) + } + + for { + if !securityGroupPager.HasNext() { + break + } + + securityGroups, err := securityGroupPager.GetNext() + if err != nil { + return nil, fmt.Errorf("error retrieving next page of security groups: %v", err) + } + + for _, sg := range securityGroups { + if *sg.Name == name { + return &sg, nil + } + } + } + + return nil, SecurityGroupByNameNotFound(name) +} + +// GetSecurityGroupRule gets a specific security group rule. +func (s *Service) GetSecurityGroupRule(options *vpcv1.GetSecurityGroupRuleOptions) (vpcv1.SecurityGroupRuleIntf, *core.DetailedResponse, error) { + return s.vpcService.GetSecurityGroupRule(options) +} + // NewService returns a new VPC Service. func NewService(svcEndpoint string) (Vpc, error) { service := &Service{} diff --git a/pkg/cloud/services/vpc/vpc.go b/pkg/cloud/services/vpc/vpc.go index f6ebffbefd..c58036dc69 100644 --- a/pkg/cloud/services/vpc/vpc.go +++ b/pkg/cloud/services/vpc/vpc.go @@ -59,4 +59,10 @@ type Vpc interface { GetVPCSubnetByName(subnetName string) (*vpcv1.Subnet, error) GetLoadBalancerByName(loadBalancerName string) (*vpcv1.LoadBalancer, error) GetSubnetAddrPrefix(vpcID, zone string) (string, error) + CreateSecurityGroup(options *vpcv1.CreateSecurityGroupOptions) (*vpcv1.SecurityGroup, *core.DetailedResponse, error) + DeleteSecurityGroup(options *vpcv1.DeleteSecurityGroupOptions) (*core.DetailedResponse, error) + ListSecurityGroups(options *vpcv1.ListSecurityGroupsOptions) (*vpcv1.SecurityGroupCollection, *core.DetailedResponse, error) + GetSecurityGroup(options *vpcv1.GetSecurityGroupOptions) (*vpcv1.SecurityGroup, *core.DetailedResponse, error) + GetSecurityGroupByName(name string) (*vpcv1.SecurityGroup, error) + GetSecurityGroupRule(options *vpcv1.GetSecurityGroupRuleOptions) (vpcv1.SecurityGroupRuleIntf, *core.DetailedResponse, error) }