From 691ee4ba04447c5902a783109eab99b680e85d48 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Thu, 12 May 2022 15:58:35 -0700 Subject: [PATCH] Upgrade to calico 1.23.0 --- Makefile | 2 +- templates/addons/calico-dual-stack.yaml | 263 +++++++++++++++-- templates/addons/calico-ipv6.yaml | 263 +++++++++++++++-- .../calico-ipv6/calico-policy-only.yaml | 278 ++++++++++++++--- templates/addons/calico.yaml | 263 +++++++++++++++-- templates/addons/calico/calico-vxlan.yaml | 279 +++++++++++++++--- templates/addons/windows/calico/calico.yaml | 6 +- ...ow-ci-version-windows-containerd-2022.yaml | 269 +++++++++++++++-- .../ci/cluster-template-prow-ci-version.yaml | 269 +++++++++++++++-- .../ci/cluster-template-prow-custom-vnet.yaml | 263 +++++++++++++++-- .../ci/cluster-template-prow-dual-stack.yaml | 263 +++++++++++++++-- ...template-prow-external-cloud-provider.yaml | 263 +++++++++++++++-- .../test/ci/cluster-template-prow-ipv6.yaml | 263 +++++++++++++++-- ...template-prow-machine-pool-ci-version.yaml | 269 +++++++++++++++-- .../cluster-template-prow-machine-pool.yaml | 269 +++++++++++++++-- .../ci/cluster-template-prow-nvidia-gpu.yaml | 263 +++++++++++++++-- .../ci/cluster-template-prow-private.yaml | 263 +++++++++++++++-- templates/test/ci/cluster-template-prow.yaml | 269 +++++++++++++++-- ...r-template-custom-builds-machine-pool.yaml | 269 +++++++++++++++-- .../dev/cluster-template-custom-builds.yaml | 269 +++++++++++++++-- 20 files changed, 4297 insertions(+), 518 deletions(-) diff --git a/Makefile b/Makefile index b51309ef8a8..8b52569ed0e 100644 --- a/Makefile +++ b/Makefile @@ -467,7 +467,7 @@ generate-addons: fetch-calico-manifests ## Generate metric-server, calico calico $(KUSTOMIZE) build $(ADDONS_DIR)/calico-dual-stack > $(ADDONS_DIR)/calico-dual-stack.yaml # When updating this, make sure to also update the Windows image version in templates/addons/windows/calico. -CALICO_VERSION := v3.22.1 +CALICO_VERSION := v3.23.0 .PHONY: fetch-calico-manifests fetch-calico-manifests: ## Get Calico release manifests and unzip them. diff --git a/templates/addons/calico-dual-stack.yaml b/templates/addons/calico-dual-stack.yaml index 1bc02c921b8..d12ce10e946 100644 --- a/templates/addons/calico-dual-stack.yaml +++ b/templates/addons/calico-dual-stack.yaml @@ -36,6 +36,12 @@ spec: 64512]' format: int32 type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. @@ -66,6 +72,37 @@ spec: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object nodeToNodeMeshEnabled: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' @@ -196,6 +233,12 @@ spec: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer password: description: Optional BGP password for the peerings generated by this BGPPeer resource. @@ -716,6 +759,11 @@ spec: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all interfaces with + BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled or Strict. [Default: Strict]' + type: string bpfExtToServiceConnmark: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local @@ -755,6 +803,51 @@ spec: logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the @@ -765,6 +858,15 @@ spec: Calico policy will be bypassed. [Default: insert]' type: string dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s]' type: string debugDisableLogDropping: type: boolean @@ -793,9 +895,14 @@ spec: routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean @@ -869,6 +976,14 @@ spec: "true" or "false" will force the feature, empty or omitted values are auto-detected. type: string + floatingIPs: + default: Disabled + description: FloatingIPs configures whether or not Felix will program + floating IP addresses. + enum: + - Enabled + - Disabled + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -906,6 +1021,9 @@ spec: disabled by setting the interval to 0. type: string ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: description: 'IPIPMTU is the MTU to set on the tunnel device. See @@ -972,6 +1090,8 @@ spec: usage. [Default: 10s]' type: string ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). type: boolean kubeNodePortRanges: description: 'KubeNodePortRanges holds list of port ranges used for @@ -985,6 +1105,12 @@ spec: pattern: ^.* x-kubernetes-int-or-string: true type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string logFilePath: description: 'LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]' @@ -1114,9 +1240,9 @@ spec: routes. - CalicoIPAM: the default - use IPAM data to construct routes.' type: string routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. properties: max: type: integer @@ -1126,6 +1252,21 @@ spec: - max - min type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting @@ -1153,12 +1294,22 @@ spec: Felix makes reports. [Default: 86400s]' type: string useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. type: boolean vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for VXLAN networking. Optional as Felix determines + this based on the existing IP pools. [Default: nil (unset)]' type: boolean vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' type: integer vxlanPort: type: integer @@ -1176,6 +1327,10 @@ spec: description: 'WireguardInterfaceName specifies the name to use for the Wireguard interface. [Default: wg.calico]' type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used by Wireguard. [Default: 51820]' @@ -1188,6 +1343,12 @@ spec: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string xdpEnabled: description: 'XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]' @@ -2256,13 +2417,25 @@ spec: resource. properties: affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. type: string allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. items: nullable: true type: integer type: array attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. items: properties: handle_id: @@ -2274,12 +2447,38 @@ spec: type: object type: array cidr: + description: The block's CIDR. type: string deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. type: boolean unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. items: type: integer type: array @@ -2450,7 +2649,7 @@ spec: type: array blockSize: description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. + this pool. Defaults to 26 for IPv4 and 122 for IPv6. type: integer cidr: description: The pool CIDR. @@ -2521,6 +2720,9 @@ status: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null name: ipreservations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2669,6 +2871,11 @@ spec: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -2779,6 +2986,11 @@ spec: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -3731,7 +3943,6 @@ rules: - apiGroups: - crd.projectcalico.org resources: - - ippools - ipreservations verbs: - list @@ -3748,6 +3959,13 @@ rules: - update - delete - watch +- apiGroups: + - crd.projectcalico.org + resources: + - ippools + verbs: + - list + - watch - apiGroups: - crd.projectcalico.org resources: @@ -3764,8 +3982,10 @@ rules: - clusterinformations verbs: - get + - list - create - update + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4018,7 +4238,7 @@ spec: value: node - name: DATASTORE_TYPE value: kubernetes - image: docker.io/calico/kube-controllers:v3.22.1 + image: docker.io/calico/kube-controllers:v3.23.0 livenessProbe: exec: command: @@ -4087,7 +4307,7 @@ spec: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/typha:v3.22.1 + image: docker.io/calico/typha:v3.23.0 livenessProbe: httpGet: host: localhost @@ -4122,7 +4342,7 @@ spec: - key: CriticalAddonsOnly operator: Exists --- -apiVersion: policy/v1beta1 +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -4135,7 +4355,7 @@ spec: matchLabels: k8s-app: calico-kube-controllers --- -apiVersion: policy/v1beta1 +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -4203,7 +4423,7 @@ spec: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/node:v3.22.1 + image: docker.io/calico/node:v3.23.0 lifecycle: preStop: exec: @@ -4278,7 +4498,7 @@ spec: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: install-cni securityContext: privileged: true @@ -4287,13 +4507,6 @@ spec: name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir - - image: docker.io/calico/pod2daemon-flexvol:v3.22.1 - name: flexvol-driver - securityContext: - privileged: true - volumeMounts: - - mountPath: /host/driver - name: flexvol-driver-host nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical @@ -4337,10 +4550,6 @@ spec: path: /var/run/nodeagent type: DirectoryOrCreate name: policysync - - hostPath: - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds - type: DirectoryOrCreate - name: flexvol-driver-host updateStrategy: rollingUpdate: maxUnavailable: 1 diff --git a/templates/addons/calico-ipv6.yaml b/templates/addons/calico-ipv6.yaml index 80ea82bfca5..09649a522af 100644 --- a/templates/addons/calico-ipv6.yaml +++ b/templates/addons/calico-ipv6.yaml @@ -36,6 +36,12 @@ spec: 64512]' format: int32 type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. @@ -66,6 +72,37 @@ spec: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object nodeToNodeMeshEnabled: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' @@ -196,6 +233,12 @@ spec: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer password: description: Optional BGP password for the peerings generated by this BGPPeer resource. @@ -716,6 +759,11 @@ spec: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all interfaces with + BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled or Strict. [Default: Strict]' + type: string bpfExtToServiceConnmark: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local @@ -755,6 +803,51 @@ spec: logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the @@ -765,6 +858,15 @@ spec: Calico policy will be bypassed. [Default: insert]' type: string dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s]' type: string debugDisableLogDropping: type: boolean @@ -793,9 +895,14 @@ spec: routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean @@ -869,6 +976,14 @@ spec: "true" or "false" will force the feature, empty or omitted values are auto-detected. type: string + floatingIPs: + default: Disabled + description: FloatingIPs configures whether or not Felix will program + floating IP addresses. + enum: + - Enabled + - Disabled + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -906,6 +1021,9 @@ spec: disabled by setting the interval to 0. type: string ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: description: 'IPIPMTU is the MTU to set on the tunnel device. See @@ -972,6 +1090,8 @@ spec: usage. [Default: 10s]' type: string ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). type: boolean kubeNodePortRanges: description: 'KubeNodePortRanges holds list of port ranges used for @@ -985,6 +1105,12 @@ spec: pattern: ^.* x-kubernetes-int-or-string: true type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string logFilePath: description: 'LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]' @@ -1114,9 +1240,9 @@ spec: routes. - CalicoIPAM: the default - use IPAM data to construct routes.' type: string routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. properties: max: type: integer @@ -1126,6 +1252,21 @@ spec: - max - min type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting @@ -1153,12 +1294,22 @@ spec: Felix makes reports. [Default: 86400s]' type: string useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. type: boolean vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for VXLAN networking. Optional as Felix determines + this based on the existing IP pools. [Default: nil (unset)]' type: boolean vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' type: integer vxlanPort: type: integer @@ -1176,6 +1327,10 @@ spec: description: 'WireguardInterfaceName specifies the name to use for the Wireguard interface. [Default: wg.calico]' type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used by Wireguard. [Default: 51820]' @@ -1188,6 +1343,12 @@ spec: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string xdpEnabled: description: 'XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]' @@ -2256,13 +2417,25 @@ spec: resource. properties: affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. type: string allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. items: nullable: true type: integer type: array attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. items: properties: handle_id: @@ -2274,12 +2447,38 @@ spec: type: object type: array cidr: + description: The block's CIDR. type: string deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. type: boolean unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. items: type: integer type: array @@ -2450,7 +2649,7 @@ spec: type: array blockSize: description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. + this pool. Defaults to 26 for IPv4 and 122 for IPv6. type: integer cidr: description: The pool CIDR. @@ -2521,6 +2720,9 @@ status: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null name: ipreservations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2669,6 +2871,11 @@ spec: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -2779,6 +2986,11 @@ spec: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -3731,7 +3943,6 @@ rules: - apiGroups: - crd.projectcalico.org resources: - - ippools - ipreservations verbs: - list @@ -3748,6 +3959,13 @@ rules: - update - delete - watch +- apiGroups: + - crd.projectcalico.org + resources: + - ippools + verbs: + - list + - watch - apiGroups: - crd.projectcalico.org resources: @@ -3764,8 +3982,10 @@ rules: - clusterinformations verbs: - get + - list - create - update + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4007,7 +4227,7 @@ spec: value: node - name: DATASTORE_TYPE value: kubernetes - image: docker.io/calico/kube-controllers:v3.22.1 + image: docker.io/calico/kube-controllers:v3.23.0 livenessProbe: exec: command: @@ -4076,7 +4296,7 @@ spec: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/typha:v3.22.1 + image: docker.io/calico/typha:v3.23.0 livenessProbe: httpGet: host: localhost @@ -4111,7 +4331,7 @@ spec: - key: CriticalAddonsOnly operator: Exists --- -apiVersion: policy/v1beta1 +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -4124,7 +4344,7 @@ spec: matchLabels: k8s-app: calico-kube-controllers --- -apiVersion: policy/v1beta1 +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -4192,7 +4412,7 @@ spec: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/node:v3.22.1 + image: docker.io/calico/node:v3.23.0 lifecycle: preStop: exec: @@ -4267,7 +4487,7 @@ spec: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: install-cni securityContext: privileged: true @@ -4276,13 +4496,6 @@ spec: name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir - - image: docker.io/calico/pod2daemon-flexvol:v3.22.1 - name: flexvol-driver - securityContext: - privileged: true - volumeMounts: - - mountPath: /host/driver - name: flexvol-driver-host nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical @@ -4326,10 +4539,6 @@ spec: path: /var/run/nodeagent type: DirectoryOrCreate name: policysync - - hostPath: - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds - type: DirectoryOrCreate - name: flexvol-driver-host updateStrategy: rollingUpdate: maxUnavailable: 1 diff --git a/templates/addons/calico-ipv6/calico-policy-only.yaml b/templates/addons/calico-ipv6/calico-policy-only.yaml index 3d7afc5112a..4f6927e1291 100644 --- a/templates/addons/calico-ipv6/calico-policy-only.yaml +++ b/templates/addons/calico-ipv6/calico-policy-only.yaml @@ -93,6 +93,12 @@ spec: 64512]' format: int32 type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. @@ -123,6 +129,37 @@ spec: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object nodeToNodeMeshEnabled: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' @@ -254,6 +291,12 @@ spec: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer password: description: Optional BGP password for the peerings generated by this BGPPeer resource. @@ -371,8 +414,6 @@ status: conditions: [] storedVersions: [] ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -780,6 +821,11 @@ spec: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all interfaces with + BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled or Strict. [Default: Strict]' + type: string bpfExtToServiceConnmark: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local @@ -819,6 +865,51 @@ spec: logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the @@ -829,6 +920,15 @@ spec: Calico policy will be bypassed. [Default: insert]' type: string dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s]' type: string debugDisableLogDropping: type: boolean @@ -857,9 +957,14 @@ spec: routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean @@ -933,6 +1038,14 @@ spec: "true" or "false" will force the feature, empty or omitted values are auto-detected. type: string + floatingIPs: + default: Disabled + description: FloatingIPs configures whether or not Felix will program + floating IP addresses. + enum: + - Enabled + - Disabled + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -970,6 +1083,9 @@ spec: disabled by setting the interval to 0. type: string ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: description: 'IPIPMTU is the MTU to set on the tunnel device. See @@ -1036,6 +1152,8 @@ spec: usage. [Default: 10s]' type: string ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). type: boolean kubeNodePortRanges: description: 'KubeNodePortRanges holds list of port ranges used for @@ -1049,6 +1167,12 @@ spec: pattern: ^.* x-kubernetes-int-or-string: true type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string logFilePath: description: 'LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]' @@ -1178,9 +1302,9 @@ spec: routes. - CalicoIPAM: the default - use IPAM data to construct routes.' type: string routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. properties: max: type: integer @@ -1190,6 +1314,21 @@ spec: - max - min type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting @@ -1217,12 +1356,22 @@ spec: Felix makes reports. [Default: 86400s]' type: string useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. type: boolean vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for VXLAN networking. Optional as Felix determines + this based on the existing IP pools. [Default: nil (unset)]' type: boolean vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' type: integer vxlanPort: type: integer @@ -1240,6 +1389,10 @@ spec: description: 'WireguardInterfaceName specifies the name to use for the Wireguard interface. [Default: wg.calico]' type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used by Wireguard. [Default: 51820]' @@ -1252,6 +1405,12 @@ spec: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string xdpEnabled: description: 'XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]' @@ -2324,8 +2483,16 @@ spec: resource. properties: affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. type: string allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. items: type: integer # TODO: This nullable is manually added in. We should update controller-gen @@ -2333,6 +2500,10 @@ spec: nullable: true type: array attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. items: properties: handle_id: @@ -2344,12 +2515,38 @@ spec: type: object type: array cidr: + description: The block's CIDR. type: string deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. type: boolean unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. items: type: integer type: array @@ -2523,19 +2720,19 @@ spec: type: array blockSize: description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. + this pool. Defaults to 26 for IPv4 and 122 for IPv6. type: integer cidr: description: The pool CIDR. type: string + disableBGPExport: + description: 'Disable exporting routes from this IP Pool''s CIDR over + BGP. [Default: false]' + type: boolean disabled: description: When disabled is true, Calico IPAM will not assign addresses from this pool. type: boolean - disableBGPExport: - description: "Disable exporting routes from this IP Pool's CIDR over - BGP. [Default: false]" - type: boolean ipip: description: 'Deprecated: this field is only used for APIv1 backwards compatibility. Setting this field is not allowed, this field is @@ -2595,6 +2792,9 @@ status: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null name: ipreservations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2744,6 +2944,11 @@ spec: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -2854,6 +3059,11 @@ spec: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -3799,10 +4009,9 @@ rules: - get - list - watch - # IPAM resources are manipulated when nodes are deleted. + # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. - apiGroups: ["crd.projectcalico.org"] resources: - - ippools - ipreservations verbs: - list @@ -3818,6 +4027,13 @@ rules: - update - delete - watch + # Pools are watched to maintain a mapping of blocks to IP pools. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + verbs: + - list + - watch # kube-controllers manages hostendpoints. - apiGroups: ["crd.projectcalico.org"] resources: @@ -3834,8 +4050,10 @@ rules: - clusterinformations verbs: - get + - list - create - update + - watch # KubeControllersConfiguration is where it gets its config - apiGroups: ["crd.projectcalico.org"] resources: @@ -4071,7 +4289,7 @@ spec: securityContext: fsGroup: 65534 containers: - - image: docker.io/calico/typha:v3.22.1 + - image: docker.io/calico/typha:v3.23.0 name: calico-typha ports: - containerPort: 5473 @@ -4131,7 +4349,7 @@ spec: # This manifest creates a Pod Disruption Budget for Typha to allow K8s Cluster Autoscaler to evict -apiVersion: policy/v1beta1 +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: calico-typha @@ -4189,7 +4407,7 @@ spec: # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 command: ["/opt/cni/bin/install"] envFrom: - configMapRef: @@ -4221,21 +4439,12 @@ spec: name: cni-net-dir securityContext: privileged: true - # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes - # to communicate with Felix over the Policy Sync API. - - name: flexvol-driver - image: docker.io/calico/pod2daemon-flexvol:v3.22.1 - volumeMounts: - - name: flexvol-driver-host - mountPath: /host/driver - securityContext: - privileged: true containers: # Runs calico-node container on each Kubernetes node. This # container programs network policy and routes on each # host. - name: calico-node - image: docker.io/calico/node:v3.22.1 + image: docker.io/calico/node:v3.23.0 envFrom: - configMapRef: # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. @@ -4378,11 +4587,6 @@ spec: hostPath: type: DirectoryOrCreate path: /var/run/nodeagent - # Used to install Flex Volume Driver - - name: flexvol-driver-host - hostPath: - type: DirectoryOrCreate - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds --- apiVersion: v1 @@ -4428,7 +4632,7 @@ spec: priorityClassName: system-cluster-critical containers: - name: calico-kube-controllers - image: docker.io/calico/kube-controllers:v3.22.1 + image: docker.io/calico/kube-controllers:v3.23.0 env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS @@ -4463,7 +4667,7 @@ metadata: # This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict -apiVersion: policy/v1beta1 +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: calico-kube-controllers diff --git a/templates/addons/calico.yaml b/templates/addons/calico.yaml index a3c61a8ba56..b8b2db72380 100644 --- a/templates/addons/calico.yaml +++ b/templates/addons/calico.yaml @@ -36,6 +36,12 @@ spec: 64512]' format: int32 type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. @@ -66,6 +72,37 @@ spec: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object nodeToNodeMeshEnabled: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' @@ -196,6 +233,12 @@ spec: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer password: description: Optional BGP password for the peerings generated by this BGPPeer resource. @@ -716,6 +759,11 @@ spec: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all interfaces with + BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled or Strict. [Default: Strict]' + type: string bpfExtToServiceConnmark: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local @@ -755,6 +803,51 @@ spec: logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the @@ -765,6 +858,15 @@ spec: Calico policy will be bypassed. [Default: insert]' type: string dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s]' type: string debugDisableLogDropping: type: boolean @@ -793,9 +895,14 @@ spec: routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean @@ -869,6 +976,14 @@ spec: "true" or "false" will force the feature, empty or omitted values are auto-detected. type: string + floatingIPs: + default: Disabled + description: FloatingIPs configures whether or not Felix will program + floating IP addresses. + enum: + - Enabled + - Disabled + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -906,6 +1021,9 @@ spec: disabled by setting the interval to 0. type: string ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: description: 'IPIPMTU is the MTU to set on the tunnel device. See @@ -972,6 +1090,8 @@ spec: usage. [Default: 10s]' type: string ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). type: boolean kubeNodePortRanges: description: 'KubeNodePortRanges holds list of port ranges used for @@ -985,6 +1105,12 @@ spec: pattern: ^.* x-kubernetes-int-or-string: true type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string logFilePath: description: 'LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]' @@ -1114,9 +1240,9 @@ spec: routes. - CalicoIPAM: the default - use IPAM data to construct routes.' type: string routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. properties: max: type: integer @@ -1126,6 +1252,21 @@ spec: - max - min type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting @@ -1153,12 +1294,22 @@ spec: Felix makes reports. [Default: 86400s]' type: string useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. type: boolean vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for VXLAN networking. Optional as Felix determines + this based on the existing IP pools. [Default: nil (unset)]' type: boolean vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' type: integer vxlanPort: type: integer @@ -1176,6 +1327,10 @@ spec: description: 'WireguardInterfaceName specifies the name to use for the Wireguard interface. [Default: wg.calico]' type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used by Wireguard. [Default: 51820]' @@ -1188,6 +1343,12 @@ spec: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string xdpEnabled: description: 'XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]' @@ -2256,13 +2417,25 @@ spec: resource. properties: affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. type: string allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. items: nullable: true type: integer type: array attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. items: properties: handle_id: @@ -2274,12 +2447,38 @@ spec: type: object type: array cidr: + description: The block's CIDR. type: string deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. type: boolean unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. items: type: integer type: array @@ -2450,7 +2649,7 @@ spec: type: array blockSize: description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. + this pool. Defaults to 26 for IPv4 and 122 for IPv6. type: integer cidr: description: The pool CIDR. @@ -2521,6 +2720,9 @@ status: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null name: ipreservations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2669,6 +2871,11 @@ spec: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -2779,6 +2986,11 @@ spec: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -3731,7 +3943,6 @@ rules: - apiGroups: - crd.projectcalico.org resources: - - ippools - ipreservations verbs: - list @@ -3748,6 +3959,13 @@ rules: - update - delete - watch +- apiGroups: + - crd.projectcalico.org + resources: + - ippools + verbs: + - list + - watch - apiGroups: - crd.projectcalico.org resources: @@ -3764,8 +3982,10 @@ rules: - clusterinformations verbs: - get + - list - create - update + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4021,7 +4241,7 @@ spec: value: node - name: DATASTORE_TYPE value: kubernetes - image: docker.io/calico/kube-controllers:v3.22.1 + image: docker.io/calico/kube-controllers:v3.23.0 livenessProbe: exec: command: @@ -4050,7 +4270,7 @@ spec: - effect: NoSchedule key: node-role.kubernetes.io/control-plane --- -apiVersion: policy/v1beta1 +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -4104,6 +4324,8 @@ spec: value: autodetect - name: CALICO_IPV4POOL_IPIP value: Never + - name: CALICO_IPV6POOL_VXLAN + value: CrossSubnet - name: FELIX_IPINIPMTU valueFrom: configMapKeyRef: @@ -4131,7 +4353,7 @@ spec: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/node:v3.22.1 + image: docker.io/calico/node:v3.23.0 lifecycle: preStop: exec: @@ -4203,7 +4425,7 @@ spec: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: upgrade-ipam securityContext: privileged: true @@ -4237,7 +4459,7 @@ spec: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: install-cni securityContext: privileged: true @@ -4246,13 +4468,6 @@ spec: name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir - - image: docker.io/calico/pod2daemon-flexvol:v3.22.1 - name: flexvol-driver - securityContext: - privileged: true - volumeMounts: - - mountPath: /host/driver - name: flexvol-driver-host nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical @@ -4299,10 +4514,6 @@ spec: path: /var/run/nodeagent type: DirectoryOrCreate name: policysync - - hostPath: - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds - type: DirectoryOrCreate - name: flexvol-driver-host updateStrategy: rollingUpdate: maxUnavailable: 1 diff --git a/templates/addons/calico/calico-vxlan.yaml b/templates/addons/calico/calico-vxlan.yaml index e83c495e44e..17ebef38ad1 100644 --- a/templates/addons/calico/calico-vxlan.yaml +++ b/templates/addons/calico/calico-vxlan.yaml @@ -94,6 +94,12 @@ spec: 64512]' format: int32 type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. @@ -124,6 +130,37 @@ spec: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object nodeToNodeMeshEnabled: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' @@ -255,6 +292,12 @@ spec: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer password: description: Optional BGP password for the peerings generated by this BGPPeer resource. @@ -372,8 +415,6 @@ status: conditions: [] storedVersions: [] ---- - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -781,6 +822,11 @@ spec: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all interfaces with + BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled or Strict. [Default: Strict]' + type: string bpfExtToServiceConnmark: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local @@ -820,6 +866,51 @@ spec: logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the @@ -830,6 +921,15 @@ spec: Calico policy will be bypassed. [Default: insert]' type: string dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s]' type: string debugDisableLogDropping: type: boolean @@ -858,9 +958,14 @@ spec: routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean @@ -934,6 +1039,14 @@ spec: "true" or "false" will force the feature, empty or omitted values are auto-detected. type: string + floatingIPs: + default: Disabled + description: FloatingIPs configures whether or not Felix will program + floating IP addresses. + enum: + - Enabled + - Disabled + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -971,6 +1084,9 @@ spec: disabled by setting the interval to 0. type: string ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: description: 'IPIPMTU is the MTU to set on the tunnel device. See @@ -1037,6 +1153,8 @@ spec: usage. [Default: 10s]' type: string ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). type: boolean kubeNodePortRanges: description: 'KubeNodePortRanges holds list of port ranges used for @@ -1050,6 +1168,12 @@ spec: pattern: ^.* x-kubernetes-int-or-string: true type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string logFilePath: description: 'LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]' @@ -1179,9 +1303,9 @@ spec: routes. - CalicoIPAM: the default - use IPAM data to construct routes.' type: string routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. properties: max: type: integer @@ -1191,6 +1315,21 @@ spec: - max - min type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting @@ -1218,12 +1357,22 @@ spec: Felix makes reports. [Default: 86400s]' type: string useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. type: boolean vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for VXLAN networking. Optional as Felix determines + this based on the existing IP pools. [Default: nil (unset)]' type: boolean vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' type: integer vxlanPort: type: integer @@ -1241,6 +1390,10 @@ spec: description: 'WireguardInterfaceName specifies the name to use for the Wireguard interface. [Default: wg.calico]' type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used by Wireguard. [Default: 51820]' @@ -1253,6 +1406,12 @@ spec: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string xdpEnabled: description: 'XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]' @@ -2325,8 +2484,16 @@ spec: resource. properties: affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. type: string allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. items: type: integer # TODO: This nullable is manually added in. We should update controller-gen @@ -2334,6 +2501,10 @@ spec: nullable: true type: array attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. items: properties: handle_id: @@ -2345,12 +2516,38 @@ spec: type: object type: array cidr: + description: The block's CIDR. type: string deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. type: boolean unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. items: type: integer type: array @@ -2524,19 +2721,19 @@ spec: type: array blockSize: description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. + this pool. Defaults to 26 for IPv4 and 122 for IPv6. type: integer cidr: description: The pool CIDR. type: string + disableBGPExport: + description: 'Disable exporting routes from this IP Pool''s CIDR over + BGP. [Default: false]' + type: boolean disabled: description: When disabled is true, Calico IPAM will not assign addresses from this pool. type: boolean - disableBGPExport: - description: "Disable exporting routes from this IP Pool's CIDR over - BGP. [Default: false]" - type: boolean ipip: description: 'Deprecated: this field is only used for APIv1 backwards compatibility. Setting this field is not allowed, this field is @@ -2596,6 +2793,9 @@ status: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null name: ipreservations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2745,6 +2945,11 @@ spec: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -2855,6 +3060,11 @@ spec: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -3800,10 +4010,9 @@ rules: - get - list - watch - # IPAM resources are manipulated when nodes are deleted. + # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. - apiGroups: ["crd.projectcalico.org"] resources: - - ippools - ipreservations verbs: - list @@ -3819,6 +4028,13 @@ rules: - update - delete - watch + # Pools are watched to maintain a mapping of blocks to IP pools. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + verbs: + - list + - watch # kube-controllers manages hostendpoints. - apiGroups: ["crd.projectcalico.org"] resources: @@ -3835,8 +4051,10 @@ rules: - clusterinformations verbs: - get + - list - create - update + - watch # KubeControllersConfiguration is where it gets its config - apiGroups: ["crd.projectcalico.org"] resources: @@ -4083,7 +4301,7 @@ spec: # It can be deleted if this is a fresh installation, or if you have already # upgraded to use calico-ipam. - name: upgrade-ipam - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 command: ["/opt/cni/bin/calico-ipam", "-upgrade"] envFrom: - configMapRef: @@ -4110,7 +4328,7 @@ spec: # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 command: ["/opt/cni/bin/install"] envFrom: - configMapRef: @@ -4148,21 +4366,12 @@ spec: name: cni-net-dir securityContext: privileged: true - # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes - # to communicate with Felix over the Policy Sync API. - - name: flexvol-driver - image: docker.io/calico/pod2daemon-flexvol:v3.22.1 - volumeMounts: - - name: flexvol-driver-host - mountPath: /host/driver - securityContext: - privileged: true containers: # Runs calico-node container on each Kubernetes node. This # container programs network policy and routes on each # host. - name: calico-node - image: docker.io/calico/node:v3.22.1 + image: docker.io/calico/node:v3.23.0 envFrom: - configMapRef: # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. @@ -4198,6 +4407,9 @@ spec: # Enable or Disable VXLAN on the default IP pool. - name: CALICO_IPV4POOL_VXLAN value: "CrossSubnet" + # Enable or Disable VXLAN on the default IPv6 IP pool. + - name: CALICO_IPV6POOL_VXLAN + value: "CrossSubnet" # Set MTU for tunnel device used if ipip is enabled - name: FELIX_IPINIPMTU valueFrom: @@ -4329,11 +4541,6 @@ spec: hostPath: type: DirectoryOrCreate path: /var/run/nodeagent - # Used to install Flex Volume Driver - - name: flexvol-driver-host - hostPath: - type: DirectoryOrCreate - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds --- apiVersion: v1 @@ -4379,7 +4586,7 @@ spec: priorityClassName: system-cluster-critical containers: - name: calico-kube-controllers - image: docker.io/calico/kube-controllers:v3.22.1 + image: docker.io/calico/kube-controllers:v3.23.0 env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS @@ -4414,7 +4621,7 @@ metadata: # This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict -apiVersion: policy/v1beta1 +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: calico-kube-controllers diff --git a/templates/addons/windows/calico/calico.yaml b/templates/addons/windows/calico/calico.yaml index 185d7a3fc97..9ace0e588a9 100644 --- a/templates/addons/windows/calico/calico.yaml +++ b/templates/addons/windows/calico/calico.yaml @@ -163,7 +163,7 @@ spec: # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni - image: sigwindowstools/calico-install:v3.22.1-hostprocess + image: sigwindowstools/calico-install:v3.23.0-hostprocess args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1"] imagePullPolicy: Always env: @@ -205,7 +205,7 @@ spec: runAsUserName: "NT AUTHORITY\\system" containers: - name: calico-node-startup - image: sigwindowstools/calico-node:v3.22.1-hostprocess + image: sigwindowstools/calico-node:v3.23.0-hostprocess args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1"] workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/" imagePullPolicy: Always @@ -232,7 +232,7 @@ spec: - name: VXLAN_VNI value: "4096" - name: calico-node-felix - image: sigwindowstools/calico-node:v3.22.1-hostprocess + image: sigwindowstools/calico-node:v3.23.0-hostprocess args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1"] imagePullPolicy: Always workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/" diff --git a/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml b/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml index c3bc9222a53..48fda05c14b 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml @@ -673,6 +673,12 @@ data: 64512]' format: int32 type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. @@ -703,6 +709,37 @@ data: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object nodeToNodeMeshEnabled: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' @@ -833,6 +870,12 @@ data: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer password: description: Optional BGP password for the peerings generated by this BGPPeer resource. @@ -1353,6 +1396,11 @@ data: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all interfaces with + BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled or Strict. [Default: Strict]' + type: string bpfExtToServiceConnmark: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local @@ -1392,6 +1440,51 @@ data: logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the @@ -1402,6 +1495,15 @@ data: Calico policy will be bypassed. [Default: insert]' type: string dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s]' type: string debugDisableLogDropping: type: boolean @@ -1430,9 +1532,14 @@ data: routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean @@ -1506,6 +1613,14 @@ data: "true" or "false" will force the feature, empty or omitted values are auto-detected. type: string + floatingIPs: + default: Disabled + description: FloatingIPs configures whether or not Felix will program + floating IP addresses. + enum: + - Enabled + - Disabled + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -1543,6 +1658,9 @@ data: disabled by setting the interval to 0. type: string ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: description: 'IPIPMTU is the MTU to set on the tunnel device. See @@ -1609,6 +1727,8 @@ data: usage. [Default: 10s]' type: string ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). type: boolean kubeNodePortRanges: description: 'KubeNodePortRanges holds list of port ranges used for @@ -1622,6 +1742,12 @@ data: pattern: ^.* x-kubernetes-int-or-string: true type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string logFilePath: description: 'LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]' @@ -1751,9 +1877,9 @@ data: routes. - CalicoIPAM: the default - use IPAM data to construct routes.' type: string routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. properties: max: type: integer @@ -1763,6 +1889,21 @@ data: - max - min type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting @@ -1790,12 +1931,22 @@ data: Felix makes reports. [Default: 86400s]' type: string useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. type: boolean vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for VXLAN networking. Optional as Felix determines + this based on the existing IP pools. [Default: nil (unset)]' type: boolean vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' type: integer vxlanPort: type: integer @@ -1813,6 +1964,10 @@ data: description: 'WireguardInterfaceName specifies the name to use for the Wireguard interface. [Default: wg.calico]' type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used by Wireguard. [Default: 51820]' @@ -1825,6 +1980,12 @@ data: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string xdpEnabled: description: 'XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]' @@ -2893,13 +3054,25 @@ data: resource. properties: affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. type: string allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. items: nullable: true type: integer type: array attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. items: properties: handle_id: @@ -2911,12 +3084,38 @@ data: type: object type: array cidr: + description: The block's CIDR. type: string deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. type: boolean unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. items: type: integer type: array @@ -3087,7 +3286,7 @@ data: type: array blockSize: description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. + this pool. Defaults to 26 for IPv4 and 122 for IPv6. type: integer cidr: description: The pool CIDR. @@ -3158,6 +3357,9 @@ data: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null name: ipreservations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -3306,6 +3508,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -3416,6 +3623,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -4368,7 +4580,6 @@ data: - apiGroups: - crd.projectcalico.org resources: - - ippools - ipreservations verbs: - list @@ -4385,6 +4596,13 @@ data: - update - delete - watch + - apiGroups: + - crd.projectcalico.org + resources: + - ippools + verbs: + - list + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4401,8 +4619,10 @@ data: - clusterinformations verbs: - get + - list - create - update + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4658,7 +4878,7 @@ data: value: node - name: DATASTORE_TYPE value: kubernetes - image: docker.io/calico/kube-controllers:v3.22.1 + image: docker.io/calico/kube-controllers:v3.23.0 livenessProbe: exec: command: @@ -4687,7 +4907,7 @@ data: - effect: NoSchedule key: node-role.kubernetes.io/control-plane --- - apiVersion: policy/v1beta1 + apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -4741,6 +4961,8 @@ data: value: autodetect - name: CALICO_IPV4POOL_IPIP value: Never + - name: CALICO_IPV6POOL_VXLAN + value: CrossSubnet - name: FELIX_IPINIPMTU valueFrom: configMapKeyRef: @@ -4768,7 +4990,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/node:v3.22.1 + image: docker.io/calico/node:v3.23.0 lifecycle: preStop: exec: @@ -4840,7 +5062,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: upgrade-ipam securityContext: privileged: true @@ -4874,7 +5096,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: install-cni securityContext: privileged: true @@ -4883,13 +5105,6 @@ data: name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir - - image: docker.io/calico/pod2daemon-flexvol:v3.22.1 - name: flexvol-driver - securityContext: - privileged: true - volumeMounts: - - mountPath: /host/driver - name: flexvol-driver-host nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical @@ -4936,10 +5151,6 @@ data: path: /var/run/nodeagent type: DirectoryOrCreate name: policysync - - hostPath: - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds - type: DirectoryOrCreate - name: flexvol-driver-host updateStrategy: rollingUpdate: maxUnavailable: 1 @@ -4991,7 +5202,7 @@ data: for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n \ - effect: NoExecute\n operator: Exists\n initContainers:\n # This container installs the CNI binaries\n # and CNI network config file - on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.22.1-hostprocess\n + on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.23.0-hostprocess\n \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n \ imagePullPolicy: Always\n env:\n # Name of the CNI config file to create.\n - name: CNI_CONF_NAME\n value: @@ -5010,7 +5221,7 @@ data: cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n \ securityContext:\n windowsOptions:\n hostProcess: true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n - \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.22.1-hostprocess\n + \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.23.0-hostprocess\n \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: @@ -5021,7 +5232,7 @@ data: name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n - \ image: sigwindowstools/calico-node:v3.22.1-hostprocess\n args: + \ image: sigwindowstools/calico-node:v3.23.0-hostprocess\n args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index 92fcf047be4..0b7c2e777a6 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -674,6 +674,12 @@ data: 64512]' format: int32 type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. @@ -704,6 +710,37 @@ data: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object nodeToNodeMeshEnabled: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' @@ -834,6 +871,12 @@ data: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer password: description: Optional BGP password for the peerings generated by this BGPPeer resource. @@ -1354,6 +1397,11 @@ data: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all interfaces with + BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled or Strict. [Default: Strict]' + type: string bpfExtToServiceConnmark: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local @@ -1393,6 +1441,51 @@ data: logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the @@ -1403,6 +1496,15 @@ data: Calico policy will be bypassed. [Default: insert]' type: string dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s]' type: string debugDisableLogDropping: type: boolean @@ -1431,9 +1533,14 @@ data: routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean @@ -1507,6 +1614,14 @@ data: "true" or "false" will force the feature, empty or omitted values are auto-detected. type: string + floatingIPs: + default: Disabled + description: FloatingIPs configures whether or not Felix will program + floating IP addresses. + enum: + - Enabled + - Disabled + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -1544,6 +1659,9 @@ data: disabled by setting the interval to 0. type: string ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: description: 'IPIPMTU is the MTU to set on the tunnel device. See @@ -1610,6 +1728,8 @@ data: usage. [Default: 10s]' type: string ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). type: boolean kubeNodePortRanges: description: 'KubeNodePortRanges holds list of port ranges used for @@ -1623,6 +1743,12 @@ data: pattern: ^.* x-kubernetes-int-or-string: true type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string logFilePath: description: 'LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]' @@ -1752,9 +1878,9 @@ data: routes. - CalicoIPAM: the default - use IPAM data to construct routes.' type: string routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. properties: max: type: integer @@ -1764,6 +1890,21 @@ data: - max - min type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting @@ -1791,12 +1932,22 @@ data: Felix makes reports. [Default: 86400s]' type: string useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. type: boolean vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for VXLAN networking. Optional as Felix determines + this based on the existing IP pools. [Default: nil (unset)]' type: boolean vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' type: integer vxlanPort: type: integer @@ -1814,6 +1965,10 @@ data: description: 'WireguardInterfaceName specifies the name to use for the Wireguard interface. [Default: wg.calico]' type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used by Wireguard. [Default: 51820]' @@ -1826,6 +1981,12 @@ data: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string xdpEnabled: description: 'XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]' @@ -2894,13 +3055,25 @@ data: resource. properties: affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. type: string allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. items: nullable: true type: integer type: array attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. items: properties: handle_id: @@ -2912,12 +3085,38 @@ data: type: object type: array cidr: + description: The block's CIDR. type: string deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. type: boolean unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. items: type: integer type: array @@ -3088,7 +3287,7 @@ data: type: array blockSize: description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. + this pool. Defaults to 26 for IPv4 and 122 for IPv6. type: integer cidr: description: The pool CIDR. @@ -3159,6 +3358,9 @@ data: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null name: ipreservations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -3307,6 +3509,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -3417,6 +3624,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -4369,7 +4581,6 @@ data: - apiGroups: - crd.projectcalico.org resources: - - ippools - ipreservations verbs: - list @@ -4386,6 +4597,13 @@ data: - update - delete - watch + - apiGroups: + - crd.projectcalico.org + resources: + - ippools + verbs: + - list + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4402,8 +4620,10 @@ data: - clusterinformations verbs: - get + - list - create - update + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4659,7 +4879,7 @@ data: value: node - name: DATASTORE_TYPE value: kubernetes - image: docker.io/calico/kube-controllers:v3.22.1 + image: docker.io/calico/kube-controllers:v3.23.0 livenessProbe: exec: command: @@ -4688,7 +4908,7 @@ data: - effect: NoSchedule key: node-role.kubernetes.io/control-plane --- - apiVersion: policy/v1beta1 + apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -4742,6 +4962,8 @@ data: value: autodetect - name: CALICO_IPV4POOL_IPIP value: Never + - name: CALICO_IPV6POOL_VXLAN + value: CrossSubnet - name: FELIX_IPINIPMTU valueFrom: configMapKeyRef: @@ -4769,7 +4991,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/node:v3.22.1 + image: docker.io/calico/node:v3.23.0 lifecycle: preStop: exec: @@ -4841,7 +5063,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: upgrade-ipam securityContext: privileged: true @@ -4875,7 +5097,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: install-cni securityContext: privileged: true @@ -4884,13 +5106,6 @@ data: name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir - - image: docker.io/calico/pod2daemon-flexvol:v3.22.1 - name: flexvol-driver - securityContext: - privileged: true - volumeMounts: - - mountPath: /host/driver - name: flexvol-driver-host nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical @@ -4937,10 +5152,6 @@ data: path: /var/run/nodeagent type: DirectoryOrCreate name: policysync - - hostPath: - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds - type: DirectoryOrCreate - name: flexvol-driver-host updateStrategy: rollingUpdate: maxUnavailable: 1 @@ -4992,7 +5203,7 @@ data: for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n \ - effect: NoExecute\n operator: Exists\n initContainers:\n # This container installs the CNI binaries\n # and CNI network config file - on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.22.1-hostprocess\n + on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.23.0-hostprocess\n \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n \ imagePullPolicy: Always\n env:\n # Name of the CNI config file to create.\n - name: CNI_CONF_NAME\n value: @@ -5011,7 +5222,7 @@ data: cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n \ securityContext:\n windowsOptions:\n hostProcess: true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n - \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.22.1-hostprocess\n + \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.23.0-hostprocess\n \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: @@ -5022,7 +5233,7 @@ data: name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n - \ image: sigwindowstools/calico-node:v3.22.1-hostprocess\n args: + \ image: sigwindowstools/calico-node:v3.23.0-hostprocess\n args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index b7053e1e4d6..82da84a0d60 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -309,6 +309,12 @@ data: 64512]' format: int32 type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. @@ -339,6 +345,37 @@ data: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object nodeToNodeMeshEnabled: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' @@ -469,6 +506,12 @@ data: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer password: description: Optional BGP password for the peerings generated by this BGPPeer resource. @@ -989,6 +1032,11 @@ data: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all interfaces with + BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled or Strict. [Default: Strict]' + type: string bpfExtToServiceConnmark: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local @@ -1028,6 +1076,51 @@ data: logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the @@ -1038,6 +1131,15 @@ data: Calico policy will be bypassed. [Default: insert]' type: string dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s]' type: string debugDisableLogDropping: type: boolean @@ -1066,9 +1168,14 @@ data: routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean @@ -1142,6 +1249,14 @@ data: "true" or "false" will force the feature, empty or omitted values are auto-detected. type: string + floatingIPs: + default: Disabled + description: FloatingIPs configures whether or not Felix will program + floating IP addresses. + enum: + - Enabled + - Disabled + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -1179,6 +1294,9 @@ data: disabled by setting the interval to 0. type: string ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: description: 'IPIPMTU is the MTU to set on the tunnel device. See @@ -1245,6 +1363,8 @@ data: usage. [Default: 10s]' type: string ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). type: boolean kubeNodePortRanges: description: 'KubeNodePortRanges holds list of port ranges used for @@ -1258,6 +1378,12 @@ data: pattern: ^.* x-kubernetes-int-or-string: true type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string logFilePath: description: 'LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]' @@ -1387,9 +1513,9 @@ data: routes. - CalicoIPAM: the default - use IPAM data to construct routes.' type: string routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. properties: max: type: integer @@ -1399,6 +1525,21 @@ data: - max - min type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting @@ -1426,12 +1567,22 @@ data: Felix makes reports. [Default: 86400s]' type: string useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. type: boolean vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for VXLAN networking. Optional as Felix determines + this based on the existing IP pools. [Default: nil (unset)]' type: boolean vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' type: integer vxlanPort: type: integer @@ -1449,6 +1600,10 @@ data: description: 'WireguardInterfaceName specifies the name to use for the Wireguard interface. [Default: wg.calico]' type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used by Wireguard. [Default: 51820]' @@ -1461,6 +1616,12 @@ data: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string xdpEnabled: description: 'XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]' @@ -2529,13 +2690,25 @@ data: resource. properties: affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. type: string allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. items: nullable: true type: integer type: array attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. items: properties: handle_id: @@ -2547,12 +2720,38 @@ data: type: object type: array cidr: + description: The block's CIDR. type: string deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. type: boolean unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. items: type: integer type: array @@ -2723,7 +2922,7 @@ data: type: array blockSize: description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. + this pool. Defaults to 26 for IPv4 and 122 for IPv6. type: integer cidr: description: The pool CIDR. @@ -2794,6 +2993,9 @@ data: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null name: ipreservations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2942,6 +3144,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -3052,6 +3259,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -4004,7 +4216,6 @@ data: - apiGroups: - crd.projectcalico.org resources: - - ippools - ipreservations verbs: - list @@ -4021,6 +4232,13 @@ data: - update - delete - watch + - apiGroups: + - crd.projectcalico.org + resources: + - ippools + verbs: + - list + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4037,8 +4255,10 @@ data: - clusterinformations verbs: - get + - list - create - update + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4294,7 +4514,7 @@ data: value: node - name: DATASTORE_TYPE value: kubernetes - image: docker.io/calico/kube-controllers:v3.22.1 + image: docker.io/calico/kube-controllers:v3.23.0 livenessProbe: exec: command: @@ -4323,7 +4543,7 @@ data: - effect: NoSchedule key: node-role.kubernetes.io/control-plane --- - apiVersion: policy/v1beta1 + apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -4377,6 +4597,8 @@ data: value: autodetect - name: CALICO_IPV4POOL_IPIP value: Never + - name: CALICO_IPV6POOL_VXLAN + value: CrossSubnet - name: FELIX_IPINIPMTU valueFrom: configMapKeyRef: @@ -4404,7 +4626,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/node:v3.22.1 + image: docker.io/calico/node:v3.23.0 lifecycle: preStop: exec: @@ -4476,7 +4698,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: upgrade-ipam securityContext: privileged: true @@ -4510,7 +4732,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: install-cni securityContext: privileged: true @@ -4519,13 +4741,6 @@ data: name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir - - image: docker.io/calico/pod2daemon-flexvol:v3.22.1 - name: flexvol-driver - securityContext: - privileged: true - volumeMounts: - - mountPath: /host/driver - name: flexvol-driver-host nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical @@ -4572,10 +4787,6 @@ data: path: /var/run/nodeagent type: DirectoryOrCreate name: policysync - - hostPath: - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds - type: DirectoryOrCreate - name: flexvol-driver-host updateStrategy: rollingUpdate: maxUnavailable: 1 diff --git a/templates/test/ci/cluster-template-prow-dual-stack.yaml b/templates/test/ci/cluster-template-prow-dual-stack.yaml index 4fae84408ce..191f726f1ef 100644 --- a/templates/test/ci/cluster-template-prow-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-dual-stack.yaml @@ -318,6 +318,12 @@ data: 64512]' format: int32 type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. @@ -348,6 +354,37 @@ data: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object nodeToNodeMeshEnabled: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' @@ -478,6 +515,12 @@ data: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer password: description: Optional BGP password for the peerings generated by this BGPPeer resource. @@ -998,6 +1041,11 @@ data: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all interfaces with + BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled or Strict. [Default: Strict]' + type: string bpfExtToServiceConnmark: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local @@ -1037,6 +1085,51 @@ data: logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the @@ -1047,6 +1140,15 @@ data: Calico policy will be bypassed. [Default: insert]' type: string dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s]' type: string debugDisableLogDropping: type: boolean @@ -1075,9 +1177,14 @@ data: routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean @@ -1151,6 +1258,14 @@ data: "true" or "false" will force the feature, empty or omitted values are auto-detected. type: string + floatingIPs: + default: Disabled + description: FloatingIPs configures whether or not Felix will program + floating IP addresses. + enum: + - Enabled + - Disabled + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -1188,6 +1303,9 @@ data: disabled by setting the interval to 0. type: string ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: description: 'IPIPMTU is the MTU to set on the tunnel device. See @@ -1254,6 +1372,8 @@ data: usage. [Default: 10s]' type: string ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). type: boolean kubeNodePortRanges: description: 'KubeNodePortRanges holds list of port ranges used for @@ -1267,6 +1387,12 @@ data: pattern: ^.* x-kubernetes-int-or-string: true type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string logFilePath: description: 'LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]' @@ -1396,9 +1522,9 @@ data: routes. - CalicoIPAM: the default - use IPAM data to construct routes.' type: string routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. properties: max: type: integer @@ -1408,6 +1534,21 @@ data: - max - min type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting @@ -1435,12 +1576,22 @@ data: Felix makes reports. [Default: 86400s]' type: string useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. type: boolean vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for VXLAN networking. Optional as Felix determines + this based on the existing IP pools. [Default: nil (unset)]' type: boolean vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' type: integer vxlanPort: type: integer @@ -1458,6 +1609,10 @@ data: description: 'WireguardInterfaceName specifies the name to use for the Wireguard interface. [Default: wg.calico]' type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used by Wireguard. [Default: 51820]' @@ -1470,6 +1625,12 @@ data: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string xdpEnabled: description: 'XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]' @@ -2538,13 +2699,25 @@ data: resource. properties: affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. type: string allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. items: nullable: true type: integer type: array attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. items: properties: handle_id: @@ -2556,12 +2729,38 @@ data: type: object type: array cidr: + description: The block's CIDR. type: string deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. type: boolean unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. items: type: integer type: array @@ -2732,7 +2931,7 @@ data: type: array blockSize: description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. + this pool. Defaults to 26 for IPv4 and 122 for IPv6. type: integer cidr: description: The pool CIDR. @@ -2803,6 +3002,9 @@ data: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null name: ipreservations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2951,6 +3153,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -3061,6 +3268,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -4013,7 +4225,6 @@ data: - apiGroups: - crd.projectcalico.org resources: - - ippools - ipreservations verbs: - list @@ -4030,6 +4241,13 @@ data: - update - delete - watch + - apiGroups: + - crd.projectcalico.org + resources: + - ippools + verbs: + - list + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4046,8 +4264,10 @@ data: - clusterinformations verbs: - get + - list - create - update + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4300,7 +4520,7 @@ data: value: node - name: DATASTORE_TYPE value: kubernetes - image: docker.io/calico/kube-controllers:v3.22.1 + image: docker.io/calico/kube-controllers:v3.23.0 livenessProbe: exec: command: @@ -4369,7 +4589,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/typha:v3.22.1 + image: docker.io/calico/typha:v3.23.0 livenessProbe: httpGet: host: localhost @@ -4404,7 +4624,7 @@ data: - key: CriticalAddonsOnly operator: Exists --- - apiVersion: policy/v1beta1 + apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -4417,7 +4637,7 @@ data: matchLabels: k8s-app: calico-kube-controllers --- - apiVersion: policy/v1beta1 + apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -4485,7 +4705,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/node:v3.22.1 + image: docker.io/calico/node:v3.23.0 lifecycle: preStop: exec: @@ -4560,7 +4780,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: install-cni securityContext: privileged: true @@ -4569,13 +4789,6 @@ data: name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir - - image: docker.io/calico/pod2daemon-flexvol:v3.22.1 - name: flexvol-driver - securityContext: - privileged: true - volumeMounts: - - mountPath: /host/driver - name: flexvol-driver-host nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical @@ -4619,10 +4832,6 @@ data: path: /var/run/nodeagent type: DirectoryOrCreate name: policysync - - hostPath: - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds - type: DirectoryOrCreate - name: flexvol-driver-host updateStrategy: rollingUpdate: maxUnavailable: 1 diff --git a/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml b/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml index 133bf939d89..9ce08f16d81 100644 --- a/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml +++ b/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml @@ -288,6 +288,12 @@ data: 64512]' format: int32 type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. @@ -318,6 +324,37 @@ data: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object nodeToNodeMeshEnabled: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' @@ -448,6 +485,12 @@ data: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer password: description: Optional BGP password for the peerings generated by this BGPPeer resource. @@ -968,6 +1011,11 @@ data: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all interfaces with + BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled or Strict. [Default: Strict]' + type: string bpfExtToServiceConnmark: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local @@ -1007,6 +1055,51 @@ data: logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the @@ -1017,6 +1110,15 @@ data: Calico policy will be bypassed. [Default: insert]' type: string dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s]' type: string debugDisableLogDropping: type: boolean @@ -1045,9 +1147,14 @@ data: routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean @@ -1121,6 +1228,14 @@ data: "true" or "false" will force the feature, empty or omitted values are auto-detected. type: string + floatingIPs: + default: Disabled + description: FloatingIPs configures whether or not Felix will program + floating IP addresses. + enum: + - Enabled + - Disabled + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -1158,6 +1273,9 @@ data: disabled by setting the interval to 0. type: string ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: description: 'IPIPMTU is the MTU to set on the tunnel device. See @@ -1224,6 +1342,8 @@ data: usage. [Default: 10s]' type: string ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). type: boolean kubeNodePortRanges: description: 'KubeNodePortRanges holds list of port ranges used for @@ -1237,6 +1357,12 @@ data: pattern: ^.* x-kubernetes-int-or-string: true type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string logFilePath: description: 'LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]' @@ -1366,9 +1492,9 @@ data: routes. - CalicoIPAM: the default - use IPAM data to construct routes.' type: string routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. properties: max: type: integer @@ -1378,6 +1504,21 @@ data: - max - min type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting @@ -1405,12 +1546,22 @@ data: Felix makes reports. [Default: 86400s]' type: string useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. type: boolean vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for VXLAN networking. Optional as Felix determines + this based on the existing IP pools. [Default: nil (unset)]' type: boolean vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' type: integer vxlanPort: type: integer @@ -1428,6 +1579,10 @@ data: description: 'WireguardInterfaceName specifies the name to use for the Wireguard interface. [Default: wg.calico]' type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used by Wireguard. [Default: 51820]' @@ -1440,6 +1595,12 @@ data: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string xdpEnabled: description: 'XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]' @@ -2508,13 +2669,25 @@ data: resource. properties: affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. type: string allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. items: nullable: true type: integer type: array attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. items: properties: handle_id: @@ -2526,12 +2699,38 @@ data: type: object type: array cidr: + description: The block's CIDR. type: string deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. type: boolean unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. items: type: integer type: array @@ -2702,7 +2901,7 @@ data: type: array blockSize: description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. + this pool. Defaults to 26 for IPv4 and 122 for IPv6. type: integer cidr: description: The pool CIDR. @@ -2773,6 +2972,9 @@ data: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null name: ipreservations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2921,6 +3123,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -3031,6 +3238,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -3983,7 +4195,6 @@ data: - apiGroups: - crd.projectcalico.org resources: - - ippools - ipreservations verbs: - list @@ -4000,6 +4211,13 @@ data: - update - delete - watch + - apiGroups: + - crd.projectcalico.org + resources: + - ippools + verbs: + - list + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4016,8 +4234,10 @@ data: - clusterinformations verbs: - get + - list - create - update + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4273,7 +4493,7 @@ data: value: node - name: DATASTORE_TYPE value: kubernetes - image: docker.io/calico/kube-controllers:v3.22.1 + image: docker.io/calico/kube-controllers:v3.23.0 livenessProbe: exec: command: @@ -4302,7 +4522,7 @@ data: - effect: NoSchedule key: node-role.kubernetes.io/control-plane --- - apiVersion: policy/v1beta1 + apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -4356,6 +4576,8 @@ data: value: autodetect - name: CALICO_IPV4POOL_IPIP value: Never + - name: CALICO_IPV6POOL_VXLAN + value: CrossSubnet - name: FELIX_IPINIPMTU valueFrom: configMapKeyRef: @@ -4383,7 +4605,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/node:v3.22.1 + image: docker.io/calico/node:v3.23.0 lifecycle: preStop: exec: @@ -4455,7 +4677,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: upgrade-ipam securityContext: privileged: true @@ -4489,7 +4711,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: install-cni securityContext: privileged: true @@ -4498,13 +4720,6 @@ data: name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir - - image: docker.io/calico/pod2daemon-flexvol:v3.22.1 - name: flexvol-driver - securityContext: - privileged: true - volumeMounts: - - mountPath: /host/driver - name: flexvol-driver-host nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical @@ -4551,10 +4766,6 @@ data: path: /var/run/nodeagent type: DirectoryOrCreate name: policysync - - hostPath: - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds - type: DirectoryOrCreate - name: flexvol-driver-host updateStrategy: rollingUpdate: maxUnavailable: 1 diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml index dbe536ecfd5..9606727507f 100644 --- a/templates/test/ci/cluster-template-prow-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ipv6.yaml @@ -339,6 +339,12 @@ data: 64512]' format: int32 type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. @@ -369,6 +375,37 @@ data: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object nodeToNodeMeshEnabled: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' @@ -499,6 +536,12 @@ data: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer password: description: Optional BGP password for the peerings generated by this BGPPeer resource. @@ -1019,6 +1062,11 @@ data: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all interfaces with + BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled or Strict. [Default: Strict]' + type: string bpfExtToServiceConnmark: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local @@ -1058,6 +1106,51 @@ data: logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the @@ -1068,6 +1161,15 @@ data: Calico policy will be bypassed. [Default: insert]' type: string dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s]' type: string debugDisableLogDropping: type: boolean @@ -1096,9 +1198,14 @@ data: routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean @@ -1172,6 +1279,14 @@ data: "true" or "false" will force the feature, empty or omitted values are auto-detected. type: string + floatingIPs: + default: Disabled + description: FloatingIPs configures whether or not Felix will program + floating IP addresses. + enum: + - Enabled + - Disabled + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -1209,6 +1324,9 @@ data: disabled by setting the interval to 0. type: string ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: description: 'IPIPMTU is the MTU to set on the tunnel device. See @@ -1275,6 +1393,8 @@ data: usage. [Default: 10s]' type: string ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). type: boolean kubeNodePortRanges: description: 'KubeNodePortRanges holds list of port ranges used for @@ -1288,6 +1408,12 @@ data: pattern: ^.* x-kubernetes-int-or-string: true type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string logFilePath: description: 'LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]' @@ -1417,9 +1543,9 @@ data: routes. - CalicoIPAM: the default - use IPAM data to construct routes.' type: string routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. properties: max: type: integer @@ -1429,6 +1555,21 @@ data: - max - min type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting @@ -1456,12 +1597,22 @@ data: Felix makes reports. [Default: 86400s]' type: string useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. type: boolean vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for VXLAN networking. Optional as Felix determines + this based on the existing IP pools. [Default: nil (unset)]' type: boolean vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' type: integer vxlanPort: type: integer @@ -1479,6 +1630,10 @@ data: description: 'WireguardInterfaceName specifies the name to use for the Wireguard interface. [Default: wg.calico]' type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used by Wireguard. [Default: 51820]' @@ -1491,6 +1646,12 @@ data: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string xdpEnabled: description: 'XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]' @@ -2559,13 +2720,25 @@ data: resource. properties: affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. type: string allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. items: nullable: true type: integer type: array attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. items: properties: handle_id: @@ -2577,12 +2750,38 @@ data: type: object type: array cidr: + description: The block's CIDR. type: string deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. type: boolean unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. items: type: integer type: array @@ -2753,7 +2952,7 @@ data: type: array blockSize: description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. + this pool. Defaults to 26 for IPv4 and 122 for IPv6. type: integer cidr: description: The pool CIDR. @@ -2824,6 +3023,9 @@ data: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null name: ipreservations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2972,6 +3174,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -3082,6 +3289,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -4034,7 +4246,6 @@ data: - apiGroups: - crd.projectcalico.org resources: - - ippools - ipreservations verbs: - list @@ -4051,6 +4262,13 @@ data: - update - delete - watch + - apiGroups: + - crd.projectcalico.org + resources: + - ippools + verbs: + - list + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4067,8 +4285,10 @@ data: - clusterinformations verbs: - get + - list - create - update + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4310,7 +4530,7 @@ data: value: node - name: DATASTORE_TYPE value: kubernetes - image: docker.io/calico/kube-controllers:v3.22.1 + image: docker.io/calico/kube-controllers:v3.23.0 livenessProbe: exec: command: @@ -4379,7 +4599,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/typha:v3.22.1 + image: docker.io/calico/typha:v3.23.0 livenessProbe: httpGet: host: localhost @@ -4414,7 +4634,7 @@ data: - key: CriticalAddonsOnly operator: Exists --- - apiVersion: policy/v1beta1 + apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -4427,7 +4647,7 @@ data: matchLabels: k8s-app: calico-kube-controllers --- - apiVersion: policy/v1beta1 + apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -4495,7 +4715,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/node:v3.22.1 + image: docker.io/calico/node:v3.23.0 lifecycle: preStop: exec: @@ -4570,7 +4790,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: install-cni securityContext: privileged: true @@ -4579,13 +4799,6 @@ data: name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir - - image: docker.io/calico/pod2daemon-flexvol:v3.22.1 - name: flexvol-driver - securityContext: - privileged: true - volumeMounts: - - mountPath: /host/driver - name: flexvol-driver-host nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical @@ -4629,10 +4842,6 @@ data: path: /var/run/nodeagent type: DirectoryOrCreate name: policysync - - hostPath: - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds - type: DirectoryOrCreate - name: flexvol-driver-host updateStrategy: rollingUpdate: maxUnavailable: 1 diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index 878f69d7b20..b4a9980c550 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -577,6 +577,12 @@ data: 64512]' format: int32 type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. @@ -607,6 +613,37 @@ data: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object nodeToNodeMeshEnabled: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' @@ -737,6 +774,12 @@ data: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer password: description: Optional BGP password for the peerings generated by this BGPPeer resource. @@ -1257,6 +1300,11 @@ data: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all interfaces with + BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled or Strict. [Default: Strict]' + type: string bpfExtToServiceConnmark: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local @@ -1296,6 +1344,51 @@ data: logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the @@ -1306,6 +1399,15 @@ data: Calico policy will be bypassed. [Default: insert]' type: string dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s]' type: string debugDisableLogDropping: type: boolean @@ -1334,9 +1436,14 @@ data: routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean @@ -1410,6 +1517,14 @@ data: "true" or "false" will force the feature, empty or omitted values are auto-detected. type: string + floatingIPs: + default: Disabled + description: FloatingIPs configures whether or not Felix will program + floating IP addresses. + enum: + - Enabled + - Disabled + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -1447,6 +1562,9 @@ data: disabled by setting the interval to 0. type: string ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: description: 'IPIPMTU is the MTU to set on the tunnel device. See @@ -1513,6 +1631,8 @@ data: usage. [Default: 10s]' type: string ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). type: boolean kubeNodePortRanges: description: 'KubeNodePortRanges holds list of port ranges used for @@ -1526,6 +1646,12 @@ data: pattern: ^.* x-kubernetes-int-or-string: true type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string logFilePath: description: 'LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]' @@ -1655,9 +1781,9 @@ data: routes. - CalicoIPAM: the default - use IPAM data to construct routes.' type: string routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. properties: max: type: integer @@ -1667,6 +1793,21 @@ data: - max - min type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting @@ -1694,12 +1835,22 @@ data: Felix makes reports. [Default: 86400s]' type: string useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. type: boolean vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for VXLAN networking. Optional as Felix determines + this based on the existing IP pools. [Default: nil (unset)]' type: boolean vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' type: integer vxlanPort: type: integer @@ -1717,6 +1868,10 @@ data: description: 'WireguardInterfaceName specifies the name to use for the Wireguard interface. [Default: wg.calico]' type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used by Wireguard. [Default: 51820]' @@ -1729,6 +1884,12 @@ data: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string xdpEnabled: description: 'XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]' @@ -2797,13 +2958,25 @@ data: resource. properties: affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. type: string allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. items: nullable: true type: integer type: array attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. items: properties: handle_id: @@ -2815,12 +2988,38 @@ data: type: object type: array cidr: + description: The block's CIDR. type: string deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. type: boolean unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. items: type: integer type: array @@ -2991,7 +3190,7 @@ data: type: array blockSize: description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. + this pool. Defaults to 26 for IPv4 and 122 for IPv6. type: integer cidr: description: The pool CIDR. @@ -3062,6 +3261,9 @@ data: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null name: ipreservations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -3210,6 +3412,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -3320,6 +3527,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -4272,7 +4484,6 @@ data: - apiGroups: - crd.projectcalico.org resources: - - ippools - ipreservations verbs: - list @@ -4289,6 +4500,13 @@ data: - update - delete - watch + - apiGroups: + - crd.projectcalico.org + resources: + - ippools + verbs: + - list + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4305,8 +4523,10 @@ data: - clusterinformations verbs: - get + - list - create - update + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4562,7 +4782,7 @@ data: value: node - name: DATASTORE_TYPE value: kubernetes - image: docker.io/calico/kube-controllers:v3.22.1 + image: docker.io/calico/kube-controllers:v3.23.0 livenessProbe: exec: command: @@ -4591,7 +4811,7 @@ data: - effect: NoSchedule key: node-role.kubernetes.io/control-plane --- - apiVersion: policy/v1beta1 + apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -4645,6 +4865,8 @@ data: value: autodetect - name: CALICO_IPV4POOL_IPIP value: Never + - name: CALICO_IPV6POOL_VXLAN + value: CrossSubnet - name: FELIX_IPINIPMTU valueFrom: configMapKeyRef: @@ -4672,7 +4894,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/node:v3.22.1 + image: docker.io/calico/node:v3.23.0 lifecycle: preStop: exec: @@ -4744,7 +4966,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: upgrade-ipam securityContext: privileged: true @@ -4778,7 +5000,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: install-cni securityContext: privileged: true @@ -4787,13 +5009,6 @@ data: name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir - - image: docker.io/calico/pod2daemon-flexvol:v3.22.1 - name: flexvol-driver - securityContext: - privileged: true - volumeMounts: - - mountPath: /host/driver - name: flexvol-driver-host nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical @@ -4840,10 +5055,6 @@ data: path: /var/run/nodeagent type: DirectoryOrCreate name: policysync - - hostPath: - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds - type: DirectoryOrCreate - name: flexvol-driver-host updateStrategy: rollingUpdate: maxUnavailable: 1 @@ -4895,7 +5106,7 @@ data: for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n \ - effect: NoExecute\n operator: Exists\n initContainers:\n # This container installs the CNI binaries\n # and CNI network config file - on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.22.1-hostprocess\n + on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.23.0-hostprocess\n \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n \ imagePullPolicy: Always\n env:\n # Name of the CNI config file to create.\n - name: CNI_CONF_NAME\n value: @@ -4914,7 +5125,7 @@ data: cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n \ securityContext:\n windowsOptions:\n hostProcess: true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n - \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.22.1-hostprocess\n + \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.23.0-hostprocess\n \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: @@ -4925,7 +5136,7 @@ data: name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n - \ image: sigwindowstools/calico-node:v3.22.1-hostprocess\n args: + \ image: sigwindowstools/calico-node:v3.23.0-hostprocess\n args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index dc321e6f8da..6330981d5bb 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -390,6 +390,12 @@ data: 64512]' format: int32 type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. @@ -420,6 +426,37 @@ data: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object nodeToNodeMeshEnabled: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' @@ -550,6 +587,12 @@ data: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer password: description: Optional BGP password for the peerings generated by this BGPPeer resource. @@ -1070,6 +1113,11 @@ data: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all interfaces with + BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled or Strict. [Default: Strict]' + type: string bpfExtToServiceConnmark: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local @@ -1109,6 +1157,51 @@ data: logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the @@ -1119,6 +1212,15 @@ data: Calico policy will be bypassed. [Default: insert]' type: string dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s]' type: string debugDisableLogDropping: type: boolean @@ -1147,9 +1249,14 @@ data: routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean @@ -1223,6 +1330,14 @@ data: "true" or "false" will force the feature, empty or omitted values are auto-detected. type: string + floatingIPs: + default: Disabled + description: FloatingIPs configures whether or not Felix will program + floating IP addresses. + enum: + - Enabled + - Disabled + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -1260,6 +1375,9 @@ data: disabled by setting the interval to 0. type: string ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: description: 'IPIPMTU is the MTU to set on the tunnel device. See @@ -1326,6 +1444,8 @@ data: usage. [Default: 10s]' type: string ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). type: boolean kubeNodePortRanges: description: 'KubeNodePortRanges holds list of port ranges used for @@ -1339,6 +1459,12 @@ data: pattern: ^.* x-kubernetes-int-or-string: true type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string logFilePath: description: 'LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]' @@ -1468,9 +1594,9 @@ data: routes. - CalicoIPAM: the default - use IPAM data to construct routes.' type: string routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. properties: max: type: integer @@ -1480,6 +1606,21 @@ data: - max - min type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting @@ -1507,12 +1648,22 @@ data: Felix makes reports. [Default: 86400s]' type: string useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. type: boolean vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for VXLAN networking. Optional as Felix determines + this based on the existing IP pools. [Default: nil (unset)]' type: boolean vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' type: integer vxlanPort: type: integer @@ -1530,6 +1681,10 @@ data: description: 'WireguardInterfaceName specifies the name to use for the Wireguard interface. [Default: wg.calico]' type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used by Wireguard. [Default: 51820]' @@ -1542,6 +1697,12 @@ data: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string xdpEnabled: description: 'XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]' @@ -2610,13 +2771,25 @@ data: resource. properties: affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. type: string allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. items: nullable: true type: integer type: array attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. items: properties: handle_id: @@ -2628,12 +2801,38 @@ data: type: object type: array cidr: + description: The block's CIDR. type: string deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. type: boolean unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. items: type: integer type: array @@ -2804,7 +3003,7 @@ data: type: array blockSize: description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. + this pool. Defaults to 26 for IPv4 and 122 for IPv6. type: integer cidr: description: The pool CIDR. @@ -2875,6 +3074,9 @@ data: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null name: ipreservations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -3023,6 +3225,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -3133,6 +3340,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -4085,7 +4297,6 @@ data: - apiGroups: - crd.projectcalico.org resources: - - ippools - ipreservations verbs: - list @@ -4102,6 +4313,13 @@ data: - update - delete - watch + - apiGroups: + - crd.projectcalico.org + resources: + - ippools + verbs: + - list + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4118,8 +4336,10 @@ data: - clusterinformations verbs: - get + - list - create - update + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4375,7 +4595,7 @@ data: value: node - name: DATASTORE_TYPE value: kubernetes - image: docker.io/calico/kube-controllers:v3.22.1 + image: docker.io/calico/kube-controllers:v3.23.0 livenessProbe: exec: command: @@ -4404,7 +4624,7 @@ data: - effect: NoSchedule key: node-role.kubernetes.io/control-plane --- - apiVersion: policy/v1beta1 + apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -4458,6 +4678,8 @@ data: value: autodetect - name: CALICO_IPV4POOL_IPIP value: Never + - name: CALICO_IPV6POOL_VXLAN + value: CrossSubnet - name: FELIX_IPINIPMTU valueFrom: configMapKeyRef: @@ -4485,7 +4707,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/node:v3.22.1 + image: docker.io/calico/node:v3.23.0 lifecycle: preStop: exec: @@ -4557,7 +4779,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: upgrade-ipam securityContext: privileged: true @@ -4591,7 +4813,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: install-cni securityContext: privileged: true @@ -4600,13 +4822,6 @@ data: name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir - - image: docker.io/calico/pod2daemon-flexvol:v3.22.1 - name: flexvol-driver - securityContext: - privileged: true - volumeMounts: - - mountPath: /host/driver - name: flexvol-driver-host nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical @@ -4653,10 +4868,6 @@ data: path: /var/run/nodeagent type: DirectoryOrCreate name: policysync - - hostPath: - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds - type: DirectoryOrCreate - name: flexvol-driver-host updateStrategy: rollingUpdate: maxUnavailable: 1 @@ -4708,7 +4919,7 @@ data: for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n \ - effect: NoExecute\n operator: Exists\n initContainers:\n # This container installs the CNI binaries\n # and CNI network config file - on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.22.1-hostprocess\n + on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.23.0-hostprocess\n \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n \ imagePullPolicy: Always\n env:\n # Name of the CNI config file to create.\n - name: CNI_CONF_NAME\n value: @@ -4727,7 +4938,7 @@ data: cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n \ securityContext:\n windowsOptions:\n hostProcess: true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n - \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.22.1-hostprocess\n + \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.23.0-hostprocess\n \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: @@ -4738,7 +4949,7 @@ data: name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n - \ image: sigwindowstools/calico-node:v3.22.1-hostprocess\n args: + \ image: sigwindowstools/calico-node:v3.23.0-hostprocess\n args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index 536394c579a..ada7ebdb576 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -4717,6 +4717,12 @@ data: 64512]' format: int32 type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. @@ -4747,6 +4753,37 @@ data: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object nodeToNodeMeshEnabled: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' @@ -4877,6 +4914,12 @@ data: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer password: description: Optional BGP password for the peerings generated by this BGPPeer resource. @@ -5397,6 +5440,11 @@ data: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all interfaces with + BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled or Strict. [Default: Strict]' + type: string bpfExtToServiceConnmark: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local @@ -5436,6 +5484,51 @@ data: logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the @@ -5446,6 +5539,15 @@ data: Calico policy will be bypassed. [Default: insert]' type: string dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s]' type: string debugDisableLogDropping: type: boolean @@ -5474,9 +5576,14 @@ data: routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean @@ -5550,6 +5657,14 @@ data: "true" or "false" will force the feature, empty or omitted values are auto-detected. type: string + floatingIPs: + default: Disabled + description: FloatingIPs configures whether or not Felix will program + floating IP addresses. + enum: + - Enabled + - Disabled + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -5587,6 +5702,9 @@ data: disabled by setting the interval to 0. type: string ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: description: 'IPIPMTU is the MTU to set on the tunnel device. See @@ -5653,6 +5771,8 @@ data: usage. [Default: 10s]' type: string ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). type: boolean kubeNodePortRanges: description: 'KubeNodePortRanges holds list of port ranges used for @@ -5666,6 +5786,12 @@ data: pattern: ^.* x-kubernetes-int-or-string: true type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string logFilePath: description: 'LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]' @@ -5795,9 +5921,9 @@ data: routes. - CalicoIPAM: the default - use IPAM data to construct routes.' type: string routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. properties: max: type: integer @@ -5807,6 +5933,21 @@ data: - max - min type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting @@ -5834,12 +5975,22 @@ data: Felix makes reports. [Default: 86400s]' type: string useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. type: boolean vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for VXLAN networking. Optional as Felix determines + this based on the existing IP pools. [Default: nil (unset)]' type: boolean vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' type: integer vxlanPort: type: integer @@ -5857,6 +6008,10 @@ data: description: 'WireguardInterfaceName specifies the name to use for the Wireguard interface. [Default: wg.calico]' type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used by Wireguard. [Default: 51820]' @@ -5869,6 +6024,12 @@ data: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string xdpEnabled: description: 'XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]' @@ -6937,13 +7098,25 @@ data: resource. properties: affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. type: string allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. items: nullable: true type: integer type: array attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. items: properties: handle_id: @@ -6955,12 +7128,38 @@ data: type: object type: array cidr: + description: The block's CIDR. type: string deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. type: boolean unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. items: type: integer type: array @@ -7131,7 +7330,7 @@ data: type: array blockSize: description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. + this pool. Defaults to 26 for IPv4 and 122 for IPv6. type: integer cidr: description: The pool CIDR. @@ -7202,6 +7401,9 @@ data: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null name: ipreservations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -7350,6 +7552,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -7460,6 +7667,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -8412,7 +8624,6 @@ data: - apiGroups: - crd.projectcalico.org resources: - - ippools - ipreservations verbs: - list @@ -8429,6 +8640,13 @@ data: - update - delete - watch + - apiGroups: + - crd.projectcalico.org + resources: + - ippools + verbs: + - list + - watch - apiGroups: - crd.projectcalico.org resources: @@ -8445,8 +8663,10 @@ data: - clusterinformations verbs: - get + - list - create - update + - watch - apiGroups: - crd.projectcalico.org resources: @@ -8702,7 +8922,7 @@ data: value: node - name: DATASTORE_TYPE value: kubernetes - image: docker.io/calico/kube-controllers:v3.22.1 + image: docker.io/calico/kube-controllers:v3.23.0 livenessProbe: exec: command: @@ -8731,7 +8951,7 @@ data: - effect: NoSchedule key: node-role.kubernetes.io/control-plane --- - apiVersion: policy/v1beta1 + apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -8785,6 +9005,8 @@ data: value: autodetect - name: CALICO_IPV4POOL_IPIP value: Never + - name: CALICO_IPV6POOL_VXLAN + value: CrossSubnet - name: FELIX_IPINIPMTU valueFrom: configMapKeyRef: @@ -8812,7 +9034,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/node:v3.22.1 + image: docker.io/calico/node:v3.23.0 lifecycle: preStop: exec: @@ -8884,7 +9106,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: upgrade-ipam securityContext: privileged: true @@ -8918,7 +9140,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: install-cni securityContext: privileged: true @@ -8927,13 +9149,6 @@ data: name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir - - image: docker.io/calico/pod2daemon-flexvol:v3.22.1 - name: flexvol-driver - securityContext: - privileged: true - volumeMounts: - - mountPath: /host/driver - name: flexvol-driver-host nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical @@ -8980,10 +9195,6 @@ data: path: /var/run/nodeagent type: DirectoryOrCreate name: policysync - - hostPath: - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds - type: DirectoryOrCreate - name: flexvol-driver-host updateStrategy: rollingUpdate: maxUnavailable: 1 diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml index 88ae8e8c134..c8a5ee78af1 100644 --- a/templates/test/ci/cluster-template-prow-private.yaml +++ b/templates/test/ci/cluster-template-prow-private.yaml @@ -318,6 +318,12 @@ data: 64512]' format: int32 type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. @@ -348,6 +354,37 @@ data: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object nodeToNodeMeshEnabled: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' @@ -478,6 +515,12 @@ data: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer password: description: Optional BGP password for the peerings generated by this BGPPeer resource. @@ -998,6 +1041,11 @@ data: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all interfaces with + BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled or Strict. [Default: Strict]' + type: string bpfExtToServiceConnmark: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local @@ -1037,6 +1085,51 @@ data: logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the @@ -1047,6 +1140,15 @@ data: Calico policy will be bypassed. [Default: insert]' type: string dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s]' type: string debugDisableLogDropping: type: boolean @@ -1075,9 +1177,14 @@ data: routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean @@ -1151,6 +1258,14 @@ data: "true" or "false" will force the feature, empty or omitted values are auto-detected. type: string + floatingIPs: + default: Disabled + description: FloatingIPs configures whether or not Felix will program + floating IP addresses. + enum: + - Enabled + - Disabled + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -1188,6 +1303,9 @@ data: disabled by setting the interval to 0. type: string ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: description: 'IPIPMTU is the MTU to set on the tunnel device. See @@ -1254,6 +1372,8 @@ data: usage. [Default: 10s]' type: string ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). type: boolean kubeNodePortRanges: description: 'KubeNodePortRanges holds list of port ranges used for @@ -1267,6 +1387,12 @@ data: pattern: ^.* x-kubernetes-int-or-string: true type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string logFilePath: description: 'LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]' @@ -1396,9 +1522,9 @@ data: routes. - CalicoIPAM: the default - use IPAM data to construct routes.' type: string routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. properties: max: type: integer @@ -1408,6 +1534,21 @@ data: - max - min type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting @@ -1435,12 +1576,22 @@ data: Felix makes reports. [Default: 86400s]' type: string useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. type: boolean vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for VXLAN networking. Optional as Felix determines + this based on the existing IP pools. [Default: nil (unset)]' type: boolean vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' type: integer vxlanPort: type: integer @@ -1458,6 +1609,10 @@ data: description: 'WireguardInterfaceName specifies the name to use for the Wireguard interface. [Default: wg.calico]' type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used by Wireguard. [Default: 51820]' @@ -1470,6 +1625,12 @@ data: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string xdpEnabled: description: 'XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]' @@ -2538,13 +2699,25 @@ data: resource. properties: affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. type: string allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. items: nullable: true type: integer type: array attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. items: properties: handle_id: @@ -2556,12 +2729,38 @@ data: type: object type: array cidr: + description: The block's CIDR. type: string deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. type: boolean unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. items: type: integer type: array @@ -2732,7 +2931,7 @@ data: type: array blockSize: description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. + this pool. Defaults to 26 for IPv4 and 122 for IPv6. type: integer cidr: description: The pool CIDR. @@ -2803,6 +3002,9 @@ data: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null name: ipreservations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -2951,6 +3153,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -3061,6 +3268,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -4013,7 +4225,6 @@ data: - apiGroups: - crd.projectcalico.org resources: - - ippools - ipreservations verbs: - list @@ -4030,6 +4241,13 @@ data: - update - delete - watch + - apiGroups: + - crd.projectcalico.org + resources: + - ippools + verbs: + - list + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4046,8 +4264,10 @@ data: - clusterinformations verbs: - get + - list - create - update + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4303,7 +4523,7 @@ data: value: node - name: DATASTORE_TYPE value: kubernetes - image: docker.io/calico/kube-controllers:v3.22.1 + image: docker.io/calico/kube-controllers:v3.23.0 livenessProbe: exec: command: @@ -4332,7 +4552,7 @@ data: - effect: NoSchedule key: node-role.kubernetes.io/control-plane --- - apiVersion: policy/v1beta1 + apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -4386,6 +4606,8 @@ data: value: autodetect - name: CALICO_IPV4POOL_IPIP value: Never + - name: CALICO_IPV6POOL_VXLAN + value: CrossSubnet - name: FELIX_IPINIPMTU valueFrom: configMapKeyRef: @@ -4413,7 +4635,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/node:v3.22.1 + image: docker.io/calico/node:v3.23.0 lifecycle: preStop: exec: @@ -4485,7 +4707,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: upgrade-ipam securityContext: privileged: true @@ -4519,7 +4741,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: install-cni securityContext: privileged: true @@ -4528,13 +4750,6 @@ data: name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir - - image: docker.io/calico/pod2daemon-flexvol:v3.22.1 - name: flexvol-driver - securityContext: - privileged: true - volumeMounts: - - mountPath: /host/driver - name: flexvol-driver-host nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical @@ -4581,10 +4796,6 @@ data: path: /var/run/nodeagent type: DirectoryOrCreate name: policysync - - hostPath: - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds - type: DirectoryOrCreate - name: flexvol-driver-host updateStrategy: rollingUpdate: maxUnavailable: 1 diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index 358ebdcef5c..f11d1a9d48f 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -471,6 +471,12 @@ data: 64512]' format: int32 type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. @@ -501,6 +507,37 @@ data: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object nodeToNodeMeshEnabled: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' @@ -631,6 +668,12 @@ data: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer password: description: Optional BGP password for the peerings generated by this BGPPeer resource. @@ -1151,6 +1194,11 @@ data: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all interfaces with + BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled or Strict. [Default: Strict]' + type: string bpfExtToServiceConnmark: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local @@ -1190,6 +1238,51 @@ data: logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the @@ -1200,6 +1293,15 @@ data: Calico policy will be bypassed. [Default: insert]' type: string dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s]' type: string debugDisableLogDropping: type: boolean @@ -1228,9 +1330,14 @@ data: routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean @@ -1304,6 +1411,14 @@ data: "true" or "false" will force the feature, empty or omitted values are auto-detected. type: string + floatingIPs: + default: Disabled + description: FloatingIPs configures whether or not Felix will program + floating IP addresses. + enum: + - Enabled + - Disabled + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -1341,6 +1456,9 @@ data: disabled by setting the interval to 0. type: string ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: description: 'IPIPMTU is the MTU to set on the tunnel device. See @@ -1407,6 +1525,8 @@ data: usage. [Default: 10s]' type: string ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). type: boolean kubeNodePortRanges: description: 'KubeNodePortRanges holds list of port ranges used for @@ -1420,6 +1540,12 @@ data: pattern: ^.* x-kubernetes-int-or-string: true type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string logFilePath: description: 'LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]' @@ -1549,9 +1675,9 @@ data: routes. - CalicoIPAM: the default - use IPAM data to construct routes.' type: string routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. properties: max: type: integer @@ -1561,6 +1687,21 @@ data: - max - min type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting @@ -1588,12 +1729,22 @@ data: Felix makes reports. [Default: 86400s]' type: string useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. type: boolean vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for VXLAN networking. Optional as Felix determines + this based on the existing IP pools. [Default: nil (unset)]' type: boolean vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' type: integer vxlanPort: type: integer @@ -1611,6 +1762,10 @@ data: description: 'WireguardInterfaceName specifies the name to use for the Wireguard interface. [Default: wg.calico]' type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used by Wireguard. [Default: 51820]' @@ -1623,6 +1778,12 @@ data: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string xdpEnabled: description: 'XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]' @@ -2691,13 +2852,25 @@ data: resource. properties: affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. type: string allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. items: nullable: true type: integer type: array attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. items: properties: handle_id: @@ -2709,12 +2882,38 @@ data: type: object type: array cidr: + description: The block's CIDR. type: string deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. type: boolean unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. items: type: integer type: array @@ -2885,7 +3084,7 @@ data: type: array blockSize: description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. + this pool. Defaults to 26 for IPv4 and 122 for IPv6. type: integer cidr: description: The pool CIDR. @@ -2956,6 +3155,9 @@ data: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null name: ipreservations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -3104,6 +3306,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -3214,6 +3421,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -4166,7 +4378,6 @@ data: - apiGroups: - crd.projectcalico.org resources: - - ippools - ipreservations verbs: - list @@ -4183,6 +4394,13 @@ data: - update - delete - watch + - apiGroups: + - crd.projectcalico.org + resources: + - ippools + verbs: + - list + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4199,8 +4417,10 @@ data: - clusterinformations verbs: - get + - list - create - update + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4456,7 +4676,7 @@ data: value: node - name: DATASTORE_TYPE value: kubernetes - image: docker.io/calico/kube-controllers:v3.22.1 + image: docker.io/calico/kube-controllers:v3.23.0 livenessProbe: exec: command: @@ -4485,7 +4705,7 @@ data: - effect: NoSchedule key: node-role.kubernetes.io/control-plane --- - apiVersion: policy/v1beta1 + apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -4539,6 +4759,8 @@ data: value: autodetect - name: CALICO_IPV4POOL_IPIP value: Never + - name: CALICO_IPV6POOL_VXLAN + value: CrossSubnet - name: FELIX_IPINIPMTU valueFrom: configMapKeyRef: @@ -4566,7 +4788,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/node:v3.22.1 + image: docker.io/calico/node:v3.23.0 lifecycle: preStop: exec: @@ -4638,7 +4860,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: upgrade-ipam securityContext: privileged: true @@ -4672,7 +4894,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: install-cni securityContext: privileged: true @@ -4681,13 +4903,6 @@ data: name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir - - image: docker.io/calico/pod2daemon-flexvol:v3.22.1 - name: flexvol-driver - securityContext: - privileged: true - volumeMounts: - - mountPath: /host/driver - name: flexvol-driver-host nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical @@ -4734,10 +4949,6 @@ data: path: /var/run/nodeagent type: DirectoryOrCreate name: policysync - - hostPath: - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds - type: DirectoryOrCreate - name: flexvol-driver-host updateStrategy: rollingUpdate: maxUnavailable: 1 @@ -4789,7 +5000,7 @@ data: for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n \ - effect: NoExecute\n operator: Exists\n initContainers:\n # This container installs the CNI binaries\n # and CNI network config file - on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.22.1-hostprocess\n + on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.23.0-hostprocess\n \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n \ imagePullPolicy: Always\n env:\n # Name of the CNI config file to create.\n - name: CNI_CONF_NAME\n value: @@ -4808,7 +5019,7 @@ data: cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n \ securityContext:\n windowsOptions:\n hostProcess: true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n - \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.22.1-hostprocess\n + \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.23.0-hostprocess\n \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: @@ -4819,7 +5030,7 @@ data: name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n - \ image: sigwindowstools/calico-node:v3.22.1-hostprocess\n args: + \ image: sigwindowstools/calico-node:v3.23.0-hostprocess\n args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml index f713517f30b..14cc0d76a57 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml @@ -480,6 +480,12 @@ data: 64512]' format: int32 type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. @@ -510,6 +516,37 @@ data: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object nodeToNodeMeshEnabled: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' @@ -640,6 +677,12 @@ data: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer password: description: Optional BGP password for the peerings generated by this BGPPeer resource. @@ -1160,6 +1203,11 @@ data: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all interfaces with + BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled or Strict. [Default: Strict]' + type: string bpfExtToServiceConnmark: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local @@ -1199,6 +1247,51 @@ data: logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the @@ -1209,6 +1302,15 @@ data: Calico policy will be bypassed. [Default: insert]' type: string dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s]' type: string debugDisableLogDropping: type: boolean @@ -1237,9 +1339,14 @@ data: routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean @@ -1313,6 +1420,14 @@ data: "true" or "false" will force the feature, empty or omitted values are auto-detected. type: string + floatingIPs: + default: Disabled + description: FloatingIPs configures whether or not Felix will program + floating IP addresses. + enum: + - Enabled + - Disabled + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -1350,6 +1465,9 @@ data: disabled by setting the interval to 0. type: string ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: description: 'IPIPMTU is the MTU to set on the tunnel device. See @@ -1416,6 +1534,8 @@ data: usage. [Default: 10s]' type: string ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). type: boolean kubeNodePortRanges: description: 'KubeNodePortRanges holds list of port ranges used for @@ -1429,6 +1549,12 @@ data: pattern: ^.* x-kubernetes-int-or-string: true type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string logFilePath: description: 'LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]' @@ -1558,9 +1684,9 @@ data: routes. - CalicoIPAM: the default - use IPAM data to construct routes.' type: string routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. properties: max: type: integer @@ -1570,6 +1696,21 @@ data: - max - min type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting @@ -1597,12 +1738,22 @@ data: Felix makes reports. [Default: 86400s]' type: string useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. type: boolean vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for VXLAN networking. Optional as Felix determines + this based on the existing IP pools. [Default: nil (unset)]' type: boolean vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' type: integer vxlanPort: type: integer @@ -1620,6 +1771,10 @@ data: description: 'WireguardInterfaceName specifies the name to use for the Wireguard interface. [Default: wg.calico]' type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used by Wireguard. [Default: 51820]' @@ -1632,6 +1787,12 @@ data: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string xdpEnabled: description: 'XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]' @@ -2700,13 +2861,25 @@ data: resource. properties: affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. type: string allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. items: nullable: true type: integer type: array attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. items: properties: handle_id: @@ -2718,12 +2891,38 @@ data: type: object type: array cidr: + description: The block's CIDR. type: string deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. type: boolean unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. items: type: integer type: array @@ -2894,7 +3093,7 @@ data: type: array blockSize: description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. + this pool. Defaults to 26 for IPv4 and 122 for IPv6. type: integer cidr: description: The pool CIDR. @@ -2965,6 +3164,9 @@ data: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null name: ipreservations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -3113,6 +3315,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -3223,6 +3430,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -4175,7 +4387,6 @@ data: - apiGroups: - crd.projectcalico.org resources: - - ippools - ipreservations verbs: - list @@ -4192,6 +4403,13 @@ data: - update - delete - watch + - apiGroups: + - crd.projectcalico.org + resources: + - ippools + verbs: + - list + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4208,8 +4426,10 @@ data: - clusterinformations verbs: - get + - list - create - update + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4465,7 +4685,7 @@ data: value: node - name: DATASTORE_TYPE value: kubernetes - image: docker.io/calico/kube-controllers:v3.22.1 + image: docker.io/calico/kube-controllers:v3.23.0 livenessProbe: exec: command: @@ -4494,7 +4714,7 @@ data: - effect: NoSchedule key: node-role.kubernetes.io/control-plane --- - apiVersion: policy/v1beta1 + apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -4548,6 +4768,8 @@ data: value: autodetect - name: CALICO_IPV4POOL_IPIP value: Never + - name: CALICO_IPV6POOL_VXLAN + value: CrossSubnet - name: FELIX_IPINIPMTU valueFrom: configMapKeyRef: @@ -4575,7 +4797,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/node:v3.22.1 + image: docker.io/calico/node:v3.23.0 lifecycle: preStop: exec: @@ -4647,7 +4869,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: upgrade-ipam securityContext: privileged: true @@ -4681,7 +4903,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: install-cni securityContext: privileged: true @@ -4690,13 +4912,6 @@ data: name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir - - image: docker.io/calico/pod2daemon-flexvol:v3.22.1 - name: flexvol-driver - securityContext: - privileged: true - volumeMounts: - - mountPath: /host/driver - name: flexvol-driver-host nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical @@ -4743,10 +4958,6 @@ data: path: /var/run/nodeagent type: DirectoryOrCreate name: policysync - - hostPath: - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds - type: DirectoryOrCreate - name: flexvol-driver-host updateStrategy: rollingUpdate: maxUnavailable: 1 @@ -4798,7 +5009,7 @@ data: for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n \ - effect: NoExecute\n operator: Exists\n initContainers:\n # This container installs the CNI binaries\n # and CNI network config file - on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.22.1-hostprocess\n + on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.23.0-hostprocess\n \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n \ imagePullPolicy: Always\n env:\n # Name of the CNI config file to create.\n - name: CNI_CONF_NAME\n value: @@ -4817,7 +5028,7 @@ data: cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n \ securityContext:\n windowsOptions:\n hostProcess: true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n - \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.22.1-hostprocess\n + \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.23.0-hostprocess\n \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: @@ -4828,7 +5039,7 @@ data: name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n - \ image: sigwindowstools/calico-node:v3.22.1-hostprocess\n args: + \ image: sigwindowstools/calico-node:v3.23.0-hostprocess\n args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index b155b66b0e4..1e8b67c3371 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -612,6 +612,12 @@ data: 64512]' format: int32 type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string communities: description: Communities is a list of BGP community values and their arbitrary names for tagging routes. @@ -642,6 +648,37 @@ data: description: 'LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]' type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object nodeToNodeMeshEnabled: description: 'NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]' @@ -772,6 +809,12 @@ data: description: Selector for the nodes that should have this peering. When this is set, the Node field must be empty. type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer password: description: Optional BGP password for the peerings generated by this BGPPeer resource. @@ -1292,6 +1335,11 @@ data: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all interfaces with + BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled or Strict. [Default: Strict]' + type: string bpfExtToServiceConnmark: description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit mark that is set on connections from an external client to a local @@ -1331,6 +1379,51 @@ data: logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. [Default: Off].' type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true chainInsertMode: description: 'ChainInsertMode controls whether Felix hooks the kernel''s top-level iptables chains by inserting a rule at the top of the @@ -1341,6 +1434,15 @@ data: Calico policy will be bypassed. [Default: insert]' type: string dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s]' type: string debugDisableLogDropping: type: boolean @@ -1369,9 +1471,14 @@ data: routes, by default this will be RTPROT_BOOT when left blank. type: integer deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. type: string disableConntrackInvalidCheck: type: boolean @@ -1445,6 +1552,14 @@ data: "true" or "false" will force the feature, empty or omitted values are auto-detected. type: string + floatingIPs: + default: Disabled + description: FloatingIPs configures whether or not Felix will program + floating IP addresses. + enum: + - Enabled + - Disabled + type: string genericXDPEnabled: description: 'GenericXDPEnabled enables Generic XDP so network cards that don''t support XDP offload or driver modes can use XDP. This @@ -1482,6 +1597,9 @@ data: disabled by setting the interval to 0. type: string ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' type: boolean ipipMTU: description: 'IPIPMTU is the MTU to set on the tunnel device. See @@ -1548,6 +1666,8 @@ data: usage. [Default: 10s]' type: string ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). type: boolean kubeNodePortRanges: description: 'KubeNodePortRanges holds list of port ranges used for @@ -1561,6 +1681,12 @@ data: pattern: ^.* x-kubernetes-int-or-string: true type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string logFilePath: description: 'LogFilePath is the full path to the Felix log. Set to none to disable file logging. [Default: /var/log/calico/felix.log]' @@ -1690,9 +1816,9 @@ data: routes. - CalicoIPAM: the default - use IPAM data to construct routes.' type: string routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. properties: max: type: integer @@ -1702,6 +1828,21 @@ data: - max - min type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array serviceLoopPrevention: description: 'When service IP advertisement is enabled, prevent routing loops to service IPs that are not in use, by dropping or rejecting @@ -1729,12 +1870,22 @@ data: Felix makes reports. [Default: 86400s]' type: string useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. type: boolean vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for VXLAN networking. Optional as Felix determines + this based on the existing IP pools. [Default: nil (unset)]' type: boolean vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' type: integer vxlanPort: type: integer @@ -1752,6 +1903,10 @@ data: description: 'WireguardInterfaceName specifies the name to use for the Wireguard interface. [Default: wg.calico]' type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string wireguardListeningPort: description: 'WireguardListeningPort controls the listening port used by Wireguard. [Default: 51820]' @@ -1764,6 +1919,12 @@ data: description: 'WireguardRoutingRulePriority controls the priority value to use for the Wireguard routing rule. [Default: 99]' type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string xdpEnabled: description: 'XDPEnabled enables XDP acceleration for suitable untracked incoming deny rules. [Default: true]' @@ -2832,13 +2993,25 @@ data: resource. properties: affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. type: string allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. items: nullable: true type: integer type: array attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. items: properties: handle_id: @@ -2850,12 +3023,38 @@ data: type: object type: array cidr: + description: The block's CIDR. type: string deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. type: boolean unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. items: type: integer type: array @@ -3026,7 +3225,7 @@ data: type: array blockSize: description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. + this pool. Defaults to 26 for IPv4 and 122 for IPv6. type: integer cidr: description: The pool CIDR. @@ -3097,6 +3296,9 @@ data: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null name: ipreservations.crd.projectcalico.org spec: group: crd.projectcalico.org @@ -3245,6 +3447,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -3355,6 +3562,11 @@ data: type: string type: object type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer etcdV3CompactionPeriod: description: 'EtcdV3CompactionPeriod is the period between etcdv3 compaction requests. Set to 0 to disable. [Default: 10m]' @@ -4307,7 +4519,6 @@ data: - apiGroups: - crd.projectcalico.org resources: - - ippools - ipreservations verbs: - list @@ -4324,6 +4535,13 @@ data: - update - delete - watch + - apiGroups: + - crd.projectcalico.org + resources: + - ippools + verbs: + - list + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4340,8 +4558,10 @@ data: - clusterinformations verbs: - get + - list - create - update + - watch - apiGroups: - crd.projectcalico.org resources: @@ -4597,7 +4817,7 @@ data: value: node - name: DATASTORE_TYPE value: kubernetes - image: docker.io/calico/kube-controllers:v3.22.1 + image: docker.io/calico/kube-controllers:v3.23.0 livenessProbe: exec: command: @@ -4626,7 +4846,7 @@ data: - effect: NoSchedule key: node-role.kubernetes.io/control-plane --- - apiVersion: policy/v1beta1 + apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: @@ -4680,6 +4900,8 @@ data: value: autodetect - name: CALICO_IPV4POOL_IPIP value: Never + - name: CALICO_IPV6POOL_VXLAN + value: CrossSubnet - name: FELIX_IPINIPMTU valueFrom: configMapKeyRef: @@ -4707,7 +4929,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/node:v3.22.1 + image: docker.io/calico/node:v3.23.0 lifecycle: preStop: exec: @@ -4779,7 +5001,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: upgrade-ipam securityContext: privileged: true @@ -4813,7 +5035,7 @@ data: - configMapRef: name: kubernetes-services-endpoint optional: true - image: docker.io/calico/cni:v3.22.1 + image: docker.io/calico/cni:v3.23.0 name: install-cni securityContext: privileged: true @@ -4822,13 +5044,6 @@ data: name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir - - image: docker.io/calico/pod2daemon-flexvol:v3.22.1 - name: flexvol-driver - securityContext: - privileged: true - volumeMounts: - - mountPath: /host/driver - name: flexvol-driver-host nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical @@ -4875,10 +5090,6 @@ data: path: /var/run/nodeagent type: DirectoryOrCreate name: policysync - - hostPath: - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds - type: DirectoryOrCreate - name: flexvol-driver-host updateStrategy: rollingUpdate: maxUnavailable: 1 @@ -4930,7 +5141,7 @@ data: for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n \ - effect: NoExecute\n operator: Exists\n initContainers:\n # This container installs the CNI binaries\n # and CNI network config file - on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.22.1-hostprocess\n + on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.23.0-hostprocess\n \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n \ imagePullPolicy: Always\n env:\n # Name of the CNI config file to create.\n - name: CNI_CONF_NAME\n value: @@ -4949,7 +5160,7 @@ data: cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n \ securityContext:\n windowsOptions:\n hostProcess: true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n - \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.22.1-hostprocess\n + \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.23.0-hostprocess\n \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: @@ -4960,7 +5171,7 @@ data: name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n - \ image: sigwindowstools/calico-node:v3.22.1-hostprocess\n args: + \ image: sigwindowstools/calico-node:v3.23.0-hostprocess\n args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n