Skip to content

Commit

Permalink
Override audit policy for conformance informing jobs.
Browse files Browse the repository at this point in the history 
Co-authored-by: Aaron Crickenberger <[email protected]>
  • Loading branch information
@hh @spiffxp
hh and spiffxp committed Oct 16, 2020
1 parent 84f802e commit 57d7311
Showing 1 changed file with 173 additions and 0 deletions.
173 changes: 173 additions & 0 deletions config/jobs/kubernetes/sig-cloud-provider/gcp/gce-conformance.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,179 @@ periodics:
requests:
cpu: 1
memory: 3Gi
env:
# cncf/apisnoop needs to consume audit logs from the cluster under test in order to
# effectively report on conformance test coverage of api operations. The default audit
# policy excludes events from being logged due to performance reasons. For this
# job we will accept the performance hit and allow events to be logged so that we can
# measure coverage of api operations that involve events.
#
# We do this by setting ENABLE_APISERVER_ADVANCED_AUDIT which tells
# cluster/gce/gci/configure-kubeapiserver.sh to override the default audit policy with the
# contents of the ADVANCED_AUDIT_POLICY env var. We've copy-pasted the default
# audit policy as output by https://github.com/kubernetes/kubernetes/blob/b1e3a2ac7a80d49ef6153457eaa35dac4fc98444/cluster/gce/gci/configure-helper.sh#L1077-L1235
# and then removed the block that says "don't log events"
#
# NOTE: if the default audit policy changes, this will need to be updated accordingly
# TODO(https://github.com/kubernetes/test-infra/issues/19613)
# ^^ ideally we could do this via a patch instead
- name: ENABLE_APISERVER_ADVANCED_AUDIT
value: "true"
- name: ADVANCED_AUDIT_POLICY
value: |-
apiVersion: audit.k8s.io/v1
kind: Policy
rules:
# The following requests were manually identified as high-volume and low-risk,
# so drop them.
- level: None
users: ["system:kube-proxy"]
verbs: ["watch"]
resources:
- group: "" # core
resources: ["endpoints", "services", "services/status"]
- level: None
# Ingress controller reads 'configmaps/ingress-uid' through the unsecured port.
# TODO(#46983): Change this to the ingress controller service account.
users: ["system:unsecured"]
namespaces: ["kube-system"]
verbs: ["get"]
resources:
- group: "" # core
resources: ["configmaps"]
- level: None
users: ["kubelet"] # legacy kubelet identity
verbs: ["get"]
resources:
- group: "" # core
resources: ["nodes", "nodes/status"]
- level: None
userGroups: ["system:nodes"]
verbs: ["get"]
resources:
- group: "" # core
resources: ["nodes", "nodes/status"]
- level: None
users:
- system:kube-controller-manager
- system:kube-scheduler
- system:serviceaccount:kube-system:endpoint-controller
verbs: ["get", "update"]
namespaces: ["kube-system"]
resources:
- group: "" # core
resources: ["endpoints"]
- level: None
users: ["system:apiserver"]
verbs: ["get"]
resources:
- group: "" # core
resources: ["namespaces", "namespaces/status", "namespaces/finalize"]
- level: None
users: ["cluster-autoscaler"]
verbs: ["get", "update"]
namespaces: ["kube-system"]
resources:
- group: "" # core
resources: ["configmaps", "endpoints"]
# Don't log HPA fetching metrics.
- level: None
users:
- system:kube-controller-manager
verbs: ["get", "list"]
resources:
- group: "metrics.k8s.io"
# Don't log these read-only URLs.
- level: None
nonResourceURLs:
- /healthz*
- /version
- /swagger*
# node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes
- level: Request
users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"]
verbs: ["update","patch"]
resources:
- group: "" # core
resources: ["nodes/status", "pods/status"]
omitStages:
- "RequestReceived"
- level: Request
userGroups: ["system:nodes"]
verbs: ["update","patch"]
resources:
- group: "" # core
resources: ["nodes/status", "pods/status"]
omitStages:
- "RequestReceived"
# deletecollection calls can be large, don't log responses for expected namespace deletions
- level: Request
users: ["system:serviceaccount:kube-system:namespace-controller"]
verbs: ["deletecollection"]
omitStages:
- "RequestReceived"
# Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data,
# so only log at the Metadata level.
- level: Metadata
resources:
- group: "" # core
resources: ["secrets", "configmaps"]
- group: authentication.k8s.io
resources: ["tokenreviews"]
omitStages:
- "RequestReceived"
# Get repsonses can be large; skip them.
- level: Request
verbs: ["get", "list", "watch"]
resources:
- group: "" # core
- group: "admissionregistration.k8s.io"
- group: "apiextensions.k8s.io"
- group: "apiregistration.k8s.io"
- group: "apps"
- group: "authentication.k8s.io"
- group: "authorization.k8s.io"
- group: "autoscaling"
- group: "batch"
- group: "certificates.k8s.io"
- group: "extensions"
- group: "metrics.k8s.io"
- group: "networking.k8s.io"
- group: "node.k8s.io"
- group: "policy"
- group: "rbac.authorization.k8s.io"
- group: "scheduling.k8s.io"
- group: "storage.k8s.io"
omitStages:
- "RequestReceived"
# Default level for known APIs
- level: RequestResponse
resources:
- group: "" # core
- group: "admissionregistration.k8s.io"
- group: "apiextensions.k8s.io"
- group: "apiregistration.k8s.io"
- group: "apps"
- group: "authentication.k8s.io"
- group: "authorization.k8s.io"
- group: "autoscaling"
- group: "batch"
- group: "certificates.k8s.io"
- group: "extensions"
- group: "metrics.k8s.io"
- group: "networking.k8s.io"
- group: "node.k8s.io"
- group: "policy"
- group: "rbac.authorization.k8s.io"
- group: "scheduling.k8s.io"
- group: "storage.k8s.io"
omitStages:
- "RequestReceived"
# Default level for all other requests.
- level: Metadata
omitStages:
- "RequestReceived"
- interval: 3h
cluster: k8s-infra-prow-build
name: ci-kubernetes-gce-conformance-latest-kubetest2
Expand Down

0 comments on commit 57d7311

Please sign in to comment.