From 89bfae96bc1c5b66818591a52d2a4b1499321a8f Mon Sep 17 00:00:00 2001 From: Michael Burke Date: Tue, 16 Apr 2019 16:40:34 -0400 Subject: [PATCH] [WIP] document how to gather the audit logs 2 --- _topic_map.yml | 2 + modules/nodes-nodes-audit-log-advanced.adoc | 140 +++++++++++++ .../nodes-nodes-audit-log-basic-config.adoc | 184 ++++++++++++++++++ .../nodes-nodes-audit-log-basic-viewing.adoc | 88 +++++++++ modules/nodes-nodes-audit-log-basic.adoc | 33 ++++ nodes/nodes/nodes-nodes-audit-log.adoc | 25 +++ 6 files changed, 472 insertions(+) create mode 100644 modules/nodes-nodes-audit-log-advanced.adoc create mode 100644 modules/nodes-nodes-audit-log-basic-config.adoc create mode 100644 modules/nodes-nodes-audit-log-basic-viewing.adoc create mode 100644 modules/nodes-nodes-audit-log-basic.adoc create mode 100644 nodes/nodes/nodes-nodes-audit-log.adoc diff --git a/_topic_map.yml b/_topic_map.yml index 4aa34e3d4d..32ad49a3e6 100644 --- a/_topic_map.yml +++ b/_topic_map.yml @@ -443,6 +443,8 @@ Topics: File: nodes-nodes-opaque-resources # - Name: Monitoring for problems in your nodes # File: nodes-nodes-problem-detector + - Name: Viewing node audit logs + File: nodes-nodes-audit-log - Name: Working with containers Dir: containers Topics: diff --git a/modules/nodes-nodes-audit-log-advanced.adoc b/modules/nodes-nodes-audit-log-advanced.adoc new file mode 100644 index 0000000000..afcfc025c7 --- /dev/null +++ b/modules/nodes-nodes-audit-log-advanced.adoc @@ -0,0 +1,140 @@ +// Module included in the following assemblies: +// +// * nodes/nodes-audit-log.adoc + +[id="nodes-nodes-audit-log-advanced-{context}"] += Advanced Audit + +*DEPRECATED for the moment* + +The advanced audit feature provides several improvements over the +basic audit functionality, including fine-grained events filtering and multiple output back ends. + +To enable the advanced audit feature, provide the following values in the +`openshift_master_audit_config` parameter: + +---- +openshift_master_audit_config={"enabled": true, "auditFilePath": "/var/lib/origin/oscp-audit.log", "maximumFileRetentionDays": 14, "maximumFileSizeMegabytes": 500, "maximumRetainedFiles": 5, "policyFile": "/etc/origin/master/adv-audit.yaml", "logFormat":"json"} +---- + +[IMPORTANT] +==== +The policy file *_/etc/origin/master/adv-audit.yaml_* must be available on each master node. +==== + + +The following table contains additional options you can use. + +.Advanced Audit Configuration Parameters + +[cols="3a,6a",options="header"] +|=== +| Parameter Name | Description + +|`policyFile` +|Path to the file that defines the audit policy configuration. + +|`policyConfiguration` +|An embedded audit policy configuration. + +|`logFormat` +|Specifies the format of the saved audit logs. Allowed values are `legacy` (the +format used in basic audit), and `json`. + +|`webHookKubeConfig` +|Path to a `.kubeconfig`-formatted file that defines the audit webhook +configuration, where the events are sent to. + +|`webHookMode` +|Specifies the strategy for sending audit events. Allowed values are `block` +(blocks processing another event until the previous has fully processed) and +`batch` (buffers events and delivers in batches). +|=== + +[IMPORTANT] +==== +To enable the advanced audit feature, you must provide either `policyFile` *or* +`policyConfiguration` describing the audit policy rules: +==== + +.Sample Audit Policy Configuration +[source,yaml] +---- +apiVersion: audit.k8s.io/v1beta1 +kind: Policy +rules: + + # Do not log watch requests by the "system:kube-proxy" on endpoints or services + - level: None <1> + users: ["system:kube-proxy"] <2> + verbs: ["watch"] <3> + resources: <4> + - group: "" + resources: ["endpoints", "services"] + + # Do not log authenticated requests to certain non-resource URL paths. + - level: None + userGroups: ["system:authenticated"] <5> + nonResourceURLs: <6> + - "/api*" # Wildcard matching. + - "/version" + + # Log the request body of configmap changes in kube-system. + - level: Request + resources: + - group: "" # core API group + resources: ["configmaps"] + # This rule only applies to resources in the "kube-system" namespace. + # The empty string "" can be used to select non-namespaced resources. + namespaces: ["kube-system"] <7> + + # Log configmap and secret changes in all other namespaces at the metadata level. + - level: Metadata + resources: + - group: "" # core API group + resources: ["secrets", "configmaps"] + + # Log all other resources in core and extensions at the request level. + - level: Request + resources: + - group: "" # core API group + - group: "extensions" # Version of group should NOT be included. + + # A catch-all rule to log all other requests at the Metadata level. + - level: Metadata <1> + + # Log login failures from the web console or CLI. Review the logs and refine your policies. + - level: Metadata + nonResourceURLs: + - /login* <8> + - /oauth* <9> +---- +<1> There are four possible levels every event can be logged at: ++ +* `None` - Do not log events that match this rule. ++ +* `Metadata` - Log request metadata (requesting user, time stamp, resource, verb, +etc.), but not request or response body. This is the same level as the one used +in basic audit. ++ +* `Request` - Log event metadata and request body, but not response body. ++ +* `RequestResponse` - Log event metadata, request, and response bodies. +<2> A list of users the rule applies to. An empty list implies every user. +<3> A list of verbs this rule applies to. An empty list implies every verb. This is + Kubernetes verb associated with API requests (including `get`, `list`, `watch`, + `create`, `update`, `patch`, `delete`, `deletecollection`, and `proxy`). +<4> A list of resources the rule applies to. An empty list implies every resource. +Each resource is specified as a group it is assigned to (for example, an empty for +Kubernetes core API, batch, build.openshift.io, etc.), and a resource list from +that group. +<5> A list of groups the rule applies to. An empty list implies every group. +<6> A list of non-resources URLs the rule applies to. +<7> A list of namespaces the rule applies to. An empty list implies every namespace. +<8> Endpoint used by the web console. +<9> Endpoint used by the CLI. + +For more information on advanced audit, see the +link:https://kubernetes.io/docs/tasks/debug-application-cluster/audit[Kubernetes +documentation] + diff --git a/modules/nodes-nodes-audit-log-basic-config.adoc b/modules/nodes-nodes-audit-log-basic-config.adoc new file mode 100644 index 0000000000..bc26871074 --- /dev/null +++ b/modules/nodes-nodes-audit-log-basic-config.adoc @@ -0,0 +1,184 @@ +// Module included in the following assemblies: +// +// * nodes/nodes-audit-log.adoc + +[id="nodes-nodes-audit-log-basic-config-{context}"] += Configuring the API Audit Log level + + +You can configure the the audit feature to set log level, retention policy, and the type of events to log. + +//This is a best guess attempt inspired by https://github.com/openshift/api/pull/279 + +.Procedure + +. Set the audit log level: + +.. Get the name of the API server Custom Resource (CR): ++ +---- +$ oc get APIServer + +NAME AGE +cluster 18h +---- + +.. Edit the API server CR: ++ +---- +$ oc edit APIServer cluster +---- ++ +[source,yaml] +---- +apiVersion: config.openshift.io/v1 +kind: APIServer +metadata: + annotations: + release.openshift.io/create-only: "true" + creationTimestamp: 2019-05-09T18:56:37Z + generation: 2 + name: cluster + resourceVersion: "562502" + selfLink: /apis/config.openshift.io/v1/apiservers/cluster + uid: 2c4e980b-728c-11e9-967d-0a973adad40e +spec: + logLevel: "Normal" <1> + operatorloglevel: "TraceAll" <2> +---- +<1> Specify the log level for an overall component. +<2> Specify the log level for the API server Operator. ++ +You can set the log level for both settings to one of the following. The setting can be different for each setting: ++ +* *Normal*. Normal is the default. Normal working log information, including helpful notices for auditing or common operations. Similar to `glog=2`. +* *Debug*. Debug is for troubleshooting problems. A greater quanitity of notices than *Normal*, but contain less information than *Trace*. Common operations might be logged. Similar to `glog=4`. +* *Trace*. Trace is for troubleshooting problems when *Debug* is not verbose enough. Logs every function call as part of a common operation, including tracing execution of a query. Similar to `glog=6`. +* *TraceAll*. TraceAll is troubleshoting at the level of API content/decoding. Contains complete body content. In production clusters, this setting causes performance degradation and results in a significant number of logs. Similar to ` glog=8`. + +//// +. Configure the audit parameters + +.. Edit the auditing configuration file on a API server pod in the `openshift-kube-apiserver` project: ++ +---- +$ oc -n openshift-kube-apiserver exec cat /etc/kubernetes/static-pod-resources/configmaps/config/config.yaml |json_reformat +---- ++ +For example: ++ +---- +$ oc -n openshift-kube-apiserver exec kube-apiserver-ip-10-0-129-10.ec2.internal cat /etc/kubernetes/static-pod-resources/configmaps/config/config.yaml |json_reformat +---- + +.. Edit the `auditConfig` stanza as needed: ++ +---- + "auditConfig": { + "auditFilePath": "/var/log/kube-apiserver/audit.log", <1> + "enabled": true, <2> + "logFormat": "json", <3> + "maximumFileSizeMegabytes": 100, <4> + "maximumRetainedFiles": 10, <5> + "policyConfiguration": { <6> + "apiVersion": "audit.k8s.io/v1beta1", + "kind": "Policy", + "omitStages": [ + "RequestReceived" + ], + "rules": [ + { + "level": "None", + "resources": [ + { + "group": "", + "resources": [ + "events" + ] + } + ] + }, + { + "level": "None", + "nonResourceURLs": [ + "/api*", + "/version", + "/healthz", + "/readyz" + ], + "userGroups": [ + "system:authenticated", + "system:unauthenticated" + ] + }, + { + "level": "Metadata", + "omitStages": [ + "RequestReceived" + ] + } + ] + } + }, + +---- +<1> File path where the requests should be logged to. If not set, logs are printed to master logs. +<2> A boolean to enable or disable audit logs. Default is `true`. +<3> Specifies maximum number of days to retain old audit log files based on the time stamp encoded in their filename. +<4> Specifies the maximum number of old audit log files to retain. +<5> Specifies maximum size in megabytes of the log file before it gets rotated. Defaults to 100MB. +<6> Configure the audit policy, described in the next step. + +.. Configure the audit policy setting: ++ +---- + "policyConfiguration": { + "apiVersion": "audit.k8s.io/v1beta1", + "kind": "Policy", + "omitStages": [ + "RequestReceived" <1> + ], + "rules": [ + { + "level": "None", <2> + "resources": [ + { + "group": "", + "resources": [ + "events" + ] + } + ] + }, + { + "level": "None", + "nonResourceURLs": [ <3> + "/api*", + "/version", + "/healthz", + "/readyz" + ], + "userGroups": [ <4> + "system:authenticated", + "system:unauthenticated" + ] + }, + { + "level": "Metadata", <5> + "omitStages": [ + "RequestReceived" + ] +---- +<1> Configure whether to generate audit events for all requests in RequestReceived stage. +<2> Configure whether to log API server events. +<3> Configure whether to log authenticated requests to certain non-resource URL paths. +<4> Configure a list of users the rule applies to. +<5> Configure a catch-all rule to log all other requests at the Metadata level. Long-running requests like watches that fall under this rule will not generate an audit event in RequestReceived. ++ +For each rule, set `level` to one of the following: +* None. Do not log events that match this rule. +* Metadata - Log request metadata: requesting user, timestamp, resource, verb, and so forth. Do not request or response body. +* Request - Log event metadata and request body but not response body. This does not apply for non-resource requests. +* RequestReceived - Log event metadata, request and response bodies. This does not apply for non-resource requests. +//// + diff --git a/modules/nodes-nodes-audit-log-basic-viewing.adoc b/modules/nodes-nodes-audit-log-basic-viewing.adoc new file mode 100644 index 0000000000..e2e8612868 --- /dev/null +++ b/modules/nodes-nodes-audit-log-basic-viewing.adoc @@ -0,0 +1,88 @@ +// Module included in the following assemblies: +// +// * nodes/nodes-audit-log.adoc + +[id="nodes-nodes-audit-log-basic-viewing-{context}"] += Viewing the API Audit Log + +You can view the basic audit log. + +.Procedure + +To view the basic audit log: + +. View the {product-title} API server logs + +.. If necessary, get the node IP and name of the log you want to view: ++ +---- +$ oc --insecure-skip-tls-verify adm node-logs --role=master --path=openshift-apiserver/ + +ip-10-0-140-97.ec2.internal audit-2019-04-09T00-12-19.834.log +ip-10-0-140-97.ec2.internal audit-2019-04-09T11-13-00.469.log +ip-10-0-140-97.ec2.internal audit.log +ip-10-0-153-35.ec2.internal audit-2019-04-09T00-11-49.835.log +ip-10-0-153-35.ec2.internal audit-2019-04-09T11-08-30.469.log +ip-10-0-153-35.ec2.internal audit.log +ip-10-0-170-165.ec2.internal audit-2019-04-09T00-13-00.128.log +ip-10-0-170-165.ec2.internal audit-2019-04-09T11-10-04.082.log +ip-10-0-170-165.ec2.internal audit.log +---- + +.. View the {product-title} API server log for a specific master node and timestamp or view all the logs for that master: ++ +---- +$ oc adm node-logs --path=openshift-apiserver/ +---- ++ +For example: ++ +---- +$ oc adm node-logs ip-10-0-140-97.ec2.internal audit-2019-04-08T13-09-01.227.log --path=openshift-apiserver/audit-2019-04-08T13-09-01.227.log +$ oc adm node-logs ip-10-0-140-97.ec2.internal audit.log --path=openshift-apiserver/audit.log +---- ++ +The output appears similar to the following: ++ +---- +ip-10-0-140-97.ec2.internal {"kind":"Event","apiVersion":"audit.k8s.io/v1beta1","metadata":{"creationTimestamp":"2019-04-09T18:52:03Z"},"level":"Metadata","timestamp":"2019-04-09T18:52:03Z","auditID":"9708b50d-8956-4c87-b9eb-a53ba054c13d","stage":"ResponseComplete","requestURI":"/","verb":"get","user":{"username":"system:anonymous","groups":["system:unauthenticated"]},"sourceIPs":["10.128.0.1"],"userAgent":"Go-http-client/2.0","responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2019-04-09T18:52:03.914638Z","stageTimestamp":"2019-04-09T18:52:03.915080Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"cluster-status-binding\" of ClusterRole \"cluster-status\" to Group \"system:unauthenticated\""}} +---- + +. View the Kubernetes API server logs: + +.. If necessary, get the node IP and name of the log you want to view: ++ +---- +$ oc --insecure-skip-tls-verify adm node-logs --role=master --path=kube-apiserver/ + +ip-10-0-140-97.ec2.internal audit-2019-04-09T14-07-27.129.log +ip-10-0-140-97.ec2.internal audit-2019-04-09T19-18-32.542.log +ip-10-0-140-97.ec2.internal audit.log +ip-10-0-153-35.ec2.internal audit-2019-04-09T19-24-22.620.log +ip-10-0-153-35.ec2.internal audit-2019-04-09T19-51-30.905.log +ip-10-0-153-35.ec2.internal audit.log +ip-10-0-170-165.ec2.internal audit-2019-04-09T18-37-07.511.log +ip-10-0-170-165.ec2.internal audit-2019-04-09T19-21-14.371.log +ip-10-0-170-165.ec2.internal audit.log +---- + +.. View the Kubernetes API server log for a specific master node and timestamp or view all the logs for that master: ++ +---- +$ oc adm node-logs --path=kube-apiserver/ +---- ++ +For example: ++ +---- +$ oc adm node-logs ip-10-0-140-97.ec2.internal audit-2019-04-09T14-07-27.129.log --path=kube-apiserver/audit-2019-04-09T14-07-27.129.log +$ oc adm node-logs ip-10-0-170-165.ec2.internal audit.log --path=kube-apiserver/audit.log +---- ++ +The output appears similar to the following: ++ +---- +ip-10-0-140-97.ec2.internal {"kind":"Event","apiVersion":"audit.k8s.io/v1beta1","metadata":{"creationTimestamp":"2019-04-09T19:56:58Z"},"level":"Metadata","timestamp":"2019-04-09T19:56:58Z","auditID":"6e96c88b-ab6f-44d2-b62e-d1413efd676b","stage":"ResponseComplete","requestURI":"/api/v1/nodes/audit-2019-04-09T14-07-27.129.log","verb":"get","user":{"username":"kube:admin","groups":["system:cluster-admins","system:authenticated"],"extra":{"scopes.authorization.openshift.io":["user:full"]}},"sourceIPs":["10.0.57.93"],"userAgent":"oc/v1.12.4+ba88cb26ba (linux/amd64) kubernetes/ba88cb2","objectRef":{"resource":"nodes","name":"audit-2019-04-09T14-07-27.129.log","apiVersion":"v1"},"responseStatus":{"metadata":{},"status":"Failure","reason":"NotFound","code":404},"requestReceivedTimestamp":"2019-04-09T19:56:58.982157Z","stageTimestamp":"2019-04-09T19:56:58.985300Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"cluster-admins\" of ClusterRole \"cluster-admin\" to Group \"system:cluster-admins\""}} +---- + + diff --git a/modules/nodes-nodes-audit-log-basic.adoc b/modules/nodes-nodes-audit-log-basic.adoc new file mode 100644 index 0000000000..2fa3e46d7a --- /dev/null +++ b/modules/nodes-nodes-audit-log-basic.adoc @@ -0,0 +1,33 @@ +// Module included in the following assemblies: +// +// * nodes/nodes-audit-log.adoc + +[id="nodes-pods-audit-log-basic-{context}"] += About the API Audit Log + +Audit works at the API server level, logging all requests coming to the server. +Each audit log contains two entries: + +. The request line containing: +.. A Unique ID allowing to match the response line (see #2) +.. The source IP of the request +.. The HTTP method being invoked +.. The original user invoking the operation +.. The impersonated user for the operation (`self` meaning himself) +.. The impersonated group for the operation (`lookup` meaning user's group) +.. The namespace of the request or +.. The URI as requested + +. The response line containing: +.. The unique ID from #1 +.. The response code + +You can view logs for the master nodes for the {product-title} API server or the Kubernetes API server. + +Example output for the Kubelet API server: + +---- +ip-10-0-140-97.ec2.internal {"kind":"Event","apiVersion":"audit.k8s.io/v1beta1","metadata":{"creationTimestamp":"2019-04-09T19:56:58Z"},"level":"Metadata","timestamp":"2019-04-09T19:56:58Z","auditID":"6e96c88b-ab6f-44d2-b62e-d1413efd676b","stage":"ResponseComplete","requestURI":"/api/v1/nodes/audit-2019-04-09T14-07-27.129.log","verb":"get","user":{"username":"kube:admin","groups":["system:cluster-admins","system:authenticated"],"extra":{"scopes.authorization.openshift.io":["user:full"]}},"sourceIPs":["10.0.57.93"],"userAgent":"oc/v1.12.4+ba88cb26ba (linux/amd64) kubernetes/ba88cb2","objectRef":{"resource":"nodes","name":"audit-2019-04-09T14-07-27.129.log","apiVersion":"v1"},"responseStatus":{"metadata":{},"status":"Failure","reason":"NotFound","code":404},"requestReceivedTimestamp":"2019-04-09T19:56:58.982157Z","stageTimestamp":"2019-04-09T19:56:58.985300Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"cluster-admins\" of ClusterRole \"cluster-admin\" to Group \"system:cluster-admins\""}} +---- + + diff --git a/nodes/nodes/nodes-nodes-audit-log.adoc b/nodes/nodes/nodes-nodes-audit-log.adoc new file mode 100644 index 0000000000..e86fb1bcde --- /dev/null +++ b/nodes/nodes/nodes-nodes-audit-log.adoc @@ -0,0 +1,25 @@ +:context: nodes-nodes-working +[id="nodes-nodes-audit-log-{context}"] += Viewing node audit logs +include::modules/common-attributes.adoc[] + +toc::[] + + +Audit provides a security-relevant chronological set of records documenting the +sequence of activities that have affected system by individual users, +administrators, or other components of the system. + + +// The following include statements pull in the module files that comprise +// the assembly. Include any combination of concept, procedure, or reference +// modules required to cover the user story. You can also include other +// assemblies. + + +include::modules/nodes-nodes-audit-log-basic.adoc[leveloffset=+1] + +include::modules/nodes-nodes-audit-log-basic-config.adoc[leveloffset=+1] + +include::modules/nodes-nodes-audit-log-basic-viewing.adoc[leveloffset=+1] +