From b65313d871cf9931596cd5aaf83c43f4f8e12eab Mon Sep 17 00:00:00 2001 From: Sara Thomas Date: Fri, 16 Aug 2024 15:08:38 -0400 Subject: [PATCH] OSDOCS-11615: TCP flags filtering + SYN flooding alerts and metrics --- ...-observability-flow-filter-parameters.adoc | 3 + ...work-observability-tcp-flag-syn-flood.adoc | 85 +++++++++++++++++++ .../metrics-alerts-dashboards.adoc | 2 + .../observing-network-traffic.adoc | 3 +- 4 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 modules/network-observability-tcp-flag-syn-flood.adoc diff --git a/modules/network-observability-flow-filter-parameters.adoc b/modules/network-observability-flow-filter-parameters.adoc index 51e10d254c..5a87178b24 100644 --- a/modules/network-observability-flow-filter-parameters.adoc +++ b/modules/network-observability-flow-filter-parameters.adoc @@ -38,6 +38,9 @@ The flow filter rules consist of required and optional parameters. |`protocol` | Defines the protocol of the flow filter rule. Possible values are `TCP`, `UDP`, `SCTP`, `ICMP`, and `ICMPv6`. +|`tcpFlags` +| Defines the TCP flags to filter flows. Possible values are `SYN`, `SYN-ACK`, `ACK`, `FIN`, `RST`, `PSH`, `URG`, `ECE`, `CWR`, `FIN-ACK`, and `RST-ACK`. + | `ports` | Defines the ports to use for filtering flows. It can be used for either source or destination ports. To filter a single port, set a single port as an integer value. For example `ports: 80`. To filter a range of ports, use a "start-end" range in string format. For example `ports: "80-100"` diff --git a/modules/network-observability-tcp-flag-syn-flood.adoc b/modules/network-observability-tcp-flag-syn-flood.adoc new file mode 100644 index 0000000000..c41dccbcec --- /dev/null +++ b/modules/network-observability-tcp-flag-syn-flood.adoc @@ -0,0 +1,85 @@ +// Module included in the following assemblies: +// +// network_observability/metrics-alerts-dashboards.adoc + +:_mod-docs-content-type: PROCEDURE +[id="network-observability-tcp-flag-syn-flood_{context}"] += Detecting SYN flooding using the FlowMetric API and TCP flags +You can create an `AlertingRule` resouce to alert for SYN flooding. + +.Procedure +. In the web console, navigate to *Operators* -> *Installed Operators*. +. In the *Provided APIs* heading for the *NetObserv Operator*, select *FlowMetric*. +. In the *Project* dropdown list, select the project of the Network Observability Operator instance. +. Click *Create FlowMetric*. +. Create `FlowMetric` resources to add the following configurations: ++ +.Configuration counting flows per destination host and resource, with TCP flags +[source,yaml] +---- +apiVersion: flows.netobserv.io/v1alpha1 +kind: FlowMetric +metadata: + name: flows-with-flags-per-destination +spec: + metricName: flows_with_flags_per_destination_total + type: Counter + labels: [SrcSubnetLabel,DstSubnetLabel,DstK8S_Name,DstK8S_Type,DstK8S_HostName,DstK8S_Namespace,Flags] +---- ++ +.Configuration counting flows per source host and resource, with TCP flags +[source,yaml] +---- +apiVersion: flows.netobserv.io/v1alpha1 +kind: FlowMetric +metadata: + name: flows-with-flags-per-source +spec: + metricName: flows_with_flags_per_source_total + type: Counter + labels: [DstSubnetLabel,SrcSubnetLabel,SrcK8S_Name,SrcK8S_Type,SrcK8S_HostName,SrcK8S_Namespace,Flags] +---- +. Deploy the following `AlertingRule` resource to alert for SYN flooding: ++ +.`AlertingRule` for SYN flooding +[source,yaml] +---- +apiVersion: monitoring.openshift.io/v1 +kind: AlertingRule +metadata: + name: netobserv-syn-alerts + namespace: openshift-monitoring +# ... + spec: + groups: + - name: NetObservSYNAlerts + rules: + - alert: NetObserv-SYNFlood-in + annotations: + message: |- + {{ $labels.job }}: incoming SYN-flood attack suspected to Host={{ $labels.DstK8S_HostName}}, Namespace={{ $labels.DstK8S_Namespace }}, Resource={{ $labels.DstK8S_Name }}. This is characterized by a high volume of SYN-only flows with different source IPs and/or ports. + summary: "Incoming SYN-flood" + expr: sum(rate(netobserv_flows_with_flags_per_destination_total{Flags="2"}[1m])) by (job, DstK8S_HostName, DstK8S_Namespace, DstK8S_Name) > 300 <1> + for: 15s + labels: + severity: warning + app: netobserv + - alert: NetObserv-SYNFlood-out + annotations: + message: |- + {{ $labels.job }}: outgoing SYN-flood attack suspected from Host={{ $labels.SrcK8S_HostName}}, Namespace={{ $labels.SrcK8S_Namespace }}, Resource={{ $labels.SrcK8S_Name }}. This is characterized by a high volume of SYN-only flows with different source IPs and/or ports. + summary: "Outgoing SYN-flood" + expr: sum(rate(netobserv_flows_with_flags_per_source_total{Flags="2"}[1m])) by (job, SrcK8S_HostName, SrcK8S_Namespace, SrcK8S_Name) > 300 <1> + for: 15s + labels: + severity: warning + app: netobserv +# ... +---- +<1> In this example, the threshold for the alert is `300`; however, you can adapt this value empirically. A threshold that is too low might produce false-positives, and if it's too high it might miss actual attacks. + +.Verification +. In the web console, click *Manage Columns* in the *Network Traffic* table view and click *TCP flags*. +. In the *Network Traffic* table view, filter on *TCP protocol SYN TCPFlag*. A large number of flows with the same *byteSize* indicates a SYN flood. +. Go to *Observe* -> *Alerting* and select the *Alerting Rules* tab. +. Filter on *netobserv-synflood-in alert*. The alert should fire when SYN flooding occurs. \ No newline at end of file diff --git a/observability/network_observability/metrics-alerts-dashboards.adoc b/observability/network_observability/metrics-alerts-dashboards.adoc index 119281744e..3c8a13b374 100644 --- a/observability/network_observability/metrics-alerts-dashboards.adoc +++ b/observability/network_observability/metrics-alerts-dashboards.adoc @@ -20,8 +20,10 @@ include::modules/network-observability-configuring-custom-metrics.adoc[leveloffs High cardinality can affect the memory usage of Prometheus. You can check whether specific labels have high cardinality in the xref:../../observability/network_observability/json-flows-format-reference.adocl#network-observability-flows-format_json_reference[Network Flows format reference]. ==== include::modules/network-observability-flowmetrics-charts.adoc[leveloffset=+1] +include::modules/network-observability-tcp-flag-syn-flood.adoc[leveloffset=+1] [role="_additional-resources"] .Additional resources +* xref:../../observability/network_observability/observing-network-traffic.adoc#network-observability-filtering-ebpf-rule_nw-observe-network-traffic[Filtering eBPF flow data using a global rule] * xref:../../observability/monitoring/managing-alerts.adoc#creating-alerting-rules-for-user-defined-projects_managing-alerts[Creating alerting rules for user-defined projects]. * xref:../../support/troubleshooting/investigating-monitoring-issues.adoc#determining-why-prometheus-is-consuming-disk-space_investigating-monitoring-issues[Troubleshooting high cardinality metrics- Determining why Prometheus is consuming a lot of disk space] diff --git a/observability/network_observability/observing-network-traffic.adoc b/observability/network_observability/observing-network-traffic.adoc index 362559bfef..1467a6ec94 100644 --- a/observability/network_observability/observing-network-traffic.adoc +++ b/observability/network_observability/observing-network-traffic.adoc @@ -65,4 +65,5 @@ Alternatively, you can access the traffic flow data in the *Network Traffic* tab [role="_additional-resources"] .Additional resources -For more information about configuring quick filters in the `FlowCollector`, see xref:../../observability/network_observability/configuring-operator.adoc#network-observability-config-quick-filters_network_observability[Configuring Quick Filters] and the xref:../../observability/network_observability/configuring-operator.adoc#network-observability-flowcollector-view_network_observability[Flow Collector sample resource]. +* xref:../../observability/network_observability/configuring-operator.adoc#network-observability-config-quick-filters_network_observability[Configuring Quick Filters] +* xref:../../observability/network_observability/configuring-operator.adoc#network-observability-flowcollector-view_network_observability[Flow Collector sample resource] \ No newline at end of file