From 702cf2eb49c0f256f3f46b6708c26c292b4bac65 Mon Sep 17 00:00:00 2001 From: Ben Hardesty Date: Thu, 3 Nov 2022 16:34:22 -0400 Subject: [PATCH] OSDOCS-3996-port-storage-book-osd-rosa --- _topic_maps/_topic_map_osd.yml | 34 +++- _topic_maps/_topic_map_rosa.yml | 30 ++-- ...ynamic-provisioning-available-plugins.adoc | 8 + .../dynamic-provisioning-aws-definition.adoc | 2 +- ...mic-provisioning-change-default-class.adoc | 14 +- ...c-provisioning-defining-storage-class.adoc | 12 +- ...provisioning-storage-class-definition.adoc | 2 +- modules/openshift-storage-common-terms.adoc | 10 ++ .../osd-persistent-storage-csi-efs-sts.adoc | 147 ++++++++++++++++++ .../osd-storage-pv-aws-config-account.adoc | 35 ----- modules/osd-storage-pv-aws-connect-pods.adoc | 38 ----- ...orage-pv-aws-create-sharedvolumes-cli.adoc | 39 ----- ...e-pv-aws-create-sharedvolumes-console.adoc | 38 ----- modules/osd-storage-pv-aws-install-efs.adoc | 14 -- modules/osd-storage-pv-aws-uninstall-efs.adoc | 20 --- modules/persistent-storage-csi-about.adoc | 1 - .../persistent-storage-csi-architecture.adoc | 2 +- ...sistent-storage-csi-drivers-supported.adoc | 9 +- ...rage-csi-dynamic-provisioning-aws-efs.adoc | 3 +- ...tent-storage-csi-dynamic-provisioning.adoc | 2 +- ...sistent-storage-csi-efs-create-volume.adoc | 1 - ...istent-storage-csi-efs-driver-install.adoc | 46 ++++++ .../persistent-storage-csi-efs-security.adoc | 1 - .../persistent-storage-csi-efs-static-pv.adoc | 1 - modules/persistent-storage-csi-efs-sts.adoc | 13 +- ...stent-storage-csi-efs-troubleshooting.adoc | 1 - ...tent-storage-csi-external-controllers.adoc | 6 +- ...orage-csi-google-cloud-file-create-sc.adoc | 2 +- ...tent-storage-csi-olm-operator-install.adoc | 39 +---- ...nt-storage-csi-olm-operator-uninstall.adoc | 3 +- ...storage-efs-csi-driver-operator-setup.adoc | 16 ++ ...wto-create-persistent-storage-aws-ebs.adoc | 22 --- modules/storage-create-storage-class-cli.adoc | 5 + .../storage-create-storage-class-console.adoc | 5 + modules/storage-create-storage-class.adoc | 1 - modules/storage-ephemeral-vols-overview.adoc | 2 + ...ersistent-storage-aws-maximum-volumes.adoc | 2 +- ...orage-persistent-storage-block-volume.adoc | 8 + ...sistent-storage-creating-volume-claim.adoc | 5 +- modules/storage-persistent-storage-pv.adoc | 28 +++- modules/storage-persistent-storage-pvc.adoc | 13 +- ...age-persistent-storage-reclaim-manual.adoc | 8 + ...rage-persistent-storage-volume-format.adoc | 4 +- .../ephemeral-storage-csi-inline.adoc | 2 + ...e-shared-resource-csi-driver-operator.adoc | 2 +- .../osd-persistent-storage-aws-efs-csi.adoc | 44 ++++-- .../persistent-storage-csi-aws-efs.adoc | 16 +- .../persistent-storage-csi-ebs.adoc | 7 +- .../persistent-storage-csi-gcp-pd.adoc | 5 +- ...sistent-storage-csi-google-cloud-file.adoc | 2 + .../persistent-storage-csi.adoc | 5 +- storage/dynamic-provisioning.adoc | 10 +- .../osd-persistent-storage-aws.adoc | 46 ------ .../persistent-storage-aws.adoc | 54 +++++-- .../persistent-storage-gce.adoc | 5 +- .../rosa-persistent-storage-aws-ebs.adoc | 49 ------ storage/understanding-ephemeral-storage.adoc | 2 + storage/understanding-persistent-storage.adoc | 4 +- 58 files changed, 499 insertions(+), 446 deletions(-) create mode 100644 modules/osd-persistent-storage-csi-efs-sts.adoc delete mode 100644 modules/osd-storage-pv-aws-config-account.adoc delete mode 100644 modules/osd-storage-pv-aws-connect-pods.adoc delete mode 100644 modules/osd-storage-pv-aws-create-sharedvolumes-cli.adoc delete mode 100644 modules/osd-storage-pv-aws-create-sharedvolumes-console.adoc delete mode 100644 modules/osd-storage-pv-aws-install-efs.adoc delete mode 100644 modules/osd-storage-pv-aws-uninstall-efs.adoc create mode 100644 modules/persistent-storage-csi-efs-driver-install.adoc create mode 100644 modules/persistent-storage-efs-csi-driver-operator-setup.adoc delete mode 100644 modules/rosa-howto-create-persistent-storage-aws-ebs.adoc rename storage/{persistent_storage => container_storage_interface}/osd-persistent-storage-aws-efs-csi.adoc (55%) delete mode 100644 storage/persistent_storage/osd-persistent-storage-aws.adoc delete mode 100644 storage/persistent_storage/rosa-persistent-storage-aws-ebs.adoc diff --git a/_topic_maps/_topic_map_osd.yml b/_topic_maps/_topic_map_osd.yml index 011191780a..0718140f97 100644 --- a/_topic_maps/_topic_map_osd.yml +++ b/_topic_maps/_topic_map_osd.yml @@ -195,14 +195,36 @@ Name: Storage Dir: storage Distros: openshift-dedicated Topics: -- Name: Persistent storage +- Name: Storage overview + File: index +- Name: Understanding ephemeral storage + File: understanding-ephemeral-storage +- Name: Understanding persistent storage + File: understanding-persistent-storage +- Name: Configuring persistent storage Dir: persistent_storage - Distros: openshift-dedicated Topics: - - Name: Persistent storage using AWS EFS - File: osd-persistent-storage-aws - - Name: AWS Elastic File Service CSI Driver Operator - File: osd-persistent-storage-aws-efs-csi + - Name: Persistent storage using AWS Elastic Block Store + File: persistent-storage-aws + - Name: Persistent storage using GCE Persistent Disk + File: persistent-storage-gce +- Name: Using Container Storage Interface (CSI) + Dir: container_storage_interface + Topics: + - Name: Configuring CSI volumes + File: persistent-storage-csi + - Name: AWS Elastic Block Store CSI Driver Operator + File: persistent-storage-csi-ebs + - Name: AWS Elastic File Service CSI Driver Operator + File: osd-persistent-storage-aws-efs-csi + - Name: GCP PD CSI Driver Operator + File: persistent-storage-csi-gcp-pd + #- Name: GCP Filestore CSI Driver Operator + # File: persistent-storage-csi-google-cloud-file +- Name: Generic ephemeral volumes + File: generic-ephemeral-vols +- Name: Dynamic provisioning + File: dynamic-provisioning --- Name: Networking Dir: networking diff --git a/_topic_maps/_topic_map_rosa.yml b/_topic_maps/_topic_map_rosa.yml index 298b9b4db9..eb05770fa4 100644 --- a/_topic_maps/_topic_map_rosa.yml +++ b/_topic_maps/_topic_map_rosa.yml @@ -279,22 +279,34 @@ Topics: - Name: Available services File: rosa-available-services --- -Name: Storage and registry +Name: Storage Dir: storage Distros: openshift-rosa Topics: -# - Name: Configuring the internal registry -# File: configuring-registry-operator -- Name: Persistent storage +- Name: Storage overview + File: index +- Name: Understanding ephemeral storage + File: understanding-ephemeral-storage +- Name: Understanding persistent storage + File: understanding-persistent-storage +- Name: Configuring persistent storage Dir: persistent_storage - Distros: openshift-rosa Topics: - Name: Persistent storage using AWS Elastic Block Store - File: rosa-persistent-storage-aws-ebs - - Name: Persistent storage using AWS Elastic File Service - File: osd-persistent-storage-aws + File: persistent-storage-aws +- Name: Using Container Storage Interface (CSI) + Dir: container_storage_interface + Topics: + - Name: Configuring CSI volumes + File: persistent-storage-csi + - Name: AWS Elastic Block Store CSI Driver Operator + File: persistent-storage-csi-ebs - Name: AWS Elastic File Service CSI Driver Operator - File: rosa-persistent-storage-aws-efs-csi + File: osd-persistent-storage-aws-efs-csi +- Name: Generic ephemeral volumes + File: generic-ephemeral-vols +- Name: Dynamic provisioning + File: dynamic-provisioning --- Name: Networking Dir: networking diff --git a/modules/dynamic-provisioning-available-plugins.adoc b/modules/dynamic-provisioning-available-plugins.adoc index 740e9811a4..32f4b02979 100644 --- a/modules/dynamic-provisioning-available-plugins.adoc +++ b/modules/dynamic-provisioning-available-plugins.adoc @@ -18,6 +18,7 @@ configured provider's API to create new storage resources: |Provisioner plugin name |Notes +ifndef::openshift-dedicated,openshift-rosa[] |{rh-openstack-first} Cinder |`kubernetes.io/cinder` | @@ -25,6 +26,7 @@ configured provider's API to create new storage resources: |{rh-openstack} Manila Container Storage Interface (CSI) |`manila.csi.openstack.org` |Once installed, the OpenStack Manila CSI Driver Operator and ManilaDriver automatically create the required storage classes for all available Manila share types needed for dynamic provisioning. +endif::openshift-dedicated,openshift-rosa[] |AWS Elastic Block Store (EBS) |`kubernetes.io/aws-ebs` @@ -32,6 +34,7 @@ configured provider's API to create new storage resources: tag each node with `Key=kubernetes.io/cluster/,Value=` where `` and `` are unique per cluster. +ifndef::openshift-dedicated,openshift-rosa[] |Azure Disk |`kubernetes.io/azure-disk` | @@ -40,12 +43,15 @@ where `` and `` are unique per cluster. |`kubernetes.io/azure-file` |The `persistent-volume-binder` service account requires permissions to create and get secrets to store the Azure storage account and keys. +endif::openshift-dedicated,openshift-rosa[] +ifndef::openshift-rosa[] |GCE Persistent Disk (gcePD) |`kubernetes.io/gce-pd` |In multi-zone configurations, it is advisable to run one {product-title} cluster per GCE project to avoid PVs from being created in zones where no node in the current cluster exists. +endif::openshift-rosa[] //|GlusterFS //|`kubernetes.io/glusterfs` @@ -59,9 +65,11 @@ no node in the current cluster exists. //|`netapp.io/trident` //|Storage orchestrator for NetApp ONTAP, SolidFire, and E-Series storage. +ifndef::openshift-dedicated,openshift-rosa[] |link:https://www.vmware.com/support/vsphere.html[VMware vSphere] |`kubernetes.io/vsphere-volume` | +endif::openshift-dedicated,openshift-rosa[] //|HPE Nimble Storage //|`hpe.com/nimble` diff --git a/modules/dynamic-provisioning-aws-definition.adoc b/modules/dynamic-provisioning-aws-definition.adoc index e7818c0c91..025245513c 100644 --- a/modules/dynamic-provisioning-aws-definition.adoc +++ b/modules/dynamic-provisioning-aws-definition.adoc @@ -22,7 +22,7 @@ parameters: fsType: ext4 <6> ---- <1> (required) Name of the storage class. The persistent volume claim uses this storage class for provisioning the associated persistent volumes. -<2> (required) Select from `io1`, `gp2`, `sc1`, `st1`. The default is `gp2`. +<2> (required) Select from `io1`, `gp3`, `sc1`, `st1`. The default is `gp3`. See the link:http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html[AWS documentation] for valid Amazon Resource Name (ARN) values. diff --git a/modules/dynamic-provisioning-change-default-class.adoc b/modules/dynamic-provisioning-change-default-class.adoc index a24519939a..e9646c911b 100644 --- a/modules/dynamic-provisioning-change-default-class.adoc +++ b/modules/dynamic-provisioning-change-default-class.adoc @@ -8,10 +8,12 @@ [id="change-default-storage-class_{context}"] = Changing the default storage class -Use the following process to change the default storage class. -For example you have two defined storage classes, `gp2` and `standard`, and you want to change the default storage class from `gp2` to `standard`. +Use this procedure to change the default storage class. +For example, you have two defined storage classes, `gp3` and `standard`, and you want to change the default storage class from `gp3` to `standard`. -. List the storage class: +.Procedure + +. List the storage classes: + [source,terminal] ---- @@ -22,7 +24,7 @@ $ oc get storageclass [source,terminal] ---- NAME TYPE -gp2 (default) kubernetes.io/aws-ebs <1> +gp3 (default) kubernetes.io/aws-ebs <1> standard kubernetes.io/aws-ebs ---- <1> `(default)` denotes the default storage class. @@ -31,7 +33,7 @@ standard kubernetes.io/aws-ebs + [source,terminal] ---- -$ oc patch storageclass gp2 -p '{"metadata": {"annotations": {"storageclass.kubernetes.io/is-default-class": "false"}}}' +$ oc patch storageclass gp3 -p '{"metadata": {"annotations": {"storageclass.kubernetes.io/is-default-class": "false"}}}' ---- . Make another storage class the default by setting the `storageclass.kubernetes.io/is-default-class` annotation to `true`: @@ -52,6 +54,6 @@ $ oc get storageclass [source,terminal] ---- NAME TYPE -gp2 kubernetes.io/aws-ebs +gp3 kubernetes.io/aws-ebs standard (default) kubernetes.io/aws-ebs ---- diff --git a/modules/dynamic-provisioning-defining-storage-class.adoc b/modules/dynamic-provisioning-defining-storage-class.adoc index 38ac67b468..38f61a8314 100644 --- a/modules/dynamic-provisioning-defining-storage-class.adoc +++ b/modules/dynamic-provisioning-defining-storage-class.adoc @@ -11,16 +11,22 @@ `StorageClass` objects are currently a globally scoped object and must be created by `cluster-admin` or `storage-admin` users. -ifndef::microshift[] +ifndef::microshift,openshift-rosa[] [IMPORTANT] ==== The Cluster Storage Operator might install a default storage class depending on the platform in use. This storage class is owned and controlled by the -operator. It cannot be deleted or modified beyond defining annotations +Operator. It cannot be deleted or modified beyond defining annotations and labels. If different behavior is desired, you must define a custom storage class. ==== -endif::microshift[] +endif::microshift,openshift-rosa[] +ifdef::openshift-rosa[] +[IMPORTANT] +==== +The Cluster Storage Operator installs a default storage class. This storage class is owned and controlled by the Operator. It cannot be deleted or modified beyond defining annotations and labels. If different behavior is desired, you must define a custom storage class. +==== +endif::openshift-rosa[] The following sections describe the basic definition for a `StorageClass` object and specific examples for each of the supported plugin types. diff --git a/modules/dynamic-provisioning-storage-class-definition.adoc b/modules/dynamic-provisioning-storage-class-definition.adoc index aaf1306b8f..436a78ed69 100644 --- a/modules/dynamic-provisioning-storage-class-definition.adoc +++ b/modules/dynamic-provisioning-storage-class-definition.adoc @@ -25,7 +25,7 @@ metadata: ... provisioner: kubernetes.io/aws-ebs <5> parameters: <6> - type: gp2 + type: gp3 ... ---- <1> (required) The API object type. diff --git a/modules/openshift-storage-common-terms.adoc b/modules/openshift-storage-common-terms.adoc index 7f8d14b539..93a32af053 100644 --- a/modules/openshift-storage-common-terms.adoc +++ b/modules/openshift-storage-common-terms.adoc @@ -15,7 +15,9 @@ Access modes:: Volume access modes describe volume capabilities. You can use acc * ReadWriteMany (RWX) * ReadWriteOncePod (RWOP) +ifndef::openshift-dedicated,openshift-rosa[] Cinder:: The Block Storage service for {rh-openstack-first} which manages the administration, security, and scheduling of all volumes. +endif::openshift-dedicated,openshift-rosa[] Config map:: A config map provides a way to inject configuration data into pods. You can reference the data stored in a config map in a volume of type `ConfigMap`. Applications running in a pod can use this data. @@ -28,14 +30,18 @@ The framework allows you to create storage volumes on-demand, eliminating the ne Ephemeral storage:: Pods and containers can require temporary or transient local storage for their operation. The lifetime of this ephemeral storage does not extend beyond the life of the individual pod, and this ephemeral storage cannot be shared across pods. +ifndef::openshift-dedicated,openshift-rosa[] Fiber channel:: A networking technology that is used to transfer data among data centers, computer servers, switches and storage. FlexVolume:: FlexVolume is an out-of-tree plugin interface that uses an exec-based model to interface with storage drivers. You must install the FlexVolume driver binaries in a pre-defined volume plugin path on each node and in some cases the control plane nodes. +endif::openshift-dedicated,openshift-rosa[] fsGroup:: The fsGroup defines a file system group ID of a pod. +ifndef::openshift-dedicated,openshift-rosa[] iSCSI:: Internet Small Computer Systems Interface (iSCSI) is an Internet Protocol-based storage networking standard for linking data storage facilities. An iSCSI volume allows an existing iSCSI (SCSI over IP) volume to be mounted into your Pod. +endif::openshift-dedicated,openshift-rosa[] hostPath:: A hostPath volume in an OpenShift Container Platform cluster mounts a file or directory from the host node’s filesystem into your pod. @@ -44,7 +50,9 @@ KMS key:: The Key Management Service (KMS) helps you achieve the required level Local volumes:: A local volume represents a mounted local storage device such as a disk, partition or directory. +ifndef::openshift-dedicated,openshift-rosa[] NFS:: A Network File System (NFS) that allows remote hosts to mount file systems over a network and interact with those file systems as though they are mounted locally. This enables system administrators to consolidate resources onto centralized servers on the network. +endif::openshift-dedicated,openshift-rosa[] OpenShift Data Foundation:: A provider of agnostic persistent storage for OpenShift Container Platform supporting file, block, and object storage, either in-house or in hybrid clouds @@ -75,4 +83,6 @@ Storage:: {product-title} supports many types of storage, both for on-premise an Storage class:: A storage class provides a way for administrators to describe the classes of storage they offer. Different classes might map to quality of service levels, backup policies, arbitrary policies determined by the cluster administrators. +ifndef::openshift-dedicated,openshift-rosa[] VMware vSphere’s Virtual Machine Disk (VMDK) volumes:: Virtual Machine Disk (VMDK) is a file format that describes containers for virtual hard disk drives that is used in virtual machines. +endif::openshift-dedicated,openshift-rosa[] \ No newline at end of file diff --git a/modules/osd-persistent-storage-csi-efs-sts.adoc b/modules/osd-persistent-storage-csi-efs-sts.adoc new file mode 100644 index 0000000000..6ee70ca383 --- /dev/null +++ b/modules/osd-persistent-storage-csi-efs-sts.adoc @@ -0,0 +1,147 @@ +// Module included in the following assemblies: +// +// * storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc + +// The OCP version of this procedure is persistent-storage-csi-efs-sts. + +:_content-type: PROCEDURE +[id="efs-sts_{context}"] += Configuring AWS EFS CSI Driver Operator with Secure Token Service + +This procedure explains how to configure the AWS EFS CSI Driver Operator with {product-title} on AWS Secure Token Service (STS). + +Perform this procedure before you have installed the AWS EFS CSI Operator, but not yet installed the AWS EFS CSI driver as part of the _Installing the AWS EFS CSI Driver Operator_ procedure. + +[IMPORTANT] +==== +If you perform this procedure after installing the driver and creating volumes, your volumes will fail to mount into pods. +==== + +.Prerequisites + +* You have access to the cluster as a user with the cluster-admin role. +* AWS account credentials +* You have installed the AWS EFS CSI Operator. + +.Procedure + +To configure the AWS EFS CSI Driver Operator with STS: + +// The on-prem version of this step is documented in the cco-ccoctl-configuring procedure. +. Extract the CCO utility (`ccoctl`) binary from the Cloud Credential Operator. + +.. Find the pod on which the Cloud Credential Operator is running. ++ +-- +[source,terminal] +---- +$ oc get pod -n openshift-cloud-credential-operator -l app=cloud-credential-operator +---- + +.Example output +[source,terminal] +---- +NAME READY STATUS RESTARTS AGE +cloud-credential-operator-78c9c575b-r6mmr 2/2 Running 0 6h33m +---- +-- + +.. Copy the `ccoctl` binary from the pod to a local directory. ++ +[source,terminal] +---- +$ oc cp -c cloud-credential-operator openshift-cloud-credential-operator/:/usr/bin/ccoctl ./ccoctl +---- + +.. Change the permissions to make `ccoctl` executable. ++ +[source,terminal] +---- +$ chmod 775 ./ccoctl +---- + +.. To verify that `ccoctl` is ready to use, display the help file: ++ +[source,terminal] +---- +$ ./ccoctl --help +---- + +. Create and save an EFS `CredentialsRequest` YAML file, such as shown in the following example: ++ +.Example +[source,yaml] +---- +apiVersion: cloudcredential.openshift.io/v1 +kind: CredentialsRequest +metadata: + name: openshift-aws-efs-csi-driver + namespace: openshift-cloud-credential-operator +spec: + providerSpec: + apiVersion: cloudcredential.openshift.io/v1 + kind: AWSProviderSpec + statementEntries: + - action: + - elasticfilesystem:* + effect: Allow + resource: '*' + secretRef: + name: aws-efs-cloud-credentials + namespace: openshift-cluster-csi-drivers + serviceAccountNames: + - aws-efs-csi-driver-operator + - aws-efs-csi-driver-controller-sa +---- + +. Run the `ccoctl` tool to generate a new IAM role in AWS, and create a YAML file for it in the local file system (`/manifests/openshift-cluster-csi-drivers-aws-efs-cloud-credentials-credentials.yaml`). ++ +-- +[source,terminal] +---- +$ ccoctl aws create-iam-roles --name= --region= --credentials-requests-dir=/credrequests --identity-provider-arn= +---- + +* `name=` is the name used to tag any cloud resources that are created for tracking. + +* `region=` is the AWS region where cloud resources are created. + +* `dir=/credrequests` is the directory containing the EFS CredentialsRequest file in previous step. + +* `` is the ARN for the OIDC provider that associates with your cluster. + +.Example +[source,terminal] +---- +$ ccoctl aws create-iam-roles --name=my-aws-efs --credentials-requests-dir= credrequests --identity-provider-arn=arn:aws:iam::123456789012:oidc-provider/example.cloudfront.net/ +---- + +.Example output +[source,terminal] +---- +2022/03/21 06:24:44 Role arn:aws:iam::123456789012:role/my-aws-efs -openshift-cluster-csi-drivers-aws-efs-cloud- created +2022/03/21 06:24:44 Saved credentials configuration to: /manifests/openshift-cluster-csi-drivers-aws-efs-cloud-credentials-credentials.yaml +2022/03/21 06:24:45 Updated Role policy for Role my-aws-efs-openshift-cluster-csi-drivers-aws-efs-cloud-credentials +---- +-- + +. Create the AWS EFS cloud credentials and secret: ++ +-- +[source,terminal] +---- +$ oc create -f /manifests/openshift-cluster-csi-drivers-aws-efs-cloud-credentials-credentials.yaml +---- + +.Example +[source,terminal] +---- +$ oc create -f /manifests/openshift-cluster-csi-drivers-aws-efs-cloud-credentials-credentials.yaml +---- + +.Example output +[source,terminal] +---- +secret/aws-efs-cloud-credentials created +---- +-- diff --git a/modules/osd-storage-pv-aws-config-account.adoc b/modules/osd-storage-pv-aws-config-account.adoc deleted file mode 100644 index 4bd9982b57..0000000000 --- a/modules/osd-storage-pv-aws-config-account.adoc +++ /dev/null @@ -1,35 +0,0 @@ -// Module included in the following assemblies: -// -// * storage/persistent_storage/osd-persistent-storage-aws.adoc - -:_content-type: PROCEDURE -[id="osd-storage-pv-aws-config-account_{context}"] -= Configuring the AWS account - -Set up your AWS account to prepare AWS EFS for use by {product-title}. - -.Procedure - -. Log in to the link:https://console.aws.amazon.com/ec2[AWS EC2 Console]. -. Select the region that matches the cluster region. -. Filter only worker EC2 instances, and select an instance. Note the VPC ID and security group ID. These values are required later in the process. -. Click the *Security* tab, and click the Security Group Name. -. From the *Actions* dropdown menu, click *Edit Inbound Rules*. Scroll to the bottom, and click *Add Rule*. -. Add an NFS rule that allows NFS traffic from the VPC private CIDR. -. Open the link:https://console.aws.amazon.com/efs/[Amazon EFS page]. To create the EFS, click *Create file system*. -. Click *Customize* and proceed through the wizard. -.. In `Step 2:`, configure the network access: -... Click the VPC of the cluster that you noted previously. -... Ensure that the private subnets are selected. -... Select the Security Group Name that you noted previously for the EC2 worker instances. -... Click *Next*. -.. In `Step 3:`, configure the client access: -... Click *Add access point*. -... Enter a unique Path such as `/access_point_1`. -... Configure the Owner fields with ownership or permissions that allow write access for your worker pods. For example, if your worker pods run with group ID `100`, you can set that ID as your `Owner Group ID` and ensure the permissions include `g+rwx`. -. Continue through the wizard steps, and click *Create File System*. -. After the file system is created: -.. Note the file system ID for later use. -.. Click *Manage client access* and note the access point ID. - -You can add more NFS rules, using steps 5-10, to create separate shared data stores. In each case, make note of the corresponding file system ID and access point ID. diff --git a/modules/osd-storage-pv-aws-connect-pods.adoc b/modules/osd-storage-pv-aws-connect-pods.adoc deleted file mode 100644 index 8e9a8b74a9..0000000000 --- a/modules/osd-storage-pv-aws-connect-pods.adoc +++ /dev/null @@ -1,38 +0,0 @@ -// Module included in the following assemblies: -// -// * storage/persistent_storage/osd-persistent-storage-aws.adoc - -:_content-type: PROCEDURE -[id="osd-storage-pv-aws-connect-pods_{context}"] -= Connecting pods - -The persistent volume claim (PVC) that was created in your project is ready for use. You can create a sample pod to test this PVC. - -.Procedure - -. Create and navigate to a project. -. Click *Workloads* -> *Pods* -> *Create Pod*. -. Enter the YAML information. Use the name of your `PersistentVolumeClaim` object under `.spec.volumes[].persistentVolumeClaim.claimName`. -+ -.Example -[source,terminal] ----- -apiVersion: v1 -kind: Pod -metadata: - name: test-efs -spec: - volumes: - - name: efs-storage-vol - persistentVolumeClaim: - claimName: pvc-sv1 - containers: - - name: test-efs - image: centos:latest - command: [ "/bin/bash", "-c", "--" ] - args: [ "while true; do touch /mnt/efs-data/verify-efs && echo 'hello efs' && sleep 30; done;" ] - volumeMounts: - - mountPath: "/mnt/efs-data" - name: efs-storage-vol ----- -. After the pods are created, click *Workloads* -> *Pods* -> *Logs* to verify the pod logs. diff --git a/modules/osd-storage-pv-aws-create-sharedvolumes-cli.adoc b/modules/osd-storage-pv-aws-create-sharedvolumes-cli.adoc deleted file mode 100644 index 344d3190ed..0000000000 --- a/modules/osd-storage-pv-aws-create-sharedvolumes-cli.adoc +++ /dev/null @@ -1,39 +0,0 @@ -// Module included in the following assemblies: -// -// * storage/persistent_storage/osd-persistent-storage-aws.adoc - - -:_content-type: PROCEDURE -[id="osd-storage-pv-aws-create-sharedvolumes-cli_{context}"] -= Creating `SharedVolume` resources using the CLI - -You must create one `SharedVolume` resource per file system:access point pair in each project from which you want pods to access it. You can create a SharedVolume manually by entering YAML or JSON definitions, or by dragging and dropping a file into an editor. - -.Procedure - -. Using the `oc` CLI, create the YAML file using the `accessPointID` and `fileSystemID` values from the EFS resources you created earlier. -+ -[source,yaml] ----- - apiVersion: aws-efs.managed.openshift.io/v1alpha1 - kind: SharedVolume - metadata: - name: sv1 - namespace: efsop2 - spec: - accessPointID: fsap-0123456789abcdef - fileSystemID: fs-0123cdef ----- - -. Apply the file to the cluster using the following command: -+ -[source,terminal] ----- -$ oc apply -f .yaml ----- -+ -The `SharedVolume` resource is created, and triggers the AWS EFS Operator to generate and associate a PersistentVolume:PersistentVolumeClaim pair with the specified EFS access point. - -. To verify that the PVC exists and is bound, navigate to *Storage* > *Persistent Volume Claims*. -+ -The PVC name is `pvc-{shared_volume_name}`. The associated PV name is `pv-{project_name}-{shared_volume_name}`. diff --git a/modules/osd-storage-pv-aws-create-sharedvolumes-console.adoc b/modules/osd-storage-pv-aws-create-sharedvolumes-console.adoc deleted file mode 100644 index 3636068df6..0000000000 --- a/modules/osd-storage-pv-aws-create-sharedvolumes-console.adoc +++ /dev/null @@ -1,38 +0,0 @@ -// Module included in the following assemblies: -// -// * storage/persistent_storage/osd-persistent-storage-aws.adoc - -:_content-type: PROCEDURE -[id="osd-storage-pv-aws-create-sharedvolumes-console_{context}"] -= Creating `SharedVolume` resources using the console - -You must create one `SharedVolume` resource per file system:access point pair in each project from which you want pods to access it. - -.Procedure - -. In the OpenShift web console, create and navigate to a project. -. Click *Operators* -> *Installed Operators*. Find the entry for AWS EFS Operator, and click *SharedVolume* under Provided APIs. -. Click *Create SharedVolume*. -. Edit the sample YAML: -.. Type a suitable value for `name`. -.. Replace the values of `accessPointID` and `fileSystemID` with the values from the EFS resources you created earlier. -+ -[source,yaml] ----- - apiVersion: aws-efs.managed.openshift.io/v1alpha1 - kind: SharedVolume - metadata: - name: sv1 - namespace: efsop2 - spec: - accessPointID: fsap-0123456789abcdef - fileSystemID: fs-0123cdef ----- - -. Click *Create*. -+ -The `SharedVolume` resource is created, and triggers the AWS EFS Operator to generate and associate a PersistentVolume:PersistentVolumeClaim pair with the specified EFS access point. - -. To verify that the persistent volume claim (PVC) exists and is bound, click *Storage* -> *Persistent Volume Claims*. -+ -The PVC name is `pvc-`. The associated PV name is `pv--`. diff --git a/modules/osd-storage-pv-aws-install-efs.adoc b/modules/osd-storage-pv-aws-install-efs.adoc deleted file mode 100644 index 2e8115ad00..0000000000 --- a/modules/osd-storage-pv-aws-install-efs.adoc +++ /dev/null @@ -1,14 +0,0 @@ -// Module included in the following assemblies: -// -// * storage/persistent_storage/osd-persistent-storage-aws.adoc - -:_content-type: PROCEDURE -[id="osd-storage-pv-aws-install-efs_{context}"] -= Installing the EFS Operator - -.Procedure - -. Log in to the OpenShift Web UI for your cluster. -. Click *Operators* -> *OperatorHub*. -. Search for and select the AWS EFS Operator. Click *Install*. -. Accept the default settings, and click *Subscribe*. diff --git a/modules/osd-storage-pv-aws-uninstall-efs.adoc b/modules/osd-storage-pv-aws-uninstall-efs.adoc deleted file mode 100644 index 7c8fdc987d..0000000000 --- a/modules/osd-storage-pv-aws-uninstall-efs.adoc +++ /dev/null @@ -1,20 +0,0 @@ -// Module included in the following assemblies: -// -// * storage/persistent_storage/osd-persistent-storage-aws.adoc - -:_content-type: PROCEDURE -[id="osd-storage-pv-aws-uninstall-efs_{context}"] -= Uninstalling the EFS Operator - -.Procedure - -To remove the Operator from your cluster: - -. Delete all of the workloads using the persistent volume claims that were generated by the Operator. -. Delete all of the shared volumes from all of the namespaces. The Operator automatically removes the associated persistent volumes and persistent volume claims. -. Uninstall the Operator: -.. Click *Operators* -> *Installed Operators*. -.. Find the entry for AWS EFS Operator, and click the menu button on the right-hand side of the Operator. -.. Click *Uninstall* and confirm the deletion. - -. Delete the shared volume CRD. This action triggers the deletion of the remaining Operator-owned resources. diff --git a/modules/persistent-storage-csi-about.adoc b/modules/persistent-storage-csi-about.adoc index 40ddc62683..3bee883a30 100644 --- a/modules/persistent-storage-csi-about.adoc +++ b/modules/persistent-storage-csi-about.adoc @@ -5,7 +5,6 @@ // * storage/container_storage_interface/persistent-storage-csi-ovirt.adoc // * storage/container_storage_interface/persistent-storage-csi-aws-efs.adoc // * storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc -// * storage/container_storage_interface/rosa-persistent-storage-aws-efs-csi.adoc :_content-type: CONCEPT [id="csi-about_{context}"] diff --git a/modules/persistent-storage-csi-architecture.adoc b/modules/persistent-storage-csi-architecture.adoc index ab2e4928eb..fdd0218716 100644 --- a/modules/persistent-storage-csi-architecture.adoc +++ b/modules/persistent-storage-csi-architecture.adoc @@ -5,7 +5,7 @@ [id="persistent-storage-csi-architecture_{context}"] -= CSI Architecture += CSI architecture CSI drivers are typically shipped as container images. These containers are not aware of {product-title} where they run. To use CSI-compatible diff --git a/modules/persistent-storage-csi-drivers-supported.adoc b/modules/persistent-storage-csi-drivers-supported.adoc index 853f613cd0..a7c5ce64cf 100644 --- a/modules/persistent-storage-csi-drivers-supported.adoc +++ b/modules/persistent-storage-csi-drivers-supported.adoc @@ -15,12 +15,16 @@ The following table describes the CSI drivers that are installed with {product-t [cols=",^v,^v,^v, width="100%",options="header"] |=== |CSI driver |CSI volume snapshots |CSI cloning |CSI resize - +ifndef::openshift-dedicated,openshift-rosa[] |AliCloud Disk | ✅ | - | ✅ +endif::openshift-dedicated,openshift-rosa[] |AWS EBS | ✅ | - | ✅ |AWS EFS | - | - | - +ifndef::openshift-rosa[] |Google Compute Platform (GCP) persistent disk (PD)| ✅ | - | ✅ |GCP Filestore | ✅ | ✅ | ✅ +endif::openshift-rosa[] +ifndef::openshift-dedicated,openshift-rosa[] |IBM VPC Block | ✅^[3]^ | - | ✅^[3]^ |Microsoft Azure Disk | ✅ | ✅ | ✅ |Microsoft Azure Stack Hub | ✅ | ✅ | ✅ @@ -30,7 +34,9 @@ The following table describes the CSI drivers that are installed with {product-t |OpenStack Manila | ✅ | - | - |Red Hat Virtualization (oVirt) | - | - | ✅ |VMware vSphere | ✅^[1]^ | - | ✅^[2]^ +endif::openshift-dedicated,openshift-rosa[] |=== +ifndef::openshift-dedicated,openshift-rosa[] -- 1. @@ -48,6 +54,7 @@ The following table describes the CSI drivers that are installed with {product-t * Does not support offline snapshots or resize. Volume must be attached to a running pod. -- +endif::openshift-dedicated,openshift-rosa[] [IMPORTANT] ==== diff --git a/modules/persistent-storage-csi-dynamic-provisioning-aws-efs.adoc b/modules/persistent-storage-csi-dynamic-provisioning-aws-efs.adoc index 6dc0b051a1..0173bbcf11 100644 --- a/modules/persistent-storage-csi-dynamic-provisioning-aws-efs.adoc +++ b/modules/persistent-storage-csi-dynamic-provisioning-aws-efs.adoc @@ -2,7 +2,6 @@ // // * storage/container_storage_interface/persistent-storage-csi-aws-efs.adoc // * storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc -// * storage/container_storage_interface/rosa-persistent-storage-aws-efs-csi.adoc :_content-type: PROCEDURE [id="csi-dynamic-provisioning-aws-efs_{context}"] @@ -30,7 +29,7 @@ Using monitoring of EFS volume sizes in AWS is strongly recommended. To enable dynamic provisioning: -* Create a PVC (or StatefulSet or Template) as usual, referring to the `StorageClass` created above. +* Create a PVC (or StatefulSet or Template) as usual, referring to the `StorageClass` created previously. + [source,yaml] ---- diff --git a/modules/persistent-storage-csi-dynamic-provisioning.adoc b/modules/persistent-storage-csi-dynamic-provisioning.adoc index 9e6b8e346f..260ac3b59b 100644 --- a/modules/persistent-storage-csi-dynamic-provisioning.adoc +++ b/modules/persistent-storage-csi-dynamic-provisioning.adoc @@ -34,4 +34,4 @@ parameters: EOF ---- <1> The name of the storage class that will be created. -<2> The name of the CSI driver that has been installed +<2> The name of the CSI driver that has been installed. diff --git a/modules/persistent-storage-csi-efs-create-volume.adoc b/modules/persistent-storage-csi-efs-create-volume.adoc index a5c1157b5a..cf1c150b2f 100644 --- a/modules/persistent-storage-csi-efs-create-volume.adoc +++ b/modules/persistent-storage-csi-efs-create-volume.adoc @@ -3,7 +3,6 @@ // * storage/persistent_storage/persistent-storage-csi-aws-efs.adoc // * storage/container_storage_interface/persistent-storage-csi-aws-efs.adoc // * storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc -// * storage/container_storage_interface/rosa-persistent-storage-aws-efs-csi.adoc :_content-type: PROCEDURE [id="efs-create-volume_{context}"] diff --git a/modules/persistent-storage-csi-efs-driver-install.adoc b/modules/persistent-storage-csi-efs-driver-install.adoc new file mode 100644 index 0000000000..83798d2b81 --- /dev/null +++ b/modules/persistent-storage-csi-efs-driver-install.adoc @@ -0,0 +1,46 @@ +// Module included in the following assemblies: +// +// * storage/container_storage_interface/persistent-storage-csi-aws-efs.adoc +// * storage/container_storage_interface/osd-persistent-storage-csi-aws-efs.adoc + +:_content-type: PROCEDURE +[id="persistent-storage-csi-efs-driver-install_{context}"] += Installing the {FeatureName} CSI Driver + +ifdef::openshift-rosa[] +After installing the {FeatureName} CSI Driver Operator and configuring it with STS, you install the {FeatureName} CSI Driver. +endif::openshift-rosa[] +ifdef::openshift-dedicated[] +After installing the {FeatureName} CSI Driver Operator, you install the {FeatureName} CSI Driver. +endif::openshift-dedicated[] + +.Prerequisites +* Access to the {product-title} web console. + +.Procedure + +. Click *Administration* -> *CustomResourceDefinitions* -> *ClusterCSIDriver*. + +. On the *Instances* tab, click *Create ClusterCSIDriver*. + +. Use the following YAML file: ++ +[source,yaml] +---- +apiVersion: operator.openshift.io/v1 +kind: ClusterCSIDriver +metadata: + name: efs.csi.aws.com +spec: + managementState: Managed +---- + +. Click *Create*. + +. Wait for the following Conditions to change to a "True" status: ++ +* AWSEFSDriverCredentialsRequestControllerAvailable + +* AWSEFSDriverNodeServiceControllerAvailable + +* AWSEFSDriverControllerServiceControllerAvailable \ No newline at end of file diff --git a/modules/persistent-storage-csi-efs-security.adoc b/modules/persistent-storage-csi-efs-security.adoc index 0bf134ca80..9e56771096 100644 --- a/modules/persistent-storage-csi-efs-security.adoc +++ b/modules/persistent-storage-csi-efs-security.adoc @@ -3,7 +3,6 @@ // * storage/persistent_storage/persistent-storage-csi-aws-efs.adoc // * storage/container_storage_interface/persistent-storage-csi-aws-efs.adoc // * storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc -// * storage/container_storage_interface/rosa-persistent-storage-aws-efs-csi.adoc [id="efs-security_{context}"] = AWS EFS security diff --git a/modules/persistent-storage-csi-efs-static-pv.adoc b/modules/persistent-storage-csi-efs-static-pv.adoc index cc8a4dbe9f..f16c125abf 100644 --- a/modules/persistent-storage-csi-efs-static-pv.adoc +++ b/modules/persistent-storage-csi-efs-static-pv.adoc @@ -3,7 +3,6 @@ // * storage/persistent_storage/persistent-storage-csi-aws-efs.adoc // * storage/container_storage_interface/persistent-storage-csi-aws-efs.adoc // * storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc -// * storage/container_storage_interface/rosa-persistent-storage-aws-efs-csi.adoc :_content-type: PROCEDURE [id="efs-create-static-pv_{context}"] diff --git a/modules/persistent-storage-csi-efs-sts.adoc b/modules/persistent-storage-csi-efs-sts.adoc index 3ba468a393..62b4133527 100644 --- a/modules/persistent-storage-csi-efs-sts.adoc +++ b/modules/persistent-storage-csi-efs-sts.adoc @@ -1,9 +1,6 @@ // Module included in the following assemblies: // -// * storage/persistent_storage/persistent-storage-csi-aws-efs.adoc // * storage/container_storage_interface/persistent-storage-csi-aws-efs.adoc -// * storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc -// * storage/container_storage_interface/rosa-persistent-storage-aws-efs-csi.adoc :_content-type: PROCEDURE [id="efs-sts_{context}"] @@ -11,11 +8,18 @@ This procedure explains how to configure the AWS EFS CSI Driver Operator with {product-title} on AWS Security Token Service (STS). -Perform this procedure after installing the AWS EFS CSI Operator, but before installing the AWS EFS CSI driver as part of _Installing the AWS EFS CSI Driver Operator_ procedure. If you perform this procedure after installing the driver and creating volumes, your volumes will fail to mount into pods. +Perform this procedure before you have installed the AWS EFS CSI Operator, but not yet installed the AWS EFS CSI driver as part of the _Installing the AWS EFS CSI Driver Operator_ procedure. + +[IMPORTANT] +==== +If you perform this procedure after installing the driver and creating volumes, your volumes will fail to mount into pods. +==== .Prerequisites +* You have access to the cluster as a user with the cluster-admin role. * AWS account credentials +* You have installed the AWS EFS CSI Operator. .Procedure @@ -26,7 +30,6 @@ To configure the AWS EFS CSI Driver Operator with STS: . Create and save an EFS `CredentialsRequest` YAML file, such as shown in the following example, and then place it in the `credrequests` directory: + .Example -+ [source, yaml] ---- apiVersion: cloudcredential.openshift.io/v1 diff --git a/modules/persistent-storage-csi-efs-troubleshooting.adoc b/modules/persistent-storage-csi-efs-troubleshooting.adoc index 7ad15f7490..6dd7ae303a 100644 --- a/modules/persistent-storage-csi-efs-troubleshooting.adoc +++ b/modules/persistent-storage-csi-efs-troubleshooting.adoc @@ -3,7 +3,6 @@ // * storage/persistent_storage/persistent-storage-csi-aws-efs.adoc // * storage/container_storage_interface/persistent-storage-csi-aws-efs.adoc // * storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc -// * storage/container_storage_interface/rosa-persistent-storage-aws-efs-csi.adoc [id="efs-troubleshooting_{context}"] = AWS EFS troubleshooting diff --git a/modules/persistent-storage-csi-external-controllers.adoc b/modules/persistent-storage-csi-external-controllers.adoc index f4a9bef2e7..b2d6d24ba1 100644 --- a/modules/persistent-storage-csi-external-controllers.adoc +++ b/modules/persistent-storage-csi-external-controllers.adoc @@ -7,7 +7,7 @@ [id="external-csi-contollers_{context}"] = External CSI controllers -External CSI Controllers is a deployment that deploys one or more pods +External CSI controllers is a deployment that deploys one or more pods with five containers: * The snapshotter container watches `VolumeSnapshot` and `VolumeSnapshotContent` objects and is responsible for the creation and deletion of `VolumeSnapshotContent` object. @@ -18,7 +18,7 @@ calls from {product-title} to respective `ControllerPublish` and * An external CSI provisioner container that translates `provision` and `delete` calls from {product-title} to respective `CreateVolume` and `DeleteVolume` calls to the CSI driver. -* A CSI driver container +* A CSI driver container. The CSI attacher and CSI provisioner containers communicate with the CSI driver container using UNIX Domain Sockets, ensuring that no CSI @@ -27,7 +27,7 @@ outside of the pod. [NOTE] ==== -`attach`, `detach`, `provision`, and `delete` operations typically require +The `attach`, `detach`, `provision`, and `delete` operations typically require the CSI driver to use credentials to the storage backend. Run the CSI controller pods on infrastructure nodes so the credentials are never leaked to user processes, even in the event of a catastrophic security breach diff --git a/modules/persistent-storage-csi-google-cloud-file-create-sc.adoc b/modules/persistent-storage-csi-google-cloud-file-create-sc.adoc index edef5f2f26..df7d5e6e00 100644 --- a/modules/persistent-storage-csi-google-cloud-file-create-sc.adoc +++ b/modules/persistent-storage-csi-google-cloud-file-create-sc.adoc @@ -6,7 +6,7 @@ [id="persistent-storage-csi-google-cloud-file-create-sc_{context}"] = Creating a storage class for GCP Filestore Storage -After installing the operator, you should create a storage class for dynamic provisioning of Google Compute Platform (GCP) Filestore volumes. +After installing the Operator, you should create a storage class for dynamic provisioning of Google Compute Platform (GCP) Filestore volumes. .Prerequisites * You are logged in to the running {product-title} cluster. diff --git a/modules/persistent-storage-csi-olm-operator-install.adoc b/modules/persistent-storage-csi-olm-operator-install.adoc index 64cdc2ed40..da7eb466a5 100644 --- a/modules/persistent-storage-csi-olm-operator-install.adoc +++ b/modules/persistent-storage-csi-olm-operator-install.adoc @@ -1,17 +1,12 @@ // Module included in the following assemblies: // // * storage/container_storage_interface/persistent-storage-csi-aws-efs.adoc -// * storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc -// * storage/container_storage_interface/rosa-persistent-storage-aws-efs-csi.adoc +// * storage/container_storage_interface/osd-persistent-storage-csi-aws-efs.adoc :_content-type: PROCEDURE [id="persistent-storage-csi-olm-operator-install_{context}"] = Installing the {FeatureName} CSI Driver Operator -ifeval::["{context}" == "persistent-storage-csi-aws-efs"] -:AWS_EFS: -endif::[] - The {FeatureName} CSI Driver Operator is not installed in {product-title} by default. Use the following procedure to install and configure the {FeatureName} CSI Driver Operator in your cluster. .Prerequisites @@ -45,35 +40,3 @@ Be sure to select the *AWS EFS CSI Driver Operator* and not the *AWS EFS Operato .. Click *Install*. + After the installation finishes, the {FeatureName} CSI Operator is listed in the *Installed Operators* section of the web console. - -ifdef::AWS_EFS[] -. If you are using {FeatureName} with AWS Security Token Service (STS), you must configure the {FeatureName} CSI Driver with STS. For more information, see "Configuring AWS EFS CSI Driver with STS". -endif::AWS_EFS[] - -. Install the {FeatureName} CSI Driver: - -.. Click *administration* -> *CustomResourceDefinitions* -> *ClusterCSIDriver*. - -.. On the *Instances* tab, click *Create ClusterCSIDriver*. - -.. Use the following YAML file: -+ -[source,yaml] ----- -apiVersion: operator.openshift.io/v1 -kind: ClusterCSIDriver -metadata: - name: efs.csi.aws.com -spec: - managementState: Managed ----- - -.. Click *Create*. - -.. Wait for the following Conditions to change to a "true" status: -+ -* AWSEFSDriverCredentialsRequestControllerAvailable - -* AWSEFSDriverNodeServiceControllerAvailable - -* AWSEFSDriverControllerServiceControllerAvailable diff --git a/modules/persistent-storage-csi-olm-operator-uninstall.adoc b/modules/persistent-storage-csi-olm-operator-uninstall.adoc index bc31c643dd..5284ed0b22 100644 --- a/modules/persistent-storage-csi-olm-operator-uninstall.adoc +++ b/modules/persistent-storage-csi-olm-operator-uninstall.adoc @@ -3,7 +3,6 @@ // * storage/container_storage_interface/persistent-storage-csi-aws-efs.adoc // * storage/container_storage_interface/persistent-storage-csi-aws-efs.adoc // * storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc -// * storage/container_storage_interface/rosa-persistent-storage-aws-efs-csi.adoc :_content-type: PROCEDURE [id="persistent-storage-csi-olm-operator-uninstall_{context}"] @@ -34,7 +33,7 @@ To uninstall the {FeatureName} CSI Driver Operator from the web console: Before you can uninstall the Operator, you must remove the CSI driver first. ==== -.. Click *administration* -> *CustomResourceDefinitions* -> *ClusterCSIDriver*. +.. Click *Administration* -> *CustomResourceDefinitions* -> *ClusterCSIDriver*. .. On the *Instances* tab, for *{provisioner}*, on the far left side, click the drop-down menu, and then click *Delete ClusterCSIDriver*. diff --git a/modules/persistent-storage-efs-csi-driver-operator-setup.adoc b/modules/persistent-storage-efs-csi-driver-operator-setup.adoc new file mode 100644 index 0000000000..209f623886 --- /dev/null +++ b/modules/persistent-storage-efs-csi-driver-operator-setup.adoc @@ -0,0 +1,16 @@ +// Module included in the following assemblies: +// +// * storage/container_storage_interface/persistent-storage-csi-aws-efs.adoc +// * storage/container_storage_interface/osd-persistent-storage-csi-aws-efs.adoc + +:_content-type: PROCEDURE +[id="persistent-storage-efs-csi-driver-operator-setup_{context}"] += Setting up the {FeatureName} CSI Driver Operator + +. Install the the {FeatureName} CSI Driver Operator. + +ifdef::openshift-rosa[] +. If you are using {FeatureName} with AWS Secure Token Service (STS), configure the {FeatureName} CSI Driver with STS. +endif::openshift-rosa[] + +. Install the {FeatureName} CSI Driver. \ No newline at end of file diff --git a/modules/rosa-howto-create-persistent-storage-aws-ebs.adoc b/modules/rosa-howto-create-persistent-storage-aws-ebs.adoc deleted file mode 100644 index ec00d23468..0000000000 --- a/modules/rosa-howto-create-persistent-storage-aws-ebs.adoc +++ /dev/null @@ -1,22 +0,0 @@ -// Module included in the following assemblies: -// -// * storage/persistent_storage/rosa-persistent-storage-aws-ebs.adoc - -:_content-type: PROCEDURE -[id="rosa-howto-create-persistent-volume-claim-aws-ebs_{context}"] -= Creating a persistent volume claim - -toc::[] - -.Prerequisites -Storage must exist in the underlying infrastructure before it can be mounted as a volume in the ROSA cluster. - -.Procedure -. In the OpenShift Cluster console, click *Storage → Persistent Volume Claims*. -. In the persistent volume claims overview, click *Create Persistent Volume Claim*. -. Define the desired options on the page that appears. - .. Select the previously created storage class from the drop-down menu. - .. Enter a unique name for the storage claim. - .. Select the access mode. This selection determines the read and write access for the storage claim. - .. Define the size of the storage claim. -. Click *Create* to create the persistent volume claim and generate a persistent volume. diff --git a/modules/storage-create-storage-class-cli.adoc b/modules/storage-create-storage-class-cli.adoc index 19eb8262fd..c3da5df068 100644 --- a/modules/storage-create-storage-class-cli.adoc +++ b/modules/storage-create-storage-class-cli.adoc @@ -1,3 +1,8 @@ +// Module included in the following assemblies: +// +// * storage/persistent_storage/persistent-storage-aws-efs-csi.adoc +// * storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc + :_content-type: PROCEDURE [id="storage-create-storage-class-cli_{context}"] = Creating the {StorageClass} storage class using the CLI diff --git a/modules/storage-create-storage-class-console.adoc b/modules/storage-create-storage-class-console.adoc index 7fe8848a6c..d99db5a5e0 100644 --- a/modules/storage-create-storage-class-console.adoc +++ b/modules/storage-create-storage-class-console.adoc @@ -1,3 +1,8 @@ +// Module included in the following assemblies: +// +// * storage/persistent_storage/persistent-storage-aws-efs-csi.adoc +// * storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc + :_content-type: PROCEDURE [id="storage-create-storage-class-console_{context}"] = Creating the {StorageClass} storage class using the console diff --git a/modules/storage-create-storage-class.adoc b/modules/storage-create-storage-class.adoc index 6b90d6a29a..47ffd27d51 100644 --- a/modules/storage-create-storage-class.adoc +++ b/modules/storage-create-storage-class.adoc @@ -9,7 +9,6 @@ // * storage/container_storage_interface/persistent-storage-csi-aws-efs.adoc // * storage/persistent_storage/rosa-persistent-storage-aws-efs-csi.adoc // * storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc -// * storage/container_storage_interface/rosa-persistent-storage-aws-efs-csi.adoc :_content-type: PROCEDURE [id="storage-create-storage-class_{context}"] diff --git a/modules/storage-ephemeral-vols-overview.adoc b/modules/storage-ephemeral-vols-overview.adoc index c511c48887..582e1601f3 100644 --- a/modules/storage-ephemeral-vols-overview.adoc +++ b/modules/storage-ephemeral-vols-overview.adoc @@ -26,9 +26,11 @@ Generic ephemeral volumes have the following features: ==== Generic ephemeral volumes do not support offline snapshots and resize. +ifndef::openshift-dedicated,openshift-rosa[] Due to this limitation, the following Container Storage Interface (CSI) drivers do not support the following features for generic ephemeral volumes: * Azure Disk CSI driver does not support resize. * Cinder CSI driver does not support snapshot. +endif::openshift-dedicated,openshift-rosa[] ==== diff --git a/modules/storage-persistent-storage-aws-maximum-volumes.adoc b/modules/storage-persistent-storage-aws-maximum-volumes.adoc index bb00e44ab6..2357e0d333 100644 --- a/modules/storage-persistent-storage-aws-maximum-volumes.adoc +++ b/modules/storage-persistent-storage-aws-maximum-volumes.adoc @@ -11,5 +11,5 @@ link:https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/volume_limits.html#linu [IMPORTANT] ==== -As a cluster administrator, you must use either in-tree or Container Storage Interface (CSI) volumes and their respective storage classes, but never both volume types at the same time. The maximum attached EBS volume number is counted separately for in-tree and CSI volumes. +As a cluster administrator, you must use either in-tree or Container Storage Interface (CSI) volumes and their respective storage classes, but never both volume types at the same time. The maximum attached EBS volume number is counted separately for in-tree and CSI volumes, which means you could have up to 39 EBS volumes of each type. ==== diff --git a/modules/storage-persistent-storage-block-volume.adoc b/modules/storage-persistent-storage-block-volume.adoc index e17c65b620..6dd28f4845 100644 --- a/modules/storage-persistent-storage-block-volume.adoc +++ b/modules/storage-persistent-storage-block-volume.adoc @@ -28,18 +28,26 @@ The following table displays which volume plugins support block volumes. |=== |Volume Plugin |Manually provisioned |Dynamically provisioned |Fully supported |AWS EBS | ✅ | ✅ | ✅ +ifndef::openshift-dedicated,openshift-rosa[] |Azure Disk | ✅ | ✅ | ✅ |Azure File | | | |Cinder | ✅ | ✅ | ✅ |Fibre Channel | ✅ | | ✅ +endif::openshift-dedicated,openshift-rosa[] +ifndef::openshift-rosa[] |GCP | ✅ | ✅ | ✅ +endif::openshift-rosa[] +ifndef::openshift-dedicated,openshift-rosa[] |HostPath | | | |iSCSI | ✅ | | ✅ |Local volume | ✅ || ✅ |NFS | | | |{rh-storage-first} | ✅ | ✅ | ✅ |VMware vSphere | ✅ | ✅ | ✅ +endif::openshift-dedicated,openshift-rosa[] |=== +ifndef::openshift-dedicated,openshift-rosa[] :FeatureName: Using any of the block volumes that can be provisioned manually, but are not provided as fully supported, include::snippets/technology-preview.adoc[leveloffset=+1] +endif::openshift-dedicated,openshift-rosa[] diff --git a/modules/storage-persistent-storage-creating-volume-claim.adoc b/modules/storage-persistent-storage-creating-volume-claim.adoc index b38b3b5a1c..712792ac56 100644 --- a/modules/storage-persistent-storage-creating-volume-claim.adoc +++ b/modules/storage-persistent-storage-creating-volume-claim.adoc @@ -18,12 +18,11 @@ Claim*. . Define the desired options on the page that appears. -.. Select the storage class created previously from the drop-down menu. +.. Select the previously-created storage class from the drop-down menu. .. Enter a unique name for the storage claim. -.. Select the access mode. This determines the read and write access for the -created storage claim. +.. Select the access mode. This selection determines the read and write access for the storage claim. .. Define the size of the storage claim. diff --git a/modules/storage-persistent-storage-pv.adoc b/modules/storage-persistent-storage-pv.adoc index a62f7b3ab2..b1fc1e8512 100644 --- a/modules/storage-persistent-storage-pv.adoc +++ b/modules/storage-persistent-storage-pv.adoc @@ -48,8 +48,10 @@ ifdef::openshift-enterprise,openshift-webscale,openshift-origin[] - Cinder - Fibre Channel endif::openshift-enterprise,openshift-webscale,openshift-origin[] +ifndef::openshift-rosa[] - GCP Persistent Disk - GCP Filestore +endif::openshift-rosa[] ifdef::openshift-enterprise,openshift-webscale,openshift-origin,openshift-aro[] - HostPath - iSCSI @@ -108,9 +110,17 @@ ifndef::microshift[] ==== Volume access modes are descriptors of volume capabilities. They are not enforced constraints. The storage provider is responsible for runtime errors resulting from invalid use of the resource. -For example, NFS offers `ReadWriteOnce` access mode. You must mark the claims as `read-only` if you want to use the volume's ROX capability. Errors in the provider show up at runtime as mount errors. +ifndef::openshift-dedicated,openshift-rosa[] +For example, NFS offers `ReadWriteOnce` access mode. You must +mark the claims as `read-only` if you want to use the volume's +ROX capability. Errors in the provider show up at runtime as mount errors. -iSCSI and Fibre Channel volumes do not currently have any fencing mechanisms. You must ensure the volumes are only used by one node at a time. In certain situations, such as draining a node, the volumes can be used simultaneously by two nodes. Before draining the node, first ensure the pods that use these volumes are deleted. +iSCSI and Fibre Channel volumes do not currently have any fencing +mechanisms. You must ensure the volumes are only used by one node at a +time. In certain situations, such as draining a node, the volumes can be +used simultaneously by two nodes. Before draining the node, first ensure +the pods that use these volumes are deleted. +endif::openshift-dedicated,openshift-rosa[] ==== endif::microshift[] @@ -127,8 +137,12 @@ ifdef::openshift-enterprise,openshift-webscale,openshift-origin[] //|CephFS | ✅ | ✅ | ✅ |Cinder | ✅ | - | - |Fibre Channel | ✅ | ✅ | - +endif::[] +ifndef::openshift-rosa[] |GCP Persistent Disk | ✅ | - | - |GCP Filestore | ✅ | ✅ | ✅ +endif::openshift-rosa[] +ifdef::openshift-enterprise,openshift-webscale,openshift-origin[] //|GlusterFS | ✅ | ✅ | ✅ |HostPath | ✅ | - | - |iSCSI | ✅ | ✅ | - @@ -144,8 +158,10 @@ endif::[] -- 1. ReadWriteOnce (RWO) volumes cannot be mounted on multiple nodes. If a node fails, the system does not allow the attached RWO volume to be mounted on a new node because it is already assigned to the failed node. If you encounter a multi-attach error message as a result, force delete the pod on a shutdown or crashed node to avoid data loss in critical workloads, such as when dynamic persistent volumes are attached. 2. Use a recreate deployment strategy for pods that rely on AWS EBS. +ifndef::openshift-dedicated,openshift-rosa[] 3. If the underlying vSphere environment supports the vSAN file service, then the vSphere Container Storage Interface (CSI) Driver Operator installed by {product-title} supports provisioning of ReadWriteMany (RWX) volumes. If you do not have vSAN file service configured, and you request RWX, the volume fails to get created and an error is logged. For more information, see "Using Container Storage Interface" -> "VMware vSphere CSI Driver Operator". +endif::openshift-dedicated,openshift-rosa[] // GCE Persistent Disks, or Openstack Cinder PVs. -- endif::microshift[] @@ -213,7 +229,6 @@ You can view the name of the PVC bound to the PV by running: $ oc get pv ---- -ifdef::openshift-enterprise,openshift-webscale,openshift-origin[] [id="pv-mount-options_{context}"] === Mount options @@ -250,10 +265,15 @@ The following PV types support mount options: // - GlusterFS // - Ceph RBD - AWS Elastic Block Store (EBS) +ifndef::openshift-dedicated,openshift-rosa[] - Azure Disk - Azure File - Cinder +endif::openshift-dedicated,openshift-rosa[] +ifndef::openshift-rosa[] - GCE Persistent Disk +endif::openshift-rosa[] +ifndef::openshift-dedicated,openshift-rosa[] - iSCSI - Local volume - NFS @@ -264,4 +284,4 @@ The following PV types support mount options: ==== Fibre Channel and HostPath PVs do not support mount options. ==== -endif::openshift-enterprise,openshift-webscale,openshift-origin[] +endif::openshift-dedicated,openshift-rosa[] diff --git a/modules/storage-persistent-storage-pvc.adoc b/modules/storage-persistent-storage-pvc.adoc index 31454e1b7e..9688508e7a 100644 --- a/modules/storage-persistent-storage-pvc.adoc +++ b/modules/storage-persistent-storage-pvc.adoc @@ -43,17 +43,22 @@ provisioners to service one or more storage classes. The cluster administrator can create a PV on demand that matches the specifications in the PVC. -ifndef::microshift[] - +ifndef::microshift,openshift-rosa[] [IMPORTANT] ==== The Cluster Storage Operator might install a default storage class depending on the platform in use. This storage class is owned and controlled by the -operator. It cannot be deleted or modified beyond defining annotations +Operator. It cannot be deleted or modified beyond defining annotations and labels. If different behavior is desired, you must define a custom storage class. ==== -endif::microshift[] +endif::microshift,openshift-rosa[] +ifdef::openshift-rosa[] +[IMPORTANT] +==== +The Cluster Storage Operator installs a default storage class. This storage class is owned and controlled by the Operator. It cannot be deleted or modified beyond defining annotations and labels. If different behavior is desired, you must define a custom storage class. +==== +endif::openshift-rosa[] The cluster administrator can also set a default storage class for all PVCs. When a default storage class is configured, the PVC must explicitly ask for diff --git a/modules/storage-persistent-storage-reclaim-manual.adoc b/modules/storage-persistent-storage-reclaim-manual.adoc index 2e6ccb36cc..7c11c96a06 100644 --- a/modules/storage-persistent-storage-reclaim-manual.adoc +++ b/modules/storage-persistent-storage-reclaim-manual.adoc @@ -26,7 +26,15 @@ To manually reclaim the PV as a cluster administrator: $ oc delete pv ---- + +ifndef::openshift-dedicated,openshift-rosa[] The associated storage asset in the external infrastructure, such as an AWS EBS, GCE PD, Azure Disk, or Cinder volume, still exists after the PV is deleted. +endif::openshift-dedicated,openshift-rosa[] +ifdef::openshift-dedicated[] +The associated storage asset in the external infrastructure, such as an AWS EBS or GCE PD volume, still exists after the PV is deleted. +endif::openshift-dedicated[] +ifdef::openshift-rosa[] +The associated storage asset in the external infrastructure, such as an AWS EBS volume, still exists after the PV is deleted. +endif::openshift-rosa[] . Clean up the data on the associated storage asset. diff --git a/modules/storage-persistent-storage-volume-format.adoc b/modules/storage-persistent-storage-volume-format.adoc index 737b2d487d..4d6555921e 100644 --- a/modules/storage-persistent-storage-volume-format.adoc +++ b/modules/storage-persistent-storage-volume-format.adoc @@ -11,12 +11,12 @@ [id="volume-format-{provider}_{context}"] = Volume format Before {product-title} mounts the volume and passes it to a container, it -checks that it contains a file system as specified by the `fsType` +checks that the volume contains a file system as specified by the `fsType` parameter in the persistent volume definition. If the device is not formatted with the file system, all data from the device is erased and the device is automatically formatted with the given file system. -This allows using unformatted {provider} volumes as persistent volumes, +This verification enables you to use unformatted {provider} volumes as persistent volumes, because {product-title} formats them before the first use. // Undefined {provider} attribute, so that any mistakes are easily spotted diff --git a/storage/container_storage_interface/ephemeral-storage-csi-inline.adoc b/storage/container_storage_interface/ephemeral-storage-csi-inline.adoc index f24eb10cb6..c7f0d2f553 100644 --- a/storage/container_storage_interface/ephemeral-storage-csi-inline.adoc +++ b/storage/container_storage_interface/ephemeral-storage-csi-inline.adoc @@ -4,6 +4,8 @@ include::_attributes/common-attributes.adoc[] :context: ephemeral-storage-csi-inline +// TP features should be excluded from OSD and ROSA. When this feature is GA, it can be included in the OSD/ROSA docs, but with a warning that it is available as of version 4.x. + toc::[] Container Storage Interface (CSI) inline ephemeral volumes allow you to define a `Pod` spec that creates inline ephemeral volumes when a pod is deployed and delete them when a pod is destroyed. diff --git a/storage/container_storage_interface/ephemeral-storage-shared-resource-csi-driver-operator.adoc b/storage/container_storage_interface/ephemeral-storage-shared-resource-csi-driver-operator.adoc index aaac18a1c1..c551a688c3 100644 --- a/storage/container_storage_interface/ephemeral-storage-shared-resource-csi-driver-operator.adoc +++ b/storage/container_storage_interface/ephemeral-storage-shared-resource-csi-driver-operator.adoc @@ -17,7 +17,7 @@ include::snippets/technology-preview.adoc[leveloffset=+1] [NOTE] ==== -To enable the Shared Resource CSI Driver, you must xref:../../nodes/clusters/nodes-cluster-enabling-features.adoc#nodes-cluster-enabling[enable features using feature gates] +To enable the Shared Resource CSI Driver, you must xref:../../nodes/clusters/nodes-cluster-enabling-features.adoc#nodes-cluster-enabling[enable features using feature gates]. ==== include::modules/persistent-storage-csi-about.adoc[leveloffset=+1] diff --git a/storage/persistent_storage/osd-persistent-storage-aws-efs-csi.adoc b/storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc similarity index 55% rename from storage/persistent_storage/osd-persistent-storage-aws-efs-csi.adoc rename to storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc index 84062b386b..043714f511 100644 --- a/storage/persistent_storage/osd-persistent-storage-aws-efs-csi.adoc +++ b/storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc @@ -5,7 +5,7 @@ include::_attributes//attributes-openshift-dedicated.adoc[] :context: osd-persistent-storage-aws-efs-csi toc::[] -//Content similar to persistent-storage-csi-aws-efs.adoc and rosa-persistent-storage-aws-efs-csi.adoc. Modules are reused. +// Content similar to persistent-storage-csi-aws-efs.adoc. Modules are reused. [IMPORTANT] ==== @@ -34,44 +34,54 @@ AWS EFS only supports regional volumes, not zonal volumes. include::modules/persistent-storage-csi-about.adoc[leveloffset=+1] :FeatureName: AWS EFS +include::modules/persistent-storage-efs-csi-driver-operator-setup.adoc[leveloffset=+1] -include::modules/persistent-storage-csi-olm-operator-install.adoc[leveloffset=+1] +include::modules/persistent-storage-csi-olm-operator-install.adoc[leveloffset=+2] +.Next steps +ifdef::openshift-rosa[] +* If you are using {FeatureName} with AWS Secure Token Service (STS), you must configure the {FeatureName} CSI Driver with STS. For more information, see xref:../../storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc#efs-sts_osd-persistent-storage-aws-efs-csi[Configuring {FeatureName} CSI Driver with STS]. +endif::openshift-rosa[] +ifdef::openshift-dedicated[] +* xref:../../storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc#persistent-storage-csi-efs-driver-install_osd-persistent-storage-aws-efs-csi[Installing the {FeatureName} CSI Driver] +endif::openshift-dedicated[] +// Separate procedure for OSD and ROSA. +ifdef::openshift-rosa[] +include::modules/osd-persistent-storage-csi-efs-sts.adoc[leveloffset=+2] [role="_additional-resources"] .Additional resources +* xref:../../storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc#persistent-storage-csi-olm-operator-install_osd-persistent-storage-aws-efs-csi[Installing the {FeatureName} CSI Driver Operator] +* xref:../../storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc#persistent-storage-csi-efs-driver-install_osd-persistent-storage-aws-efs-csi[Installing the {FeatureName} CSI Driver] +endif::openshift-rosa[] -* xref:../../storage/persistent_storage/osd-persistent-storage-aws-efs-csi.adoc#efs-sts_osd-persistent-storage-aws-efs-csi[Configuring AWS EFS CSI Driver with STS] - -include::modules/persistent-storage-csi-efs-sts.adoc[leveloffset=+1] - -[role="_additional-resources"] -.Additional resources - -* xref:../../storage/persistent_storage/osd-persistent-storage-aws-efs-csi.adoc#persistent-storage-csi-olm-operator-install_osd-persistent-storage-aws-efs-csi[Installing the AWS EFS CSI Driver Operator] - -* link:https://access.redhat.com/documentation/en-us/openshift_container_platform/4.12/html-single/authentication_and_authorization/index#cco-ccoctl-configuring_cco-mode-sts[Configuring the Cloud Credential Operator utility] +include::modules/persistent-storage-csi-efs-driver-install.adoc[leveloffset=+2] :StorageClass: AWS EFS :Provisioner: efs.csi.aws.com - include::modules/storage-create-storage-class.adoc[leveloffset=+1] +include::modules/storage-create-storage-class-console.adoc[leveloffset=+2] +include::modules/storage-create-storage-class-cli.adoc[leveloffset=+2] include::modules/persistent-storage-csi-efs-create-volume.adoc[leveloffset=+1] include::modules/persistent-storage-csi-dynamic-provisioning-aws-efs.adoc[leveloffset=+1] +If you have problems setting up dynamic provisioning, see xref:../../storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc#efs-troubleshooting_osd-persistent-storage-aws-efs-csi[{FeatureName} troubleshooting]. +[role="_additional-resources"] +.Additional resources +* xref:../../storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc#efs-create-volume_osd-persistent-storage-aws-efs-csi[Creating and configuring access to {FeatureName} volume(s)] +* xref:../../storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc#storage-create-storage-class_osd-persistent-storage-aws-efs-csi[Creating the {FeatureName} storage class] -If you have problems setting up dynamic provisioning, see xref:../../storage/persistent_storage/osd-persistent-storage-aws-efs-csi.adoc#efs-troubleshooting_osd-persistent-storage-aws-efs-csi[AWS EFS troubleshooting]. +// Undefine {StorageClass} attribute, so that any mistakes are easily spotted +:!StorageClass: include::modules/persistent-storage-csi-efs-static-pv.adoc[leveloffset=+1] - -If you have problems setting up static PVs, see xref:../../storage/persistent_storage/osd-persistent-storage-aws-efs-csi.adoc#efs-troubleshooting_osd-persistent-storage-aws-efs-csi[AWS EFS troubleshooting]. +If you have problems setting up static PVs, see xref:../../storage/container_storage_interface/osd-persistent-storage-aws-efs-csi.adoc#efs-troubleshooting_osd-persistent-storage-aws-efs-csi[{FeatureName} troubleshooting]. include::modules/persistent-storage-csi-efs-security.adoc[leveloffset=+1] include::modules/persistent-storage-csi-efs-troubleshooting.adoc[leveloffset=+1] :FeatureName: AWS EFS - include::modules/persistent-storage-csi-olm-operator-uninstall.adoc[leveloffset=+1] [role="_additional-resources"] diff --git a/storage/container_storage_interface/persistent-storage-csi-aws-efs.adoc b/storage/container_storage_interface/persistent-storage-csi-aws-efs.adoc index 1568025a31..3fd0c113f8 100644 --- a/storage/container_storage_interface/persistent-storage-csi-aws-efs.adoc +++ b/storage/container_storage_interface/persistent-storage-csi-aws-efs.adoc @@ -6,6 +6,8 @@ include::_attributes/common-attributes.adoc[] toc::[] +// Content similar to osd-persistent-storage-csi-aws-efs.adoc and rosa-persistent-storage-aws-efs-csi.adoc. Modules are reused. + == Overview {product-title} is capable of provisioning persistent volumes (PVs) using the Container Storage Interface (CSI) driver for AWS Elastic File Service (EFS). @@ -28,16 +30,20 @@ AWS EFS only supports regional volumes, not zonal volumes. include::modules/persistent-storage-csi-about.adoc[leveloffset=+1] :FeatureName: AWS EFS -include::modules/persistent-storage-csi-olm-operator-install.adoc[leveloffset=+1] -[role="_additional-resources"] -.Additional resources -* xref:../../storage/container_storage_interface//persistent-storage-csi-aws-efs.adoc#efs-sts_persistent-storage-csi-aws-efs[Configuring AWS EFS CSI Driver with STS] +include::modules/persistent-storage-efs-csi-driver-operator-setup.adoc[leveloffset=+1] -include::modules/persistent-storage-csi-efs-sts.adoc[leveloffset=+1] +include::modules/persistent-storage-csi-olm-operator-install.adoc[leveloffset=+2] +.Next steps +* If you are using {FeatureName} with AWS Secure Token Service (STS), you must configure the {FeatureName} CSI Driver with STS. For more information, see xref:../../storage/container_storage_interface/persistent-storage-csi-aws-efs.adoc#efs-sts_persistent-storage-csi-aws-efs[Configuring AWS EFS CSI Driver with STS]. + +include::modules/persistent-storage-csi-efs-sts.adoc[leveloffset=+2] [role="_additional-resources"] .Additional resources * xref:../../storage/container_storage_interface/persistent-storage-csi-aws-efs.adoc#persistent-storage-csi-olm-operator-install_persistent-storage-csi-aws-efs[Installing the AWS EFS CSI Driver Operator] * xref:../../authentication/managing_cloud_provider_credentials/cco-mode-sts.adoc#cco-ccoctl-configuring_cco-mode-sts[Configuring the Cloud Credential Operator utility] +* xref:../../storage/container_storage_interface/persistent-storage-csi-aws-efs.adoc#persistent-storage-csi-efs-driver-install_persistent-storage-csi-aws-efs[Installing the {FeatureName} CSI Driver] + +include::modules/persistent-storage-csi-efs-driver-install.adoc[leveloffset=+2] :StorageClass: AWS EFS :Provisioner: efs.csi.aws.com diff --git a/storage/container_storage_interface/persistent-storage-csi-ebs.adoc b/storage/container_storage_interface/persistent-storage-csi-ebs.adoc index e6ee17542f..e6df5b3077 100644 --- a/storage/container_storage_interface/persistent-storage-csi-ebs.adoc +++ b/storage/container_storage_interface/persistent-storage-csi-ebs.adoc @@ -18,10 +18,12 @@ To create CSI-provisioned PVs that mount to AWS EBS storage assets, {product-tit * The _AWS EBS CSI driver_ enables you to create and mount AWS EBS PVs. +ifndef::openshift-rosa,openshift-dedicated[] [NOTE] ==== If you installed the AWS EBS CSI Operator and driver on an {product-title} 4.5 cluster, you must uninstall the 4.5 Operator and driver before you update to {product-title} {product-version}. ==== +endif::openshift-rosa,openshift-dedicated[] include::modules/persistent-storage-csi-about.adoc[leveloffset=+1] @@ -29,7 +31,10 @@ include::modules/persistent-storage-csi-about.adoc[leveloffset=+1] ==== {product-title} defaults to using an in-tree (non-CSI) plugin to provision AWS EBS storage. -In future {product-title} versions, volumes provisioned using existing in-tree plugins are planned for migration to their equivalent CSI driver. CSI automatic migration should be seamless. Migration does not change how you use all existing API objects, such as persistent volumes, persistent volume claims, and storage classes. For more information about migration, see xref:../../storage/container_storage_interface/persistent-storage-csi-migration.adoc#persistent-storage-csi-migration[CSI automatic migration]. +In future {product-title} versions, volumes provisioned using existing in-tree plug-ins are planned for migration to their equivalent CSI driver. CSI automatic migration should be seamless. Migration does not change how you use all existing API objects, such as persistent volumes, persistent volume claims, and storage classes. +ifndef::openshift-rosa,openshift-dedicated[] +For more information about migration, see xref:../../storage/container_storage_interface/persistent-storage-csi-migration.adoc#persistent-storage-csi-migration[CSI automatic migration]. +endif::openshift-rosa,openshift-dedicated[] After full migration, in-tree plugins will eventually be removed in future versions of {product-title}. ==== diff --git a/storage/container_storage_interface/persistent-storage-csi-gcp-pd.adoc b/storage/container_storage_interface/persistent-storage-csi-gcp-pd.adoc index 910a2febf8..65727bc2ec 100644 --- a/storage/container_storage_interface/persistent-storage-csi-gcp-pd.adoc +++ b/storage/container_storage_interface/persistent-storage-csi-gcp-pd.adoc @@ -24,7 +24,10 @@ To create CSI-provisioned persistent volumes (PVs) that mount to GCP PD storage ==== {product-title} defaults to using an in-tree (non-CSI) plugin to provision GCP PD storage. -In future {product-title} versions, volumes provisioned using existing in-tree plugins are planned for migration to their equivalent CSI driver. CSI automatic migration should be seamless. Migration does not change how you use all existing API objects, such as persistent volumes, persistent volume claims, and storage classes. For more information about migration, see xref:../../storage/container_storage_interface/persistent-storage-csi-migration.adoc#persistent-storage-csi-migration[CSI automatic migration]. +In future {product-title} versions, volumes provisioned using existing in-tree plug-ins are planned for migration to their equivalent CSI driver. CSI automatic migration should be seamless. Migration does not change how you use all existing API objects, such as persistent volumes, persistent volume claims, and storage classes. +ifndef::openshift-dedicated[] +For more information about migration, see xref:../../storage/container_storage_interface/persistent-storage-csi-migration.adoc#persistent-storage-csi-migration[CSI automatic migration]. +endif::openshift-dedicated[] After full migration, in-tree plugins will eventually be removed in future versions of {product-title}. ==== diff --git a/storage/container_storage_interface/persistent-storage-csi-google-cloud-file.adoc b/storage/container_storage_interface/persistent-storage-csi-google-cloud-file.adoc index f0567d1a82..d73114b13c 100644 --- a/storage/container_storage_interface/persistent-storage-csi-google-cloud-file.adoc +++ b/storage/container_storage_interface/persistent-storage-csi-google-cloud-file.adoc @@ -5,6 +5,8 @@ include::_attributes/common-attributes.adoc[] toc::[] +// TP features should be excluded from OSD and ROSA. When this feature is GA, it can be included in the OSD/ROSA docs, but with a warning that it is available as of version 4.x. + [id="persistent-storage-csi-google-cloud-file-overview"] == Overview diff --git a/storage/container_storage_interface/persistent-storage-csi.adoc b/storage/container_storage_interface/persistent-storage-csi.adoc index 726d149f28..b8ea0ff6b7 100644 --- a/storage/container_storage_interface/persistent-storage-csi.adoc +++ b/storage/container_storage_interface/persistent-storage-csi.adoc @@ -28,6 +28,9 @@ include::modules/persistent-storage-csi-dynamic-provisioning.adoc[leveloffset=+1 include::modules/persistent-storage-csi-mysql-example.adoc[leveloffset=+1] +// TP features should be excluded from OSD and ROSA. When this feature is GA, it can be included in the OSD/ROSA docs, but with a warning that it is available as of version 4.x. +ifndef::openshift-dedicated,openshift-rosa[] include::modules/persistent-storage-csi-vol-populator.adoc[leveloffset=+1] -For more information about volume populators, see link:https://kubernetes.io/blog/2022/05/16/volume-populators-beta/[Kubernetes volume populators]. \ No newline at end of file +For more information about volume populators, see link:https://kubernetes.io/blog/2022/05/16/volume-populators-beta/[Kubernetes volume populators]. +endif::openshift-dedicated,openshift-rosa[] \ No newline at end of file diff --git a/storage/dynamic-provisioning.adoc b/storage/dynamic-provisioning.adoc index 9d02c810ce..4838b14b7c 100644 --- a/storage/dynamic-provisioning.adoc +++ b/storage/dynamic-provisioning.adoc @@ -16,25 +16,31 @@ include::modules/dynamic-provisioning-storage-class-definition.adoc[leveloffset= include::modules/dynamic-provisioning-annotations.adoc[leveloffset=+2] +ifndef::openshift-dedicated,openshift-rosa[] include::modules/dynamic-provisioning-cinder-definition.adoc[leveloffset=+2] include::modules/dynamic-provisioning-manila-csi-definition.adoc[leveloffset=+2] +endif::openshift-dedicated,openshift-rosa[] include::modules/dynamic-provisioning-aws-definition.adoc[leveloffset=+2] +ifndef::openshift-dedicated,openshift-rosa[] include::modules/dynamic-provisioning-azure-disk-definition.adoc[leveloffset=+2] include::modules/dynamic-provisioning-azure-file-definition.adoc[leveloffset=+2] include::modules/dynamic-provisioning-azure-file-considerations.adoc[leveloffset=+3] +endif::openshift-dedicated,openshift-rosa[] +ifndef::openshift-rosa[] include::modules/dynamic-provisioning-gce-definition.adoc[leveloffset=+2] - +endif::openshift-rosa[] // include::modules/dynamic-provisioning-gluster-definition.adoc[leveloffset=+2] // include::modules/dynamic-provisioning-ceph-rbd-definition.adoc[leveloffset=+2] +ifndef::openshift-dedicated,openshift-rosa[] include::modules/dynamic-provisioning-vsphere-definition.adoc[leveloffset=+2] - +endif::openshift-dedicated,openshift-rosa[] include::modules/dynamic-provisioning-change-default-class.adoc[leveloffset=+1] diff --git a/storage/persistent_storage/osd-persistent-storage-aws.adoc b/storage/persistent_storage/osd-persistent-storage-aws.adoc deleted file mode 100644 index 4c14c97e1e..0000000000 --- a/storage/persistent_storage/osd-persistent-storage-aws.adoc +++ /dev/null @@ -1,46 +0,0 @@ -:_content-type: ASSEMBLY -[id="osd-persistent-storage-aws"] -= Setting up AWS EFS for {product-title} -include::_attributes/attributes-openshift-dedicated.adoc[] -:context: osd-persistent-storage-aws - -toc::[] - -ifdef::openshift-rosa[] -[WARNING] -==== -This procedure is specific to the Amazon Web Services Elastic File System (AWS EFS) community Operator, which is only applicable up to {product-title} 4.9. -==== -endif::openshift-rosa[] - -The Amazon Web Services Elastic File System (AWS EFS) is a Network File System (NFS) that can be provisioned on {product-title} clusters. AWS also provides and supports a CSI EFS Driver to be used with Kubernetes that allows Kubernetes workloads to leverage this shared file storage. - -This document describes the basic steps needed to set up your AWS account to prepare EFS to be used by {product-title}. For more information about AWS EFS, see the link:https://docs.aws.amazon.com/efs/index.html[AWS EFS documentation]. - -[IMPORTANT] -==== -Red Hat does not provide official support for this feature, including backup and recovery. The customer is responsible for backing up the EFS data and recovering it in the event of an outage or data loss. -==== - -The high-level process to enable EFS on a cluster is: - -. Create an AWS EFS in the AWS account used by the cluster. -. Install the AWS EFS Operator from OperatorHub. -. Create `SharedVolume` custom resources. -. Use the generated persistent volume claims in pod `spec.volumes`. - -== Prerequisites -ifdef::openshift-dedicated[] -* Customer Cloud Subscription (CCS) for an {product-title} cluster -endif::[] -ifdef::openshift-rosa[] -* A {product-title} cluster -endif::[] -* Administrator access to the AWS account of that cluster - -include::modules/osd-storage-pv-aws-config-account.adoc[leveloffset=+1] -include::modules/osd-storage-pv-aws-install-efs.adoc[leveloffset=+1] -include::modules/osd-storage-pv-aws-create-sharedvolumes-console.adoc[leveloffset=+1] -include::modules/osd-storage-pv-aws-create-sharedvolumes-cli.adoc[leveloffset=+1] -include::modules/osd-storage-pv-aws-connect-pods.adoc[leveloffset=+1] -include::modules/osd-storage-pv-aws-uninstall-efs.adoc[leveloffset=+1] diff --git a/storage/persistent_storage/persistent-storage-aws.adoc b/storage/persistent_storage/persistent-storage-aws.adoc index 6b375529d6..98c2d56480 100644 --- a/storage/persistent_storage/persistent-storage-aws.adoc +++ b/storage/persistent_storage/persistent-storage-aws.adoc @@ -6,24 +6,47 @@ include::_attributes/common-attributes.adoc[] toc::[] -{product-title} supports AWS Elastic Block Store volumes (EBS). You can -provision your {product-title} cluster with persistent storage by using link:https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html[Amazon EC2]. -Some familiarity with Kubernetes and AWS is assumed. +ifdef::openshift-rosa,openshift-dedicated[] +{product-title} clusters are prebuilt with four storage classes that use AWS Elastic Block Store (EBS) volumes. These storage classes are ready to use and some familiarity with Kubernetes and AWS is assumed. -The Kubernetes persistent volume framework allows administrators to provision a -cluster with persistent storage and gives users a way to request those -resources without having any knowledge of the underlying infrastructure. -AWS Elastic Block Store volumes can be provisioned dynamically. -Persistent volumes are not bound to a single project or namespace; they can be -shared across the {product-title} cluster. -Persistent volume claims are specific to a project or namespace and can be -requested by users. You can define a KMS key to encrypt container-persistent volumes on AWS. +Following are the four prebuilt storage classes: +[options="header"] + +|=== + +| Name | Provisioner + +| gp2 | kubernetes.io/aws-ebs + +| gp2-csi | ebs.csi.aws.com + +| gp3 (default) | kubernetes.io/aws-ebs + +| gp3-csi | ebs.csi.aws.com + +|=== +The gp3 storage class is set as default; however, you can select any of the storage classes as the default storage class. +endif::openshift-rosa,openshift-dedicated[] + +ifndef::openshift-rosa,openshift-dedicated[] +{product-title} supports AWS Elastic Block Store volumes (EBS). +You can provision your {product-title} cluster with persistent storage by using link:https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html[Amazon EC2]. +endif::openshift-rosa,openshift-dedicated[] + +The Kubernetes persistent volume framework allows administrators to provision a cluster with persistent storage and gives users a way to request those resources without having any knowledge of the underlying infrastructure. +You can dynamically provision AWS EBS volumes. +Persistent volumes are not bound to a single project or namespace; they can be shared across the {product-title} cluster. +Persistent volume claims are specific to a project or namespace and can be requested by users. +You can define a KMS key to encrypt container-persistent volumes on AWS. [IMPORTANT] ==== -{product-title} defaults to using an in-tree (non-CSI) plugin to provision AWS EBS storage. +{product-title} defaults to using an in-tree, or non-Container Storage Interface (CSI) plug-in to provision AWS EBS storage. In future {product-title} versions, volumes provisioned using existing in-tree plug-ins are planned for migration to their equivalent CSI driver. -In future {product-title} versions, volumes provisioned using existing in-tree plugins are planned for migration to their equivalent CSI driver. CSI automatic migration should be seamless. Migration does not change how you use all existing API objects, such as persistent volumes, persistent volume claims, and storage classes. For more information about migration, see xref:../../storage/container_storage_interface/persistent-storage-csi-migration.adoc#persistent-storage-csi-migration[CSI automatic migration]. +CSI automatic migration should be seamless. Migration does not change how you use all existing API objects, such as persistent volumes, persistent volume claims, and storage classes. +ifndef::openshift-rosa,openshift-dedicated[] +For more information about migration, see xref:../../storage/container_storage_interface/persistent-storage-csi-migration.adoc#persistent-storage-csi-migration[CSI automatic migration]. +endif::openshift-rosa,openshift-dedicated[] After full migration, in-tree plugins will eventually be removed in future versions of {product-title}. ==== @@ -34,9 +57,10 @@ High-availability of storage in the infrastructure is left to the underlying storage provider. ==== +ifndef::openshift-rosa,openshift-dedicated[] For {product-title}, automatic migration from AWS EBS in-tree to the Container Storage Interface (CSI) driver is available as a Technology Preview (TP) feature. With migration enabled, volumes provisioned using the existing in-tree driver are automatically migrated to use the AWS EBS CSI driver. For more information, see xref:../../storage/container_storage_interface/persistent-storage-csi-migration.adoc#persistent-storage-csi-migration[CSI automatic migration feature]. - +endif::openshift-rosa,openshift-dedicated[] // Defining attributes required by the next module :StorageClass: EBS @@ -52,6 +76,8 @@ include::modules/storage-persistent-storage-volume-format.adoc[leveloffset=+1] include::modules/storage-persistent-storage-aws-maximum-volumes.adoc[leveloffset=+1] +For information about accessing additional storage options, such as volume snapshots, that are not possible with in-tree volume plug-ins, see xref:../../storage/container_storage_interface/persistent-storage-csi-ebs.adoc#persistent-storage-csi-ebs[AWS Elastic Block Store CSI Driver Operator]. + include::modules/storage-persistent-storage-volume-encrypt-with-kms-key.adoc[leveloffset=+1] [id="additional-resources_persistent-storage-aws"] diff --git a/storage/persistent_storage/persistent-storage-gce.adoc b/storage/persistent_storage/persistent-storage-gce.adoc index 170fc1f685..592c1f137d 100644 --- a/storage/persistent_storage/persistent-storage-gce.adoc +++ b/storage/persistent_storage/persistent-storage-gce.adoc @@ -27,7 +27,10 @@ requested by users. ==== {product-title} defaults to using an in-tree (non-CSI) plugin to provision gcePD storage. -In future {product-title} versions, volumes provisioned using existing in-tree plugins are planned for migration to their equivalent CSI driver. CSI automatic migration should be seamless. Migration does not change how you use all existing API objects, such as persistent volumes, persistent volume claims, and storage classes. For more information about migration, see xref:../../storage/container_storage_interface/persistent-storage-csi-migration.adoc#persistent-storage-csi-migration[CSI automatic migration]. +In future {product-title} versions, volumes provisioned using existing in-tree plug-ins are planned for migration to their equivalent CSI driver. CSI automatic migration should be seamless. Migration does not change how you use all existing API objects, such as persistent volumes, persistent volume claims, and storage classes. +ifndef::openshift-dedicated[] +For more information about migration, see xref:../../storage/container_storage_interface/persistent-storage-csi-migration.adoc#persistent-storage-csi-migration[CSI automatic migration]. +endif::openshift-dedicated[] After full migration, in-tree plugins will eventually be removed in future versions of {product-title}. ==== diff --git a/storage/persistent_storage/rosa-persistent-storage-aws-ebs.adoc b/storage/persistent_storage/rosa-persistent-storage-aws-ebs.adoc deleted file mode 100644 index 11a1fccd16..0000000000 --- a/storage/persistent_storage/rosa-persistent-storage-aws-ebs.adoc +++ /dev/null @@ -1,49 +0,0 @@ -:_content-type: ASSEMBLY -[id="rosa-persistent-storage-aws-ebs"] -= Persistent storage using AWS Elastic Block Store (EBS) -include::_attributes/attributes-openshift-dedicated.adoc[] -:context: rosa-persistent-storage-aws-ebs - -toc::[] - -{product-title} (ROSA) clusters are prebuilt with two storage classes that use AWS Elastic Block Store (EBS) volumes. These storage classes are ready to use and some familiarity with Kubernetes and AWS is assumed. - -Following are the two prebuilt storage classes: -[options="header"] - -|=== - -| Name | Provisioner - -| gp3 | kubernetes.io/aws-ebs - -| gp3-csi | ebs.csi.aws.com - -|=== -The gp3 storage class is set as default; however, you can select either one as the default storage class. - -The Kubernetes persistent volume framework enables administrators to provision a cluster with persistent storage and gives users a way to request those resources without having any knowledge of the underlying infrastructure. You can dynamically provision AWS EBS volumes. Persistent volumes are not bound to a single project or namespace; therefore, the volumes can be shared across ROSA clusters. Persistent volume claims are specific to a project or namespace and can be requested by users. - -[IMPORTANT] -==== -* ROSA defaults to using an in-tree, or non-Container Storage Interface (CSI), plugin to provision AWS EBS storage. In future ROSA versions, volumes provisioned using existing in-tree plugins are planned for migration to their equivalent CSI driver. After full migration, the in-tree plugins are planned to be removed from the future versions of ROSA. -* High-availability of storage in the infrastructure is left to the underlying storage provider. -==== - -[id="format-persistent-storage-aws-ebs"] -== Format of persistent volumes -Before a ROSA cluster mounts the volume and passes it to a container, the cluster checks that the volume contains a file system as specified by the *fsType* parameter in the persistent volume definition. If the device is not formatted with a file system, all data from the device is erased and the device is automatically formatted with the given file system. -This verification enables you to use unformatted AWS volumes as persistent volumes, as the ROSA cluster formats the AWS volumes before the first use. - -[id="capacity-persistent-storage-aws-ebs"] -== Capacity of EBS volumes on a node -By default, a ROSA cluster supports a maximum of 39 EBS volumes attached to one node. This limit is consistent with the link:https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/volume_limits.html#linux-specific-volume-limits[AWS volume limits]. The volume limit depends on the instance type. - -[IMPORTANT] -==== -You must use either in-tree or CSI volumes and their respective storage classes, but never both volume types at the same time. The maximum attached EBS volume number is counted separately for in-tree and CSI volumes, so you could have up to 39 EBS volumes of each type. -==== - -For information about accessing additional storage options, such as volume snapshots, that are not possible with in-tree volume plugins, see link:https://docs.openshift.com/container-platform/4.9/storage/container_storage_interface/persistent-storage-csi-ebs.html#persistent-storage-csi-ebs[Elastic Block Store CSI Driver Operator]. - -include::modules/rosa-howto-create-persistent-storage-aws-ebs.adoc[leveloffset=+1] diff --git a/storage/understanding-ephemeral-storage.adoc b/storage/understanding-ephemeral-storage.adoc index b2cb4415c1..97fefcb1d0 100644 --- a/storage/understanding-ephemeral-storage.adoc +++ b/storage/understanding-ephemeral-storage.adoc @@ -12,6 +12,8 @@ include::modules/storage-ephemeral-storage-types.adoc[leveloffset=+1] include::modules/storage-ephemeral-storage-manage.adoc[leveloffset=+1] +ifndef::openshift-dedicated,openshift-rosa[] For information about defining quotas for projects, see xref:../applications/quotas/quotas-setting-per-project.adoc[Quota setting per project]. +endif::openshift-dedicated,openshift-rosa[] include::modules/storage-ephemeral-storage-monitoring.adoc[leveloffset=+1] diff --git a/storage/understanding-persistent-storage.adoc b/storage/understanding-persistent-storage.adoc index a10c61aea6..9c7d7ff70f 100644 --- a/storage/understanding-persistent-storage.adoc +++ b/storage/understanding-persistent-storage.adoc @@ -16,13 +16,14 @@ include::modules/storage-persistent-storage-reclaim.adoc[leveloffset=+2] include::modules/storage-persistent-storage-pv.adoc[leveloffset=+1] +ifndef::openshift-dedicated,openshift-rosa[] [role="_additional-resources"] .Additional resources * xref:../storage/container_storage_interface/persistent-storage-csi-vsphere.adoc#persistent-storage-csi-vsphere-rwx_persistent-storage-csi-vsphere[ReadWriteMany vSphere volume support] +endif::openshift-dedicated,openshift-rosa[] include::modules/storage-persistent-storage-pvc.adoc[leveloffset=+1] -ifdef::openshift-enterprise,openshift-webscale,openshift-origin[] include::modules/storage-persistent-storage-block-volume.adoc[leveloffset=+1] // As these volumes have transitioned to being tech preview per plugin, @@ -31,6 +32,5 @@ include::modules/storage-persistent-storage-block-volume.adoc[leveloffset=+1] // include::snippets/technology-preview.adoc[leveloffset=+1] include::modules/storage-persistent-storage-block-volume-examples.adoc[leveloffset=+2] -endif::openshift-enterprise,openshift-webscale,openshift-origin[] include::modules/storage-persistent-storage-fsGroup.adoc[leveloffset=+1]