CEPH RGW with CSI

https://github.com/yandex-cloud/k8s-csi-s3

https://access.redhat.com/documentation/zh-cn/red_hat_ceph_storage/6/html/developer_guide/s3-bucket-operations#s3-bucket-policies_dev

https://access.redhat.com/documentation/zh-cn/red_hat_ceph_storage/6/html-single/object_gateway_guide/index#deploying-the-ceph-object-gateway-using-the-command-line-interface_rgw

https://access.redhat.com/documentation/zh-cn/red_hat_ceph_storage/3/html/object_gateway_guide_for_red_hat_enterprise_linux/rgw-administration-rgw#administrative-data-storage-rgw

Step 1 Deploy Ceph Rgw

# rgw rgw moduleceph mgr module enable rgw 

# use command create rgwceph orch apply rgw default-realm default-zone --placement="2  ceph01" --port=8000

注意3.2. 使用命令行界面部署 Ceph 对象网关 Red Hat Ceph Storage 5 | Red Hat Customer Portal

NUMBER_OF_DAEMONS 控制每个主机上部署的 Ceph 对象网关数量。要在不产生额外成本的情况下获得最高性能，请将此值设置为 2。

# use yml create rgw[root@ceph01 ~]# cat rgw.yml service_type: rgwservice_id: defaultplacement:  count_per_host: 2  hosts:   - ceph01spec:  rgw_realm: default-realm  rgw_zone: default-zone  rgw_frontend_port: 1234

# create radonsgw userradosgw-admin user create --uid=s3 --display-name="object_storage" --system# get access_key, secret_keyradosgw-admin user info --uid=s3 --display-name="object_storage" --system

# create custom bucket, use s3cmdyum -y install s3cmd# set access_key, secret_key, host_base, host_buckets3cmd --configurecat /root/.s3cfg[default]access_key = HIHPOUURDQ6SIEBVKTGOaccess_token = add_encoding_exts = add_headers = bucket_location = USca_certs_file = cache_file = check_ssl_certificate = Truecheck_ssl_hostname = Truecloudfront_host = cloudfront.amazonaws.comconnection_max_age = 5connection_pooling = Truecontent_disposition = content_type = default_mime_type = binary/octet-streamdelay_updates = Falsedelete_after = Falsedelete_after_fetch = Falsedelete_removed = Falsedry_run = Falseenable_multipart = Trueencoding = UTF-8encrypt = Falseexpiry_date = expiry_days = expiry_prefix = follow_symlinks = Falseforce = Falseget_continue = Falsegpg_command = /usr/bin/gpggpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)sgpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)sgpg_passphrase = guess_mime_type = Truehost_base = http://10.220.9.54:80#host_bucket = %(bucket)s.s3.amazonaws.comhost_bucket = http://10.220.9.54:80human_readable_sizes = Falseinvalidate_default_index_on_cf = Falseinvalidate_default_index_root_on_cf = Trueinvalidate_on_cf = Falsekeep_dirs = Falsekms_key = limit = -1limitrate = 0list_allow_unordered = Falselist_md5 = Falselog_target_prefix = long_listing = Falsemax_delete = -1max_retries = 5mime_type = multipart_chunk_size_mb = 15multipart_copy_chunk_size_mb = 1024multipart_max_chunks = 10000preserve_attrs = Trueprogress_meter = Trueproxy_host = proxy_port = 0public_url_use_https = Falseput_continue = Falserecursive = Falserecv_chunk = 65536reduced_redundancy = Falserequester_pays = Falserestore_days = 1restore_priority = Standardsecret_key = WYF14K7MJb6hOt4AALLvxWfN43rnSaYnz9CoT8Ymsend_chunk = 65536server_side_encryption = Falsesignature_v2 = Falsesignurl_use_https = Falsesimpledb_host = sdb.amazonaws.comskip_destination_validation = Falseskip_existing = Falsesocket_timeout = 300ssl_client_cert_file = ssl_client_key_file = stats = Falsestop_on_error = Falsestorage_class = throttle_max = 100upload_id = urlencoding_mode = normaluse_http_expect = Falseuse_https = Falseuse_mime_magic = Trueverbosity = WARNINGwebsite_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/website_error = website_index = index.html

# create default buckets3cmd mb s3://my-new-bucket --host=localhost --host-bucket=localhost

Step 2 Install k8s-csi-s3

# Download git projectgit clone https://github.com/yandex-cloud/k8s-csi-s3.git1. Create a secret with your S3 credentialsapiVersion: v1kind: Secretmetadata:  name: csi-s3-secret  # Namespace depends on the configuration in the storageclass.yaml  namespace: kube-systemstringData:  accessKeyID:   secretAccessKey:   # For AWS set it to "https://s3..amazonaws.com", for example https://s3.eu-central-1.amazonaws.com  endpoint: https://storage.yandexcloud.net  # For AWS set it to AWS region  #region: ""The region can be empty if you are using some other S3 compatible storage.2. Deploy the drivercd deploy/kuberneteskubectl create -f provisioner.yamlkubectl create -f driver.yamlkubectl create -f csi-s3.yaml# use pvc-maunal.yaml deployapiVersion: v1kind: PersistentVolumemetadata:  name: manualbucket-with-pathspec:  storageClassName: csi-s3  capacity:    storage: 10Gi  accessModes:    - ReadWriteMany  claimRef:    namespace: default    name: csi-s3-manual-pvc  csi:    driver: ru.yandex.s3.csi    controllerPublishSecretRef:      name: csi-s3-secret      namespace: kube-system    nodePublishSecretRef:      name: csi-s3-secret      namespace: kube-system    nodeStageSecretRef:      name: csi-s3-secret      namespace: kube-system    volumeAttributes:      capacity: 10Gi      mounter: geesefs      options: --memory-limit 1000 --dir-mode 0777 --file-mode 0666     #options: --memory-limit 1000 --dir-mode 0777 --file-mode 0666 --stat-cache-ttl 0s # --stat-cache-ttl 0s:"no cache,default 1m0s"    volumeHandle: manualbucket/path # default_bucket---apiVersion: v1kind: PersistentVolumeClaimmetadata:  name: csi-s3-manual-pvcspec:  # Empty storage class disables dynamic provisioning  storageClassName: ""  accessModes:    - ReadWriteMany  resources:    requests:      storage: 10Gi

Step 3 Test & Check

Check if the PVC has been bound:$ kubectl get pvc csi-s3-pvcNAME         STATUS    VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGEcsi-s3-pvc   Bound     pvc-c5d4634f-8507-11e8-9f33-0e243832354b   5Gi        RWO            csi-s3         9sCreate a test pod which mounts your volume:kubectl create -f examples/pod.yamlIf the pod can start, everything should be working.Test the mount$ kubectl exec -ti csi-s3-test-nginx bash$ mount | grep fusepvc-035763df-0488-4941-9a34-f637292eb95c: on /usr/share/nginx/html/s3 type fuse.geesefs (rw,nosuid,nodev,relatime,user_id=65534,group_id=0,default_permissions,allow_other)$ touch /usr/share/nginx/html/s3/hello_world.txt$ ls /usr/share/nginx/html/s3123  ansible.cfg  hello-world.txt[root@ceph01 ~]# s3cmd ls s3://my-new-bucket/2024-04-26 04:47            0  s3://my-new-bucket/1232024-04-26 03:36          869  s3://my-new-bucket/ansible.cfg2024-04-26 03:59           17  s3://my-new-bucket/hello-world.txt

Set Bucket Policy

[root@sd-ceph01 ~]# cat aps-bucket-policy {  "Version": "2012-10-17",  "Statement": [{    "Effect": "Allow",    "Principal": {"AWS": ["arn:aws:iam::usfolks:user/aps"]},    "Action": "s3:*",     "Resource": [      "arn:aws:s3:::aps-bucket/*"    ]  }]}[root@sd-ceph01 ~]# s3cmd setpolicy aps-bucket-policy s3://aps-buckets3://aps-bucket/: Policy updated