Simulate tiering usage in tenant

Issue

https://github.com/miniohq/engineering/issues/584

Clone master branch of minio operator

git clone https://github.com/minio/operator.git
cd operator

Setup

kind delete clusters kind
kind create cluster --config testing/kind-config.yaml
cd "testing/.." && TAG=minio/operator:noop make docker
kind load docker-image minio/operator:noop
kubectl apply -k "testing/../testing/dev"

Create an ubuntu pod for warm storage

cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: ubuntu
  namespace: tenant-lite
  labels:
    app: ubuntu
spec:
  containers:
  - image: ubuntu
    command:
      - "sleep"
      - "604800"
    imagePullPolicy: IfNotPresent
    name: ubuntu
  restartPolicy: Always
EOF

k -n tenant-lite exec -it ubuntu -- /bin/bash

On ubuntu pod, terminal 1

### Install minio and mc
apt update
apt install wget -y

wget https://dl.min.io/server/minio/release/linux-arm64/minio
chmod +x minio

wget https://dl.min.io/client/mc/release/linux-arm64/mc
chmod +x mc

Start minio

./minio server /data

On ubuntu pod, terminal 2

Setup warm storage bucket

./mc alias set k8s-minio-dev http://localhost:9000 minioadmin minioadmin
./mc mb k8s-minio-dev/warm-bucket

On local machine, interact with k8s

kubectl port-forward svc/storage-lite-hl -n tenant-lite 9000
sudo mc alias set k8s-minio-dev http://localhost:9000 minioadmin minioadmin --insecure
#Added `k8s-minio-dev` successfully.
sudo mc mb k8s-minio-dev/hot-bucket --insecure
#Bucket created successfully `k8s-minio-dev/hot-bucket`.
sudo mc ilm tier add minio k8s-minio-dev WARM-MINIO-TIER \
                               --endpoint http://10.244.3.6:9000 \
                               --access-key minioadmin \
                               --secret-key minioadmin \
                               --bucket warm-bucket \
                               --insecure
#Added remote tier WARM-MINIO-TIER of type minio
sudo mc ilm rule add --transition-days 0 --transition-tier "WARM-MINIO-TIER" k8s-minio-dev/hot-bucket --insecure
#Lifecycle configuration rule added with ID `cfru1f2401g8a12bb2v0` to k8s-minio-dev/hot-bucket.
sudo mc cp /Users/allanreid/Documents/MinIO/demo/pics/42489388690_1f48eec1cc_o.jpg k8s-minio-dev/hot-bucket/42489388690_1f48eec1cc_o.jpg --insecure
#...o/pics/42489388690_1f48eec1cc_o.jpg: 78.52 KiB / 78.52 KiB ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.16 MiB/s 0s
sudo mc ls k8s-minio-dev/hot-bucket --insecure
#[2023-02-23 13:56:29 PST]  78KiB WARM-MINIO-TIER 42489388690_1f48eec1cc_o.jpg

Back on ubuntu pod, terminal 2

./mc ls k8s-minio-dev/warm-bucket --insecure
#[2023-02-23 21:58:34 UTC]     0B 8d3a0fd4-3efb-4d0d-a1ae-70897443b323/

On local machine, view tiering in tenant status

k -n tenant-lite describe tenant/storage-lite