Skip to content

minio‐20091

Jump to bottom
Allan Roger Reid edited this page Aug 1, 2024 · 2 revisions

i.- Reference material - Setup minio using docker

See https://github.com/minio/minio/blob/master/docs/orchestration/docker-compose/README.md

1.- Pre-requisite - Install docker

See below install docker script

2.- Create the minio cluster. Run docker compose

See below docker-compose.yml and nginx.conf

docker compose --file docker-compose.yml up

3a.- Create an alias to the deployment and run reproducer.go (see below)

In this specific case, I run several workers (10 MiB) with part size 5 MiB to more easily provoke the issue. Observe example error

400 InvalidPart: (1: try #0, obj: obj0): One or more of the specified parts could not be found.  The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.
image

3b.- In parallel, bounce minio nodes in sequence, while waiting for write quorum after each bounce

See below ./minio_wait.sh minio-20091 where minio-20091 is the minio deployment alias

image

install docker script

# https://docs.docker.com/engine/install/ubuntu/
# Uninstall all conflicting packages
for pkg in docker.io docker-doc docker-compose docker-compose-v2 podman-docker containerd runc; do sudo apt-get remove $pkg; done

# Install using the apt repository
# Add Docker's official GPG key:
sudo apt-get update
sudo apt-get install ca-certificates curl gnupg
sudo install -m 0755 -d /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
sudo chmod a+r /etc/apt/keyrings/docker.gpg

# Add the repository to Apt sources:
echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
  $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
sudo apt-get update -y

sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y

sudo docker run hello-world
# docker run hello-world

# Manage Docker as a non-root user
sudo groupadd docker
sudo usermod -aG docker $USER
newgrp docker

# docker run hello-world

docker-compose.yml

version: '3.7'

# Settings and configurations that are common for all containers
x-minio-common: &minio-common
  image: quay.io/minio/minio:RELEASE.2024-07-29T22-14-52Z
  command: server --address ":9000" --console-address ":9090" http://minio{1...4}/data{1...2}
  expose:
    - "9000"
    - "9090"
  environment:
    MINIO_ROOT_USER: minioadmin
    MINIO_ROOT_PASSWORD: minioadmin
  # healthcheck:
  #  test: ["CMD", "mc", "ready", "local"]
  #  interval: 5s
  #  timeout: 5s
  #  retries: 5

# starts 4 docker containers running minio server instances.
# using nginx reverse proxy, load balancing, you can access
# it through port 9000.
services:
  minio1:
    <<: *minio-common
    hostname: minio1
    volumes:
      - data1-1:/data1
      - data1-2:/data2

  minio2:
    <<: *minio-common
    hostname: minio2
    volumes:
      - data2-1:/data1
      - data2-2:/data2

  minio3:
    <<: *minio-common
    hostname: minio3
    volumes:
      - data3-1:/data1
      - data3-2:/data2

  minio4:
    <<: *minio-common
    hostname: minio4
    volumes:
      - data4-1:/data1
      - data4-2:/data2

  nginx:
    image: nginx:1.19.2-alpine
    hostname: nginx
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf:ro
    ports:
      - "10011:9000"
      - "10019:9090"
    depends_on:
      - minio1
      - minio2
      - minio3
      - minio4

  sidekick:
    command: --health-path=/minio/health/ready --address :8000 http://minio{1...4}:9000
    image: quay.io/minio/sidekick:v7.0.0
    hostname: sidekick
    expose:
    - "8000"
    ports:
      - "10080:8000"
    depends_on:
      - minio1
      - minio2
      - minio3
      - minio4
      - nginx

## By default this config uses default local driver,
## For custom volumes replace with volume driver configuration.
volumes:
  data1-1:
  data1-2:
  data2-1:
  data2-2:
  data3-1:
  data3-2:
  data4-1:
  data4-2:

nginx.conf

user  nginx;
worker_processes  auto;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;

events {
    worker_connections  4096;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;
    sendfile        on;
    keepalive_timeout  65;

    # include /etc/nginx/conf.d/*.conf;

    upstream minio {
        server minio1:9000;
        server minio2:9000;
        server minio3:9000;
        server minio4:9000;
    }

    upstream console {
        ip_hash;
        server minio1:9090;
        server minio2:9090;
        server minio3:9090;
        server minio4:9090;
    }

    upstream sidekick {
        server sidekick:8000;
    }

    server {
        listen       9000;
        listen  [::]:9000;
        server_name  localhost;

        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;
        proxy_request_buffering off;

        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;

            proxy_connect_timeout 300;
            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            chunked_transfer_encoding off;

            proxy_pass http://minio;
        }
    }

    server {
        listen       9090;
        listen  [::]:9090;
        server_name  localhost;

        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;
        proxy_request_buffering off;

        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_set_header X-NginX-Proxy true;

            # This is necessary to pass the correct IP to be hashed
            real_ip_header X-Real-IP;

            proxy_connect_timeout 300;
            
            # To support websocket
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "upgrade";
            
            chunked_transfer_encoding off;

            proxy_pass http://console;
        }
    }

    server {
        listen       8000;
        listen  [::]:8000;
        server_name  localhost;

        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;
        proxy_request_buffering off;

        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;

            proxy_connect_timeout 300;
            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            chunked_transfer_encoding off;

            proxy_pass http://sidekick;
        }
    }
}

reproducer.go

package main

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"log"
	"math/rand"
	"os"
	"os/signal"
	"sync"
	"time"

	mclient "github.com/minio/minio-go/v7"
	mcreds "github.com/minio/minio-go/v7/pkg/credentials"
)

const (
	Host      = "<my host>"
	AccessKey = "minioadmin"
	SecretKey = "minioadmin"
)

const (
	NWorkers         = 10
	ReqSize          = 1 << 20 * 10
	DisableMultipart = false
)

func main() {
	mc, err := mclient.New(Host, &mclient.Options{
		Creds:  mcreds.NewStaticV4(AccessKey, SecretKey, ""),
		Secure: false,
	})
	if err != nil {
		log.Fatal(err)
	}

	ctx := context.Background()
	ctx, stop := signal.NotifyContext(ctx, os.Interrupt)
	defer stop()

	msgs := make(chan string, 100)

	var wg sync.WaitGroup
	for i := 0; i < NWorkers; i++ {
		wg.Add(1)

		// start request worker
		go func(i int) {
			defer wg.Done()
			data := make([]byte, ReqSize)
			if _, err := rand.Read(data); err != nil {
				log.Fatalf("data prep err: %v", err)
			}
			buf := bytes.NewReader(data)
			name := fmt.Sprintf("obj%d", i)

			opts := mclient.PutObjectOptions{
				//PartSize:         (1 << 20) * 100,
				DisableMultipart: DisableMultipart,
				PartSize:         (1 << 20) * 5,
			}
			var err error
			for {
				if ctx.Err() != nil {
					return // cancelled
				}

				// retry certain error types
				for tries := 0; tries < 3; tries++ {
					localCtx, done := context.WithTimeout(ctx, 1500*time.Second)
					_, err = mc.PutObject(localCtx, "testing", name, buf, int64(len(data)), opts)
					done()
					if err == nil {
						break
					}

					if errors.Is(err, context.DeadlineExceeded) {
						msgs <- fmt.Sprintf("Deadline (%d: try #%d): %v", i+1, tries, err)
						continue
					}
					eResp := mclient.ToErrorResponse(err)
					if eResp.StatusCode == 400 && eResp.Code == "InvalidPart" {
						msgs <- fmt.Sprintf("400 InvalidPart: (%d: try #%d, obj: %v): %v", i+1, tries, name, err)
						continue
					}
					if eResp.StatusCode == 404 && eResp.Code == "NoSuchUpload" {
						msgs <- fmt.Sprintf("404 NoSuchUpload: (%d: try #%d): %v", i+1, tries, err)
						continue
					}
					if eResp.StatusCode == 503 && eResp.Code == "SlowDownRead" {
						time.Sleep(2 * time.Second)
						msgs <- fmt.Sprintf("503 SlowDownRead: (%d: try #%d): %v", i+1, tries, err)
						continue
					}
					msgs <- fmt.Sprintf("Error: (%d: try #%d): %v", i+1, tries, err)
					break
				}

				if errors.Is(err, context.Canceled) {
					return
				}
				if errors.Is(err, context.DeadlineExceeded) {
					msgs <- fmt.Sprintf("Deadline (%d): %v: skipping request", i+1, err)
					continue
				}
				if err != nil {
					eResp := mclient.ToErrorResponse(err)
					log.Fatalf("Copy error: %#v", eResp)
				}

				msgs <- fmt.Sprintf("OK (%d)", i+1)
				r := rand.Intn(400) + 50
				time.Sleep(time.Duration(r) * time.Millisecond)
			}

		}(i)
	}

	// sync handle messages from workers
	go func() {
		for {
			select {
			case <-ctx.Done():
				return
			case m := <-msgs:
				fmt.Println(m)
			}
		}
	}()

	wg.Wait()
}

minio_wait.sh

#!/bin/bash

alias=$1

minio_wait() {
  mc ready $1
}

bounce() {
  docker stop $1; sleep 1; docker start $1; minio_wait $alias; 
}

while true; do 
  bounce ubuntu-minio1-1;
  bounce ubuntu-minio2-1;
  bounce ubuntu-minio3-1;
  bounce ubuntu-minio4-1;
done
Clone this wiki locally