diff --git a/build/images/base/Dockerfile b/build/images/base/Dockerfile index 54ed163ff2e..0c76841450f 100644 --- a/build/images/base/Dockerfile +++ b/build/images/base/Dockerfile @@ -22,3 +22,10 @@ RUN curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \ && tar -C /usr/local -xzf golang.tar.gz \ && rm golang.tar.gz \ && go install github.com/go-delve/delve/cmd/dlv@v1.7.0 + +# Create none root user and group +ARG USER_NAME=dragonfly +ARG USER_UID=1000 +ARG GROUP_NAME=dragonfly +ARG GROUP_GID=1000 +RUN groupadd -g $GROUP_GID $GROUP_NAME && useradd -u $USER_UID -g $GROUP_GID -m -s /bin/bash $USER_NAME diff --git a/build/images/dfdaemon/Dockerfile b/build/images/dfdaemon/Dockerfile index 7b58f9699c1..c8bc9025cd0 100644 --- a/build/images/dfdaemon/Dockerfile +++ b/build/images/dfdaemon/Dockerfile @@ -29,11 +29,28 @@ RUN if [ "$(uname -m)" = "ppc64le" ]; then \ FROM ${BASE_IMAGE} +ARG USER_NAME=dragonfly +ARG USER_UID=1000 +ARG GROUP_NAME=dragonfly +ARG GROUP_GID=1000 + ENV PATH=/opt/dragonfly/bin:$PATH RUN echo "hosts: files dns" > /etc/nsswitch.conf -COPY --from=builder /opt/dragonfly/bin/dfget /opt/dragonfly/bin/dfget -COPY --from=health /bin/grpc_health_probe /bin/grpc_health_probe +# NOTE: Command works for alpine linux, for other distro please update run statement or create user/group in base image +RUN if [ getent group "$GROUP_NAME" ]; then \ + echo "group exists"; \ + else \ + addgroup -S -g $GROUP_GID $GROUP_NAME; \ + fi +RUN if [ getent passwd "$USER_NAME" ]; then \ + echo "user exists"; \ + else \ + adduser -S -u $USER_UID -G $GROUP_NAME -D $USER_NAME; \ + fi + +COPY --from=builder --chown=$GROUP_NAME:$USER_NAME /opt/dragonfly/bin/dfget /opt/dragonfly/bin/dfget +COPY --from=health --chown=$GROUP_NAME:$USER_NAME /bin/grpc_health_probe /bin/grpc_health_probe EXPOSE 65001 diff --git a/build/images/manager/Dockerfile b/build/images/manager/Dockerfile index 01aaedc7d2e..5f3eec3d185 100644 --- a/build/images/manager/Dockerfile +++ b/build/images/manager/Dockerfile @@ -43,6 +43,11 @@ RUN if [ "$(uname -m)" = "ppc64le" ]; then \ FROM ${BASE_IMAGE} +ARG USER_NAME=dragonfly +ARG USER_UID=1000 +ARG GROUP_NAME=dragonfly +ARG GROUP_GID=1000 + WORKDIR /opt/dragonfly ENV PATH=/opt/dragonfly/bin:$PATH @@ -50,8 +55,20 @@ ENV PATH=/opt/dragonfly/bin:$PATH RUN mkdir -p /opt/dragonfly/bin/manager/console \ && echo "hosts: files dns" > /etc/nsswitch.conf -COPY --from=server-builder /opt/dragonfly/bin/manager /opt/dragonfly/bin/server -COPY --from=health /bin/grpc_health_probe /bin/grpc_health_probe +# NOTE: Command works for alpine linux, for other distro please update run statement or create user/group in base image +RUN if [ getent group "$GROUP_NAME" ]; then \ + echo "group exists"; \ + else \ + addgroup -S -g $GROUP_GID $GROUP_NAME; \ + fi +RUN if [ getent passwd "$USER_NAME" ]; then \ + echo "user exists"; \ + else \ + adduser -S -u $USER_UID -G $GROUP_NAME -D $USER_NAME; \ + fi + +COPY --from=server-builder --chown=$GROUP_NAME:$USER_NAME /opt/dragonfly/bin/manager /opt/dragonfly/bin/server +COPY --from=health --chown=$GROUP_NAME:$USER_NAME /bin/grpc_health_probe /bin/grpc_health_probe EXPOSE 8080 65003 diff --git a/build/images/scheduler/Dockerfile b/build/images/scheduler/Dockerfile index b8e831b2a0e..644ce2d3cc0 100644 --- a/build/images/scheduler/Dockerfile +++ b/build/images/scheduler/Dockerfile @@ -29,11 +29,28 @@ RUN if [ "$(uname -m)" = "ppc64le" ]; then \ FROM ${BASE_IMAGE} +ARG USER_NAME=dragonfly +ARG USER_UID=1000 +ARG GROUP_NAME=dragonfly +ARG GROUP_GID=1000 + ENV PATH=/opt/dragonfly/bin:$PATH RUN echo "hosts: files dns" > /etc/nsswitch.conf -COPY --from=builder /opt/dragonfly/bin/scheduler /opt/dragonfly/bin/scheduler -COPY --from=health /bin/grpc_health_probe /bin/grpc_health_probe +# NOTE: Command works for alpine linux, for other distro please update run statement or create user/group in base image +RUN if [ getent group "$GROUP_NAME" ]; then \ + echo "group exists"; \ + else \ + addgroup -S -g $GROUP_GID $GROUP_NAME; \ + fi +RUN if [ getent passwd "$USER_NAME" ]; then \ + echo "user exists"; \ + else \ + adduser -S -u $USER_UID -G $GROUP_NAME -D $USER_NAME; \ + fi + +COPY --from=builder --chown=$GROUP_NAME:$USER_NAME /opt/dragonfly/bin/scheduler /opt/dragonfly/bin/scheduler +COPY --from=health --chown=$GROUP_NAME:$USER_NAME /bin/grpc_health_probe /bin/grpc_health_probe EXPOSE 8002 diff --git a/build/images/trainer/Dockerfile b/build/images/trainer/Dockerfile index 76648aa5e5d..e7fec3d5993 100644 --- a/build/images/trainer/Dockerfile +++ b/build/images/trainer/Dockerfile @@ -29,11 +29,28 @@ RUN if [ "$(uname -m)" = "ppc64le" ]; then \ FROM ${BASE_IMAGE} +ARG USER_NAME=dragonfly +ARG USER_UID=1000 +ARG GROUP_NAME=dragonfly +ARG GROUP_GID=1000 + ENV PATH=/opt/dragonfly/bin:$PATH RUN echo "hosts: files dns" > /etc/nsswitch.conf -COPY --from=builder /opt/dragonfly/bin/trainer /opt/dragonfly/bin/trainer -COPY --from=health /bin/grpc_health_probe /bin/grpc_health_probe +# NOTE: Command works for alpine linux, for other distro please update run statement or create user/group in base image +RUN if [ getent group "$GROUP_NAME" ]; then \ + echo "group exists"; \ + else \ + addgroup -S -g $GROUP_GID $GROUP_NAME; \ + fi +RUN if [ getent passwd "$USER_NAME" ]; then \ + echo "user exists"; \ + else \ + adduser -S -u $USER_UID -G $GROUP_NAME -D $USER_NAME; \ + fi + +COPY --from=builder --chown=$GROUP_NAME:$USER_NAME /opt/dragonfly/bin/trainer /opt/dragonfly/bin/trainer +COPY --from=health --chown=$GROUP_NAME:$USER_NAME /bin/grpc_health_probe /bin/grpc_health_probe EXPOSE 9090 diff --git a/cmd/manager/cmd/root.go b/cmd/manager/cmd/root.go index e42951a1f16..07721054b84 100644 --- a/cmd/manager/cmd/root.go +++ b/cmd/manager/cmd/root.go @@ -107,6 +107,10 @@ func initDfpath(cfg *config.ServerConfig) (dfpath.Dfpath, error) { options = append(options, dfpath.WithPluginDir(cfg.PluginDir)) } + if cfg.DataDir != "" { + options = append(options, dfpath.WithDataDir(cfg.DataDir)) + } + return dfpath.New(options...) } diff --git a/deploy/docker-compose/README.md b/deploy/docker-compose/README.md index 7e6fd952815..51e6b2f4596 100644 --- a/deploy/docker-compose/README.md +++ b/deploy/docker-compose/README.md @@ -2,6 +2,15 @@ > Currently, docker compose deploying is tested just in single host, no HA support. +## Build local images for Docker Compose + +Try the command below in the dragonfly root directory. + +```shell +export D7Y_REGISTRY=dragonflyoss +make docker-build +``` + ## Deploy with Docker Compose The `run.sh` script will generate config and deploy all components with `docker-compose`. diff --git a/deploy/docker-compose/docker-compose.yaml b/deploy/docker-compose/docker-compose.yaml index 19f263dff42..a1160798c25 100644 --- a/deploy/docker-compose/docker-compose.yaml +++ b/deploy/docker-compose/docker-compose.yaml @@ -44,6 +44,9 @@ services: interval: 1s timeout: 2s retries: 30 + # Enable when running in none root mode + # user: "1000:1000" + command: ["--verbose", "--console"] ports: - 65003:65003 - 8080:8080 @@ -61,6 +64,9 @@ services: interval: 1s timeout: 2s retries: 30 + command: ["--verbose", "--console"] + # Enable when running in none root mode + # user: "1000:1000" volumes: - ./log/peer:/var/log/dragonfly/daemon - ./config/dfget.yaml:/etc/dragonfly/dfget.yaml:ro @@ -83,6 +89,9 @@ services: volumes: - ./log/scheduler:/var/log/dragonfly/scheduler - ./config/scheduler.yaml:/etc/dragonfly/scheduler.yaml:ro + # Enable when running in none root mode + # user: "1000:1000" + command: ["--verbose", "--console"] ports: - 8002:8002 @@ -101,6 +110,9 @@ services: volumes: - ./log/seed-peer:/var/log/dragonfly/daemon - ./config/seed-peer.yaml:/etc/dragonfly/dfget.yaml:ro + # Enable when running in none root mode + # user: "1000:1000" + command: ["--verbose", "--console"] ports: - 65006:65006 - 65007:65007 diff --git a/deploy/docker-compose/template/dfget.template.yaml b/deploy/docker-compose/template/dfget.template.yaml index 445d42afeb6..26cb5cb5465 100644 --- a/deploy/docker-compose/template/dfget.template.yaml +++ b/deploy/docker-compose/template/dfget.template.yaml @@ -6,27 +6,32 @@ aliveTime: 0s gcInterval: 1m0s # WorkHome is working directory. -# In linux, default value is /usr/local/dragonfly. +# In linux, default value is /usr/local/dragonfly, +# And when running within none root, the path should locate to homedir, for example: /home/dragonfly/.dragonfly # In macos(just for testing), default value is /Users/$USER/.dragonfly. workHome: '' # logDir is the log directory. -# In linux, default value is /var/log/dragonfly. +# In linux, default value is /var/log/dragonfly, +# And when running within none root, the path should locate to homedir, for example: /home/dragonfly/.dragonfly/logs # In macos(just for testing), default value is /Users/$USER/.dragonfly/logs. logDir: '' # cacheDir is dynconfig cache directory. -# In linux, default value is /var/cache/dragonfly. +# In linux, default value is /var/cache/dragonfly, +# And when running within none root, the path should locate to homedir, for example: /home/dragonfly/.dragonfly/cache # In macos(just for testing), default value is /Users/$USER/.dragonfly/cache. cacheDir: '' # pluginDir is the plugin directory. -# In linux, default value is /usr/local/dragonfly/plugins. +# In linux, default value is /usr/local/dragonfly/plugins, +# And when running within none root, the path should locate to homedir, for example: /home/dragonfly/.dragonfly/plugins # In macos(just for testing), default value is /Users/$USER/.dragonfly/plugins. pluginDir: '' # dataDir is the download data directory. -# In linux, default value is /var/lib/dragonfly. +# In linux, default value is /var/lib/dragonfly, +# And when running within none root, the path should locate to homedir, for example: /home/dragonfly/.dragonfly/data # In macos(just for testing), default value is /Users/$USER/.dragonfly/data. dataDir: '' diff --git a/deploy/docker-compose/template/manager.template.yaml b/deploy/docker-compose/template/manager.template.yaml index 4a193e7e9ed..ae9088a0727 100644 --- a/deploy/docker-compose/template/manager.template.yaml +++ b/deploy/docker-compose/template/manager.template.yaml @@ -16,22 +16,35 @@ server: # REST server address addr: :8080 # WorkHome is working directory. - # In linux, default value is /usr/local/dragonfly. + # In linux, default value is /usr/local/dragonfly, + # And when running within none root, the path should locate to homedir, for example: /home/dragonfly/.dragonfly # In macos(just for testing), default value is /Users/$USER/.dragonfly. workHome: '' + # logDir is the log directory. - # In linux, default value is /var/log/dragonfly. + # In linux, default value is /var/log/dragonfly, + # And when running within none root, the path should locate to homedir, for example: /home/dragonfly/.dragonfly/logs # In macos(just for testing), default value is /Users/$USER/.dragonfly/logs. logDir: '' + # cacheDir is dynconfig cache directory. - # In linux, default value is /var/cache/dragonfly. + # In linux, default value is /var/cache/dragonfly, + # And when running within none root, the path should locate to homedir, for example: /home/dragonfly/.dragonfly/cache # In macos(just for testing), default value is /Users/$USER/.dragonfly/cache. cacheDir: '' + # pluginDir is the plugin directory. - # In linux, default value is /usr/local/dragonfly/plugins. + # In linux, default value is /usr/local/dragonfly/plugins, + # And when running within none root, the path should locate to homedir, for example: /home/dragonfly/.dragonfly/plugins # In macos(just for testing), default value is /Users/$USER/.dragonfly/plugins. pluginDir: '' + # dataDir is the download data directory. + # In linux, default value is /var/lib/dragonfly, + # And when running within none root, the path should locate to homedir, for example: /home/dragonfly/.dragonfly/data + # In macos(just for testing), default value is /Users/$USER/.dragonfly/data. + dataDir: '' + auth: jwt: # Realm name to display to the user, default value is Dragonfly. diff --git a/deploy/docker-compose/template/scheduler.template.yaml b/deploy/docker-compose/template/scheduler.template.yaml index c7e5d33c692..69970432314 100644 --- a/deploy/docker-compose/template/scheduler.template.yaml +++ b/deploy/docker-compose/template/scheduler.template.yaml @@ -9,23 +9,28 @@ server: # # Server host. # host: localhost # WorkHome is working directory. - # In linux, default value is /usr/local/dragonfly. + # In linux, default value is /usr/local/dragonfly, + # And when running within none root, the path should locate to homedir, for example: /home/dragonfly/.dragonfly # In macos(just for testing), default value is /Users/$USER/.dragonfly. workHome: '' # logDir is the log directory. - # In linux, default value is /var/log/dragonfly. + # In linux, default value is /var/log/dragonfly, + # And when running within none root, the path should locate to homedir, for example: /home/dragonfly/.dragonfly/logs # In macos(just for testing), default value is /Users/$USER/.dragonfly/logs. logDir: '' # cacheDir is dynconfig cache directory. - # In linux, default value is /var/cache/dragonfly. + # In linux, default value is /var/cache/dragonfly, + # And when running within none root, the path should locate to homedir, for example: /home/dragonfly/.dragonfly/cache # In macos(just for testing), default value is /Users/$USER/.dragonfly/cache. cacheDir: '' # pluginDir is the plugin directory. - # In linux, default value is /usr/local/dragonfly/plugins. + # In linux, default value is /usr/local/dragonfly/plugins, + # And when running within none root, the path should locate to homedir, for example: /home/dragonfly/.dragonfly/plugins # In macos(just for testing), default value is /Users/$USER/.dragonfly/plugins. pluginDir: '' - # dataDir is the directory. - # In linux, default value is /var/lib/dragonfly. + # dataDir is the download data directory. + # In linux, default value is /var/lib/dragonfly, + # And when running within none root, the path should locate to homedir, for example: /home/dragonfly/.dragonfly/data # In macos(just for testing), default value is /Users/$USER/.dragonfly/data. dataDir: '' diff --git a/deploy/docker-compose/template/seed-peer.template.yaml b/deploy/docker-compose/template/seed-peer.template.yaml index 181e61d2da3..969b98ebcd1 100644 --- a/deploy/docker-compose/template/seed-peer.template.yaml +++ b/deploy/docker-compose/template/seed-peer.template.yaml @@ -6,27 +6,32 @@ aliveTime: 0s gcInterval: 1m0s # WorkHome is working directory. -# In linux, default value is /usr/local/dragonfly. +# In linux, default value is /usr/local/dragonfly, +# And when running within none root, the path should locate to homedir, for example: /home/dragonfly/.dragonfly # In macos(just for testing), default value is /Users/$USER/.dragonfly. workHome: '' # logDir is the log directory. -# In linux, default value is /var/log/dragonfly. +# In linux, default value is /var/log/dragonfly, +# And when running within none root, the path should locate to homedir, for example: /home/dragonfly/.dragonfly/logs # In macos(just for testing), default value is /Users/$USER/.dragonfly/logs. logDir: '' # cacheDir is dynconfig cache directory. -# In linux, default value is /var/cache/dragonfly. +# In linux, default value is /var/cache/dragonfly, +# And when running within none root, the path should locate to homedir, for example: /home/dragonfly/.dragonfly/cache # In macos(just for testing), default value is /Users/$USER/.dragonfly/cache. cacheDir: '' # pluginDir is the plugin directory. -# In linux, default value is /usr/local/dragonfly/plugins. +# In linux, default value is /usr/local/dragonfly/plugins, +# And when running within none root, the path should locate to homedir, for example: /home/dragonfly/.dragonfly/plugins # In macos(just for testing), default value is /Users/$USER/.dragonfly/plugins. pluginDir: '' # dataDir is the download data directory. -# In linux, default value is /var/lib/dragonfly. +# In linux, default value is /var/lib/dragonfly, +# # And when running within none root, the path should locate to homedir, for example: /home/dragonfly/.dragonfly/data # In macos(just for testing), default value is /Users/$USER/.dragonfly/data. dataDir: '' diff --git a/manager/config/config.go b/manager/config/config.go index dd9fb7ebbbf..98733674b11 100644 --- a/manager/config/config.go +++ b/manager/config/config.go @@ -81,6 +81,9 @@ type ServerConfig struct { // Server plugin directory. PluginDir string `yaml:"pluginDir" mapstructure:"pluginDir"` + // Server storage data directory. + DataDir string `yaml:"dataDir" mapstructure:"dataDir"` + // GRPC server configuration. GRPC GRPCConfig `yaml:"grpc" mapstructure:"grpc"`