refactor: refact the pouch hook plugin

Refact the pouch hook plugin, add plugins code into binary when
building, instead of hook_plugin.so.

Add the Cri plugin.

More detail you can see: `docs/features/pouch_with_plugin.md`

Signed-off-by: Rudy Zhang <[email protected]>

.gitignore

@@ -3,6 +3,7 @@ pouchd
 pouch
 *.patch
 volume_build.go
+hook_plugins_build.go
 coverage.txt
 .*.swp
 .*.swo

Makefile

@@ -71,7 +71,7 @@ COVERAGE_PACKAGES_LIST=$(shell echo $(COVERAGE_PACKAGES) | tr " " ",")
 
 build: build-daemon build-cli ## build PouchContainer both daemon and cli binaries
 
-build-daemon: modules ## build PouchContainer daemon binary
+build-daemon: modules plugin ## build PouchContainer daemon binary
 	@echo "$@: bin/${DAEMON_BINARY_NAME}"
 	@mkdir -p bin
 	@GOOS=linux go build -ldflags ${DEFAULT_LDFLAGS} -o bin/${DAEMON_BINARY_NAME} -tags 'selinux'
@@ -81,14 +81,14 @@ build-cli: ## build PouchContainer cli binary
 	@mkdir -p bin
 	@go build -o bin/${CLI_BINARY_NAME} github.com/alibaba/pouch/cli
 
-build-daemon-integration: modules ## build PouchContainer daemon integration testing binary
+build-daemon-integration: modules plugin ## build PouchContainer daemon integration testing binary
 	@echo $@
 	@mkdir -p bin
 	go test -c ${TEST_FLAGS} \
 		-cover -covermode=atomic -coverpkg ${COVERAGE_PACKAGES_LIST} \
 		-o bin/${DAEMON_INTEGRATION_BINARY_NAME}
 
-build-integration-test: modules ## build PouchContainer integration test-case binary
+build-integration-test: modules plugin ## build PouchContainer integration test-case binary
 	@echo $@
 	@mkdir -p bin
 	go test -c \
@@ -151,7 +151,7 @@ gometalinter: ## run gometalinter for go source code
 
 
 .PHONY: unit-test
-unit-test: modules ## run go unit-test
+unit-test: modules plugin ## run go unit-test
 	@echo $@
 	@mkdir -p coverage
 	@( for pkg in ${COVERAGE_PACKAGES}; do \
@@ -193,6 +193,13 @@ coverage: ## combine coverage after test
 	@echo $@
 	@gocovmerge coverage/* > coverage.txt
 
+.PHONY: plugin
+plugin: ## build hook plugin
+	@echo "build $@"
+	@./hack/module --add-plugin=github.com/alibaba/pouch/hookplugins/containerplugin
+	@./hack/module --add-plugin=github.com/alibaba/pouch/hookplugins/daemonplugin
+	@./hack/module --add-plugin=github.com/alibaba/pouch/hookplugins/criplugin
+	@./hack/module --add-plugin=github.com/alibaba/pouch/hookplugins/volumeplugin
 
 .PHONY: help
 help: ## this help

apis/server/server.go

@@ -9,10 +9,10 @@ import (
 	"sync"
 	"time"
 
-	"github.com/alibaba/pouch/apis/plugins"
 	"github.com/alibaba/pouch/cri/stream"
 	"github.com/alibaba/pouch/daemon/config"
 	"github.com/alibaba/pouch/daemon/mgr"
+	"github.com/alibaba/pouch/hookplugins"
 	"github.com/alibaba/pouch/pkg/httputils"
 	"github.com/alibaba/pouch/pkg/netutils"
 
@@ -29,7 +29,7 @@ type Server struct {
 	NetworkMgr       mgr.NetworkMgr
 	StreamRouter     stream.Router
 	listeners        []net.Listener
-	ContainerPlugin  plugins.ContainerPlugin
+	ContainerPlugin  hookplugins.ContainerPlugin
 	ManagerWhiteList map[string]struct{}
 	lock             sync.RWMutex
 }

cri/criservice.go

@@ -10,12 +10,13 @@ import (
 	servicev1alpha2 "github.com/alibaba/pouch/cri/v1alpha2/service"
 	"github.com/alibaba/pouch/daemon/config"
 	"github.com/alibaba/pouch/daemon/mgr"
+	"github.com/alibaba/pouch/hookplugins"
 
 	"github.com/sirupsen/logrus"
 )
 
 // RunCriService start cri service if pouchd is specified with --enable-cri.
-func RunCriService(daemonconfig *config.Config, containerMgr mgr.ContainerMgr, imageMgr mgr.ImageMgr, volumeMgr mgr.VolumeMgr, streamRouterCh chan stream.Router, stopCh chan error, readyCh chan bool) {
+func RunCriService(daemonconfig *config.Config, containerMgr mgr.ContainerMgr, imageMgr mgr.ImageMgr, volumeMgr mgr.VolumeMgr, criPlugin hookplugins.CriPlugin, streamRouterCh chan stream.Router, stopCh chan error, readyCh chan bool) {
 	var err error
 
 	defer func() {
@@ -32,7 +33,7 @@ func RunCriService(daemonconfig *config.Config, containerMgr mgr.ContainerMgr, i
 	case "v1alpha1":
 		err = runv1alpha1(daemonconfig, containerMgr, imageMgr, streamRouterCh, readyCh)
 	case "v1alpha2":
-		err = runv1alpha2(daemonconfig, containerMgr, imageMgr, volumeMgr, streamRouterCh, readyCh)
+		err = runv1alpha2(daemonconfig, containerMgr, imageMgr, volumeMgr, criPlugin, streamRouterCh, readyCh)
 	default:
 		streamRouterCh <- nil
 		readyCh <- false
@@ -92,9 +93,9 @@ func runv1alpha1(daemonconfig *config.Config, containerMgr mgr.ContainerMgr, ima
 }
 
 // Start CRI service with CRI version: v1alpha2
-func runv1alpha2(daemonconfig *config.Config, containerMgr mgr.ContainerMgr, imageMgr mgr.ImageMgr, volumeMgr mgr.VolumeMgr, streamRouterCh chan stream.Router, readyCh chan bool) error {
+func runv1alpha2(daemonconfig *config.Config, containerMgr mgr.ContainerMgr, imageMgr mgr.ImageMgr, volumeMgr mgr.VolumeMgr, criPlugin hookplugins.CriPlugin, streamRouterCh chan stream.Router, readyCh chan bool) error {
 	logrus.Infof("Start CRI service with CRI version: v1alpha2")
-	criMgr, err := criv1alpha2.NewCriManager(daemonconfig, containerMgr, imageMgr, volumeMgr)
+	criMgr, err := criv1alpha2.NewCriManager(daemonconfig, containerMgr, imageMgr, volumeMgr, criPlugin)
 	if err != nil {
 		streamRouterCh <- nil
 		readyCh <- false

cri/v1alpha2/cri.go

@@ -23,6 +23,7 @@ import (
 	criutils "github.com/alibaba/pouch/cri/utils"
 	"github.com/alibaba/pouch/daemon/config"
 	"github.com/alibaba/pouch/daemon/mgr"
+	"github.com/alibaba/pouch/hookplugins"
 	"github.com/alibaba/pouch/pkg/errtypes"
 	"github.com/alibaba/pouch/pkg/meta"
 	"github.com/alibaba/pouch/pkg/reference"
@@ -104,6 +105,7 @@ type CriManager struct {
 	ImageMgr     mgr.ImageMgr
 	VolumeMgr    mgr.VolumeMgr
 	CniMgr       cni.CniMgr
+	CriPlugin    hookplugins.CriPlugin
 
 	// StreamServer is the stream server of CRI serves container streaming request.
 	StreamServer Server
@@ -125,7 +127,7 @@ type CriManager struct {
 }
 
 // NewCriManager creates a brand new cri manager.
-func NewCriManager(config *config.Config, ctrMgr mgr.ContainerMgr, imgMgr mgr.ImageMgr, volumeMgr mgr.VolumeMgr) (CriMgr, error) {
+func NewCriManager(config *config.Config, ctrMgr mgr.ContainerMgr, imgMgr mgr.ImageMgr, volumeMgr mgr.VolumeMgr, criPlugin hookplugins.CriPlugin) (CriMgr, error) {
 	var streamServerAddress string
 	streamServerPort := config.CriConfig.StreamServerPort
 	// If stream server reuse the pouchd's port, extract the ip and port from pouchd's listening addresses.
@@ -147,6 +149,7 @@ func NewCriManager(config *config.Config, ctrMgr mgr.ContainerMgr, imgMgr mgr.Im
 		ContainerMgr:   ctrMgr,
 		ImageMgr:       imgMgr,
 		VolumeMgr:      volumeMgr,
+		CriPlugin:      criPlugin,
 		StreamServer:   streamServer,
 		SandboxBaseDir: path.Join(config.HomeDir, "sandboxes"),
 		SandboxImage:   config.CriConfig.SandboxImage,
@@ -683,6 +686,19 @@ func (c *CriManager) CreateContainer(ctx context.Context, r *runtime.CreateConta
 	sandboxConfig := r.GetSandboxConfig()
 	podSandboxID := r.GetPodSandboxId()
 
+	// get sandbox
+	sandbox, err := c.ContainerMgr.Get(ctx, podSandboxID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get sandbox %q: %v", podSandboxID, err)
+	}
+
+	res, err := c.SandboxStore.Get(podSandboxID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get metadata of %q from SandboxStore: %v", podSandboxID, err)
+	}
+	sandboxMeta := res.(*SandboxMeta)
+	sandboxMeta.NetNS = containerNetns(sandbox)
+
 	labels := makeLabels(config.GetLabels(), config.GetAnnotations())
 	// Apply the container type lable.
 	labels[containerTypeLabelKey] = containerTypeLabelContainer
@@ -723,7 +739,8 @@ func (c *CriManager) CreateContainer(ctx context.Context, r *runtime.CreateConta
 		},
 		NetworkingConfig: &apitypes.NetworkingConfig{},
 	}
-	err := c.updateCreateConfig(createConfig, config, sandboxConfig, podSandboxID)
+
+	err = c.updateCreateConfig(createConfig, config, sandboxConfig, sandboxMeta)
 	if err != nil {
 		return nil, err
 	}
@@ -744,6 +761,13 @@ func (c *CriManager) CreateContainer(ctx context.Context, r *runtime.CreateConta
 
 	containerName := makeContainerName(sandboxConfig, config)
 
+	// call cri plugin to update create config
+	if c.CriPlugin != nil {
+		if err := c.CriPlugin.PreCreateContainer(createConfig, sandboxMeta); err != nil {
+			return nil, err
+		}
+	}
+
 	createResp, err := c.ContainerMgr.Create(ctx, containerName, createConfig)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create container for sandbox %q: %v", podSandboxID, err)

cri/v1alpha2/cri_types.go

@@ -20,6 +20,9 @@ type SandboxMeta struct {
 
 	// Runtime whether to enable lxcfs for a container
 	LxcfsEnabled bool
+
+	// Netns is the sandbox's network namespace
+	NetNS string
 }
 
 // Key returns sandbox's id.

cri/v1alpha2/cri_utils.go

@@ -730,13 +730,8 @@ func applyContainerSecurityContext(lc *runtime.LinuxContainerConfig, podSandboxI
 }
 
 // Apply Linux-specific options if applicable.
-func (c *CriManager) updateCreateConfig(createConfig *apitypes.ContainerCreateConfig, config *runtime.ContainerConfig, sandboxConfig *runtime.PodSandboxConfig, podSandboxID string) error {
+func (c *CriManager) updateCreateConfig(createConfig *apitypes.ContainerCreateConfig, config *runtime.ContainerConfig, sandboxConfig *runtime.PodSandboxConfig, sandboxMeta *SandboxMeta) error {
 	// Apply runtime options.
-	res, err := c.SandboxStore.Get(podSandboxID)
-	if err != nil {
-		return fmt.Errorf("failed to get metadata of %q from SandboxStore: %v", podSandboxID, err)
-	}
-	sandboxMeta := res.(*SandboxMeta)
 	if sandboxMeta.Runtime != "" {
 		createConfig.HostConfig.Runtime = sandboxMeta.Runtime
 	}
@@ -755,7 +750,7 @@ func (c *CriManager) updateCreateConfig(createConfig *apitypes.ContainerCreateCo
 		}
 
 		// Apply security context.
-		if err := applyContainerSecurityContext(lc, podSandboxID, &createConfig.ContainerConfig, createConfig.HostConfig); err != nil {
+		if err := applyContainerSecurityContext(lc, sandboxMeta.ID, &createConfig.ContainerConfig, createConfig.HostConfig); err != nil {
 			return fmt.Errorf("failed to apply container security context for container %q: %v", config.Metadata.Name, err)
 		}
 	}