From c692952e5a882e0bfd021359ddd7fdca87695802 Mon Sep 17 00:00:00 2001 From: Daniel Canter Date: Wed, 16 Mar 2022 14:00:09 -0700 Subject: [PATCH] Add Go bindflt/silo definitions This change adds a couple Go bindings for calls and constants from bindfltapi.dll as well as some silo flags for job objects. Together these allow file bindings (think bind mounts on Linux) to be performed for a specific job object. The bindings are only viewable from processes within that specific job. This change additionally adds a couple unit tests for this behavior. Signed-off-by: Daniel Canter --- internal/exec/exec_test.go | 5 +- internal/jobobject/jobobject.go | 92 +++++++++++++++++++ internal/jobobject/jobobject_test.go | 70 ++++++++++++++ internal/winapi/bindflt.go | 20 ++++ internal/winapi/jobobject.go | 1 + internal/winapi/winapi.go | 2 +- internal/winapi/zsyscall_windows.go | 30 ++++-- .../hcsshim/internal/winapi/bindflt.go | 20 ++++ .../hcsshim/internal/winapi/jobobject.go | 1 + .../hcsshim/internal/winapi/winapi.go | 2 +- .../internal/winapi/zsyscall_windows.go | 30 ++++-- 11 files changed, 256 insertions(+), 17 deletions(-) create mode 100644 internal/winapi/bindflt.go create mode 100644 test/vendor/github.com/Microsoft/hcsshim/internal/winapi/bindflt.go diff --git a/internal/exec/exec_test.go b/internal/exec/exec_test.go index 7afdf34ad6..7b44fe9d18 100644 --- a/internal/exec/exec_test.go +++ b/internal/exec/exec_test.go @@ -11,6 +11,7 @@ import ( "github.com/Microsoft/hcsshim/internal/conpty" "github.com/Microsoft/hcsshim/internal/jobobject" + "golang.org/x/sys/windows" ) func TestExec(t *testing.T) { @@ -139,6 +140,7 @@ func TestExecsWithJob(t *testing.T) { WithJobObject(job), WithStdio(false, false, false), WithEnv(os.Environ()), + WithProcessFlags(windows.CREATE_NEW_PROCESS_GROUP), ) if err != nil { t.Fatal(err) @@ -156,6 +158,7 @@ func TestExecsWithJob(t *testing.T) { WithJobObject(job), WithStdio(false, false, false), WithEnv(os.Environ()), + WithProcessFlags(windows.CREATE_NEW_PROCESS_GROUP), ) if err != nil { t.Fatal(err) @@ -177,7 +180,7 @@ func TestExecsWithJob(t *testing.T) { } if len(pids) != 2 { - t.Fatalf("should be two pids in job object, got: %d", len(pids)) + t.Fatalf("should be two pids in job object, got: %d. Pids: %+v", len(pids), pids) } for _, pid := range pids { diff --git a/internal/jobobject/jobobject.go b/internal/jobobject/jobobject.go index b8d586b204..5a2a9f499f 100644 --- a/internal/jobobject/jobobject.go +++ b/internal/jobobject/jobobject.go @@ -3,6 +3,8 @@ package jobobject import ( "context" "fmt" + "os" + "path/filepath" "sync" "unsafe" @@ -25,6 +27,7 @@ import ( // of the job and a mutex for synchronized handle access. type JobObject struct { handle windows.Handle + isAppSilo bool mq *queue.MessageQueue handleLock sync.RWMutex } @@ -56,6 +59,7 @@ const ( var ( ErrAlreadyClosed = errors.New("the handle has already been closed") ErrNotRegistered = errors.New("job is not registered to receive notifications") + ErrNotSilo = errors.New("job is not a silo") ) // Options represents the set of configurable options when making or opening a job object. @@ -68,6 +72,9 @@ type Options struct { // `UseNTVariant` specifies if we should use the `Nt` variant of Open/CreateJobObject. // Defaults to false. UseNTVariant bool + // `Silo` specifies to promote the job to a silo. This additionally sets the flag JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE as it is required for + // the upgrade to complete. + Silo bool } // Create creates a job object. @@ -134,6 +141,16 @@ func Create(ctx context.Context, options *Options) (_ *JobObject, err error) { job.mq = mq } + if options.Silo { + // This is a required setting for upgrading to a silo. + if err := job.SetTerminateOnLastHandleClose(); err != nil { + return nil, err + } + if err := job.PromoteToSilo(); err != nil { + return nil, err + } + } + return job, nil } @@ -436,3 +453,78 @@ func (job *JobObject) QueryStorageStats() (*winapi.JOBOBJECT_BASIC_AND_IO_ACCOUN } return &info, nil } + +// ApplyFileBinding makes a file binding using the Bind Filter from target to root. If the job has not been upgraded to a silo +// this call will fail. The binding is only applied and visible for processes running in the job, any processes on the host or in another job +// will not be able to see the binding. +func (job *JobObject) ApplyFileBinding(root, target string, merged bool) error { + job.handleLock.RLock() + defer job.handleLock.RUnlock() + + if !job.isAppSilo { + return ErrNotSilo + } + + // The parent directory needs to exist for the bind to work. + if _, err := os.Stat(filepath.Dir(root)); os.IsNotExist(err) { + if err := os.MkdirAll(filepath.Dir(root), 0); err != nil { + return err + } + } + + rootPtr, err := windows.UTF16PtrFromString(root) + if err != nil { + return err + } + + targetPtr, err := windows.UTF16PtrFromString(target) + if err != nil { + return err + } + + flags := winapi.BINDFLT_FLAG_USE_CURRENT_SILO_MAPPING + if merged { + flags |= winapi.BINDFLT_FLAG_MERGED_BIND_MAPPING + } + + if err := winapi.BfSetupFilterEx( + flags, + job.handle, + nil, + rootPtr, + targetPtr, + nil, + 0, + ); err != nil { + return fmt.Errorf("failed to bind target %q to root %q for job object: %w", target, root, err) + } + return nil +} + +// PromoteToSilo promotes a job object to a silo. There must be no running processess in the job for this to succeed. If the job is already a +// silo this is a no-op. +func (job *JobObject) PromoteToSilo() error { + job.handleLock.RLock() + defer job.handleLock.RUnlock() + + if job.handle == 0 { + return ErrAlreadyClosed + } + + if job.isAppSilo { + return nil + } + + _, err := windows.SetInformationJobObject( + job.handle, + winapi.JobObjectCreateSilo, + 0, + 0, + ) + if err != nil { + return fmt.Errorf("failed to promote job to silo: %w", err) + } + + job.isAppSilo = true + return nil +} diff --git a/internal/jobobject/jobobject_test.go b/internal/jobobject/jobobject_test.go index c1baa033c7..642790176c 100644 --- a/internal/jobobject/jobobject_test.go +++ b/internal/jobobject/jobobject_test.go @@ -2,7 +2,9 @@ package jobobject import ( "context" + "os" "os/exec" + "path/filepath" "syscall" "testing" "time" @@ -293,3 +295,71 @@ func TestVerifyPidCount(t *testing.T) { t.Fatal(err) } } + +func TestSilo(t *testing.T) { + // Test asking for a silo in the options. + options := &Options{ + Name: "test", + Notifications: true, + Silo: true, + } + job, err := Create(context.Background(), options) + if err != nil { + t.Fatal(err) + } + defer job.Close() +} + +func TestSiloFileBinding(t *testing.T) { + // Can't use osversion as the binary needs to be manifested for it to work. Just stat for the bindflt dll. + if _, err := os.Stat(`C:\windows\system32\bindfltapi.dll`); err != nil { + t.Skip("Bindflt not present on RS5 or lower, skipping.") + } + // Test upgrading to a silo and binding a file only the silo can see. + options := &Options{ + Name: "test", + Notifications: true, + Silo: true, + } + job, err := Create(context.Background(), options) + if err != nil { + t.Fatal(err) + } + defer job.Close() + + target := t.TempDir() + hostPath := filepath.Join(target, "bind-test.txt") + f, err := os.Create(hostPath) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + root := t.TempDir() + siloPath := filepath.Join(root, "silo-path.txt") + if err := job.ApplyFileBinding(siloPath, hostPath, false); err != nil { + t.Fatal(err) + } + + // First check that we can't see the file on the host. + if _, err := os.Stat(siloPath); err == nil { + t.Fatalf("expected to not be able to see %q on the host", siloPath) + } + + // Now check that we can see it in the silo. Couple second timeout (ping something) so we can be relatively sure the process has been assigned to + // the job before we go to check on the file. Unfortunately we can't use our internal/exec package that has support for assigning a process to a job + // at creation time as it causes a cyclical import. + cmd := exec.Command("cmd", "/c", "ping", "localhost", "&&", "dir", siloPath) + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + + if err := job.Assign(uint32(cmd.Process.Pid)); err != nil { + t.Fatal(err) + } + + // Process will have an exit code of 1 if dir couldn't find the file; if we get no error here we should be A-OK. + if err := cmd.Wait(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/winapi/bindflt.go b/internal/winapi/bindflt.go new file mode 100644 index 0000000000..ab434a75b5 --- /dev/null +++ b/internal/winapi/bindflt.go @@ -0,0 +1,20 @@ +package winapi + +const ( + BINDFLT_FLAG_READ_ONLY_MAPPING uint32 = 0x00000001 + BINDFLT_FLAG_MERGED_BIND_MAPPING uint32 = 0x00000002 + BINDFLT_FLAG_USE_CURRENT_SILO_MAPPING uint32 = 0x00000004 +) + +// HRESULT +// BfSetupFilterEx( +// _In_ ULONG Flags, +// _In_opt_ HANDLE JobHandle, +// _In_opt_ PSID Sid, +// _In_ LPCWSTR VirtualizationRootPath, +// _In_ LPCWSTR VirtualizationTargetPath, +// _In_reads_opt_( VirtualizationExceptionPathCount ) LPCWSTR* VirtualizationExceptionPaths, +// _In_opt_ ULONG VirtualizationExceptionPathCount +// ); +// +//sys BfSetupFilterEx(flags uint32, jobHandle windows.Handle, sid *windows.SID, virtRootPath *uint16, virtTargetPath *uint16, virtExceptions **uint16, virtExceptionPathCount uint32) (hr error) = bindfltapi.BfSetupFilterEx? diff --git a/internal/winapi/jobobject.go b/internal/winapi/jobobject.go index b2ca47c02b..44371b3886 100644 --- a/internal/winapi/jobobject.go +++ b/internal/winapi/jobobject.go @@ -52,6 +52,7 @@ const ( JobObjectLimitViolationInformation uint32 = 13 JobObjectMemoryUsageInformation uint32 = 28 JobObjectNotificationLimitInformation2 uint32 = 33 + JobObjectCreateSilo uint32 = 35 JobObjectIoAttribution uint32 = 42 ) diff --git a/internal/winapi/winapi.go b/internal/winapi/winapi.go index d2cc9d9fba..298ff838ac 100644 --- a/internal/winapi/winapi.go +++ b/internal/winapi/winapi.go @@ -2,4 +2,4 @@ // be thought of as an extension to golang.org/x/sys/windows. package winapi -//go:generate go run ..\..\mksyscall_windows.go -output zsyscall_windows.go user.go console.go system.go net.go path.go thread.go jobobject.go logon.go memory.go process.go processor.go devices.go filesystem.go errors.go +//go:generate go run ..\..\mksyscall_windows.go -output zsyscall_windows.go bindflt.go user.go console.go system.go net.go path.go thread.go jobobject.go logon.go memory.go process.go processor.go devices.go filesystem.go errors.go diff --git a/internal/winapi/zsyscall_windows.go b/internal/winapi/zsyscall_windows.go index fc2732a64a..935219b063 100644 --- a/internal/winapi/zsyscall_windows.go +++ b/internal/winapi/zsyscall_windows.go @@ -37,13 +37,15 @@ func errnoErr(e syscall.Errno) error { } var ( - modnetapi32 = windows.NewLazySystemDLL("netapi32.dll") - modkernel32 = windows.NewLazySystemDLL("kernel32.dll") - modntdll = windows.NewLazySystemDLL("ntdll.dll") - modiphlpapi = windows.NewLazySystemDLL("iphlpapi.dll") - modadvapi32 = windows.NewLazySystemDLL("advapi32.dll") - modcfgmgr32 = windows.NewLazySystemDLL("cfgmgr32.dll") - + modbindfltapi = windows.NewLazySystemDLL("bindfltapi.dll") + modnetapi32 = windows.NewLazySystemDLL("netapi32.dll") + modkernel32 = windows.NewLazySystemDLL("kernel32.dll") + modntdll = windows.NewLazySystemDLL("ntdll.dll") + modiphlpapi = windows.NewLazySystemDLL("iphlpapi.dll") + modadvapi32 = windows.NewLazySystemDLL("advapi32.dll") + modcfgmgr32 = windows.NewLazySystemDLL("cfgmgr32.dll") + + procBfSetupFilterEx = modbindfltapi.NewProc("BfSetupFilterEx") procNetLocalGroupGetInfo = modnetapi32.NewProc("NetLocalGroupGetInfo") procNetUserAdd = modnetapi32.NewProc("NetUserAdd") procNetUserDel = modnetapi32.NewProc("NetUserDel") @@ -77,6 +79,20 @@ var ( procRtlNtStatusToDosError = modntdll.NewProc("RtlNtStatusToDosError") ) +func BfSetupFilterEx(flags uint32, jobHandle windows.Handle, sid *windows.SID, virtRootPath *uint16, virtTargetPath *uint16, virtExceptions **uint16, virtExceptionPathCount uint32) (hr error) { + if hr = procBfSetupFilterEx.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall9(procBfSetupFilterEx.Addr(), 7, uintptr(flags), uintptr(jobHandle), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(virtRootPath)), uintptr(unsafe.Pointer(virtTargetPath)), uintptr(unsafe.Pointer(virtExceptions)), uintptr(virtExceptionPathCount), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + func netLocalGroupGetInfo(serverName *uint16, groupName *uint16, level uint32, bufptr **byte) (status error) { r0, _, _ := syscall.Syscall6(procNetLocalGroupGetInfo.Addr(), 4, uintptr(unsafe.Pointer(serverName)), uintptr(unsafe.Pointer(groupName)), uintptr(level), uintptr(unsafe.Pointer(bufptr)), 0, 0) if r0 != 0 { diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/bindflt.go b/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/bindflt.go new file mode 100644 index 0000000000..ab434a75b5 --- /dev/null +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/bindflt.go @@ -0,0 +1,20 @@ +package winapi + +const ( + BINDFLT_FLAG_READ_ONLY_MAPPING uint32 = 0x00000001 + BINDFLT_FLAG_MERGED_BIND_MAPPING uint32 = 0x00000002 + BINDFLT_FLAG_USE_CURRENT_SILO_MAPPING uint32 = 0x00000004 +) + +// HRESULT +// BfSetupFilterEx( +// _In_ ULONG Flags, +// _In_opt_ HANDLE JobHandle, +// _In_opt_ PSID Sid, +// _In_ LPCWSTR VirtualizationRootPath, +// _In_ LPCWSTR VirtualizationTargetPath, +// _In_reads_opt_( VirtualizationExceptionPathCount ) LPCWSTR* VirtualizationExceptionPaths, +// _In_opt_ ULONG VirtualizationExceptionPathCount +// ); +// +//sys BfSetupFilterEx(flags uint32, jobHandle windows.Handle, sid *windows.SID, virtRootPath *uint16, virtTargetPath *uint16, virtExceptions **uint16, virtExceptionPathCount uint32) (hr error) = bindfltapi.BfSetupFilterEx? diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/jobobject.go b/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/jobobject.go index b2ca47c02b..44371b3886 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/jobobject.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/jobobject.go @@ -52,6 +52,7 @@ const ( JobObjectLimitViolationInformation uint32 = 13 JobObjectMemoryUsageInformation uint32 = 28 JobObjectNotificationLimitInformation2 uint32 = 33 + JobObjectCreateSilo uint32 = 35 JobObjectIoAttribution uint32 = 42 ) diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/winapi.go b/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/winapi.go index d2cc9d9fba..298ff838ac 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/winapi.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/winapi.go @@ -2,4 +2,4 @@ // be thought of as an extension to golang.org/x/sys/windows. package winapi -//go:generate go run ..\..\mksyscall_windows.go -output zsyscall_windows.go user.go console.go system.go net.go path.go thread.go jobobject.go logon.go memory.go process.go processor.go devices.go filesystem.go errors.go +//go:generate go run ..\..\mksyscall_windows.go -output zsyscall_windows.go bindflt.go user.go console.go system.go net.go path.go thread.go jobobject.go logon.go memory.go process.go processor.go devices.go filesystem.go errors.go diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/zsyscall_windows.go b/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/zsyscall_windows.go index fc2732a64a..935219b063 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/zsyscall_windows.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/zsyscall_windows.go @@ -37,13 +37,15 @@ func errnoErr(e syscall.Errno) error { } var ( - modnetapi32 = windows.NewLazySystemDLL("netapi32.dll") - modkernel32 = windows.NewLazySystemDLL("kernel32.dll") - modntdll = windows.NewLazySystemDLL("ntdll.dll") - modiphlpapi = windows.NewLazySystemDLL("iphlpapi.dll") - modadvapi32 = windows.NewLazySystemDLL("advapi32.dll") - modcfgmgr32 = windows.NewLazySystemDLL("cfgmgr32.dll") - + modbindfltapi = windows.NewLazySystemDLL("bindfltapi.dll") + modnetapi32 = windows.NewLazySystemDLL("netapi32.dll") + modkernel32 = windows.NewLazySystemDLL("kernel32.dll") + modntdll = windows.NewLazySystemDLL("ntdll.dll") + modiphlpapi = windows.NewLazySystemDLL("iphlpapi.dll") + modadvapi32 = windows.NewLazySystemDLL("advapi32.dll") + modcfgmgr32 = windows.NewLazySystemDLL("cfgmgr32.dll") + + procBfSetupFilterEx = modbindfltapi.NewProc("BfSetupFilterEx") procNetLocalGroupGetInfo = modnetapi32.NewProc("NetLocalGroupGetInfo") procNetUserAdd = modnetapi32.NewProc("NetUserAdd") procNetUserDel = modnetapi32.NewProc("NetUserDel") @@ -77,6 +79,20 @@ var ( procRtlNtStatusToDosError = modntdll.NewProc("RtlNtStatusToDosError") ) +func BfSetupFilterEx(flags uint32, jobHandle windows.Handle, sid *windows.SID, virtRootPath *uint16, virtTargetPath *uint16, virtExceptions **uint16, virtExceptionPathCount uint32) (hr error) { + if hr = procBfSetupFilterEx.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall9(procBfSetupFilterEx.Addr(), 7, uintptr(flags), uintptr(jobHandle), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(virtRootPath)), uintptr(unsafe.Pointer(virtTargetPath)), uintptr(unsafe.Pointer(virtExceptions)), uintptr(virtExceptionPathCount), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + func netLocalGroupGetInfo(serverName *uint16, groupName *uint16, level uint32, bufptr **byte) (status error) { r0, _, _ := syscall.Syscall6(procNetLocalGroupGetInfo.Addr(), 4, uintptr(unsafe.Pointer(serverName)), uintptr(unsafe.Pointer(groupName)), uintptr(level), uintptr(unsafe.Pointer(bufptr)), 0, 0) if r0 != 0 {