Pluggable authentication

src/libstore/auth-tunnel.cc

@@ -0,0 +1,148 @@
+#include "auth-tunnel.hh"
+#include "serialise.hh"
+#include "auth.hh"
+#include "store-api.hh"
+#include "unix-domain-socket.hh"
+
+#include <sys/socket.h>
+
+namespace nix {
+
+AuthTunnel::AuthTunnel(StoreDirConfig & storeConfig, WorkerProto::Version clientVersion)
+    : clientVersion(clientVersion)
+{
+    auto sockets = socketPair();
+    serverFd = std::move(sockets.first);
+    clientFd = std::move(sockets.second);
+
+    serverThread = std::thread([this, clientVersion, &storeConfig]() {
+        try {
+            FdSource fromSource(serverFd.get());
+            WorkerProto::ReadConn from{
+                .from = fromSource,
+                .version = clientVersion,
+            };
+            FdSink toSource(serverFd.get());
+            WorkerProto::WriteConn to{
+                .to = toSource,
+                .version = clientVersion,
+            };
+
+            while (true) {
+                auto op = (WorkerProto::CallbackOp) readInt(from.from);
+
+                switch (op) {
+
+                case WorkerProto::CallbackOp::FillAuth: {
+                    auto authRequest = WorkerProto::Serialise<auth::AuthData>::read(storeConfig, from);
+                    bool required;
+                    from.from >> required;
+                    debug("tunneling auth request: %s", authRequest);
+                    // FIXME: handle exceptions
+                    auto authData = auth::getAuthenticator()->fill(authRequest, required);
+                    if (authData)
+                        debug("tunneling auth response: %s", *authData);
+                    to.to << 1;
+                    WorkerProto::Serialise<std::optional<auth::AuthData>>::write(storeConfig, to, authData);
+                    toSource.flush();
+                    break;
+                }
+
+                case WorkerProto::CallbackOp::RejectAuth: {
+                    auto authData = WorkerProto::Serialise<auth::AuthData>::read(storeConfig, from);
+                    debug("tunneling auth data erase: %s", authData);
+                    auth::getAuthenticator()->reject(authData);
+                    to.to << 1;
+                    toSource.flush();
+                    break;
+                }
+
+                default:
+                    throw Error("invalid callback operation %1%", (int) op);
+                }
+            }
+        } catch (EndOfFile &) {
+        } catch (...) {
+            ignoreException();
+        }
+    });
+}
+
+AuthTunnel::~AuthTunnel()
+{
+    if (serverFd)
+        shutdown(serverFd.get(), SHUT_RDWR);
+
+    if (serverThread.joinable())
+        serverThread.join();
+}
+
+struct TunneledAuthSource : auth::AuthSource
+{
+    struct State
+    {
+        /**
+         * File descriptor to send requests to the client.
+         */
+        AutoCloseFD fd;
+
+        FdSource from;
+        FdSink to;
+
+        WorkerProto::ReadConn fromConn;
+        WorkerProto::WriteConn toConn;
+
+        State(WorkerProto::Version clientVersion, AutoCloseFD && fd)
+            : fd(std::move(fd))
+            , from(this->fd.get())
+            , to(this->fd.get())
+            , fromConn{.from = from, .version = clientVersion}
+            , toConn{.to = to, .version = clientVersion}
+        {
+        }
+    };
+
+    Sync<State> state_;
+
+    ref<StoreDirConfig> storeConfig;
+
+    TunneledAuthSource(ref<StoreDirConfig> storeConfig, WorkerProto::Version clientVersion, AutoCloseFD && fd)
+        : state_(clientVersion, std::move(fd))
+        , storeConfig(storeConfig)
+    {
+    }
+
+    std::optional<auth::AuthData> get(const auth::AuthData & request, bool required) override
+    {
+        auto state(state_.lock());
+
+        state->to << (int) WorkerProto::CallbackOp::FillAuth;
+        WorkerProto::Serialise<auth::AuthData>::write(*storeConfig, state->toConn, request);
+        state->to << required;
+        state->to.flush();
+
+        if (readInt(state->from))
+            return WorkerProto::Serialise<std::optional<auth::AuthData>>::read(*storeConfig, state->fromConn);
+        else
+            return std::nullopt;
+    }
+
+    void erase(const auth::AuthData & authData) override
+    {
+        auto state(state_.lock());
+
+        state->to << (int) WorkerProto::CallbackOp::RejectAuth;
+        WorkerProto::Serialise<auth::AuthData>::write(*storeConfig, state->toConn, authData);
+        state->to.flush();
+
+        readInt(state->from);
+    }
+};
+
+ref<auth::AuthSource>
+makeTunneledAuthSource(ref<StoreDirConfig> storeConfig, WorkerProto::Version clientVersion, AutoCloseFD && clientFd)
+{
+    return make_ref<TunneledAuthSource>(storeConfig, clientVersion, std::move(clientFd));
+}
+
+}

src/libstore/auth-tunnel.hh

@@ -0,0 +1,27 @@
+#pragma once
+
+#include "file-descriptor.hh"
+#include "worker-protocol.hh"
+#include "ref.hh"
+
+#include <thread>
+
+namespace nix {
+
+struct AuthTunnel
+{
+    AutoCloseFD clientFd, serverFd;
+    std::thread serverThread;
+    const WorkerProto::Version clientVersion;
+    AuthTunnel(StoreDirConfig & storeConfig, WorkerProto::Version clientVersion);
+    ~AuthTunnel();
+};
+
+namespace auth {
+struct AuthSource;
+}
+
+ref<auth::AuthSource>
+makeTunneledAuthSource(ref<StoreDirConfig> storeConfig, WorkerProto::Version clientVersion, AutoCloseFD && clientFd);
+
+}

src/libstore/builtins.hh

@@ -5,11 +5,13 @@
 
 namespace nix {
 
+namespace auth { class Authenticator; }
+
 // TODO: make pluggable.
 void builtinFetchurl(
     const BasicDerivation & drv,
     const std::map<std::string, Path> & outputs,
-    const std::string & netrcData);
+    ref<auth::Authenticator> authenticator);
 
 void builtinUnpackChannel(
     const BasicDerivation & drv,

src/libstore/builtins/fetchurl.cc

@@ -9,16 +9,8 @@ namespace nix {
 void builtinFetchurl(
     const BasicDerivation & drv,
     const std::map<std::string, Path> & outputs,
-    const std::string & netrcData)
+    ref<auth::Authenticator> authenticator)
 {
-    /* Make the host's netrc data available. Too bad curl requires
-       this to be stored in a file. It would be nice if we could just
-       pass a pointer to the data. */
-    if (netrcData != "") {
-        settings.netrcFile = "netrc";
-        writeFile(settings.netrcFile, netrcData, 0600);
-    }
-
     auto out = get(drv.outputs, "out");
     if (!out)
         throw Error("'builtin:fetchurl' requires an 'out' output");
@@ -41,6 +33,7 @@ void builtinFetchurl(
             /* No need to do TLS verification, because we check the hash of
                the result anyway. */
             FileTransferRequest request(url);
+            request.authenticator = authenticator;
             request.verifyTLS = false;
             request.decompress = false;
 

src/libstore/daemon.cc

@@ -15,6 +15,10 @@
 #include "derivations.hh"
 #include "args.hh"
 #include "git.hh"
+#include "auth.hh"
+#include "auth-tunnel.hh"
+
+#include <sys/socket.h>
 
 #ifndef _WIN32 // TODO need graceful async exit support on Windows?
 # include "monitor-fd.hh"
@@ -271,7 +275,7 @@ struct ClientSettings
 
 static void performOp(TunnelLogger * logger, ref<Store> store,
     TrustedFlag trusted, RecursiveFlag recursive, WorkerProto::Version clientVersion,
-    Source & from, BufferedSink & to, WorkerProto::Op op)
+    FdSource & from, BufferedSink & to, WorkerProto::Op op)
 {
     WorkerProto::ReadConn rconn {
         .from = from,
@@ -1014,6 +1018,43 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case WorkerProto::Op::ClearFailedPaths:
         throw Error("Removed operation %1%", op);
 
+    case WorkerProto::Op::InitCallback: {
+        // Indicate that we're ready to receive the file descriptor.
+        to << 0;
+        to.flush();
+
+        struct msghdr msg = {0};
+
+        char msgData[256];
+        struct iovec io = { .iov_base = msgData, .iov_len = sizeof(msgData) };
+        msg.msg_iov = &io;
+        msg.msg_iovlen = 1;
+
+        char controlData[256];
+        msg.msg_control = controlData;
+        msg.msg_controllen = sizeof(controlData);
+
+        if (recvmsg(from.fd, &msg, 0) < 0)
+            throw SysError("receiving callback socket");
+
+        AutoCloseFD fd(*((int *) CMSG_DATA(CMSG_FIRSTHDR(&msg))));
+        debug("received file descriptor %d from client", fd.get());
+
+        logger->startWork();
+
+        if (experimentalFeatureSettings.isEnabled(Xp::AuthForwarding)
+            && ((auth::authSettings.authForwarding == auth::AuthForwarding::TrustedUsers && trusted)
+                || (auth::authSettings.authForwarding == auth::AuthForwarding::AllUsers)))
+            auth::getAuthenticator()->addAuthSource(
+                makeTunneledAuthSource(store, clientVersion, std::move(fd)));
+
+        logger->stopWork();
+        to << 1;
+        to.flush();
+
+        break;
+    }
+
     default:
         throw Error("invalid operation %1%", op);
     }

src/libstore/filetransfer.cc

@@ -7,6 +7,8 @@
 #include "finally.hh"
 #include "callback.hh"
 #include "signals.hh"
+#include "auth.hh"
+#include "url.hh"
 
 #if ENABLE_S3
 #include <aws/core/client/ClientConfiguration.h>
@@ -38,6 +40,12 @@ FileTransferSettings fileTransferSettings;
 
 static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings);
 
+FileTransferRequest::FileTransferRequest(std::string_view uri)
+    : uri(uri)
+    , parentAct(getCurActivity())
+    , authenticator(auth::getAuthenticator())
+{ }
+
 struct curlFileTransfer : public FileTransfer
 {
     CURLM * curlm = 0;
@@ -71,6 +79,8 @@ struct curlFileTransfer : public FileTransfer
 
         curl_off_t writtenToSink = 0;
 
+        std::optional<auth::AuthData> authData;
+
         inline static const std::set<long> successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */};
         /* Get the HTTP status code, or 0 for other protocols. */
         long getHTTPStatus()
@@ -359,10 +369,23 @@ struct curlFileTransfer : public FileTransfer
             curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
             curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get());
 
-            /* If no file exist in the specified path, curl continues to work
-               anyway as if netrc support was disabled. */
-            curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str());
-            curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
+            auto url = parseURL(request.uri);
+            auth::AuthData authRequest = {
+                .protocol = url.scheme,
+                .host = url.authority,
+                .path = request.authPath.value_or(url.path),
+                // FIXME: add username
+            };
+            authData = request.authenticator->fill(authRequest, request.requireAuth);
+
+            if (authData) {
+                if (authData->userName)
+                    curl_easy_setopt(req, CURLOPT_USERNAME, authData->userName->c_str());
+                if (authData->password)
+                    curl_easy_setopt(req, CURLOPT_PASSWORD, authData->password->c_str());
+            }
+            else
+                debug("no auth data for '%s'", request.uri);
 
             if (writtenToSink)
                 curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink);
@@ -418,7 +441,17 @@ struct curlFileTransfer : public FileTransfer
                 if (httpStatus == 404 || httpStatus == 410 || code == CURLE_FILE_COULDNT_READ_FILE) {
                     // The file is definitely not there
                     err = NotFound;
-                } else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) {
+                } else if (httpStatus == 401) {
+                    if (authData)
+                        /* This authentication data didn't work, so
+                           erase it. */
+                        request.authenticator->reject(*authData);
+                    if (authData || request.requireAuth)
+                        // FIXME: call erase() on the auth and retry.
+                        err = Forbidden;
+                    else
+                        request.requireAuth = true;
+                } else if (httpStatus == 403 || httpStatus == 407) {
                     // Don't retry on authentication/authorization failures
                     err = Forbidden;
                 } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408 && httpStatus != 429) {
@@ -490,7 +523,10 @@ struct curlFileTransfer : public FileTransfer
                         || writtenToSink == 0
                         || (acceptRanges && encoding.empty())))
                 {
-                    int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937));
+                    int ms =
+                        httpStatus == 401
+                        ? 0
+                        : request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937));
                     if (writtenToSink)
                         warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms);
                     else