master performance: 264 build new instances from a base image; improve

multi user usage

Doil now builds on the first create call a base image. This image is the
template for following instances. Tihs will minimize the build time for round
about 3 minutes.

Multi User Support

If you add a new doil user, doil also create a .config/composer folder.
This is necessary for mounting it into the container.

Doil create checks for an .ssh folder in your home directory.

Doil forbids global instances below the home directory.

Doil adds or append a gitconfig file with safe directories for each
global instance.

README.md

@@ -51,6 +51,7 @@ however **doil** needs [Docker](https://www.docker.com/) in order to work:
 * php*.*-dom
 * composer version = depending on installed php version
 * git
+* .ssh folder in your home directory. **doil** will mount it into the container. **doil** needs this to have access to any private git repositories that may be used. 
 
 Additional dependencies, but these are installed automatically during setup.
 

app/src/App.php

@@ -9,7 +9,7 @@
 
 class App extends Application
 {
-    const NAME = "Doil Version 20230329 - build 2023-03-29";
+    const NAME = "Doil Version 20230524 - build 2023-05-24";
 
     public function __construct(Command ...$commands)
     {

app/src/Commands/Instances/CreateCommand.php

@@ -25,6 +25,7 @@
 
 class CreateCommand extends Command
 {
+    protected const DEBIAN_TAG = "11";
     protected const LOCAL_REPO_PATH = "/.doil/repositories";
     protected const GLOBAL_REPO_PATH = "/usr/local/share/doil/repositories";
     protected const LOCAL_INSTANCES_PATH = "/.doil/instances";
@@ -101,6 +102,8 @@ public function execute(InputInterface $input, OutputInterface $output) : int
         $suffix = $options["global"] ? "global" : "local";
         $instance_name = $options["name"] . "_" . $suffix;
         $instance_salt_name = $options["name"] . "." . $suffix;
+        $user_name = $this->posix->getCurrentUserName();
+        $home_dir = $this->posix->getHomeDirectory($this->posix->getUserId());
 
         if ($this->filesystem->exists($instance_path)) {
             $this->writer->error(
@@ -111,6 +114,22 @@ public function execute(InputInterface $input, OutputInterface $output) : int
             return Command::FAILURE;
         }
 
+        if (! $this->filesystem->exists($home_dir . "/.ssh")) {
+            $this->writer->error(
+                $output,
+                "Folder $home_dir/.ssh not found."
+            );
+            return Command::FAILURE;
+        }
+
+        if ($suffix == "global" && stristr($instance_path, "/home/") !== false) {
+            $this->writer->error(
+                $output,
+                "Global instances must not be created below /home directory."
+            );
+            return Command::FAILURE;
+        }
+
         $this->writer->beginBlock($output, "Creating instance " . $options['name']);
 
         if (isset($options["repo_path"]) && ! $this->filesystem->exists($options["repo_path"])) {
@@ -126,10 +145,15 @@ public function execute(InputInterface $input, OutputInterface $output) : int
             }
         }
 
-        // update debian image
-        $this->writer->beginBlock($output, "Updating debian image");
-        $this->docker->pull("debian");
-        $this->writer->endBlock();
+        // install base image if not exists
+        if ($this->docker->getImageIdsByName("doil/base_global")[0] == "") {
+            $this->writer->beginBlock($output, "Update debian image");
+            $this->docker->pull("debian", self::DEBIAN_TAG);
+            $this->writer->endBlock();
+            $this->writer->beginBlock($output, "Install base image");
+            $this->docker->build("/usr/local/share/doil/templates/base/Dockerfile", "base_global");
+            $this->writer->endBlock();
+        }
 
         // create basic folders
         $this->writer->beginBlock($output, "Create basic folders");
@@ -143,7 +167,6 @@ public function execute(InputInterface $input, OutputInterface $output) : int
         if ($options["global"]) {
             $this->filesystem->symlink($instance_path, self::GLOBAL_INSTANCES_PATH . "/" . $options["name"]);
         } else {
-            $home_dir = $this->posix->getHomeDirectory($this->posix->getUserId());
             $this->filesystem->symlink($instance_path, $home_dir . self::LOCAL_INSTANCES_PATH . "/" . $options["name"]);
         }
         $this->writer->endBlock();
@@ -153,15 +176,15 @@ public function execute(InputInterface $input, OutputInterface $output) : int
         if ($options["global"]) {
             $this->filesystem->chownRecursive(
                 $instance_path,
-                $this->posix->getCurrentUserName(),
+                $user_name,
                 "doil"
             );
             $this->filesystem->chmod($instance_path, 02775);
         } else {
             $this->filesystem->chownRecursive(
                 $instance_path,
-                $this->posix->getCurrentUserName(),
-                $this->posix->getCurrentUserName()
+                $user_name,
+                $user_name
             );
         }
         $this->writer->endBlock();
@@ -172,10 +195,6 @@ public function execute(InputInterface $input, OutputInterface $output) : int
             "/usr/local/share/doil/templates/minion/run-supervisor.sh",
             $instance_path . "/conf/run-supervisor.sh"
         );
-        $this->filesystem->copy(
-            "/usr/local/share/doil/templates/minion/Dockerfile",
-            $instance_path . "/Dockerfile"
-        );
         $this->filesystem->copy(
             "/usr/local/share/doil/templates/minion/salt-minion.conf",
             $instance_path . "/conf/salt-minion.conf"
@@ -206,7 +225,6 @@ public function execute(InputInterface $input, OutputInterface $output) : int
         $this->writer->beginBlock($output, "Copying ilias to target");
         $git_path = Filesystem::DOIL_PATH_SHARE . "/repositories/" . $options["repo"];
         if (! $options["use_global_repo"]) {
-            $home_dir = $this->posix->getHomeDirectory($this->posix->getUserId());
             $git_path = $home_dir . "/.doil/repositories/" . $options["repo"];
         }
 
@@ -237,23 +255,15 @@ public function execute(InputInterface $input, OutputInterface $output) : int
             "%TPL_PROJECT_DOMAINNAME%",
             $suffix
         );
-        $this->filesystem->replaceStringInFile(
-            $instance_path . "/Dockerfile",
-            "%USER_ID%",
-            (string) $this->posix->getUserId()
-        );
-        $this->filesystem->replaceStringInFile(
-            $instance_path . "/Dockerfile",
-            "%GROUP_ID%",
-            (string) $this->posix->getGroupId()
-        );
         $this->writer->endBlock();
 
         // building minion image
         $this->writer->beginBlock($output, "Building minion image. This will take a while. Please be patient");
-        $this->docker->build($instance_path, $instance_name);
         $this->docker->runContainer($instance_name);
-        $this->docker->executeDockerCommand($instance_name, "apt install -y mariadb-server python3-mysqldb");
+        $usr_id = (string) $this->posix->getUserId();
+        $group_id = (string) $this->posix->getGroupId();
+        $this->docker->executeDockerCommand($instance_name, "usermod -u $usr_id www-data");
+        $this->docker->executeDockerCommand($instance_name, "groupmod -g $group_id www-data");
         $this->docker->executeDockerCommand($instance_name, "/etc/init.d/mariadb start");
         sleep(5);
         $this->docker->executeDockerCommand($instance_name, "/etc/init.d/mariadb stop");
@@ -280,7 +290,6 @@ public function execute(InputInterface $input, OutputInterface $output) : int
             $this->docker->executeDockerCommand($instance_name, "rm -rf /var/lib/salt/pki/minion/*");
             $this->docker->executeDockerCommand("doil_saltmain", "salt-key -d " . $instance_salt_name . " -y");
             $this->docker->executeDockerCommand($instance_name, "salt-minion -d");
-
             sleep(5);
             $salt_keys = $this->docker->getSaltAcceptedKeys();
         }
@@ -401,7 +410,6 @@ public function execute(InputInterface $input, OutputInterface $output) : int
         }
 
         $output->writeln("Please start the created instance by <fg=gray>doil up {$options["name"]}$flag</>.");
-
         return Command::SUCCESS;
     }
 

app/src/Commands/User/AddCommand.php

@@ -8,6 +8,7 @@
 use CaT\Doil\Lib\Posix\Posix;
 use CaT\Doil\Lib\ConsoleOutput\Writer;
 use CaT\Doil\Lib\FileSystem\Filesystem;
+use CaT\Doil\Commands\Repo\RepoManager;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
@@ -23,13 +24,15 @@ class AddCommand extends Command
     protected Linux $linux;
     protected Filesystem $filesystem;
     protected Writer $writer;
+    protected RepoManager $repo_manager;
 
     public function __construct(
         UserManager $user_manager,
         Posix $posix,
         Linux $linux,
         Filesystem $filesystem,
-        Writer $writer
+        Writer $writer,
+        RepoManager $repo_manager
     ) {
         parent::__construct();
 
@@ -38,6 +41,7 @@ public function __construct(
         $this->linux = $linux;
         $this->filesystem = $filesystem;
         $this->writer = $writer;
+        $this->repo_manager = $repo_manager;
     }
 
     public function configure() : void
@@ -83,9 +87,14 @@ public function execute(InputInterface $input, OutputInterface $output) : int
         $this->writer->beginBlock($output, "Add user {$user->getName()} to doil");
         $this->user_manager->addUser($user);
         $this->user_manager->createFileInfrastructure($home_dir, $user->getName());
+        $this->user_manager->ensureGlobalReposAreGitSafe($home_dir, $this->repo_manager->getGlobalRepos());
+        $this->linux->initComposer($user->getName());
         $this->linux->addUserToGroup($user->getName(), "doil");
         $this->writer->endBlock();
 
+        $output->writeln("\nPlease ensure that {$user->getName()} is also member of the docker group!");
+        $output->writeln("\t<fg=gray>sudo usermod -aG docker {$user->getName()}</>");
+
         return Command::SUCCESS;
     }
 }

app/src/Commands/User/UserManager.php

@@ -4,6 +4,7 @@
 
 namespace CaT\Doil\Commands\User;
 
+use CaT\Doil\Commands\Repo\Repo;
 use CaT\Doil\Lib\FileSystem\Filesystem;
 
 class UserManager
@@ -78,6 +79,16 @@ public function deleteUser(User $user) : void
         $this->saveUsers($users);
     }
 
+    /**
+     * @param array<Repo>  $repos
+     */
+    public function ensureGlobalReposAreGitSafe(string $home_dir, array $repos) : void
+    {
+        foreach ($repos as $repo) {
+            $this->filesystem->addToGitConfig($home_dir, "safe", "directory = /usr/local/share/doil/repositories/" . $repo->getName());
+        }
+    }
+
     /**
      * @param User[]
      */

app/src/Lib/Docker/Docker.php

@@ -28,7 +28,7 @@ public function commit(string $instance_name, ?string $image_name = null) : void
     public function applyState(string $name, string $state) : void;
     public function copy(string $instance_name, string $from, string $to) : void;
     public function listContainerDirectory(string $container_name, string $path) : array;
-    public function pull(string $name) : void;
+    public function pull(string $name, string $tag) : void;
     public function build(string $path, string $name) : void;
     public function runContainer(string $name) : void;
     public function stop(string $name) : void;

app/src/Lib/Docker/DockerShell.php

@@ -345,16 +345,16 @@ public function listContainerDirectory(string $container_name, string $path) : a
         return explode("\n", trim($this->run($cmd, $logger)));
     }
 
-    public function pull(string $name) : void
+    public function pull(string $name, string $tag) : void
     {
         $cmd = [
             "docker",
             "pull",
-            $name . ":stable"
+            $name . ":" . $tag
         ];
 
         $logger = $this->logger->getDoilLogger("DOCKER");
-        $logger->info("Pull image $name:stable");
+        $logger->info("Pull image $name:$tag");
         $this->run($cmd, $logger);
     }
 
@@ -365,7 +365,9 @@ public function build(string $path, string $name) : void
             "build",
             "-t",
             "doil/" . $name . ":stable",
-            $path
+            "-f",
+            $path,
+            "."
         ];
 
         $logger = $this->logger->getDoilLogger($name);
@@ -381,7 +383,7 @@ public function runContainer(string $name) : void
             "-d",
             "--name",
             $name,
-            "doil/" . $name . ":stable"
+            "doil/base_global:stable"
         ];
 
         $logger = $this->logger->getDoilLogger($name);

app/src/Lib/FileSystem/Filesystem.php

@@ -34,4 +34,5 @@ public function getLineInFile(string $path, string $needle) : ?string;
     public function saveToJsonFile(string $path, array $objects) : void;
     public function readFromJsonFile(string $path) : array;
     public function grepMysqlPasswordFromFile(string $path) : string;
+    public function addToGitConfig(string $path, string $section, string $line) : void;
 }

app/src/Lib/FileSystem/FilesystemShell.php

@@ -201,4 +201,19 @@ public function grepMysqlPasswordFromFile(string $path) : string
 
         return "";
     }
+
+    public function addToGitConfig(string $path, string $section, string $line) : void
+    {
+        $path = $path . "/.gitconfig";
+        if (! $this->exists($path)) {
+            $this->symfony_file_system->touch($path);
+        }
+
+        if (! is_null($this->getLineInFile($path, $line))) {
+            return;
+        }
+
+        $this->symfony_file_system->appendToFile($path, "[$section]\n");
+        $this->symfony_file_system->appendToFile($path, "\t" . $line . "\n");
+    }
 }

app/src/Lib/Linux/Linux.php

@@ -10,4 +10,5 @@ public function addUserToGroup(string $user, string $group) : void;
     public function removeUserFromGroup(string $user, string $group) : void;
     public function deleteGroup(string $name) : void;
     public function isWSL() : bool;
+    public function initComposer(string $user) : void;
 }

app/src/Lib/Linux/LinuxShell.php

@@ -67,4 +67,17 @@ public function isWSL() : bool
         $wsl = $this->run($cmd, $this->logger);
         return (bool) stristr($wsl, "microsoft");
     }
+
+    public function initComposer(string $user) : void
+    {
+        $cmd = [
+            "su",
+            $user,
+            "-c",
+            "$(which composer)"
+        ];
+
+        $this->logger->info("Init composer for user '$user'.");
+        $this->run($cmd, $this->logger);
+    }
 }

app/src/Lib/Logger/LoggerFactory.php

@@ -17,7 +17,7 @@ class LoggerFactory
     public function getDoilLogger(string $channel) : LoggerInterface
     {
         $logger = new Logger($channel);
-        $stream = new StreamHandler(self::DOIL_LOG_PATH, Logger::DEBUG);
+        $stream = new StreamHandler(self::DOIL_LOG_PATH, Logger::DEBUG, true, 0777);
         $form = new LineFormatter(null,null,true,false,true);
         $form->setJsonPrettyPrint(true);
         $stream->setFormatter($form);
@@ -28,7 +28,7 @@ public function getDoilLogger(string $channel) : LoggerInterface
     public function getSaltLogger(string $channel) : LoggerInterface
     {
         $logger = new Logger($channel);
-        $stream = new StreamHandler(self::SALT_LOG_PATH, Logger::DEBUG);
+        $stream = new StreamHandler(self::SALT_LOG_PATH, Logger::DEBUG, true, 0777);
         $form = new LineFormatter(null,null,true,false,true);
         $form->setJsonPrettyPrint(true);
         $stream->setFormatter($form);

app/src/cli.php

@@ -403,7 +403,8 @@ function buildContainerForApp() : Container
             $c["posix.shell"],
             $c["linux.shell"],
             $c["filesystem.shell"],
-            $c["command.writer"]
+            $c["command.writer"],
+            $c["repo.manager"]
         );
     };
 

app/tests/Commands/Instances/CreateCommandTest.php

@@ -308,7 +308,17 @@ public function test_execute() : void
             ->method("getSaltAcceptedKeys")
             ->willReturn(["1232.local"])
         ;
+        $docker
+            ->expects($this->once())
+            ->method("getImageIdsByName")
+            ->willReturn([""])
+        ;
 
+        $filesystem
+            ->expects($this->exactly(3))
+            ->method("exists")
+            ->willReturn(false, true, true)
+        ;
         $filesystem
             ->expects($this->exactly(2))
             ->method("getLineInFile")