master performance: 264 build new instances from a base image; improve

multi user usage Doil now builds on the first create call a base image. This image is the template for following instances. Tihs will minimize the build time for round about 3 minutes. Multi User Support If you add a new doil user, doil also create a .config/composer folder. This is necessary for mounting it into the container. Doil create checks for an .ssh folder in your home directory. Doil forbids global instances below the home directory. Doil adds or append a gitconfig file with safe directories for each global instance.
conceptsandtraining · May 24, 2023 · f5c109b · f5c109b
1 parent 8cd87e7
commit f5c109b
Show file tree

Hide file tree

Showing 22 changed files with 128 additions and 52 deletions.
diff --git a/README.md b/README.md
@@ -51,6 +51,7 @@ however **doil** needs [Docker](https://www.docker.com/) in order to work:
 * php*.*-dom
 * composer version = depending on installed php version
 * git
+* .ssh folder in your home directory. **doil** will mount it into the container. 
 
 Additional dependencies, but these are installed automatically during setup.
 

diff --git a/app/src/App.php b/app/src/App.php
@@ -9,7 +9,7 @@
 
 class App extends Application
 {
-    const NAME = "Doil Version 20230329 - build 2023-03-29";
+    const NAME = "Doil Version 20230524 - build 2023-05-24";
 
     public function __construct(Command ...$commands)
     {

diff --git a/app/src/Commands/Instances/CreateCommand.php b/app/src/Commands/Instances/CreateCommand.php
@@ -101,6 +101,8 @@ public function execute(InputInterface $input, OutputInterface $output) : int
         $suffix = $options["global"] ? "global" : "local";
         $instance_name = $options["name"] . "_" . $suffix;
         $instance_salt_name = $options["name"] . "." . $suffix;
+        $user_name = $this->posix->getCurrentUserName();
+        $home_dir = $this->posix->getHomeDirectory($this->posix->getUserId());
 
         if ($this->filesystem->exists($instance_path)) {
             $this->writer->error(
@@ -111,6 +113,22 @@ public function execute(InputInterface $input, OutputInterface $output) : int
             return Command::FAILURE;
         }
 
+        if (! $this->filesystem->exists($home_dir . "/.ssh")) {
+            $this->writer->error(
+                $output,
+                "Folder $home_dir/.ssh not found."
+            );
+            return Command::FAILURE;
+        }
+
+        if ($suffix == "global" && stristr($instance_path, "/home/") !== false) {
+            $this->writer->error(
+                $output,
+                "Global instances must not be created below /home directory."
+            );
+            return Command::FAILURE;
+        }
+
         $this->writer->beginBlock($output, "Creating instance " . $options['name']);
 
         if (isset($options["repo_path"]) && ! $this->filesystem->exists($options["repo_path"])) {
@@ -126,10 +144,15 @@ public function execute(InputInterface $input, OutputInterface $output) : int
             }
         }
 
-        // update debian image
-        $this->writer->beginBlock($output, "Updating debian image");
-        $this->docker->pull("debian");
-        $this->writer->endBlock();
+        // install base image if not exists
+        if ($this->docker->getImageIdsByName("doil/base_global")[0] == "") {
+            $this->writer->beginBlock($output, "Updating debian image");
+            $this->docker->pull("debian");
+            $this->writer->endBlock();
+            $this->writer->beginBlock($output, "Install base image");
+            $this->docker->build("/usr/local/share/doil/templates/base/Dockerfile", "base_global");
+            $this->writer->endBlock();
+        }
 
         // create basic folders
         $this->writer->beginBlock($output, "Create basic folders");
@@ -143,7 +166,6 @@ public function execute(InputInterface $input, OutputInterface $output) : int
         if ($options["global"]) {
             $this->filesystem->symlink($instance_path, self::GLOBAL_INSTANCES_PATH . "/" . $options["name"]);
         } else {
-            $home_dir = $this->posix->getHomeDirectory($this->posix->getUserId());
             $this->filesystem->symlink($instance_path, $home_dir . self::LOCAL_INSTANCES_PATH . "/" . $options["name"]);
         }
         $this->writer->endBlock();
@@ -153,15 +175,15 @@ public function execute(InputInterface $input, OutputInterface $output) : int
         if ($options["global"]) {
             $this->filesystem->chownRecursive(
                 $instance_path,
-                $this->posix->getCurrentUserName(),
+                $user_name,
                 "doil"
             );
             $this->filesystem->chmod($instance_path, 02775);
         } else {
             $this->filesystem->chownRecursive(
                 $instance_path,
-                $this->posix->getCurrentUserName(),
-                $this->posix->getCurrentUserName()
+                $user_name,
+                $user_name
             );
         }
         $this->writer->endBlock();
@@ -172,10 +194,6 @@ public function execute(InputInterface $input, OutputInterface $output) : int
             "/usr/local/share/doil/templates/minion/run-supervisor.sh",
             $instance_path . "/conf/run-supervisor.sh"
         );
-        $this->filesystem->copy(
-            "/usr/local/share/doil/templates/minion/Dockerfile",
-            $instance_path . "/Dockerfile"
-        );
         $this->filesystem->copy(
             "/usr/local/share/doil/templates/minion/salt-minion.conf",
             $instance_path . "/conf/salt-minion.conf"
@@ -206,7 +224,6 @@ public function execute(InputInterface $input, OutputInterface $output) : int
         $this->writer->beginBlock($output, "Copying ilias to target");
         $git_path = Filesystem::DOIL_PATH_SHARE . "/repositories/" . $options["repo"];
         if (! $options["use_global_repo"]) {
-            $home_dir = $this->posix->getHomeDirectory($this->posix->getUserId());
             $git_path = $home_dir . "/.doil/repositories/" . $options["repo"];
         }
 
@@ -237,23 +254,15 @@ public function execute(InputInterface $input, OutputInterface $output) : int
             "%TPL_PROJECT_DOMAINNAME%",
             $suffix
         );
-        $this->filesystem->replaceStringInFile(
-            $instance_path . "/Dockerfile",
-            "%USER_ID%",
-            (string) $this->posix->getUserId()
-        );
-        $this->filesystem->replaceStringInFile(
-            $instance_path . "/Dockerfile",
-            "%GROUP_ID%",
-            (string) $this->posix->getGroupId()
-        );
         $this->writer->endBlock();
 
         // building minion image
         $this->writer->beginBlock($output, "Building minion image. This will take a while. Please be patient");
-        $this->docker->build($instance_path, $instance_name);
         $this->docker->runContainer($instance_name);
-        $this->docker->executeDockerCommand($instance_name, "apt install -y mariadb-server python3-mysqldb");
+        $usr_id = (string) $this->posix->getUserId();
+        $group_id = (string) $this->posix->getGroupId();
+        $this->docker->executeDockerCommand($instance_name, "usermod -u $usr_id www-data");
+        $this->docker->executeDockerCommand($instance_name, "groupmod -g $group_id www-data");
         $this->docker->executeDockerCommand($instance_name, "/etc/init.d/mariadb start");
         sleep(5);
         $this->docker->executeDockerCommand($instance_name, "/etc/init.d/mariadb stop");
@@ -401,7 +410,6 @@ public function execute(InputInterface $input, OutputInterface $output) : int
         }
 
         $output->writeln("Please start the created instance by <fg=gray>doil up {$options["name"]}$flag</>.");
-
         return Command::SUCCESS;
     }
 

diff --git a/app/src/Commands/User/AddCommand.php b/app/src/Commands/User/AddCommand.php
@@ -8,6 +8,7 @@
 use CaT\Doil\Lib\Posix\Posix;
 use CaT\Doil\Lib\ConsoleOutput\Writer;
 use CaT\Doil\Lib\FileSystem\Filesystem;
+use CaT\Doil\Commands\Repo\RepoManager;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
@@ -23,13 +24,15 @@ class AddCommand extends Command
     protected Linux $linux;
     protected Filesystem $filesystem;
     protected Writer $writer;
+    protected RepoManager $repo_manager;
 
     public function __construct(
         UserManager $user_manager,
         Posix $posix,
         Linux $linux,
         Filesystem $filesystem,
-        Writer $writer
+        Writer $writer,
+        RepoManager $repo_manager
     ) {
         parent::__construct();
 
@@ -38,6 +41,7 @@ public function __construct(
         $this->linux = $linux;
         $this->filesystem = $filesystem;
         $this->writer = $writer;
+        $this->repo_manager = $repo_manager;
     }
 
     public function configure() : void
@@ -83,9 +87,14 @@ public function execute(InputInterface $input, OutputInterface $output) : int
         $this->writer->beginBlock($output, "Add user {$user->getName()} to doil");
         $this->user_manager->addUser($user);
         $this->user_manager->createFileInfrastructure($home_dir, $user->getName());
+        $this->user_manager->ensureGlobalReposAreGitSafe($home_dir, $this->repo_manager->getGlobalRepos());
+        $this->linux->initComposer($user->getName());
         $this->linux->addUserToGroup($user->getName(), "doil");
         $this->writer->endBlock();
 
+        $output->writeln("\nPlease ensure that {$user->getName()} is also member of the docker group!");
+        $output->writeln("\t<fg=gray>sudo usermod -aG docker {$user->getName()}</>");
+
         return Command::SUCCESS;
     }
 }
diff --git a/app/src/Commands/User/UserManager.php b/app/src/Commands/User/UserManager.php
@@ -4,6 +4,7 @@
 
 namespace CaT\Doil\Commands\User;
 
+use CaT\Doil\Commands\Repo\Repo;
 use CaT\Doil\Lib\FileSystem\Filesystem;
 
 class UserManager
@@ -78,6 +79,16 @@ public function deleteUser(User $user) : void
         $this->saveUsers($users);
     }
 
+    /**
+     * @param array<Repo>  $repos
+     */
+    public function ensureGlobalReposAreGitSafe(string $home_dir, array $repos) : void
+    {
+        foreach ($repos as $repo) {
+            $this->filesystem->addToGitConfig($home_dir, "safe", "directory = /usr/local/share/doil/repositories/" . $repo->getName());
+        }
+    }
+
     /**
      * @param User[]
      */

diff --git a/app/src/Lib/Docker/DockerShell.php b/app/src/Lib/Docker/DockerShell.php
@@ -350,7 +350,7 @@ public function pull(string $name) : void
         $cmd = [
             "docker",
             "pull",
-            $name . ":stable"
+            $name . ":11"
         ];
 
         $logger = $this->logger->getDoilLogger("DOCKER");
@@ -365,7 +365,9 @@ public function build(string $path, string $name) : void
             "build",
             "-t",
             "doil/" . $name . ":stable",
-            $path
+            "-f",
+            $path,
+            "."
         ];
 
         $logger = $this->logger->getDoilLogger($name);
@@ -381,7 +383,7 @@ public function runContainer(string $name) : void
             "-d",
             "--name",
             $name,
-            "doil/" . $name . ":stable"
+            "doil/base_global:stable"
         ];
 
         $logger = $this->logger->getDoilLogger($name);

diff --git a/app/src/Lib/FileSystem/Filesystem.php b/app/src/Lib/FileSystem/Filesystem.php
@@ -34,4 +34,5 @@ public function getLineInFile(string $path, string $needle) : ?string;
     public function saveToJsonFile(string $path, array $objects) : void;
     public function readFromJsonFile(string $path) : array;
     public function grepMysqlPasswordFromFile(string $path) : string;
+    public function addToGitConfig(string $path, string $section, string $line) : void;
 }
diff --git a/app/src/Lib/FileSystem/FilesystemShell.php b/app/src/Lib/FileSystem/FilesystemShell.php
@@ -201,4 +201,19 @@ public function grepMysqlPasswordFromFile(string $path) : string
 
         return "";
     }
+
+    public function addToGitConfig(string $path, string $section, string $line) : void
+    {
+        $path = $path . "/.gitconfig";
+        if (! $this->exists($path)) {
+            $this->symfony_file_system->touch($path);
+        }
+
+        if (! is_null($this->getLineInFile($path, $line))) {
+            return;
+        }
+
+        $this->symfony_file_system->appendToFile($path, "[$section]\n");
+        $this->symfony_file_system->appendToFile($path, "\t" . $line . "\n");
+    }
 }
diff --git a/app/src/Lib/Linux/Linux.php b/app/src/Lib/Linux/Linux.php
@@ -10,4 +10,5 @@ public function addUserToGroup(string $user, string $group) : void;
     public function removeUserFromGroup(string $user, string $group) : void;
     public function deleteGroup(string $name) : void;
     public function isWSL() : bool;
+    public function initComposer(string $user) : void;
 }
diff --git a/app/src/Lib/Linux/LinuxShell.php b/app/src/Lib/Linux/LinuxShell.php
@@ -67,4 +67,17 @@ public function isWSL() : bool
         $wsl = $this->run($cmd, $this->logger);
         return (bool) stristr($wsl, "microsoft");
     }
+
+    public function initComposer(string $user) : void
+    {
+        $cmd = [
+            "su",
+            $user,
+            "-c",
+            "$(which composer)"
+        ];
+
+        $this->logger->info("Init composer for user '$user'.");
+        $this->run($cmd, $this->logger);
+    }
 }
diff --git a/app/src/Lib/Logger/LoggerFactory.php b/app/src/Lib/Logger/LoggerFactory.php
@@ -17,7 +17,7 @@ class LoggerFactory
     public function getDoilLogger(string $channel) : LoggerInterface
     {
         $logger = new Logger($channel);
-        $stream = new StreamHandler(self::DOIL_LOG_PATH, Logger::DEBUG);
+        $stream = new StreamHandler(self::DOIL_LOG_PATH, Logger::DEBUG, true, 0777);
         $form = new LineFormatter(null,null,true,false,true);
         $form->setJsonPrettyPrint(true);
         $stream->setFormatter($form);
@@ -28,7 +28,7 @@ public function getDoilLogger(string $channel) : LoggerInterface
     public function getSaltLogger(string $channel) : LoggerInterface
     {
         $logger = new Logger($channel);
-        $stream = new StreamHandler(self::SALT_LOG_PATH, Logger::DEBUG);
+        $stream = new StreamHandler(self::SALT_LOG_PATH, Logger::DEBUG, true, 0777);
         $form = new LineFormatter(null,null,true,false,true);
         $form->setJsonPrettyPrint(true);
         $stream->setFormatter($form);

diff --git a/app/src/cli.php b/app/src/cli.php
@@ -403,7 +403,8 @@ function buildContainerForApp() : Container
             $c["posix.shell"],
             $c["linux.shell"],
             $c["filesystem.shell"],
-            $c["command.writer"]
+            $c["command.writer"],
+            $c["repo.manager"]
         );
     };
 

diff --git a/app/tests/Commands/Instances/CreateCommandTest.php b/app/tests/Commands/Instances/CreateCommandTest.php
@@ -308,6 +308,11 @@ public function test_execute() : void
             ->method("getSaltAcceptedKeys")
             ->willReturn(["1232.local"])
         ;
+        $docker
+            ->expects($this->once())
+            ->method("getImageIdsByName")
+            ->willReturn([""])
+        ;
 
         $filesystem
             ->expects($this->exactly(2))