diff --git a/bin/sspak b/bin/sspak index 5a1cffc..757c109 100755 --- a/bin/sspak +++ b/bin/sspak @@ -11,6 +11,8 @@ require_once(PACKAGE_ROOT . 'src/FilesystemEntity.php'); require_once(PACKAGE_ROOT . 'src/SSPakFile.php'); require_once(PACKAGE_ROOT . 'src/Webroot.php'); +require_once(PACKAGE_ROOT . 'vendor/autoload.php'); + $argObj = new Args($_SERVER['argv']); /* // Special case for self-extracting sspaks @@ -43,4 +45,4 @@ if(isset($allowedActions[$action])) { echo "Unrecognised action '" . $action . "'.\n"; $ssPak->help($argObj); exit(3); -} \ No newline at end of file +} diff --git a/composer.json b/composer.json index 5d2e6a3..152c48e 100644 --- a/composer.json +++ b/composer.json @@ -1,18 +1,18 @@ { - "name": "silverstripe/sspak", - "description": "CLI tool for saving & loading data in SilverStripe installations", - "license": "BSD-3-Clause", - "authors": [ - { - "name": "Sam Minnee", - "email": "sam@silverstripe.com" - } - ], - "autoload": { - "classmap": [ "src/" ] - }, - "require": {}, - "require-dev": { - "phpunit/phpunit": "^3.7" - } - } + "name": "silverstripe/sspak", + "description": "CLI tool for saving & loading data in SilverStripe installations", + "license": "BSD-3-Clause", + "authors": [ + { + "name": "Sam Minnee", + "email": "sam@silverstripe.com" + } + ], + "autoload": { + "classmap": [ "src/" ] + }, + "require": {}, + "require-dev": { + "phpunit/phpunit": "^3.7" + } +} diff --git a/composer.lock b/composer.lock index 56a115f..6a581bc 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "This file is @generated automatically" ], - "hash": "4d3cf135125c8dc81d955e67cb7fd3c4", + "hash": "e8f637aaa0931f73f8b15e558734a4dc", "content-hash": "e649c0660b9ca5d326f6a0e52e30031e", "packages": [], "packages-dev": [ diff --git a/src/DataExtractor/CsvTableReader.php b/src/DataExtractor/CsvTableReader.php new file mode 100644 index 0000000..3e7fc34 --- /dev/null +++ b/src/DataExtractor/CsvTableReader.php @@ -0,0 +1,70 @@ +filename = $filename; + } + + function getColumns() { + if (!$this->columns) { + $this->initColumns(); + } + return $this->columns; + } + + function getIterator() { + $this->columns = null; + $this->initColumns(); + + while(($row = $this->getRow()) !== false) { + yield $this->mapToColumns($row); + } + + $this->close(); + } + + private function mapToColumns($row) { + $record = []; + foreach($row as $i => $value) + { + if(isset($this->columns[$i])) { + $record[$this->columns[$i]] = $value; + } else { + throw new \LogicException("Row contains invalid column #$i\n" . var_export($row, true)); + } + } + return $record; + } + + private function initColumns() { + $this->open(); + $this->columns = $this->getRow(); + } + + private function getRow() { + return fgetcsv($this->handle); + } + + private function open() { + if ($this->handle) { + fclose($this->handle); + $this->handle = null; + } + $this->handle = fopen($this->filename, 'r'); + } + + private function close() { + if ($this->handle) { + fclose($this->handle); + $this->handle = null; + } + } +} diff --git a/src/DataExtractor/CsvTableWriter.php b/src/DataExtractor/CsvTableWriter.php new file mode 100644 index 0000000..06a9ca7 --- /dev/null +++ b/src/DataExtractor/CsvTableWriter.php @@ -0,0 +1,64 @@ +filename = $filename; + } + + function start($columns) { + $this->open(); + $this->putRow($columns); + $this->columns = $columns; + } + + function finish() { + $this->close(); + } + + function writeRecord($record) { + if (!$this->columns) { + $this->start(array_keys($record)); + } + + $this->putRow($this->mapFromColumns($record)); + } + + private function mapFromColumns($record) { + $row = []; + foreach($this->columns as $i => $column) + { + $row[$i] = isset($record[$column]) ? $record[$column] : null; + } + return $row; + } + + private function putRow($row) { + return fputcsv($this->handle, $row); + } + + private function open() { + if ($this->handle) { + fclose($this->handle); + $this->handle = null; + } + $this->handle = fopen($this->filename, 'w'); + if (!$this->handle) { + throw new \LogicException("Can't open $this->filename for writing."); + } + } + + private function close() { + if ($this->handle) { + fclose($this->handle); + $this->handle = null; + } + } +} diff --git a/src/DataExtractor/DatabaseConnector.php b/src/DataExtractor/DatabaseConnector.php new file mode 100644 index 0000000..3624052 --- /dev/null +++ b/src/DataExtractor/DatabaseConnector.php @@ -0,0 +1,129 @@ +basePath = $basePath; + } + + function connect() { + if ($this->isConnected) { + return; + } + + $this->isConnected = true; + + // Necessary for SilverStripe's _ss_environment.php loader to work + $_SERVER['SCRIPT_FILENAME'] = $this->basePath . '/dummy.php'; + + global $databaseConfig; + + // require composers autoloader + if (file_exists($this->basePath . '/vendor/autoload.php')) { + require_once $this->basePath . '/vendor/autoload.php'; + } + + if (file_exists($this->basePath . '/framework/core/Core.php')) { + require_once($this->basePath . '/framework/core/Core.php'); + } elseif (file_exists($this->basePath . '/sapphire/core/Core.php')) { + require_once($this->basePath . '/sapphire/core/Core.php'); + } else { + throw new \LogicException("No framework/core/Core.php or sapphire/core/Core.php included in project. Perhaps $this->basePath is not a SilverStripe project?"); + } + + // Connect to database + require_once('model/DB.php'); + + if ($databaseConfig) { + DB::connect($databaseConfig); + } else { + throw new \LogicException("No \$databaseConfig found"); + } + } + + function getDatabase() { + $this->connect(); + + if(method_exists('DB', 'get_conn')) { + return DB::get_conn(); + } else { + return DB::getConn(); + } + } + + /** + * Get a list of tables from the database + */ + function getTables() { + $this->connect(); + + if(method_exists('DB', 'table_list')) { + return DB::table_list(); + } else { + return DB::tableList(); + } + } + + /** + * Get a list of tables from the database + */ + function getFieldsForTable($tableName) { + $this->connect(); + + if(method_exists('DB', 'field_list')) { + return DB::field_list($tableName); + } else { + return DB::fieldList($tableName); + } + } + + /** + * Save the named table to the given table write + */ + function saveTable($tableName, TableWriter $writer) { + $query = $this->getDatabase()->query("SELECT * FROM \"$tableName\""); + + foreach ($query as $record) { + $writer->writeRecord($record); + } + + $writer->finish(); + } + + /** + * Save the named table to the given table write + */ + function loadTable($tableName, TableReader $reader) { + $this->getDatabase()->clearTable($tableName); + + $fields = $this->getFieldsForTable($tableName); + + foreach($reader as $record) { + foreach ($record as $k => $v) { + if (!isset($fields[$k])) { + unset($record[$k]); + } + } + // TODO: Batch records + $manipulation = [ + $tableName => [ + 'command' => 'insert', + 'fields' => $record, + ], + ]; + DB::manipulate($manipulation); + } + } +} diff --git a/src/DataExtractor/TableReader.php b/src/DataExtractor/TableReader.php new file mode 100644 index 0000000..4ed0af5 --- /dev/null +++ b/src/DataExtractor/TableReader.php @@ -0,0 +1,18 @@ + value data + */ + function writeRecord($record); + + /** + * Finish writing. + * writeRecord() must not be called after this + */ + function finish(); +} diff --git a/src/SSPak.php b/src/SSPak.php index 06f223b..f246a79 100644 --- a/src/SSPak.php +++ b/src/SSPak.php @@ -1,5 +1,9 @@ array("sspak file", "destination path"), "method" => "extract" ), + "listtables" => array( + "description" => "List tables in the database", + "unnamedArgs" => array("webroot"), + "method" => "listTables" + ), + + "savecsv" => array( + "description" => "Save tables in the database to a collection of CSV files", + "unnamedArgs" => array("webroot", "output-path"), + "method" => "saveCsv" + ), + + "loadcsv" => array( + "description" => "Load tables from collection of CSV files to a webroot", + "unnamedArgs" => array("input-path", "webroot"), + "method" => "loadCsv" + ), /* + "install" => array( "description" => "Install a .sspak file into a new environment.", "unnamedArgs" => array("sspak file", "new webroot"), @@ -134,6 +156,66 @@ function extract($args) { $phar->extractTo($dest); } + function listTables($args) { + $args->requireUnnamed(array('webroot')); + $unnamedArgs = $args->getUnnamedArgs(); + $webroot = $unnamedArgs[0]; + + $db = new DatabaseConnector($webroot); + + print_r($db->getTables()); + } + + function saveCsv($args) { + $args->requireUnnamed(array('webroot', 'path')); + $unnamedArgs = $args->getUnnamedArgs(); + $webroot = $unnamedArgs[0]; + $destPath = $unnamedArgs[1]; + + if (!file_exists($destPath)) { + mkdir($destPath) || die("Can't create $destPath"); + } + if (!is_dir($destPath)) { + die("$destPath isn't a directory"); + } + + $db = new DatabaseConnector($webroot); + + foreach($db->getTables() as $table) { + $filename = $destPath . '/' . $table . '.csv'; + echo $filename . "...\n"; + touch($filename); + $writer = new CsvTableWriter($filename); + $db->saveTable($table, $writer); + } + echo "Done!"; + } + + function loadCsv($args) { + $args->requireUnnamed(array('input-path', 'webroot')); + $unnamedArgs = $args->getUnnamedArgs(); + + $srcPath = $unnamedArgs[0]; + $webroot = $unnamedArgs[1]; + + if (!is_dir($srcPath)) { + die("$srcPath isn't a directory"); + } + + $db = new DatabaseConnector($webroot); + + foreach($db->getTables() as $table) { + $filename = $srcPath . '/' . $table . '.csv'; + if(file_exists($filename)) { + echo $filename . "...\n"; + $reader = new CsvTableReader($filename); + $db->loadTable($table, $reader); + } else { + echo "$filename doesn't exist; skipping.\n"; + } + } + echo "Done!"; + } /** * Save a .sspak.phar file */ diff --git a/src/SSPakFile.php b/src/SSPakFile.php index 5c8a1a6..42b3c6d 100644 --- a/src/SSPakFile.php +++ b/src/SSPakFile.php @@ -12,7 +12,7 @@ function __construct($path, $executor, $pharAlias = 'sspak.phar') { $this->pharAlias = $pharAlias; $this->pharPath = $path; - + // Executable Phar version if(substr($path,-5) === '.phar') { $this->phar = new Phar($path, FilesystemIterator::CURRENT_AS_FILEINFO | FilesystemIterator::KEY_AS_FILENAME, @@ -38,15 +38,28 @@ function makeExecutable() { throw new Exception("Please set phar.readonly to false in your php.ini."); } + passthru("composer install -d " . escapeshellarg(PACKAGE_ROOT) . " --no-dev"); + $root = PACKAGE_ROOT; - $srcRoot = PACKAGE_ROOT . 'src/'; + $srcRoots = [ + 'src/', + 'vendor/', + ]; // Add the bin file, but strip of the #! exec header. $this->phar['bin/sspak'] = preg_replace("/^#!\/usr\/bin\/env php\n/", '', file_get_contents($root . "bin/sspak")); - foreach(scandir($srcRoot) as $file) { - if($file[0] == '.') continue; - $this->phar['src/'.$file] = file_get_contents($srcRoot . $file); + foreach($srcRoots as $srcRoot) { + foreach(new RecursiveIteratorIterator(new RecursiveDirectoryIterator($root . $srcRoot)) as $fileObj) { + if($fileObj->isFile()) { + $file = $fileObj->getRealPath(); + + $relativeFile = str_replace($root, '', $file); + + echo "Adding $relativeFile\n"; + $this->phar[$relativeFile] = file_get_contents($file); + } + } } $stub = <<phar->setStub($stub); chmod($this->path, 0775); + + passthru("composer install -d " . escapeshellarg(PACKAGE_ROOT)); } /** diff --git a/tests/DataExtractor/CsvTableReaderTest.php b/tests/DataExtractor/CsvTableReaderTest.php new file mode 100644 index 0000000..2ad6b2f --- /dev/null +++ b/tests/DataExtractor/CsvTableReaderTest.php @@ -0,0 +1,26 @@ +assertEquals(['Col1', 'Col2', 'Col3'], $csv->getColumns()); + + $extractedData = []; + foreach($csv as $record) { + $extractedData[] = $record; + } + + $this->assertEquals( + [ + [ 'Col1' => 'One', 'Col2' => 2, 'Col3' => 'Three' ], + [ 'Col1' => 'Hello, Sam', 'Col2' => 5, 'Col3' => "Nice to meet you\nWhat is your name?" ] + ], + $extractedData + ); + + } +} diff --git a/tests/DataExtractor/CsvTableWriterTest.php b/tests/DataExtractor/CsvTableWriterTest.php new file mode 100644 index 0000000..2e5a145 --- /dev/null +++ b/tests/DataExtractor/CsvTableWriterTest.php @@ -0,0 +1,48 @@ +start(['Col1', 'Col2', 'Col3']); + $csv->writeRecord([ 'Col1' => 'One', 'Col2' => 2, 'Col3' => 'Three' ]); + $csv->writeRecord([ 'Col1' => 'Hello, Sam', 'Col2' => 5, 'Col3' => "Nice to meet you\nWhat is your name?" ]); + $csv->finish(); + + $csvContent = file_get_contents('/tmp/output.csv'); + unlink('/tmp/output.csv'); + + $fixture = file_get_contents(__DIR__ . '/fixture/input.csv'); + + $this->assertEquals($fixture, $csvContent); + } + + function testNoStartCall() { + + if (file_exists('/tmp/output.csv')) { + unlink('/tmp/output.csv'); + } + + $csv = new CsvTableWriter('/tmp/output.csv'); + + $csv->writeRecord([ 'Col1' => 'One', 'Col2' => 2, 'Col3' => 'Three' ]); + $csv->writeRecord([ 'Col1' => 'Hello, Sam', 'Col2' => 5, 'Col3' => "Nice to meet you\nWhat is your name?" ]); + $csv->finish(); + + $csvContent = file_get_contents('/tmp/output.csv'); + unlink('/tmp/output.csv'); + + $fixture = file_get_contents(__DIR__ . '/fixture/input.csv'); + + $this->assertEquals($fixture, $csvContent); + } + +} diff --git a/tests/DataExtractor/fixture/input.csv b/tests/DataExtractor/fixture/input.csv new file mode 100644 index 0000000..a26d877 --- /dev/null +++ b/tests/DataExtractor/fixture/input.csv @@ -0,0 +1,4 @@ +Col1,Col2,Col3 +One,2,Three +"Hello, Sam",5,"Nice to meet you +What is your name?"