wbrawner/server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #3928 from nextcloud/downstream-17978

Browse source 
Adjust 4 byte MySQL code to upstream
This commit is contained in:
Morris Jobke 2017-03-23 23:45:43 -06:00 committed by GitHub
commit 1397b84777
13 changed files with 256 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

92
core/Command/Db/ConvertMysqlToMB4.php Normal file
View file

@ -0,0 +1,92 @@
<?php
/**
* @author Thomas Müller <thomas.mueller@tmit.eu>
*
* @copyright Copyright (c) 2017, ownCloud GmbH
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OC\Core\Command\Db;
use Doctrine\DBAL\Platforms\MySqlPlatform;
use OC\Migration\ConsoleOutput;
use OC\Repair\Collation;
use OCP\IConfig;
use OCP\IDBConnection;
use OCP\ILogger;
use OCP\IURLGenerator;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
class ConvertMysqlToMB4 extends Command {
/** @var IConfig */
private $config;
/** @var IDBConnection */
private $connection;
/** @var IURLGenerator */
private $urlGenerator;
/** @var ILogger */
private $logger;
/**
* @param IConfig $config
* @param IDBConnection $connection
* @param IURLGenerator $urlGenerator
* @param ILogger $logger
*/
public function __construct(IConfig $config, IDBConnection $connection, IURLGenerator $urlGenerator, ILogger $logger) {
$this->config = $config;
$this->connection = $connection;
$this->urlGenerator = $urlGenerator;
$this->logger = $logger;
parent::__construct();
}
protected function configure() {
$this
->setName('db:convert-mysql-charset')
->setDescription('Convert charset of MySQL/MariaDB to use utf8mb4');
}
protected function execute(InputInterface $input, OutputInterface $output) {
if (!$this->connection->getDatabasePlatform() instanceof MySqlPlatform) {
$output->writeln("This command is only valid for MySQL/MariaDB databases.");
return 1;
}
$oldValue = $this->config->getSystemValue('mysql.utf8mb4', false);
// enable charset
$this->config->setSystemValue('mysql.utf8mb4', true);
if (!$this->connection->supports4ByteText()) {
$url = $this->urlGenerator->linkToDocs('admin-mysql-utf8mb4');
$output->writeln("The database is not properly setup to use the charset utf8mb4.");
$output->writeln("For more information please read the documentation at $url");
$this->config->setSystemValue('mysql.utf8mb4', $oldValue);
return 1;
}
// run conversion
$coll = new Collation($this->config, $this->logger, $this->connection, false);
$coll->run(new ConsoleOutput($output));
return 0;
}
}

1
core/register_command.php
View file

@ -84,6 +84,7 @@ if (\OC::$server->getConfig()->getSystemValue('installed', false)) {
$application->add(new OC\Core\Command\Db\GenerateChangeScript());
$application->add(new OC\Core\Command\Db\ConvertType(\OC::$server->getConfig(), new \OC\DB\ConnectionFactory(\OC::$server->getSystemConfig())));
$application->add(new OC\Core\Command\Db\ConvertMysqlToMB4(\OC::$server->getConfig(), \OC::$server->getDatabaseConnection(), \OC::$server->getURLGenerator(), \OC::$server->getLogger()));
$application->add(new OC\Core\Command\Encryption\Disable(\OC::$server->getConfig()));
$application->add(new OC\Core\Command\Encryption\Enable(\OC::$server->getConfig(), \OC::$server->getEncryptionManager()));

2
lib/composer/composer/autoload_classmap.php
View file

@ -389,6 +389,7 @@ return array(
'OC\\Core\\Command\\Config\\System\\DeleteConfig' => $baseDir . '/core/Command/Config/System/DeleteConfig.php',
'OC\\Core\\Command\\Config\\System\\GetConfig' => $baseDir . '/core/Command/Config/System/GetConfig.php',
'OC\\Core\\Command\\Config\\System\\SetConfig' => $baseDir . '/core/Command/Config/System/SetConfig.php',
'OC\\Core\\Command\\Db\\ConvertMysqlToMB4' => $baseDir . '/core/Command/Db/ConvertMysqlToMB4.php',
'OC\\Core\\Command\\Db\\ConvertType' => $baseDir . '/core/Command/Db/ConvertType.php',
'OC\\Core\\Command\\Db\\GenerateChangeScript' => $baseDir . '/core/Command/Db/GenerateChangeScript.php',
'OC\\Core\\Command\\Encryption\\ChangeKeyStorageRoot' => $baseDir . '/core/Command/Encryption/ChangeKeyStorageRoot.php',
@ -642,6 +643,7 @@ return array(
'OC\\Memcache\\Redis' => $baseDir . '/lib/private/Memcache/Redis.php',
'OC\\Memcache\\XCache' => $baseDir . '/lib/private/Memcache/XCache.php',
'OC\\Migration\\BackgroundRepair' => $baseDir . '/lib/private/Migration/BackgroundRepair.php',
'OC\\Migration\\ConsoleOutput' => $baseDir . '/lib/private/Migration/ConsoleOutput.php',
'OC\\NaturalSort' => $baseDir . '/lib/private/NaturalSort.php',
'OC\\NaturalSort_DefaultCollator' => $baseDir . '/lib/private/NaturalSort_DefaultCollator.php',
'OC\\NavigationManager' => $baseDir . '/lib/private/NavigationManager.php',

2
lib/composer/composer/autoload_static.php
View file

@ -419,6 +419,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
'OC\\Core\\Command\\Config\\System\\DeleteConfig' => __DIR__ . '/../../..' . '/core/Command/Config/System/DeleteConfig.php',
'OC\\Core\\Command\\Config\\System\\GetConfig' => __DIR__ . '/../../..' . '/core/Command/Config/System/GetConfig.php',
'OC\\Core\\Command\\Config\\System\\SetConfig' => __DIR__ . '/../../..' . '/core/Command/Config/System/SetConfig.php',
'OC\\Core\\Command\\Db\\ConvertMysqlToMB4' => __DIR__ . '/../../..' . '/core/Command/Db/ConvertMysqlToMB4.php',
'OC\\Core\\Command\\Db\\ConvertType' => __DIR__ . '/../../..' . '/core/Command/Db/ConvertType.php',
'OC\\Core\\Command\\Db\\GenerateChangeScript' => __DIR__ . '/../../..' . '/core/Command/Db/GenerateChangeScript.php',
'OC\\Core\\Command\\Encryption\\ChangeKeyStorageRoot' => __DIR__ . '/../../..' . '/core/Command/Encryption/ChangeKeyStorageRoot.php',
@ -672,6 +673,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
'OC\\Memcache\\Redis' => __DIR__ . '/../../..' . '/lib/private/Memcache/Redis.php',
'OC\\Memcache\\XCache' => __DIR__ . '/../../..' . '/lib/private/Memcache/XCache.php',
'OC\\Migration\\BackgroundRepair' => __DIR__ . '/../../..' . '/lib/private/Migration/BackgroundRepair.php',
'OC\\Migration\\ConsoleOutput' => __DIR__ . '/../../..' . '/lib/private/Migration/ConsoleOutput.php',
'OC\\NaturalSort' => __DIR__ . '/../../..' . '/lib/private/NaturalSort.php',
'OC\\NaturalSort_DefaultCollator' => __DIR__ . '/../../..' . '/lib/private/NaturalSort_DefaultCollator.php',
'OC\\NavigationManager' => __DIR__ . '/../../..' . '/lib/private/NavigationManager.php',

5
lib/private/DB/Connection.php
View file

@ -416,6 +416,9 @@ class Connection extends \Doctrine\DBAL\Connection implements IDBConnection {
* @since 11.0.0
*/
public function supports4ByteText() {
return ! ($this->getDatabasePlatform() instanceof MySqlPlatform && $this->getParams()['charset'] !== 'utf8mb4');
if (!$this->getDatabasePlatform() instanceof MySqlPlatform) {
return true;
}
return $this->getParams()['charset'] === 'utf8mb4';
}
}

15
lib/private/DB/ConnectionFactory.php
View file

@ -201,6 +201,21 @@ class ConnectionFactory {
$connectionParams['driverOptions'] = $driverOptions;
}
// set default table creation options
$connectionParams['defaultTableOptions'] = [
'collate' => 'utf8_bin',
'tablePrefix' => $connectionParams['tablePrefix']
];
if($this->config->getValue('mysql.utf8mb4', false)) {
$connectionParams['defaultTableOptions'] = [
'collate' => 'utf8mb4_bin',
'charset' => 'utf8mb4',
'row_format' => 'compressed',
'tablePrefix' => $connectionParams['tablePrefix']
];
}
return $connectionParams;
}
}

10
lib/private/DB/MDB2SchemaManager.php
View file

@ -33,6 +33,7 @@ use Doctrine\DBAL\Platforms\MySqlPlatform;
use Doctrine\DBAL\Platforms\OraclePlatform;
use Doctrine\DBAL\Platforms\PostgreSqlPlatform;
use Doctrine\DBAL\Platforms\SqlitePlatform;
use Doctrine\DBAL\Schema\Schema;
use OCP\IDBConnection;
class MDB2SchemaManager {
@ -66,7 +67,8 @@ class MDB2SchemaManager {
*/
public function createDbFromStructure($file) {
$schemaReader = new MDB2SchemaReader(\OC::$server->getConfig(), $this->conn->getDatabasePlatform());
$toSchema = $schemaReader->loadSchemaFromFile($file);
$toSchema = new Schema([], [], $this->conn->getSchemaManager()->createSchemaConfig());
$toSchema = $schemaReader->loadSchemaFromFile($file, $toSchema);
return $this->executeSchemaChange($toSchema);
}
@ -100,7 +102,8 @@ class MDB2SchemaManager {
private function readSchemaFromFile($file) {
$platform = $this->conn->getDatabasePlatform();
$schemaReader = new MDB2SchemaReader(\OC::$server->getConfig(), $platform);
return $schemaReader->loadSchemaFromFile($file);
$toSchema = new Schema([], [], $this->conn->getSchemaManager()->createSchemaConfig());
return $schemaReader->loadSchemaFromFile($file, $toSchema);
}
/**
@ -137,7 +140,8 @@ class MDB2SchemaManager {
*/
public function removeDBStructure($file) {
$schemaReader = new MDB2SchemaReader(\OC::$server->getConfig(), $this->conn->getDatabasePlatform());
$fromSchema = $schemaReader->loadSchemaFromFile($file);
$toSchema = new Schema([], [], $this->conn->getSchemaManager()->createSchemaConfig());
$fromSchema = $schemaReader->loadSchemaFromFile($file, $toSchema);
$toSchema = clone $fromSchema;
/** @var $table \Doctrine\DBAL\Schema\Table */
foreach ($toSchema->getTables() as $table) {

27
lib/private/DB/MDB2SchemaReader.php
View file

@ -38,10 +38,6 @@ use Doctrine\DBAL\Schema\Schema;
use OCP\IConfig;
class MDB2SchemaReader {
/**
* @var string $DBNAME
*/
protected $DBNAME;
/**
* @var string $DBTABLEPREFIX
@ -53,9 +49,6 @@ class MDB2SchemaReader {
*/
protected $platform;
/** @var \Doctrine\DBAL\Schema\SchemaConfig $schemaConfig */
protected $schemaConfig;
/** @var IConfig */
protected $config;
@ -66,23 +59,16 @@ class MDB2SchemaReader {
public function __construct(IConfig $config, AbstractPlatform $platform) {
$this->platform = $platform;
$this->config = $config;
$this->DBNAME = $config->getSystemValue('dbname', 'owncloud');
$this->DBTABLEPREFIX = $config->getSystemValue('dbtableprefix', 'oc_');
// Oracle does not support longer index names then 30 characters.
// We use this limit for all DBs to make sure it does not cause a
// problem.
$this->schemaConfig = new SchemaConfig();
$this->schemaConfig->setMaxIdentifierLength(30);
}
/**
* @param string $file
* @param Schema $schema
* @return Schema
* @throws \DomainException
*/
public function loadSchemaFromFile($file) {
$schema = new \Doctrine\DBAL\Schema\Schema();
public function loadSchemaFromFile($file, Schema $schema) {
$loadEntities = libxml_disable_entity_loader(false);
$xml = simplexml_load_file($file);
libxml_disable_entity_loader($loadEntities);
@ -124,15 +110,6 @@ class MDB2SchemaReader {
$name = str_replace('*dbprefix*', $this->DBTABLEPREFIX, $name);
$name = $this->platform->quoteIdentifier($name);
$table = $schema->createTable($name);
$table->setSchemaConfig($this->schemaConfig);
if($this->platform instanceof MySqlPlatform && $this->config->getSystemValue('mysql.utf8mb4', false)) {
$table->addOption('charset', 'utf8mb4');
$table->addOption('collate', 'utf8mb4_bin');
$table->addOption('row_format', 'compressed');
} else {
$table->addOption('collate', 'utf8_bin');
}
break;
case 'create':
case 'overwrite':

93
lib/private/Migration/ConsoleOutput.php Normal file
View file

@ -0,0 +1,93 @@
<?php
/**
* @author Thomas Müller <thomas.mueller@tmit.eu>
*
* @copyright Copyright (c) 2017, ownCloud GmbH
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OC\Migration;
use OCP\Migration\IOutput;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Output\OutputInterface;
/**
* Class SimpleOutput
*
* Just a simple IOutput implementation with writes messages to the log file.
* Alternative implementations will write to the console or to the web ui (web update case)
*
* @package OC\Migration
*/
class ConsoleOutput implements IOutput {
/** @var OutputInterface */
private $output;
/** @var ProgressBar */
private $progressBar;
public function __construct(OutputInterface $output) {
$this->output = $output;
}
/**
* @param string $message
*/
public function info($message) {
$this->output->writeln("<info>$message</info>");
}
/**
* @param string $message
*/
public function warning($message) {
$this->output->writeln("<comment>$message</comment>");
}
/**
* @param int $max
*/
public function startProgress($max = 0) {
if (!is_null($this->progressBar)) {
$this->progressBar->finish();
}
$this->progressBar = new ProgressBar($this->output);
$this->progressBar->start($max);
}
/**
* @param int $step
* @param string $description
*/
public function advance($step = 1, $description = '') {
if (!is_null($this->progressBar)) {
$this->progressBar = new ProgressBar($this->output);
$this->progressBar->start();
}
$this->progressBar->advance($step);
}
public function finishProgress() {
if (is_null($this->progressBar)) {
return;
}
$this->progressBar->finish();
}
}

38
lib/private/Repair/Collation.php
View file

@ -88,6 +88,11 @@ class Collation implements IRepairStep {
}
$output->info("Change collation for $table ...");
if ($characterSet === 'utf8mb4') {
// need to set row compression first
$query = $this->connection->prepare('ALTER TABLE `' . $table . '` ROW_FORMAT=COMPRESSED;');
$query->execute();
}
$query = $this->connection->prepare('ALTER TABLE `' . $table . '` CONVERT TO CHARACTER SET ' . $characterSet . ' COLLATE ' . $characterSet . '_bin;');
try {
$query->execute();
@ -99,16 +104,21 @@ class Collation implements IRepairStep {
}
}
}
if (empty($tables)) {
$output->info('All tables already have the correct collation -> nothing to do');
}
}
/**
* @param \Doctrine\DBAL\Connection $connection
* @param IDBConnection $connection
* @return string[]
*/
protected function getAllNonUTF8BinTables($connection) {
protected function getAllNonUTF8BinTables(IDBConnection $connection) {
$dbName = $this->config->getSystemValue("dbname");
$characterSet = $this->config->getSystemValue('mysql.utf8mb4', false) ? 'utf8mb4' : 'utf8';
$rows = $connection->fetchAll(
// fetch tables by columns
$statement = $connection->executeQuery(
"SELECT DISTINCT(TABLE_NAME) AS `table`" .
" FROM INFORMATION_SCHEMA . COLUMNS" .
" WHERE TABLE_SCHEMA = ?" .
@ -116,11 +126,27 @@ class Collation implements IRepairStep {
" AND TABLE_NAME LIKE \"*PREFIX*%\"",
array($dbName)
);
$result = array();
$rows = $statement->fetchAll();
$result = [];
foreach ($rows as $row) {
$result[] = $row['table'];
$result[$row['table']] = true;
}
return $result;
// fetch tables by collation
$statement = $connection->executeQuery(
"SELECT DISTINCT(TABLE_NAME) AS `table`" .
" FROM INFORMATION_SCHEMA . TABLES" .
" WHERE TABLE_SCHEMA = ?" .
" AND TABLE_COLLATION <> '" . $characterSet . "_bin'" .
" AND TABLE_NAME LIKE \"*PREFIX*%\"",
[$dbName]
);
$rows = $statement->fetchAll();
foreach ($rows as $row) {
$result[$row['table']] = true;
}
return array_keys($result);
}
}

2
tests/lib/DB/MDB2SchemaReaderTest.php
View file

@ -47,7 +47,7 @@ class MDB2SchemaReaderTest extends TestCase {
public function testRead() {
$reader = new MDB2SchemaReader($this->getConfig(), new MySqlPlatform());
$schema = $reader->loadSchemaFromFile(__DIR__ . '/testschema.xml');
$schema = $reader->loadSchemaFromFile(__DIR__ . '/testschema.xml', new Schema());
$this->assertCount(1, $schema->getTables());
$table = $schema->getTable('test_table');

4
tests/lib/DB/SchemaDiffTest.php
View file

@ -21,6 +21,7 @@
namespace Test\DB;
use Doctrine\DBAL\Schema\Schema;
use Doctrine\DBAL\Schema\SchemaDiff;
use OC\DB\MDB2SchemaManager;
use OC\DB\MDB2SchemaReader;
@ -79,7 +80,8 @@ class SchemaDiffTest extends TestCase {
$this->manager->createDbFromStructure($schemaFile);
$schemaReader = new MDB2SchemaReader($this->config, $this->connection->getDatabasePlatform());
$endSchema = $schemaReader->loadSchemaFromFile($schemaFile);
$toSchema = new Schema([], [], $this->connection->getSchemaManager()->createSchemaConfig());
$endSchema = $schemaReader->loadSchemaFromFile($schemaFile, $toSchema);
// get the diff
/** @var SchemaDiff $diff */

4
tests/lib/Repair/RepairCollationTest.php
View file

@ -18,10 +18,10 @@ use Test\TestCase;
class TestCollationRepair extends Collation {
/**
* @param \Doctrine\DBAL\Connection $connection
* @param IDBConnection $connection
* @return string[]
*/
public function getAllNonUTF8BinTables($connection) {
public function getAllNonUTF8BinTables(IDBConnection $connection) {
return parent::getAllNonUTF8BinTables($connection);
}
}