wbrawner/server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Chunk the queries to make sure they don't time out

Browse source
This commit is contained in:
Joas Schilling 2016-02-10 12:01:55 +01:00
parent 9a2c517ca8
commit 0ebb205010
3 changed files with 70 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
apps/files/command/deleteorphanedfiles.php
View file

@ -33,6 +33,8 @@ use Symfony\Component\Console\Output\OutputInterface;
*/
class DeleteOrphanedFiles extends Command {
const CHUNK_SIZE = 200;
/**
* @var IDBConnection
*/
@ -50,13 +52,31 @@ class DeleteOrphanedFiles extends Command {
}
public function execute(InputInterface $input, OutputInterface $output) {
$deletedEntries = 0;
$sql =
'DELETE FROM `*PREFIX*filecache` ' .
'WHERE NOT EXISTS ' .
'(SELECT 1 FROM `*PREFIX*storages` WHERE `storage` = `numeric_id`)';
$query = $this->connection->getQueryBuilder();
$query->select('fc.fileid')
->from('filecache', 'fc')
->where($query->expr()->isNull('s.numeric_id'))
->leftJoin('fc', 'storages', 's', $query->expr()->eq('fc.storage', 's.numeric_id'))
->setMaxResults(self::CHUNK_SIZE);
$deleteQuery = $this->connection->getQueryBuilder();
$deleteQuery->delete('filecache')
->where($deleteQuery->expr()->eq('fileid', $deleteQuery->createParameter('objectid')));
$deletedInLastChunk = self::CHUNK_SIZE;
while ($deletedInLastChunk === self::CHUNK_SIZE) {
$deletedInLastChunk = 0;
$result = $query->execute();
while ($row = $result->fetch()) {
$deletedInLastChunk++;
$deletedEntries += $deleteQuery->setParameter('objectid', (int) $row['fileid'])
->execute();
}
$result->closeCursor();
}
$deletedEntries = $this->connection->executeUpdate($sql);
$output->writeln("$deletedEntries orphaned file cache entries deleted");
}

39
apps/files/lib/backgroundjob/deleteorphaneditems.php
View file

@ -29,6 +29,8 @@ use OCP\DB\QueryBuilder\IQueryBuilder;
*/
class DeleteOrphanedItems extends TimedJob {
const CHUNK_SIZE = 200;
/** @var \OCP\IDBConnection */
protected $connection;
@ -66,19 +68,38 @@ class DeleteOrphanedItems extends TimedJob {
/**
* Deleting orphaned system tag mappings
*
* @param string $table
* @param string $idCol
* @param string $typeCol
* @return int Number of deleted entries
*/
protected function cleanUp($table, $idCol, $typeCol) {
$subQuery = $this->connection->getQueryBuilder();
$subQuery->select($subQuery->expr()->literal('1'))
->from('filecache', 'f')
->where($subQuery->expr()->eq($idCol, 'f.fileid'));
$deletedEntries = 0;
$query = $this->connection->getQueryBuilder();
$deletedEntries = $query->delete($table)
$query->select('t1.' . $idCol)
->from($table, 't1')
->where($query->expr()->eq($typeCol, $query->expr()->literal('files')))
->andWhere($query->expr()->isNull($query->createFunction('(' . $subQuery->getSql() . ')')))
->andWhere($query->expr()->isNull('t2.fileid'))
->leftJoin('t1', 'filecache', 't2', $query->expr()->eq($query->expr()->castColumn('t1.' . $idCol, IQueryBuilder::PARAM_INT), 't2.fileid'))
->groupBy('t1.' . $idCol)
->setMaxResults(self::CHUNK_SIZE);
$deleteQuery = $this->connection->getQueryBuilder();
$deleteQuery->delete($table)
->where($deleteQuery->expr()->eq($idCol, $deleteQuery->createParameter('objectid')));
$deletedInLastChunk = self::CHUNK_SIZE;
while ($deletedInLastChunk === self::CHUNK_SIZE) {
$result = $query->execute();
$deletedInLastChunk = 0;
while ($row = $result->fetch()) {
$deletedInLastChunk++;
$deletedEntries += $deleteQuery->setParameter('objectid', (int) $row[$idCol])
->execute();
}
$result->closeCursor();
}
return $deletedEntries;
}
@ -111,8 +132,7 @@ class DeleteOrphanedItems extends TimedJob {
* @return int Number of deleted entries
*/
protected function cleanComments() {
$qb = $this->connection->getQueryBuilder();
$deletedEntries = $this->cleanUp('comments', $qb->expr()->castColumn('object_id', IQueryBuilder::PARAM_INT), 'object_type');
$deletedEntries = $this->cleanUp('comments', 'object_id', 'object_type');
$this->logger->debug("$deletedEntries orphaned comments deleted", ['app' => 'DeleteOrphanedItems']);
return $deletedEntries;
}
@ -123,8 +143,7 @@ class DeleteOrphanedItems extends TimedJob {
* @return int Number of deleted entries
*/
protected function cleanCommentMarkers() {
$qb = $this->connection->getQueryBuilder();
$deletedEntries = $this->cleanUp('comments_read_markers', $qb->expr()->castColumn('object_id', IQueryBuilder::PARAM_INT), 'object_type');
$deletedEntries = $this->cleanUp('comments_read_markers', 'object_id', 'object_type');
$this->logger->debug("$deletedEntries orphaned comment read marks deleted", ['app' => 'DeleteOrphanedItems']);
return $deletedEntries;
}

12
lib/private/repair/repairinvalidshares.php
View file

@ -30,6 +30,8 @@ use OC\Hooks\BasicEmitter;
*/
class RepairInvalidShares extends BasicEmitter implements \OC\RepairStep {
const CHUNK_SIZE = 200;
/**
* @var \OCP\IConfig
*/
@ -83,18 +85,24 @@ class RepairInvalidShares extends BasicEmitter implements \OC\RepairStep {
->where($query->expr()->isNotNull('s1.parent'))
->andWhere($query->expr()->isNull('s2.id'))
->leftJoin('s1', 'share', 's2', $query->expr()->eq('s1.parent', 's2.id'))
->groupBy('s1.parent');
->groupBy('s1.parent')
->setMaxResults(self::CHUNK_SIZE);
$deleteQuery = $this->connection->getQueryBuilder();
$deleteQuery->delete('share')
->where($query->expr()->eq('parent', $deleteQuery->createParameter('parent')));
->where($deleteQuery->expr()->eq('parent', $deleteQuery->createParameter('parent')));
$deletedInLastChunk = self::CHUNK_SIZE;
while ($deletedInLastChunk === self::CHUNK_SIZE) {
$deletedInLastChunk = 0;
$result = $query->execute();
while ($row = $result->fetch()) {
$deletedInLastChunk++;
$deletedEntries += $deleteQuery->setParameter('parent', (int) $row['parent'])
->execute();
}
$result->closeCursor();
}
if ($deletedEntries) {
$this->emit('\OC\Repair', 'info', array('Removed ' . $deletedEntries . ' shares where the parent did not exist'));