Merge pull request #10922 from owncloud/explicit-scan-transactions

Use bigger transactions when doing explicit file system scans

apps/files/ajax/scan.php

@@ -20,7 +20,7 @@ $listener = new ScanListener($eventSource);
 
 foreach ($users as $user) {
 	$eventSource->send('user', $user);
-	$scanner = new \OC\Files\Utils\Scanner($user);
+	$scanner = new \OC\Files\Utils\Scanner($user, \OC::$server->getDatabaseConnection());
 	$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', array($listener, 'file'));
 	$scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', array($listener, 'folder'));
 	if ($force) {

apps/files/command/scan.php

@@ -46,7 +46,7 @@ class Scan extends Command {
 	}
 
 	protected function scanFiles($user, OutputInterface $output) {
-		$scanner = new \OC\Files\Utils\Scanner($user);
+		$scanner = new \OC\Files\Utils\Scanner($user, \OC::$server->getDatabaseConnection());
 		$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function ($path) use ($output) {
 			$output->writeln("Scanning <info>$path</info>");
 		});

lib/private/files/cache/scanner.php

@@ -44,6 +44,11 @@ class Scanner extends BasicEmitter {
 	 */
 	protected $cacheActive;
 
+	/**
+	 * @var bool $useTransactions whether to use transactions
+	 */
+	protected $useTransactions = true;
+
 	const SCAN_RECURSIVE = true;
 	const SCAN_SHALLOW = false;
 
@@ -57,6 +62,16 @@ class Scanner extends BasicEmitter {
 		$this->cacheActive = !Config::getSystemValue('filesystem_cache_readonly', false);
 	}
 
+	/**
+	 * Whether to wrap the scanning of a folder in a database transaction
+	 * On default transactions are used
+	 *
+	 * @param bool $useTransactions
+	 */
+	public function setUseTransactions($useTransactions) {
+		$this->useTransactions = $useTransactions;
+	}
+
 	/**
 	 * get all the metadata of a file or folder
 	 * *
@@ -234,7 +249,9 @@ class Scanner extends BasicEmitter {
 		$newChildren = array();
 		if ($this->storage->is_dir($path) && ($dh = $this->storage->opendir($path))) {
 			$exceptionOccurred = false;
+			if ($this->useTransactions) {
 				\OC_DB::beginTransaction();
+			}
 			if (is_resource($dh)) {
 				while (($file = readdir($dh)) !== false) {
 					$child = ($path) ? $path . '/' . $file : $file;
@@ -266,7 +283,9 @@ class Scanner extends BasicEmitter {
 				$child = ($path) ? $path . '/' . $childName : $childName;
 				$this->removeFromCache($child);
 			}
+			if ($this->useTransactions) {
 				\OC_DB::commit();
+			}
 			if ($exceptionOccurred) {
 				// It might happen that the parallel scan process has already
 				// inserted mimetypes but those weren't available yet inside the transaction

lib/private/files/utils/scanner.php

@@ -35,11 +35,18 @@ class Scanner extends PublicEmitter {
 	protected $propagator;
 
 	/**
-	 * @param string $user
+	 * @var \OCP\IDBConnection
 	 */
-	public function __construct($user) {
+	protected $db;
+
+	/**
+	 * @param string $user
+	 * @param \OCP\IDBConnection $db
+	 */
+	public function __construct($user, $db) {
 		$this->user = $user;
 		$this->propagator = new ChangePropagator(new View(''));
+		$this->db = $db;
 	}
 
 	/**
@@ -121,8 +128,11 @@ class Scanner extends PublicEmitter {
 				throw new ForbiddenException();
 			}
 			$scanner = $storage->getScanner();
+			$scanner->useTransactions(false);
 			$this->attachListener($mount);
+			$this->db->beginTransaction();
 			$scanner->scan('', \OC\Files\Cache\Scanner::SCAN_RECURSIVE, \OC\Files\Cache\Scanner::REUSE_ETAG | \OC\Files\Cache\Scanner::REUSE_SIZE);
+			$this->db->commit();
 		}
 		$this->propagator->propagateChanges(time());
 	}

tests/lib/files/etagtest.php

@@ -62,7 +62,7 @@ class EtagTest extends \PHPUnit_Framework_TestCase {
 		$files = array('/foo.txt', '/folder/bar.txt', '/folder/subfolder', '/folder/subfolder/qwerty.txt');
 		$originalEtags = $this->getEtags($files);
 
-		$scanner = new \OC\Files\Utils\Scanner($user1);
+		$scanner = new \OC\Files\Utils\Scanner($user1, \OC::$server->getDatabaseConnection());
 		$scanner->backgroundScan('/');
 
 		$newEtags = $this->getEtags($files);