wbrawner/server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #3764 from owncloud/scanfolder-existing

Browse source 
Improve re-using existing data in the filescanner
This commit is contained in:
icewind1991 2013-06-19 04:30:55 -07:00
commit cb41b88520
4 changed files with 70 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
apps/files/ajax/scan.php
View file

@ -24,7 +24,7 @@ foreach ($mountPoints as $mountPoint) {
ScanListener::$mountPoints[$storage->getId()] = $mountPoint;
$scanner = $storage->getScanner();
if ($force) {
$scanner->scan('');
$scanner->scan('', \OC\Files\Cache\Scanner::SCAN_RECURSIVE, \OC\Files\Cache\Scanner::REUSE_ETAG);
} else {
$scanner->backgroundScan();
}

79
lib/files/cache/scanner.php vendored
View file

@ -8,6 +8,8 @@
namespace OC\Files\Cache;
use OC\Files\Filesystem;
class Scanner {
/**
* @var \OC\Files\Storage\Storage $storage
@ -27,6 +29,9 @@ class Scanner {
const SCAN_RECURSIVE = true;
const SCAN_SHALLOW = false;
const REUSE_ETAG = 1;
const REUSE_SIZE = 2;
public function __construct(\OC\Files\Storage\Storage $storage) {
$this->storage = $storage;
$this->storageId = $this->storage->getId();
@ -59,12 +64,12 @@ class Scanner {
* scan a single file and store it in the cache
*
* @param string $file
* @param bool $checkExisting check existing folder sizes in the cache instead of always using -1 for folder size
* @param int $reuseExisting
* @return array with metadata of the scanned file
*/
public function scanFile($file, $checkExisting = false) {
if ( ! self::isPartialFile($file)
and ! \OC\Files\Filesystem::isFileBlacklisted($file)
public function scanFile($file, $reuseExisting = 0) {
if (!self::isPartialFile($file)
and !Filesystem::isFileBlacklisted($file)
) {
\OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', array('path' => $file, 'storage' => $this->storageId));
$data = $this->getData($file);
@ -79,20 +84,22 @@ class Scanner {
}
}
$newData = $data;
if ($cacheData = $this->cache->get($file)) {
if ($checkExisting && $data['size'] === -1) {
$data['size'] = $cacheData['size'];
}
if ($data['mtime'] === $cacheData['mtime'] &&
$data['size'] === $cacheData['size']) {
$data['etag'] = $cacheData['etag'];
if ($reuseExisting and $cacheData = $this->cache->get($file)) {
// only reuse data if the file hasn't explicitly changed
if ($data['mtime'] === $cacheData['mtime']) {
if (($reuseExisting & self::REUSE_SIZE) && ($data['size'] === -1)) {
$data['size'] = $cacheData['size'];
}
if ($reuseExisting & self::REUSE_ETAG) {
$data['etag'] = $cacheData['etag'];
}
}
// Only update metadata that has changed
$newData = array_diff($data, $cacheData);
}
if (!empty($newData)) {
$this->cache->put($file, $newData);
}
}
if (!empty($newData)) {
$this->cache->put($file, $newData);
}
return $data;
}
@ -100,38 +107,50 @@ class Scanner {
}
/**
* scan all the files in a folder and store them in the cache
* scan a folder and all it's children
*
* @param string $path
* @param bool $recursive
* @param bool $onlyChilds
* @param int $reuse
* @return int the size of the scanned folder or -1 if the size is unknown at this stage
*/
public function scan($path, $recursive = self::SCAN_RECURSIVE, $onlyChilds = false) {
\OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_folder', array('path' => $path, 'storage' => $this->storageId));
$childQueue = array();
if (!$onlyChilds) {
$this->scanFile($path);
public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1) {
if ($reuse === -1) {
$reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : 0;
}
$this->scanFile($path, $reuse);
return $this->scanChildren($path, $recursive, $reuse);
}
/**
* scan all the files and folders in a folder
*
* @param string $path
* @param bool $recursive
* @param int $reuse
* @return int the size of the scanned folder or -1 if the size is unknown at this stage
*/
public function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1) {
if ($reuse === -1) {
$reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : 0;
}
\OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_folder', array('path' => $path, 'storage' => $this->storageId));
$size = 0;
$childQueue = array();
if ($this->storage->is_dir($path) && ($dh = $this->storage->opendir($path))) {
\OC_DB::beginTransaction();
while ($file = readdir($dh)) {
$child = ($path) ? $path . '/' . $file : $file;
if (!\OC\Files\Filesystem::isIgnoredDir($file)) {
$data = $this->scanFile($child, $recursive === self::SCAN_SHALLOW);
if (!Filesystem::isIgnoredDir($file)) {
$data = $this->scanFile($child, $reuse);
if ($data) {
if ($data['size'] === -1) {
if ($recursive === self::SCAN_RECURSIVE) {
$childQueue[] = $child;
$data['size'] = 0;
} else {
$size = -1;
}
}
if ($size !== -1) {
} else if ($size !== -1) {
$size += $data['size'];
}
}
@ -139,16 +158,14 @@ class Scanner {
}
\OC_DB::commit();
foreach ($childQueue as $child) {
$childSize = $this->scan($child, self::SCAN_RECURSIVE, true);
$childSize = $this->scanChildren($child, self::SCAN_RECURSIVE);
if ($childSize === -1) {
$size = -1;
} else {
$size += $childSize;
}
}
if ($size !== -1) {
$this->cache->put($path, array('size' => $size));
}
$this->cache->put($path, array('size' => $size));
}
return $size;
}

20
tests/lib/files/cache/scanner.php vendored
View file

@ -104,7 +104,7 @@ class Scanner extends \PHPUnit_Framework_TestCase {
$this->assertNotEquals($cachedDataFolder['size'], -1);
}
function testBackgroundScan(){
function testBackgroundScan() {
$this->fillTestFolders();
$this->storage->mkdir('folder2');
$this->storage->file_put_contents('folder2/bar.txt', 'foobar');
@ -126,6 +126,24 @@ class Scanner extends \PHPUnit_Framework_TestCase {
$this->assertFalse($this->cache->getIncomplete());
}
public function testReuseExisting() {
$this->fillTestFolders();
$this->scanner->scan('');
$oldData = $this->cache->get('');
$this->storage->unlink('folder/bar.txt');
$this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW, \OC\Files\Cache\Scanner::REUSE_SIZE);
$newData = $this->cache->get('');
$this->assertNotEquals($oldData['etag'], $newData['etag']);
$this->assertEquals($oldData['size'], $newData['size']);
$oldData = $newData;
$this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW, \OC\Files\Cache\Scanner::REUSE_ETAG);
$newData = $this->cache->get('');
$this->assertEquals($oldData['etag'], $newData['etag']);
$this->assertEquals(-1, $newData['size']);
}
function setUp() {
$this->storage = new \OC\Files\Storage\Temporary(array());
$this->scanner = new \OC\Files\Cache\Scanner($this->storage);

5
tests/lib/files/cache/updater.php vendored
View file

@ -69,6 +69,7 @@ class Updater extends \PHPUnit_Framework_TestCase {
public function testWrite() {
$textSize = strlen("dummy file data\n");
$imageSize = filesize(\OC::$SERVERROOT . '/core/img/logo.png');
$this->cache->put('foo.txt', array('mtime' => 100));
$rootCachedData = $this->cache->get('');
$this->assertEquals(3 * $textSize + $imageSize, $rootCachedData['size']);
@ -77,11 +78,9 @@ class Updater extends \PHPUnit_Framework_TestCase {
$cachedData = $this->cache->get('foo.txt');
$this->assertEquals(3, $cachedData['size']);
$this->assertNotEquals($fooCachedData['etag'], $cachedData['etag']);
$mtime = $cachedData['mtime'];
$cachedData = $this->cache->get('');
$this->assertEquals(2 * $textSize + $imageSize + 3, $cachedData['size']);
$this->assertNotEquals($rootCachedData['etag'], $cachedData['etag']);
$this->assertGreaterThanOrEqual($rootCachedData['mtime'], $mtime);
$rootCachedData = $cachedData;
$this->assertFalse($this->cache->inCache('bar.txt'));
@ -264,4 +263,4 @@ class Updater extends \PHPUnit_Framework_TestCase {
$this->assertEquals($time, $cachedData['mtime']);
}
}
}