wbrawner/server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #1052 from owncloud/filesystem-etags

Browse source 
ETags stored in the file cache
This commit is contained in:
Michael Gapczynski 2013-01-11 18:11:22 -08:00
commit 3a9ec45272
13 changed files with 69 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
db_structure.xml
View file

@ -225,6 +225,14 @@
<length>4</length>
</field>
<field>
<name>etag</name>
<type>text</type>
<default></default>
<notnull>true</notnull>
<length>40</length>
</field>
<index>
<name>fs_storage_path_hash</name>
<unique>true</unique>

2
lib/connector/sabre/directory.php
View file

@ -121,8 +121,8 @@ class OC_Connector_Sabre_Directory extends OC_Connector_Sabre_Node implements Sa
$paths = array();
foreach($folder_content as $info) {
$paths[] = $this->path.'/'.$info['name'];
$properties[$this->path.'/'.$info['name']][self::GETETAG_PROPERTYNAME] = $info['etag'];
}
$properties = array_fill_keys($paths, array());
if(count($paths)>0) {
//
// the number of arguments within IN conditions are limited in most databases

11
lib/connector/sabre/file.php
View file

@ -98,16 +98,7 @@ class OC_Connector_Sabre_File extends OC_Connector_Sabre_Node implements Sabre_D
if (isset($properties[self::GETETAG_PROPERTYNAME])) {
return $properties[self::GETETAG_PROPERTYNAME];
}
return $this->getETagPropertyForPath($this->path);
}
/**
* Creates a ETag for this path.
* @param string $path Path of the file
* @return string|null Returns null if the ETag can not effectively be determined
*/
static protected function createETag($path) {
return \OC\Files\Filesystem::hash('md5', $path);
return null;
}
/**

36
lib/connector/sabre/node.php
View file

@ -190,6 +190,7 @@ abstract class OC_Connector_Sabre_Node implements Sabre_DAV_INode, Sabre_DAV_IPr
while( $row = $result->fetchRow()) {
$this->property_cache[$row['propertyname']] = $row['propertyvalue'];
}
$this->property_cache[self::GETETAG_PROPERTYNAME] = $this->getETagPropertyForPath($this->path);
}
// if the array was empty, we need to return everything
@ -210,38 +211,11 @@ abstract class OC_Connector_Sabre_Node implements Sabre_DAV_INode, Sabre_DAV_IPr
* @return string|null Returns null if the ETag can not effectively be determined
*/
static public function getETagPropertyForPath($path) {
$tag = \OC\Files\Filesystem::getETag($path);
if (empty($tag)) {
return null;
$data = \OC\Files\Filesystem::getFileInfo($path);
if (isset($data['etag'])) {
return '"'.$data['etag'].'"';
}
$etag = '"'.$tag.'"';
$query = OC_DB::prepare( 'INSERT INTO `*PREFIX*properties` (`userid`,`propertypath`,`propertyname`,`propertyvalue`) VALUES(?,?,?,?)' );
$query->execute( array( OC_User::getUser(), $path, self::GETETAG_PROPERTYNAME, $etag ));
return $etag;
return null;
}
/**
* @brief Remove the ETag from the cache.
* @param string $path Path of the file
*/
static public function removeETagPropertyForPath($path) {
// remove tags from this and parent paths
$paths = array();
while ($path != '/' && $path != '.' && $path != '' && $path != '\\') {
$paths[] = $path;
$path = dirname($path);
}
if (empty($paths)) {
return;
}
$paths[] = $path;
$path_placeholders = join(',', array_fill(0, count($paths), '?'));
$query = OC_DB::prepare( 'DELETE FROM `*PREFIX*properties`'
.' WHERE `userid` = ?'
.' AND `propertyname` = ?'
.' AND `propertypath` IN ('.$path_placeholders.')'
);
$vals = array( OC_User::getUser(), self::GETETAG_PROPERTYNAME );
$query->execute(array_merge( $vals, $paths ));
}
}

11
lib/files/cache/cache.php vendored
View file

@ -114,7 +114,7 @@ class Cache {
$params = array($file);
}
$query = \OC_DB::prepare(
'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`
'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag`
FROM `*PREFIX*filecache` ' . $where);
$result = $query->execute($params);
$data = $result->fetchRow();
@ -147,7 +147,7 @@ class Cache {
$fileId = $this->getId($folder);
if ($fileId > -1) {
$query = \OC_DB::prepare(
'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`
'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag`
FROM `*PREFIX*filecache` WHERE parent = ? ORDER BY `name` ASC');
$result = $query->execute(array($fileId));
$files = $result->fetchAll();
@ -225,8 +225,7 @@ class Cache {
* @return array
*/
function buildParts(array $data) {
$fields = array('path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'encrypted');
$fields = array('path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'encrypted', 'etag');
$params = array();
$queryParts = array();
foreach ($data as $name => $value) {
@ -379,7 +378,7 @@ class Cache {
*/
public function search($pattern) {
$query = \OC_DB::prepare('
SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`
SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag`
FROM `*PREFIX*filecache` WHERE `name` LIKE ? AND `storage` = ?'
);
$result = $query->execute(array($pattern, $this->numericId));
@ -405,7 +404,7 @@ class Cache {
$where = '`mimepart` = ?';
}
$query = \OC_DB::prepare('
SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`
SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag`
FROM `*PREFIX*filecache` WHERE ' . $where . ' AND `storage` = ?'
);
$mimetype = $this->getMimetypeId($mimetype);

1
lib/files/cache/scanner.php vendored
View file

@ -50,6 +50,7 @@ class Scanner {
} else {
$data['size'] = $this->storage->filesize($path);
}
$data['etag'] = $this->storage->getETag($path);
return $data;
}

24
lib/files/cache/updater.php vendored
View file

@ -35,6 +35,7 @@ class Updater {
$scanner = $storage->getScanner($internalPath);
$scanner->scan($internalPath, Scanner::SCAN_SHALLOW);
$cache->correctFolderSize($internalPath);
self::eTagUpdate($path);
}
}
@ -48,6 +49,29 @@ class Updater {
$cache = $storage->getCache($internalPath);
$cache->remove($internalPath);
$cache->correctFolderSize($internalPath);
self::eTagUpdate($path);
}
}
static public function eTagUpdate($path) {
if ($path !== '' && $path !== '/') {
$parent = dirname($path);
if ($parent === '.') {
$parent = '';
}
/**
* @var \OC\Files\Storage\Storage $storage
* @var string $internalPath
*/
list($storage, $internalPath) = self::resolvePath($parent);
if ($storage) {
$cache = $storage->getCache();
$id = $cache->getId($internalPath);
if ($id !== -1) {
$cache->update($id, array('etag' => $storage->getETag($internalPath)));
self::eTagUpdate($parent);
}
}
}
}

21
lib/files/filesystem.php
View file

@ -585,23 +585,6 @@ class Filesystem {
return self::$defaultInstance->hasUpdated($path, $time);
}
static public function removeETagHook($params, $root = false) {
if (isset($params['path'])) {
$path = $params['path'];
} else {
$path = $params['oldpath'];
}
if ($root) { // reduce path to the required part of it (no 'username/files')
$fakeRootView = new View($root);
$count = 1;
$path = str_replace(\OC_App::getStorage("files")->getAbsolutePath(''), "", $fakeRootView->getAbsolutePath($path), $count);
}
$path = self::normalizePath($path);
\OC_Connector_Sabre_Node::removeETagPropertyForPath($path);
}
/**
* normalize a path
*
@ -685,10 +668,6 @@ class Filesystem {
}
}
\OC_Hook::connect('OC_Filesystem', 'post_write', 'OC_Filesystem', 'removeETagHook');
\OC_Hook::connect('OC_Filesystem', 'post_delete', 'OC_Filesystem', 'removeETagHook');
\OC_Hook::connect('OC_Filesystem', 'post_rename', 'OC_Filesystem', 'removeETagHook');
\OC_Hook::connect('OC_Filesystem', 'post_write', '\OC\Files\Cache\Updater', 'writeHook');
\OC_Hook::connect('OC_Filesystem', 'post_delete', '\OC\Files\Cache\Updater', 'deleteHook');
\OC_Hook::connect('OC_Filesystem', 'post_rename', '\OC\Files\Cache\Updater', 'renameHook');

2
lib/files/storage/common.php
View file

@ -274,7 +274,7 @@ abstract class Common implements \OC\Files\Storage\Storage {
$hash = call_user_func($ETagFunction, $path);
return $hash;
}else{
return uniqid('', true);
return uniqid();
}
}
}

2
lib/files/view.php
View file

@ -462,8 +462,6 @@ class View {
Filesystem::signal_post_write,
array(Filesystem::signal_param_path => $path2)
);
} else { // no real copy, file comes from somewhere else, e.g. version rollback -> just update the file cache and the webdav properties without all the other post_write actions
Filesystem::removeETagHook(array("path" => $path2), $this->fakeRoot);
}
return $result;
} else {

7
lib/filesystem.php
View file

@ -398,13 +398,6 @@ class OC_Filesystem {
return \OC\Files\Filesystem::hasUpdated($path, $time);
}
/**
* @deprecated OC_Filesystem is replaced by \OC\Files\Filesystem
*/
static public function removeETagHook($params, $root = false) {
\OC\Files\Filesystem::removeETagHook($params, $root);
}
/**
* normalize a path
*

2
lib/util.php
View file

@ -74,7 +74,7 @@ class OC_Util {
*/
public static function getVersion() {
// hint: We only can count up. So the internal version number of ownCloud 4.5 will be 4.90.0. This is not visible to the user
return array(4,91,06);
return array(4,91,07);
}
/**

28
tests/lib/files/cache/updater.php vendored
View file

@ -70,14 +70,18 @@ class Updater extends \PHPUnit_Framework_TestCase {
public function testWrite() {
$textSize = strlen("dummy file data\n");
$imageSize = filesize(\OC::$SERVERROOT . '/core/img/logo.png');
$cachedData = $this->cache->get('');
$this->assertEquals(3 * $textSize + $imageSize, $cachedData['size']);
$rootCachedData = $this->cache->get('');
$this->assertEquals(3 * $textSize + $imageSize, $rootCachedData['size']);
$fooCachedData = $this->cache->get('foo.txt');
Filesystem::file_put_contents('foo.txt', 'asd');
$cachedData = $this->cache->get('foo.txt');
$this->assertEquals(3, $cachedData['size']);
$this->assertNotEquals($fooCachedData['etag'], $cachedData['etag']);
$cachedData = $this->cache->get('');
$this->assertEquals(2 * $textSize + $imageSize + 3, $cachedData['size']);
$this->assertNotEquals($rootCachedData['etag'], $cachedData['etag']);
$rootCachedData = $cachedData;
$this->assertFalse($this->cache->inCache('bar.txt'));
Filesystem::file_put_contents('bar.txt', 'asd');
@ -86,38 +90,50 @@ class Updater extends \PHPUnit_Framework_TestCase {
$this->assertEquals(3, $cachedData['size']);
$cachedData = $this->cache->get('');
$this->assertEquals(2 * $textSize + $imageSize + 2 * 3, $cachedData['size']);
$this->assertNotEquals($rootCachedData['etag'], $cachedData['etag']);
}
public function testDelete() {
$textSize = strlen("dummy file data\n");
$imageSize = filesize(\OC::$SERVERROOT . '/core/img/logo.png');
$cachedData = $this->cache->get('');
$this->assertEquals(3 * $textSize + $imageSize, $cachedData['size']);
$rootCachedData = $this->cache->get('');
$this->assertEquals(3 * $textSize + $imageSize, $rootCachedData['size']);
$this->assertTrue($this->cache->inCache('foo.txt'));
Filesystem::unlink('foo.txt', 'asd');
$this->assertFalse($this->cache->inCache('foo.txt'));
$cachedData = $this->cache->get('');
$this->assertEquals(2 * $textSize + $imageSize, $cachedData['size']);
$this->assertNotEquals($rootCachedData['etag'], $cachedData['etag']);
$rootCachedData = $cachedData;
Filesystem::mkdir('bar_folder');
$this->assertTrue($this->cache->inCache('bar_folder'));
$cachedData = $this->cache->get('');
$this->assertNotEquals($rootCachedData['etag'], $cachedData['etag']);
$rootCachedData = $cachedData;
Filesystem::rmdir('bar_folder');
$this->assertFalse($this->cache->inCache('bar_folder'));
$cachedData = $this->cache->get('');
$this->assertNotEquals($rootCachedData['etag'], $cachedData['etag']);
}
public function testRename() {
$textSize = strlen("dummy file data\n");
$imageSize = filesize(\OC::$SERVERROOT . '/core/img/logo.png');
$cachedData = $this->cache->get('');
$this->assertEquals(3 * $textSize + $imageSize, $cachedData['size']);
$rootCachedData = $this->cache->get('');
$this->assertEquals(3 * $textSize + $imageSize, $rootCachedData['size']);
$this->assertTrue($this->cache->inCache('foo.txt'));
$fooCachedData = $this->cache->get('foo.txt');
$this->assertFalse($this->cache->inCache('bar.txt'));
Filesystem::rename('foo.txt', 'bar.txt');
$this->assertFalse($this->cache->inCache('foo.txt'));
$this->assertTrue($this->cache->inCache('bar.txt'));
$cachedData = $this->cache->get('foo.txt');
$this->assertNotEquals($fooCachedData['etag'], $cachedData['etag']);
$cachedData = $this->cache->get('');
$this->assertEquals(3 * $textSize + $imageSize, $cachedData['size']);
$this->assertNotEquals($rootCachedData['etag'], $cachedData['etag']);
}
}