From 2c23e143d33e231e62dccb450bf1f1b0f3938ccd Mon Sep 17 00:00:00 2001 From: Michael Gapczynski Date: Sun, 30 Dec 2012 16:32:55 -0500 Subject: [PATCH 01/12] Store etags in the file cache --- db_structure.xml | 8 ++++++++ lib/files/cache/cache.php | 10 +++++----- lib/files/cache/scanner.php | 1 + lib/util.php | 2 +- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/db_structure.xml b/db_structure.xml index aa0916264c..7c67ca78f4 100644 --- a/db_structure.xml +++ b/db_structure.xml @@ -189,6 +189,14 @@ 4 + + etag + text + + true + 250 + + storage_path_hash true diff --git a/lib/files/cache/cache.php b/lib/files/cache/cache.php index 3ebae9baa5..d105f865eb 100644 --- a/lib/files/cache/cache.php +++ b/lib/files/cache/cache.php @@ -76,7 +76,7 @@ class Cache { $params = array($file); } $query = \OC_DB::prepare( - 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted` + 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag` FROM `*PREFIX*filecache` ' . $where); $result = $query->execute($params); $data = $result->fetchRow(); @@ -107,7 +107,7 @@ class Cache { $fileId = $this->getId($folder); if ($fileId > -1) { $query = \OC_DB::prepare( - 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted` + 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag` FROM `*PREFIX*filecache` WHERE parent = ? ORDER BY `name` ASC'); $result = $query->execute(array($fileId)); return $result->fetchAll(); @@ -180,7 +180,7 @@ class Cache { * @return array */ static function buildParts(array $data) { - $fields = array('path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'encrypted'); + $fields = array('path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'encrypted', 'etag'); $params = array(); $queryParts = array(); @@ -333,7 +333,7 @@ class Cache { */ public function search($pattern) { $query = \OC_DB::prepare(' - SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted` + SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag` FROM `*PREFIX*filecache` WHERE `name` LIKE ? AND `storage` = ?' ); $result = $query->execute(array($pattern, $this->numericId)); @@ -357,7 +357,7 @@ class Cache { $where = '`mimepart` = ?'; } $query = \OC_DB::prepare(' - SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted` + SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag` FROM `*PREFIX*filecache` WHERE ' . $where . ' AND `storage` = ?' ); $result = $query->execute(array($mimetype, $this->numericId)); diff --git a/lib/files/cache/scanner.php b/lib/files/cache/scanner.php index 4c0ec9617f..e7bfb1898e 100644 --- a/lib/files/cache/scanner.php +++ b/lib/files/cache/scanner.php @@ -58,6 +58,7 @@ class Scanner { $data['size'] = $this->storage->filesize($path); $data['permissions'] = $this->storage->getPermissions($path); } + $data['etag'] = $this->storage->getETag($path); return $data; } diff --git a/lib/util.php b/lib/util.php index c5a495234d..df26a825d1 100755 --- a/lib/util.php +++ b/lib/util.php @@ -74,7 +74,7 @@ class OC_Util { */ public static function getVersion() { // hint: We only can count up. So the internal version number of ownCloud 4.5 will be 4.90.0. This is not visible to the user - return array(4,91,04); + return array(4,91,05); } /** From a5cb7363a5031d25d3e4cbd46817930c19893143 Mon Sep 17 00:00:00 2001 From: Michael Gapczynski Date: Sun, 30 Dec 2012 19:18:05 -0500 Subject: [PATCH 02/12] Use etags from file cache in SabreDAV connector --- lib/connector/sabre/directory.php | 2 +- lib/connector/sabre/node.php | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/lib/connector/sabre/directory.php b/lib/connector/sabre/directory.php index 8d4dd92a3d..a720157936 100644 --- a/lib/connector/sabre/directory.php +++ b/lib/connector/sabre/directory.php @@ -121,8 +121,8 @@ class OC_Connector_Sabre_Directory extends OC_Connector_Sabre_Node implements Sa $paths = array(); foreach($folder_content as $info) { $paths[] = $this->path.'/'.$info['name']; + $properties[$this->path.'/'.$info['name']][self::GETETAG_PROPERTYNAME] = $info['etag']; } - $properties = array_fill_keys($paths, array()); if(count($paths)>0) { // // the number of arguments within IN conditions are limited in most databases diff --git a/lib/connector/sabre/node.php b/lib/connector/sabre/node.php index ad08bd434f..d4023dfad7 100644 --- a/lib/connector/sabre/node.php +++ b/lib/connector/sabre/node.php @@ -190,6 +190,7 @@ abstract class OC_Connector_Sabre_Node implements Sabre_DAV_INode, Sabre_DAV_IPr while( $row = $result->fetchRow()) { $this->property_cache[$row['propertyname']] = $row['propertyvalue']; } + $this->property_cache[self::GETETAG_PROPERTYNAME] = $this->getETagPropertyForPath($this->path); } // if the array was empty, we need to return everything @@ -210,14 +211,11 @@ abstract class OC_Connector_Sabre_Node implements Sabre_DAV_INode, Sabre_DAV_IPr * @return string|null Returns null if the ETag can not effectively be determined */ static public function getETagPropertyForPath($path) { - $tag = \OC\Files\Filesystem::getETag($path); - if (empty($tag)) { - return null; + $data = \OC\Files\Filesystem::getFileInfo($path); + if (isset($data['etag'])) { + return $data['etag']; } - $etag = '"'.$tag.'"'; - $query = OC_DB::prepare( 'INSERT INTO `*PREFIX*properties` (`userid`,`propertypath`,`propertyname`,`propertyvalue`) VALUES(?,?,?,?)' ); - $query->execute( array( OC_User::getUser(), $path, self::GETETAG_PROPERTYNAME, $etag )); - return $etag; + return null; } /** From 83064aca51db7d8382282743ade2ab9da2a6e1b0 Mon Sep 17 00:00:00 2001 From: Michael Gapczynski Date: Sun, 30 Dec 2012 19:23:31 -0500 Subject: [PATCH 03/12] Remove old etag code --- lib/connector/sabre/node.php | 24 ------------------------ lib/files/filesystem.php | 21 --------------------- lib/files/view.php | 2 -- lib/filesystem.php | 7 ------- 4 files changed, 54 deletions(-) diff --git a/lib/connector/sabre/node.php b/lib/connector/sabre/node.php index d4023dfad7..dd8ae152ff 100644 --- a/lib/connector/sabre/node.php +++ b/lib/connector/sabre/node.php @@ -218,28 +218,4 @@ abstract class OC_Connector_Sabre_Node implements Sabre_DAV_INode, Sabre_DAV_IPr return null; } - /** - * @brief Remove the ETag from the cache. - * @param string $path Path of the file - */ - static public function removeETagPropertyForPath($path) { - // remove tags from this and parent paths - $paths = array(); - while ($path != '/' && $path != '.' && $path != '' && $path != '\\') { - $paths[] = $path; - $path = dirname($path); - } - if (empty($paths)) { - return; - } - $paths[] = $path; - $path_placeholders = join(',', array_fill(0, count($paths), '?')); - $query = OC_DB::prepare( 'DELETE FROM `*PREFIX*properties`' - .' WHERE `userid` = ?' - .' AND `propertyname` = ?' - .' AND `propertypath` IN ('.$path_placeholders.')' - ); - $vals = array( OC_User::getUser(), self::GETETAG_PROPERTYNAME ); - $query->execute(array_merge( $vals, $paths )); - } } diff --git a/lib/files/filesystem.php b/lib/files/filesystem.php index 8183b8ff99..d62b5186cb 100644 --- a/lib/files/filesystem.php +++ b/lib/files/filesystem.php @@ -582,23 +582,6 @@ class Filesystem { return self::$defaultInstance->hasUpdated($path, $time); } - static public function removeETagHook($params, $root = false) { - if (isset($params['path'])) { - $path = $params['path']; - } else { - $path = $params['oldpath']; - } - - if ($root) { // reduce path to the required part of it (no 'username/files') - $fakeRootView = new View($root); - $count = 1; - $path = str_replace(\OC_App::getStorage("files")->getAbsolutePath(''), "", $fakeRootView->getAbsolutePath($path), $count); - } - - $path = self::normalizePath($path); - \OC_Connector_Sabre_Node::removeETagPropertyForPath($path); - } - /** * normalize a path * @@ -682,10 +665,6 @@ class Filesystem { } } -\OC_Hook::connect('OC_Filesystem', 'post_write', 'OC_Filesystem', 'removeETagHook'); -\OC_Hook::connect('OC_Filesystem', 'post_delete', 'OC_Filesystem', 'removeETagHook'); -\OC_Hook::connect('OC_Filesystem', 'post_rename', 'OC_Filesystem', 'removeETagHook'); - \OC_Hook::connect('OC_Filesystem', 'post_write', '\OC\Files\Cache\Updater', 'writeHook'); \OC_Hook::connect('OC_Filesystem', 'post_delete', '\OC\Files\Cache\Updater', 'deleteHook'); \OC_Hook::connect('OC_Filesystem', 'post_rename', '\OC\Files\Cache\Updater', 'renameHook'); diff --git a/lib/files/view.php b/lib/files/view.php index 592c484a21..77146895e6 100644 --- a/lib/files/view.php +++ b/lib/files/view.php @@ -462,8 +462,6 @@ class View { Filesystem::signal_post_write, array(Filesystem::signal_param_path => $path2) ); - } else { // no real copy, file comes from somewhere else, e.g. version rollback -> just update the file cache and the webdav properties without all the other post_write actions - Filesystem::removeETagHook(array("path" => $path2), $this->fakeRoot); } return $result; } else { diff --git a/lib/filesystem.php b/lib/filesystem.php index 20b5ab2790..57cca90230 100644 --- a/lib/filesystem.php +++ b/lib/filesystem.php @@ -398,13 +398,6 @@ class OC_Filesystem { return \OC\Files\Filesystem::hasUpdated($path, $time); } - /** - * @deprecated OC_Filesystem is replaced by \OC\Files\Filesystem - */ - static public function removeETagHook($params, $root = false) { - \OC\Files\Filesystem::removeETagHook($params, $root); - } - /** * normalize a path * From 77f12c526bf0010219e363844b588ffae27f1251 Mon Sep 17 00:00:00 2001 From: Michael Gapczynski Date: Sun, 30 Dec 2012 19:54:51 -0500 Subject: [PATCH 04/12] Update etags in parent folders --- lib/files/cache/updater.php | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lib/files/cache/updater.php b/lib/files/cache/updater.php index fb9783023e..d57e4a1abe 100644 --- a/lib/files/cache/updater.php +++ b/lib/files/cache/updater.php @@ -34,6 +34,7 @@ class Updater { $scanner = new Scanner($storage); $scanner->scan($internalPath, Scanner::SCAN_SHALLOW); $cache->correctFolderSize($internalPath); + self::eTagUpdate($path); } static public function deleteUpdate($path) { @@ -45,6 +46,29 @@ class Updater { $cache = new Cache($storage); $cache->remove($internalPath); $cache->correctFolderSize($internalPath); + self::eTagUpdate($path); + } + + static public function eTagUpdate($path) { + if ($path !== '') { + $parent = dirname($path); + if ($parent === '.') { + $parent = ''; + } + /** + * @var \OC\Files\Storage\Storage $storage + * @var string $internalPath + */ + list($storage, $internalPath) = self::resolvePath($parent); + if ($storage) { + $cache = $storage->getCache(); + $id = $cache->getId($internalPath); + if ($id !== -1) { + $cache->update($id, array('etag' => $storage->getETag($internalPath))); + self::updateFolderETags($parent); + } + } + } } /** From 96e08a1d962b42782d03721009970cbab352eec2 Mon Sep 17 00:00:00 2001 From: Michael Gapczynski Date: Sun, 30 Dec 2012 21:23:17 -0500 Subject: [PATCH 05/12] Fix function name --- lib/files/cache/updater.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/files/cache/updater.php b/lib/files/cache/updater.php index d57e4a1abe..7fc9bd382f 100644 --- a/lib/files/cache/updater.php +++ b/lib/files/cache/updater.php @@ -65,7 +65,7 @@ class Updater { $id = $cache->getId($internalPath); if ($id !== -1) { $cache->update($id, array('etag' => $storage->getETag($internalPath))); - self::updateFolderETags($parent); + self::eTagUpdate($parent); } } } From 29b82ccdf32f8a13723e7f49be0c3cbf7e64a404 Mon Sep 17 00:00:00 2001 From: Michael Gapczynski Date: Sun, 30 Dec 2012 21:23:54 -0500 Subject: [PATCH 06/12] Change length of etag field to 40 --- db_structure.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db_structure.xml b/db_structure.xml index 7c67ca78f4..527b53040d 100644 --- a/db_structure.xml +++ b/db_structure.xml @@ -194,7 +194,7 @@ text true - 250 + 40 From d0a50fae8317e4b4871027ee4b2940ab5443961f Mon Sep 17 00:00:00 2001 From: Michael Gapczynski Date: Mon, 31 Dec 2012 18:16:44 -0500 Subject: [PATCH 07/12] Fix eTagUpdate and add tests --- lib/files/cache/updater.php | 2 +- tests/lib/files/cache/updater.php | 28 ++++++++++++++++++++++------ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/lib/files/cache/updater.php b/lib/files/cache/updater.php index 2604159009..cfc1ec731e 100644 --- a/lib/files/cache/updater.php +++ b/lib/files/cache/updater.php @@ -54,7 +54,7 @@ class Updater { } static public function eTagUpdate($path) { - if ($path !== '') { + if ($path !== '' && $path !== '/') { $parent = dirname($path); if ($parent === '.') { $parent = ''; diff --git a/tests/lib/files/cache/updater.php b/tests/lib/files/cache/updater.php index d19966c191..cad3d9d46f 100644 --- a/tests/lib/files/cache/updater.php +++ b/tests/lib/files/cache/updater.php @@ -70,14 +70,18 @@ class Updater extends \PHPUnit_Framework_TestCase { public function testWrite() { $textSize = strlen("dummy file data\n"); $imageSize = filesize(\OC::$SERVERROOT . '/core/img/logo.png'); - $cachedData = $this->cache->get(''); - $this->assertEquals(3 * $textSize + $imageSize, $cachedData['size']); + $rootCachedData = $this->cache->get(''); + $this->assertEquals(3 * $textSize + $imageSize, $rootCachedData['size']); + $fooCachedData = $this->cache->get('foo.txt'); Filesystem::file_put_contents('foo.txt', 'asd'); $cachedData = $this->cache->get('foo.txt'); $this->assertEquals(3, $cachedData['size']); + $this->assertNotEquals($fooCachedData['etag'], $cachedData['etag']); $cachedData = $this->cache->get(''); $this->assertEquals(2 * $textSize + $imageSize + 3, $cachedData['size']); + $this->assertNotEquals($rootCachedData['etag'], $cachedData['etag']); + $rootCachedData = $cachedData; $this->assertFalse($this->cache->inCache('bar.txt')); Filesystem::file_put_contents('bar.txt', 'asd'); @@ -86,38 +90,50 @@ class Updater extends \PHPUnit_Framework_TestCase { $this->assertEquals(3, $cachedData['size']); $cachedData = $this->cache->get(''); $this->assertEquals(2 * $textSize + $imageSize + 2 * 3, $cachedData['size']); + $this->assertNotEquals($rootCachedData['etag'], $cachedData['etag']); } public function testDelete() { $textSize = strlen("dummy file data\n"); $imageSize = filesize(\OC::$SERVERROOT . '/core/img/logo.png'); - $cachedData = $this->cache->get(''); - $this->assertEquals(3 * $textSize + $imageSize, $cachedData['size']); + $rootCachedData = $this->cache->get(''); + $this->assertEquals(3 * $textSize + $imageSize, $rootCachedData['size']); $this->assertTrue($this->cache->inCache('foo.txt')); Filesystem::unlink('foo.txt', 'asd'); $this->assertFalse($this->cache->inCache('foo.txt')); $cachedData = $this->cache->get(''); $this->assertEquals(2 * $textSize + $imageSize, $cachedData['size']); + $this->assertNotEquals($rootCachedData['etag'], $cachedData['etag']); + $rootCachedData = $cachedData; Filesystem::mkdir('bar_folder'); $this->assertTrue($this->cache->inCache('bar_folder')); + $cachedData = $this->cache->get(''); + $this->assertNotEquals($rootCachedData['etag'], $cachedData['etag']); + $rootCachedData = $cachedData; Filesystem::rmdir('bar_folder'); $this->assertFalse($this->cache->inCache('bar_folder')); + $cachedData = $this->cache->get(''); + $this->assertNotEquals($rootCachedData['etag'], $cachedData['etag']); } public function testRename() { $textSize = strlen("dummy file data\n"); $imageSize = filesize(\OC::$SERVERROOT . '/core/img/logo.png'); - $cachedData = $this->cache->get(''); - $this->assertEquals(3 * $textSize + $imageSize, $cachedData['size']); + $rootCachedData = $this->cache->get(''); + $this->assertEquals(3 * $textSize + $imageSize, $rootCachedData['size']); $this->assertTrue($this->cache->inCache('foo.txt')); + $fooCachedData = $this->cache->get('foo.txt'); $this->assertFalse($this->cache->inCache('bar.txt')); Filesystem::rename('foo.txt', 'bar.txt'); $this->assertFalse($this->cache->inCache('foo.txt')); $this->assertTrue($this->cache->inCache('bar.txt')); + $cachedData = $this->cache->get('foo.txt'); + $this->assertNotEquals($fooCachedData['etag'], $cachedData['etag']); $cachedData = $this->cache->get(''); $this->assertEquals(3 * $textSize + $imageSize, $cachedData['size']); + $this->assertNotEquals($rootCachedData['etag'], $cachedData['etag']); } } From 8a63bcc1e85499335ade1df459f0c9f467161404 Mon Sep 17 00:00:00 2001 From: Michael Gapczynski Date: Fri, 11 Jan 2013 20:56:36 -0500 Subject: [PATCH 08/12] Don't use more entropy for etags --- lib/files/storage/common.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/files/storage/common.php b/lib/files/storage/common.php index e859d447f3..591803f044 100644 --- a/lib/files/storage/common.php +++ b/lib/files/storage/common.php @@ -274,7 +274,7 @@ abstract class Common implements \OC\Files\Storage\Storage { $hash = call_user_func($ETagFunction, $path); return $hash; }else{ - return uniqid('', true); + return uniqid(); } } } From 9e2f3a53244e353cb75f9927e1c69ef40f589db7 Mon Sep 17 00:00:00 2001 From: Michael Gapczynski Date: Fri, 11 Jan 2013 20:59:53 -0500 Subject: [PATCH 09/12] Remove old create etag function --- lib/connector/sabre/file.php | 9 --------- 1 file changed, 9 deletions(-) diff --git a/lib/connector/sabre/file.php b/lib/connector/sabre/file.php index 1770b49128..e0723e1230 100644 --- a/lib/connector/sabre/file.php +++ b/lib/connector/sabre/file.php @@ -101,15 +101,6 @@ class OC_Connector_Sabre_File extends OC_Connector_Sabre_Node implements Sabre_D return $this->getETagPropertyForPath($this->path); } - /** - * Creates a ETag for this path. - * @param string $path Path of the file - * @return string|null Returns null if the ETag can not effectively be determined - */ - static protected function createETag($path) { - return \OC\Files\Filesystem::hash('md5', $path); - } - /** * Returns the mime-type for a file * From a00b9e0a03a7cdb1103415d33d2c3f22e86edffb Mon Sep 17 00:00:00 2001 From: Michael Gapczynski Date: Fri, 11 Jan 2013 21:01:28 -0500 Subject: [PATCH 10/12] Bump version --- lib/util.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/util.php b/lib/util.php index e814a3a32d..faae962859 100755 --- a/lib/util.php +++ b/lib/util.php @@ -74,7 +74,7 @@ class OC_Util { */ public static function getVersion() { // hint: We only can count up. So the internal version number of ownCloud 4.5 will be 4.90.0. This is not visible to the user - return array(4,91,06); + return array(4,91,07); } /** From b30648cb7dc258208fad4dadb297aa1d792fa7ef Mon Sep 17 00:00:00 2001 From: Michael Gapczynski Date: Fri, 11 Jan 2013 21:09:01 -0500 Subject: [PATCH 11/12] Don't waste time making another call since we know it doesn't exist --- lib/connector/sabre/file.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/connector/sabre/file.php b/lib/connector/sabre/file.php index e0723e1230..1c18a39174 100644 --- a/lib/connector/sabre/file.php +++ b/lib/connector/sabre/file.php @@ -98,7 +98,7 @@ class OC_Connector_Sabre_File extends OC_Connector_Sabre_Node implements Sabre_D if (isset($properties[self::GETETAG_PROPERTYNAME])) { return $properties[self::GETETAG_PROPERTYNAME]; } - return $this->getETagPropertyForPath($this->path); + return null; } /** From 094a852bff378c03c873b6e8fd94587d2a2c784e Mon Sep 17 00:00:00 2001 From: Michael Gapczynski Date: Fri, 11 Jan 2013 21:09:58 -0500 Subject: [PATCH 12/12] Wrap the etag in double quotes --- lib/connector/sabre/node.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/connector/sabre/node.php b/lib/connector/sabre/node.php index dd8ae152ff..93d8fc477d 100644 --- a/lib/connector/sabre/node.php +++ b/lib/connector/sabre/node.php @@ -213,7 +213,7 @@ abstract class OC_Connector_Sabre_Node implements Sabre_DAV_INode, Sabre_DAV_IPr static public function getETagPropertyForPath($path) { $data = \OC\Files\Filesystem::getFileInfo($path); if (isset($data['etag'])) { - return $data['etag']; + return '"'.$data['etag'].'"'; } return null; }