Merge pull request #1052 from owncloud/filesystem-etags

ETags stored in the file cache

db_structure.xml

@@ -225,6 +225,14 @@
 				<length>4</length>
 			</field>
 
+			<field>
+				<name>etag</name>
+				<type>text</type>
+				<default></default>
+				<notnull>true</notnull>
+				<length>40</length>
+			</field>
+
 			<index>
 				<name>fs_storage_path_hash</name>
 				<unique>true</unique>

lib/connector/sabre/directory.php

@@ -121,8 +121,8 @@ class OC_Connector_Sabre_Directory extends OC_Connector_Sabre_Node implements Sa
 		$paths = array();
 		foreach($folder_content as $info) {
 			$paths[] = $this->path.'/'.$info['name'];
+			$properties[$this->path.'/'.$info['name']][self::GETETAG_PROPERTYNAME] = $info['etag'];
 		}
-		$properties = array_fill_keys($paths, array());
 		if(count($paths)>0) {
 			//
 			// the number of arguments within IN conditions are limited in most databases

lib/connector/sabre/file.php

@@ -98,16 +98,7 @@ class OC_Connector_Sabre_File extends OC_Connector_Sabre_Node implements Sabre_D
 		if (isset($properties[self::GETETAG_PROPERTYNAME])) {
 			return $properties[self::GETETAG_PROPERTYNAME];
 		}
-		return $this->getETagPropertyForPath($this->path);
+		return null;
-	}
-
-	/**
-	 * Creates a ETag for this path.
-	 * @param string $path Path of the file
-	 * @return string|null Returns null if the ETag can not effectively be determined
-	 */
-	static protected function createETag($path) {
-		return \OC\Files\Filesystem::hash('md5', $path);
 	}
 
 	/**

lib/connector/sabre/node.php

@@ -190,6 +190,7 @@ abstract class OC_Connector_Sabre_Node implements Sabre_DAV_INode, Sabre_DAV_IPr
 			while( $row = $result->fetchRow()) {
 				$this->property_cache[$row['propertyname']] = $row['propertyvalue'];
 			}
+			$this->property_cache[self::GETETAG_PROPERTYNAME] = $this->getETagPropertyForPath($this->path);
 		}
 
 		// if the array was empty, we need to return everything
@@ -210,38 +211,11 @@ abstract class OC_Connector_Sabre_Node implements Sabre_DAV_INode, Sabre_DAV_IPr
 	 * @return string|null Returns null if the ETag can not effectively be determined
 	 */
 	static public function getETagPropertyForPath($path) {
-		$tag = \OC\Files\Filesystem::getETag($path);
+		$data = \OC\Files\Filesystem::getFileInfo($path);
-		if (empty($tag)) {
+		if (isset($data['etag'])) {
-			return null;
+			return '"'.$data['etag'].'"';
 		}
-		$etag = '"'.$tag.'"';
+		return null;
-		$query = OC_DB::prepare( 'INSERT INTO `*PREFIX*properties` (`userid`,`propertypath`,`propertyname`,`propertyvalue`) VALUES(?,?,?,?)' );
-		$query->execute( array( OC_User::getUser(), $path, self::GETETAG_PROPERTYNAME, $etag ));
-		return $etag;
 	}
 
-	/**
-	 * @brief Remove the ETag from the cache.
-	 * @param string $path Path of the file
-	 */
-	static public function removeETagPropertyForPath($path) {
-		// remove tags from this and parent paths
-		$paths = array();
-		while ($path != '/' && $path != '.' && $path != '' && $path != '\\') {
-			$paths[] = $path;
-			$path = dirname($path);
-		}
-		if (empty($paths)) {
-			return;
-		}
-		$paths[] = $path;
-		$path_placeholders = join(',', array_fill(0, count($paths), '?'));
-		$query = OC_DB::prepare( 'DELETE FROM `*PREFIX*properties`'
-			.' WHERE `userid` = ?'
-			.' AND `propertyname` = ?'
-			.' AND `propertypath` IN ('.$path_placeholders.')'
-			);
-		$vals = array( OC_User::getUser(), self::GETETAG_PROPERTYNAME );
-		$query->execute(array_merge( $vals, $paths ));
-	}
 }

lib/files/cache/cache.php

@@ -114,7 +114,7 @@ class Cache {
 			$params = array($file);
 		}
 		$query = \OC_DB::prepare(
-			'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`
+			'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag`
 			 FROM `*PREFIX*filecache` ' . $where);
 		$result = $query->execute($params);
 		$data = $result->fetchRow();
@@ -147,7 +147,7 @@ class Cache {
 		$fileId = $this->getId($folder);
 		if ($fileId > -1) {
 			$query = \OC_DB::prepare(
-				'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`
+				'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag`
 			 	 FROM `*PREFIX*filecache` WHERE parent = ? ORDER BY `name` ASC');
 			$result = $query->execute(array($fileId));
 			$files = $result->fetchAll();
@@ -225,8 +225,7 @@ class Cache {
 	 * @return array
 	 */
 	function buildParts(array $data) {
-		$fields = array('path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'encrypted');
+		$fields = array('path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'encrypted', 'etag');
-
 		$params = array();
 		$queryParts = array();
 		foreach ($data as $name => $value) {
@@ -379,7 +378,7 @@ class Cache {
 	 */
 	public function search($pattern) {
 		$query = \OC_DB::prepare('
-			SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`
+			SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag`
 			FROM `*PREFIX*filecache` WHERE `name` LIKE ? AND `storage` = ?'
 		);
 		$result = $query->execute(array($pattern, $this->numericId));
@@ -405,7 +404,7 @@ class Cache {
 			$where = '`mimepart` = ?';
 		}
 		$query = \OC_DB::prepare('
-			SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`
+			SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `etag`
 			FROM `*PREFIX*filecache` WHERE ' . $where . ' AND `storage` = ?'
 		);
 		$mimetype = $this->getMimetypeId($mimetype);

lib/files/cache/scanner.php

@@ -50,6 +50,7 @@ class Scanner {
 		} else {
 			$data['size'] = $this->storage->filesize($path);
 		}
+		$data['etag'] = $this->storage->getETag($path);
 		return $data;
 	}
 

lib/files/cache/updater.php

@@ -35,6 +35,7 @@ class Updater {
 			$scanner = $storage->getScanner($internalPath);
 			$scanner->scan($internalPath, Scanner::SCAN_SHALLOW);
 			$cache->correctFolderSize($internalPath);
+			self::eTagUpdate($path);
 		}
 	}
 
@@ -48,6 +49,29 @@ class Updater {
 			$cache = $storage->getCache($internalPath);
 			$cache->remove($internalPath);
 			$cache->correctFolderSize($internalPath);
+			self::eTagUpdate($path);
+		}
+	}
+
+	static public function eTagUpdate($path) {
+		if ($path !== '' && $path !== '/') {
+			$parent = dirname($path);
+			if ($parent === '.') {
+				$parent = '';
+			}
+			/**
+			* @var \OC\Files\Storage\Storage $storage
+			* @var string $internalPath
+			*/
+			list($storage, $internalPath) = self::resolvePath($parent);
+			if ($storage) {
+				$cache = $storage->getCache();
+				$id = $cache->getId($internalPath);
+				if ($id !== -1) {
+					$cache->update($id, array('etag' => $storage->getETag($internalPath)));
+					self::eTagUpdate($parent);
+				}
+			}
 		}
 	}
 

lib/files/filesystem.php

@@ -585,23 +585,6 @@ class Filesystem {
 		return self::$defaultInstance->hasUpdated($path, $time);
 	}
 
-	static public function removeETagHook($params, $root = false) {
-		if (isset($params['path'])) {
-			$path = $params['path'];
-		} else {
-			$path = $params['oldpath'];
-		}
-
-		if ($root) { // reduce path to the required part of it (no 'username/files')
-			$fakeRootView = new View($root);
-			$count = 1;
-			$path = str_replace(\OC_App::getStorage("files")->getAbsolutePath(''), "", $fakeRootView->getAbsolutePath($path), $count);
-		}
-
-		$path = self::normalizePath($path);
-		\OC_Connector_Sabre_Node::removeETagPropertyForPath($path);
-	}
-
 	/**
 	 * normalize a path
 	 *
@@ -685,10 +668,6 @@ class Filesystem {
 	}
 }
 
-\OC_Hook::connect('OC_Filesystem', 'post_write', 'OC_Filesystem', 'removeETagHook');
-\OC_Hook::connect('OC_Filesystem', 'post_delete', 'OC_Filesystem', 'removeETagHook');
-\OC_Hook::connect('OC_Filesystem', 'post_rename', 'OC_Filesystem', 'removeETagHook');
-
 \OC_Hook::connect('OC_Filesystem', 'post_write', '\OC\Files\Cache\Updater', 'writeHook');
 \OC_Hook::connect('OC_Filesystem', 'post_delete', '\OC\Files\Cache\Updater', 'deleteHook');
 \OC_Hook::connect('OC_Filesystem', 'post_rename', '\OC\Files\Cache\Updater', 'renameHook');

lib/files/storage/common.php

@@ -274,7 +274,7 @@ abstract class Common implements \OC\Files\Storage\Storage {
 			$hash = call_user_func($ETagFunction, $path);
 			return $hash;
 		}else{
-			return uniqid('', true);
+			return uniqid();
 		}
 	}
 }

lib/files/view.php

@@ -462,8 +462,6 @@ class View {
 						Filesystem::signal_post_write,
 						array(Filesystem::signal_param_path => $path2)
 					);
-				} else { // no real copy, file comes from somewhere else, e.g. version rollback -> just update the file cache and the webdav properties without all the other post_write actions
-					Filesystem::removeETagHook(array("path" => $path2), $this->fakeRoot);
 				}
 				return $result;
 			} else {

lib/filesystem.php

@@ -398,13 +398,6 @@ class OC_Filesystem {
 		return \OC\Files\Filesystem::hasUpdated($path, $time);
 	}
 
-	/**
-	 * @deprecated OC_Filesystem is replaced by \OC\Files\Filesystem
-	 */
-	static public function removeETagHook($params, $root = false) {
-		\OC\Files\Filesystem::removeETagHook($params, $root);
-	}
-
 	/**
 	 * normalize a path
 	 *

lib/util.php

@@ -74,7 +74,7 @@ class OC_Util {
 	 */
 	public static function getVersion() {
 		// hint: We only can count up. So the internal version number of ownCloud 4.5 will be 4.90.0. This is not visible to the user
-		return array(4,91,06);
+		return array(4,91,07);
 	}
 
 	/**

tests/lib/files/cache/updater.php

@@ -70,14 +70,18 @@ class Updater extends \PHPUnit_Framework_TestCase {
 	public function testWrite() {
 		$textSize = strlen("dummy file data\n");
 		$imageSize = filesize(\OC::$SERVERROOT . '/core/img/logo.png');
-		$cachedData = $this->cache->get('');
+		$rootCachedData = $this->cache->get('');
-		$this->assertEquals(3 * $textSize + $imageSize, $cachedData['size']);
+		$this->assertEquals(3 * $textSize + $imageSize, $rootCachedData['size']);
 
+		$fooCachedData = $this->cache->get('foo.txt');
 		Filesystem::file_put_contents('foo.txt', 'asd');
 		$cachedData = $this->cache->get('foo.txt');
 		$this->assertEquals(3, $cachedData['size']);
+		$this->assertNotEquals($fooCachedData['etag'], $cachedData['etag']);
 		$cachedData = $this->cache->get('');
 		$this->assertEquals(2 * $textSize + $imageSize + 3, $cachedData['size']);
+		$this->assertNotEquals($rootCachedData['etag'], $cachedData['etag']);
+		$rootCachedData = $cachedData;
 
 		$this->assertFalse($this->cache->inCache('bar.txt'));
 		Filesystem::file_put_contents('bar.txt', 'asd');
@@ -86,38 +90,50 @@ class Updater extends \PHPUnit_Framework_TestCase {
 		$this->assertEquals(3, $cachedData['size']);
 		$cachedData = $this->cache->get('');
 		$this->assertEquals(2 * $textSize + $imageSize + 2 * 3, $cachedData['size']);
+		$this->assertNotEquals($rootCachedData['etag'], $cachedData['etag']);
 	}
 
 	public function testDelete() {
 		$textSize = strlen("dummy file data\n");
 		$imageSize = filesize(\OC::$SERVERROOT . '/core/img/logo.png');
-		$cachedData = $this->cache->get('');
+		$rootCachedData = $this->cache->get('');
-		$this->assertEquals(3 * $textSize + $imageSize, $cachedData['size']);
+		$this->assertEquals(3 * $textSize + $imageSize, $rootCachedData['size']);
 
 		$this->assertTrue($this->cache->inCache('foo.txt'));
 		Filesystem::unlink('foo.txt', 'asd');
 		$this->assertFalse($this->cache->inCache('foo.txt'));
 		$cachedData = $this->cache->get('');
 		$this->assertEquals(2 * $textSize + $imageSize, $cachedData['size']);
+		$this->assertNotEquals($rootCachedData['etag'], $cachedData['etag']);
+		$rootCachedData = $cachedData;
 
 		Filesystem::mkdir('bar_folder');
 		$this->assertTrue($this->cache->inCache('bar_folder'));
+		$cachedData = $this->cache->get('');
+		$this->assertNotEquals($rootCachedData['etag'], $cachedData['etag']);
+		$rootCachedData = $cachedData;
 		Filesystem::rmdir('bar_folder');
 		$this->assertFalse($this->cache->inCache('bar_folder'));
+		$cachedData = $this->cache->get('');
+		$this->assertNotEquals($rootCachedData['etag'], $cachedData['etag']);
 	}
 
 	public function testRename() {
 		$textSize = strlen("dummy file data\n");
 		$imageSize = filesize(\OC::$SERVERROOT . '/core/img/logo.png');
-		$cachedData = $this->cache->get('');
+		$rootCachedData = $this->cache->get('');
-		$this->assertEquals(3 * $textSize + $imageSize, $cachedData['size']);
+		$this->assertEquals(3 * $textSize + $imageSize, $rootCachedData['size']);
 
 		$this->assertTrue($this->cache->inCache('foo.txt'));
+		$fooCachedData = $this->cache->get('foo.txt');
 		$this->assertFalse($this->cache->inCache('bar.txt'));
 		Filesystem::rename('foo.txt', 'bar.txt');
 		$this->assertFalse($this->cache->inCache('foo.txt'));
 		$this->assertTrue($this->cache->inCache('bar.txt'));
+		$cachedData = $this->cache->get('foo.txt');
+		$this->assertNotEquals($fooCachedData['etag'], $cachedData['etag']);
 		$cachedData = $this->cache->get('');
 		$this->assertEquals(3 * $textSize + $imageSize, $cachedData['size']);
+		$this->assertNotEquals($rootCachedData['etag'], $cachedData['etag']);
 	}
 }