initial version of a local storage implementation which will use unique slugified filename on the local filesystem.

This implementation will only be enabled on windows based system to solve the issues around UTF-8 file names with php on windows.
This commit is contained in:
Thomas Mueller 2013-02-06 23:36:38 +01:00
parent 223f538bb8
commit fd8cb9974b
6 changed files with 614 additions and 4 deletions

View file

@ -94,6 +94,50 @@
</table>
<table>
<name>*dbprefix*file_map</name>
<declaration>
<field>
<name>logic_path</name>
<type>text</type>
<default></default>
<notnull>true</notnull>
<length>512</length>
</field>
<field>
<name>physic_path</name>
<type>text</type>
<default></default>
<notnull>true</notnull>
<length>512</length>
</field>
<index>
<name>file_map_lp_index</name>
<unique>true</unique>
<field>
<name>logic_path</name>
<sorting>ascending</sorting>
</field>
</index>
<index>
<name>file_map_pp_index</name>
<unique>true</unique>
<field>
<name>physic_path</name>
<sorting>ascending</sorting>
</field>
</index>
</declaration>
</table>
<table>
<name>*dbprefix*mimetypes</name>

216
lib/files/mapper.php Normal file
View file

@ -0,0 +1,216 @@
<?php
namespace OC\Files;
/**
* class Mapper is responsible to translate logical paths to physical paths and reverse
*/
class Mapper
{
/**
* @param string $logicPath
* @param bool $create indicates if the generated physical name shall be stored in the database or not
* @return string the physical path
*/
public function logicToPhysical($logicPath, $create) {
$physicalPath = $this->resolveLogicPath($logicPath);
if ($physicalPath !== null) {
return $physicalPath;
}
return $this->create($logicPath, $create);
}
/**
* @param string $physicalPath
* @return string|null
*/
public function physicalToLogic($physicalPath) {
$logicPath = $this->resolvePhysicalPath($physicalPath);
if ($logicPath !== null) {
return $logicPath;
}
$this->insert($physicalPath, $physicalPath);
return $physicalPath;
}
/**
* @param string $path
* @param bool $isLogicPath indicates if $path is logical or physical
* @param $recursive
*/
public function removePath($path, $isLogicPath, $recursive) {
if ($recursive) {
$path=$path.'%';
}
if ($isLogicPath) {
$query = \OC_DB::prepare('DELETE FROM `*PREFIX*file_map` WHERE `logic_path` LIKE ?');
$query->execute(array($path));
} else {
$query = \OC_DB::prepare('DELETE FROM `*PREFIX*file_map` WHERE `physic_path` LIKE ?');
$query->execute(array($path));
}
}
/**
* @param $path1
* @param $path2
* @throws \Exception
*/
public function copy($path1, $path2)
{
$path1 = $this->stripLast($path1);
$path2 = $this->stripLast($path2);
$physicPath1 = $this->logicToPhysical($path1, true);
$physicPath2 = $this->logicToPhysical($path2, true);
$query = \OC_DB::prepare('SELECT * FROM `*PREFIX*file_map` WHERE `logic_path` LIKE ?');
$result = $query->execute(array($path1.'%'));
$updateQuery = \OC_DB::prepare('UPDATE `*PREFIX*file_map`'
.' SET `logic_path` = ?'
.' AND `physic_path` = ?'
.' WHERE `logic_path` = ?');
while( $row = $result->fetchRow()) {
$currentLogic = $row['logic_path'];
$currentPhysic = $row['physic_path'];
$newLogic = $path2.$this->stripRootFolder($currentLogic, $path1);
$newPhysic = $physicPath2.$this->stripRootFolder($currentPhysic, $physicPath1);
if ($path1 !== $currentLogic) {
try {
$updateQuery->execute(array($newLogic, $newPhysic, $currentLogic));
} catch (\Exception $e) {
error_log('Mapper::Copy failed '.$currentLogic.' -> '.$newLogic.'\n'.$e);
throw $e;
}
}
}
}
/**
* @param $path
* @param $root
* @return bool|string
*/
public function stripRootFolder($path, $root) {
if (strpos($path, $root) !== 0) {
// throw exception ???
return false;
}
if (strlen($path) > strlen($root)) {
return substr($path, strlen($root));
}
return '';
}
private function stripLast($path) {
if (substr($path, -1) == '/') {
$path = substr_replace($path ,'',-1);
}
return $path;
}
private function resolveLogicPath($logicPath) {
$logicPath = $this->stripLast($logicPath);
$query = \OC_DB::prepare('SELECT * FROM `*PREFIX*file_map` WHERE `logic_path` = ?');
$result = $query->execute(array($logicPath));
$result = $result->fetchRow();
return $result['physic_path'];
}
private function resolvePhysicalPath($physicalPath) {
$physicalPath = $this->stripLast($physicalPath);
$query = \OC_DB::prepare('SELECT * FROM `*PREFIX*file_map` WHERE `physic_path` = ?');
$result = $query->execute(array($physicalPath));
$result = $result->fetchRow();
return $result['logic_path'];
}
private function create($logicPath, $store) {
$logicPath = $this->stripLast($logicPath);
$index = 0;
// create the slugified path
$physicalPath = $this->slugifyPath($logicPath);
// detect duplicates
while ($this->resolvePhysicalPath($physicalPath) !== null) {
$physicalPath = $this->slugifyPath($physicalPath, $index++);
}
// insert the new path mapping if requested
if ($store) {
$this->insert($logicPath, $physicalPath);
}
return $physicalPath;
}
private function insert($logicPath, $physicalPath) {
$query = \OC_DB::prepare('INSERT INTO `*PREFIX*file_map`(`logic_path`,`physic_path`) VALUES(?,?)');
$query->execute(array($logicPath, $physicalPath));
}
private function slugifyPath($path, $index=null) {
$pathElements = explode('/', $path);
$sluggedElements = array();
// skip slugging the drive letter on windows - TODO: test if local path
if (strpos(strtolower(php_uname('s')), 'win') !== false) {
$sluggedElements[]= $pathElements[0];
array_shift($pathElements);
}
foreach ($pathElements as $pathElement) {
// TODO: remove file ext before slugify on last element
$sluggedElements[] = self::slugify($pathElement);
}
//
// TODO: add the index before the file extension
//
if ($index !== null) {
$last= end($sluggedElements);
array_pop($sluggedElements);
array_push($sluggedElements, $last.'-'.$index);
}
return implode(DIRECTORY_SEPARATOR, $sluggedElements);
}
/**
* Modifies a string to remove all non ASCII characters and spaces.
*
* @param string $text
* @return string
*/
private function slugify($text)
{
// replace non letter or digits by -
$text = preg_replace('~[^\\pL\d]+~u', '-', $text);
// trim
$text = trim($text, '-');
// transliterate
if (function_exists('iconv')) {
$text = iconv('utf-8', 'us-ascii//TRANSLIT', $text);
}
// lowercase
$text = strtolower($text);
// remove unwanted characters
$text = preg_replace('~[^-\w]+~', '', $text);
if (empty($text))
{
// TODO: we better generate a guid in this case
return 'n-a';
}
return $text;
}
}

View file

@ -8,6 +8,10 @@
namespace OC\Files\Storage;
if (\OC_Util::runningOnWindows()) {
require_once 'mappedlocal.php';
} else {
/**
* for local filestore, we only have to map the paths
*/
@ -245,3 +249,4 @@ class Local extends \OC\Files\Storage\Common{
return $this->filemtime($path)>$time;
}
}
}

View file

@ -0,0 +1,335 @@
<?php
/**
* Copyright (c) 2012 Robin Appelman <icewind@owncloud.com>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/
namespace OC\Files\Storage;
/**
* for local filestore, we only have to map the paths
*/
class Local extends \OC\Files\Storage\Common{
protected $datadir;
private $mapper;
public function __construct($arguments) {
$this->datadir=$arguments['datadir'];
if(substr($this->datadir, -1)!=='/') {
$this->datadir.='/';
}
$this->mapper= new \OC\Files\Mapper();
}
public function __destruct() {
if (defined('PHPUNIT_RUN')) {
$this->mapper->removePath($this->datadir, true, true);
}
}
public function getId(){
return 'local::'.$this->datadir;
}
public function mkdir($path) {
return @mkdir($this->buildPath($path));
}
public function rmdir($path) {
if ($result = @rmdir($this->buildPath($path))) {
$this->cleanMapper($path);
}
return $result;
}
public function opendir($path) {
$files = array('.', '..');
$physicalPath= $this->buildPath($path);
$logicalPath = $this->mapper->physicalToLogic($physicalPath);
$dh = opendir($physicalPath);
while ($file = readdir($dh)) {
if ($file === '.' or $file === '..') {
continue;
}
$logicalFilePath = $this->mapper->physicalToLogic($physicalPath.DIRECTORY_SEPARATOR.$file);
$file= $this->mapper->stripRootFolder($logicalFilePath, $logicalPath);
$file = $this->stripLeading($file);
$files[]= $file;
}
\OC\Files\Stream\Dir::register('local-win32'.$path, $files);
return opendir('fakedir://local-win32'.$path);
}
public function is_dir($path) {
if(substr($path,-1)=='/') {
$path=substr($path, 0, -1);
}
return is_dir($this->buildPath($path));
}
public function is_file($path) {
return is_file($this->buildPath($path));
}
public function stat($path) {
$fullPath = $this->buildPath($path);
$statResult = stat($fullPath);
if ($statResult['size'] < 0) {
$size = self::getFileSizeFromOS($fullPath);
$statResult['size'] = $size;
$statResult[7] = $size;
}
return $statResult;
}
public function filetype($path) {
$filetype=filetype($this->buildPath($path));
if($filetype=='link') {
$filetype=filetype(realpath($this->buildPath($path)));
}
return $filetype;
}
public function filesize($path) {
if($this->is_dir($path)) {
return 0;
}else{
$fullPath = $this->buildPath($path);
$fileSize = filesize($fullPath);
if ($fileSize < 0) {
return self::getFileSizeFromOS($fullPath);
}
return $fileSize;
}
}
public function isReadable($path) {
return is_readable($this->buildPath($path));
}
public function isUpdatable($path) {
return is_writable($this->buildPath($path));
}
public function file_exists($path) {
return file_exists($this->buildPath($path));
}
public function filemtime($path) {
return filemtime($this->buildPath($path));
}
public function touch($path, $mtime=null) {
// sets the modification time of the file to the given value.
// If mtime is nil the current time is set.
// note that the access time of the file always changes to the current time.
if(!is_null($mtime)) {
$result=touch( $this->buildPath($path), $mtime );
}else{
$result=touch( $this->buildPath($path));
}
if( $result ) {
clearstatcache( true, $this->buildPath($path) );
}
return $result;
}
public function file_get_contents($path) {
return file_get_contents($this->buildPath($path));
}
public function file_put_contents($path, $data) {//trigger_error("$path = ".var_export($path, 1));
return file_put_contents($this->buildPath($path), $data);
}
public function unlink($path) {
return $this->delTree($path);
}
public function rename($path1, $path2) {
if (!$this->isUpdatable($path1)) {
\OC_Log::write('core','unable to rename, file is not writable : '.$path1,\OC_Log::ERROR);
return false;
}
if(! $this->file_exists($path1)) {
\OC_Log::write('core','unable to rename, file does not exists : '.$path1,\OC_Log::ERROR);
return false;
}
$physicPath1 = $this->buildPath($path1);
$physicPath2 = $this->buildPath($path2);
if($return=rename($physicPath1, $physicPath2)) {
// mapper needs to create copies or all children
$this->copyMapping($path1, $path2);
$this->cleanMapper($physicPath1, false, true);
}
return $return;
}
public function copy($path1, $path2) {
if($this->is_dir($path2)) {
if(!$this->file_exists($path2)) {
$this->mkdir($path2);
}
$source=substr($path1, strrpos($path1, '/')+1);
$path2.=$source;
}
if($return=copy($this->buildPath($path1), $this->buildPath($path2))) {
// mapper needs to create copies or all children
$this->copyMapping($path1, $path2);
}
return $return;
}
public function fopen($path, $mode) {
if($return=fopen($this->buildPath($path), $mode)) {
switch($mode) {
case 'r':
break;
case 'r+':
case 'w+':
case 'x+':
case 'a+':
break;
case 'w':
case 'x':
case 'a':
break;
}
}
return $return;
}
public function getMimeType($path) {
if($this->isReadable($path)) {
return \OC_Helper::getMimeType($this->buildPath($path));
}else{
return false;
}
}
private function delTree($dir, $isLogicPath=true) {
$dirRelative=$dir;
if ($isLogicPath) {
$dir=$this->buildPath($dir);
}
if (!file_exists($dir)) {
return true;
}
if (!is_dir($dir) || is_link($dir)) {
if($return=unlink($dir)) {
$this->cleanMapper($dir, false);
return $return;
}
}
foreach (scandir($dir) as $item) {
if ($item == '.' || $item == '..') {
continue;
}
if(is_file($dir.'/'.$item)) {
if(unlink($dir.'/'.$item)) {
$this->cleanMapper($dir.'/'.$item, false);
}
}elseif(is_dir($dir.'/'.$item)) {
if (!$this->delTree($dir. "/" . $item, false)) {
return false;
};
}
}
if($return=rmdir($dir)) {
$this->cleanMapper($dir, false);
}
return $return;
}
private static function getFileSizeFromOS($fullPath) {
$name = strtolower(php_uname('s'));
// Windows OS: we use COM to access the filesystem
if (strpos($name, 'win') !== false) {
if (class_exists('COM')) {
$fsobj = new \COM("Scripting.FileSystemObject");
$f = $fsobj->GetFile($fullPath);
return $f->Size;
}
} else if (strpos($name, 'bsd') !== false) {
if (\OC_Helper::is_function_enabled('exec')) {
return (float)exec('stat -f %z ' . escapeshellarg($fullPath));
}
} else if (strpos($name, 'linux') !== false) {
if (\OC_Helper::is_function_enabled('exec')) {
return (float)exec('stat -c %s ' . escapeshellarg($fullPath));
}
} else {
\OC_Log::write('core', 'Unable to determine file size of "'.$fullPath.'". Unknown OS: '.$name, \OC_Log::ERROR);
}
return 0;
}
public function hash($path, $type, $raw=false) {
return hash_file($type, $this->buildPath($path), $raw);
}
public function free_space($path) {
return @disk_free_space($this->buildPath($path));
}
public function search($query) {
return $this->searchInDir($query);
}
public function getLocalFile($path) {
return $this->buildPath($path);
}
public function getLocalFolder($path) {
return $this->buildPath($path);
}
protected function searchInDir($query, $dir='', $isLogicPath=true) {
$files=array();
$physicalDir = $this->buildPath($dir);
foreach (scandir($physicalDir) as $item) {
if ($item == '.' || $item == '..')
continue;
$physicalItem = $this->mapper->physicalToLogic($physicalDir.DIRECTORY_SEPARATOR.$item);
$item = substr($physicalItem, strlen($physicalDir)+1);
if(strstr(strtolower($item), strtolower($query)) !== false) {
$files[]=$dir.'/'.$item;
}
if(is_dir($physicalItem)) {
$files=array_merge($files, $this->searchInDir($query, $physicalItem, false));
}
}
return $files;
}
/**
* check if a file or folder has been updated since $time
* @param string $path
* @param int $time
* @return bool
*/
public function hasUpdated($path, $time) {
return $this->filemtime($path)>$time;
}
private function buildPath($path, $create=true) {
$path = $this->stripLeading($path);
$fullPath = $this->datadir.$path;
return $this->mapper->logicToPhysical($fullPath, $create);
}
private function cleanMapper($path, $isLogicPath=true, $recursive=true) {
$fullPath = $path;
if ($isLogicPath) {
$fullPath = $this->datadir.$path;
}
$this->mapper->removePath($fullPath, $isLogicPath, $recursive);
}
private function copyMapping($path1, $path2) {
$path1 = $this->stripLeading($path1);
$path2 = $this->stripLeading($path2);
$fullPath1 = $this->datadir.$path1;
$fullPath2 = $this->datadir.$path2;
$this->mapper->copy($fullPath1, $fullPath2);
}
private function stripLeading($path) {
if(strpos($path, '/') === 0) {
$path = substr($path, 1);
}
return $path;
}
}

View file

@ -21,6 +21,7 @@ class Temporary extends Local{
}
public function __destruct() {
parent::__destruct();
$this->cleanUp();
}
}

View file

@ -146,10 +146,19 @@ abstract class Storage extends \PHPUnit_Framework_TestCase {
$localFolder = $this->instance->getLocalFolder('/folder');
$this->assertTrue(is_dir($localFolder));
$this->assertTrue(file_exists($localFolder . '/lorem.txt'));
$this->assertEquals(file_get_contents($localFolder . '/lorem.txt'), file_get_contents($textFile));
$this->assertEquals(file_get_contents($localFolder . '/bar.txt'), 'asd');
$this->assertEquals(file_get_contents($localFolder . '/recursive/file.txt'), 'foo');
// test below require to use instance->getLocalFile because the physical storage might be different
$localFile = $this->instance->getLocalFile('/folder/lorem.txt');
$this->assertTrue(file_exists($localFile));
$this->assertEquals(file_get_contents($localFile), file_get_contents($textFile));
$localFile = $this->instance->getLocalFile('/folder/bar.txt');
$this->assertTrue(file_exists($localFile));
$this->assertEquals(file_get_contents($localFile), 'asd');
$localFile = $this->instance->getLocalFile('/folder/recursive/file.txt');
$this->assertTrue(file_exists($localFile));
$this->assertEquals(file_get_contents($localFile), 'foo');
}
public function testStat() {