607 lines
14 KiB
PHP
607 lines
14 KiB
PHP
<?php
|
|
|
|
/**
|
|
* ownCloud
|
|
*
|
|
* @author Michael Gapczynski
|
|
* @author Christian Berendt
|
|
* @copyright 2012 Michael Gapczynski mtgap@owncloud.com
|
|
* @copyright 2013 Christian Berendt berendt@b1-systems.de
|
|
*
|
|
* This library is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
|
|
* License as published by the Free Software Foundation; either
|
|
* version 3 of the License, or any later version.
|
|
*
|
|
* This library is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU AFFERO GENERAL PUBLIC LICENSE for more details.
|
|
*
|
|
* You should have received a copy of the GNU Affero General Public
|
|
* License along with this library. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
namespace OC\Files\Storage;
|
|
|
|
set_include_path(get_include_path() . PATH_SEPARATOR .
|
|
\OC_App::getAppPath('files_external') . '/3rdparty/aws-sdk-php');
|
|
require 'aws-autoloader.php';
|
|
|
|
use Aws\S3\S3Client;
|
|
use Aws\S3\Exception\S3Exception;
|
|
|
|
class AmazonS3 extends \OC\Files\Storage\Common {
|
|
|
|
/**
|
|
* @var \Aws\S3\S3Client
|
|
*/
|
|
private $connection;
|
|
/**
|
|
* @var string
|
|
*/
|
|
private $bucket;
|
|
/**
|
|
* @var array
|
|
*/
|
|
private static $tmpFiles = array();
|
|
/**
|
|
* @var bool
|
|
*/
|
|
private $test = false;
|
|
/**
|
|
* @var int
|
|
*/
|
|
private $timeout = 15;
|
|
/**
|
|
* @var int in seconds
|
|
*/
|
|
private $rescanDelay = 10;
|
|
|
|
/**
|
|
* @param string $path
|
|
* @return string correctly encoded path
|
|
*/
|
|
private function normalizePath($path) {
|
|
$path = trim($path, '/');
|
|
|
|
if (!$path) {
|
|
$path = '.';
|
|
}
|
|
|
|
return $path;
|
|
}
|
|
|
|
/**
|
|
* when running the tests wait to let the buckets catch up
|
|
*/
|
|
private function testTimeout() {
|
|
if ($this->test) {
|
|
sleep($this->timeout);
|
|
}
|
|
}
|
|
|
|
private function isRoot($path) {
|
|
return $path === '.';
|
|
}
|
|
|
|
private function cleanKey($path) {
|
|
if ($this->isRoot($path)) {
|
|
return '/';
|
|
}
|
|
return $path;
|
|
}
|
|
|
|
public function __construct($params) {
|
|
if (empty($params['key']) || empty($params['secret']) || empty($params['bucket'])) {
|
|
throw new \Exception("Access Key, Secret and Bucket have to be configured.");
|
|
}
|
|
|
|
$this->id = 'amazon::' . $params['bucket'];
|
|
$this->updateLegacyId($params);
|
|
|
|
$this->bucket = $params['bucket'];
|
|
$scheme = ($params['use_ssl'] === 'false') ? 'http' : 'https';
|
|
$this->test = isset($params['test']);
|
|
$this->timeout = (!isset($params['timeout'])) ? 15 : $params['timeout'];
|
|
$this->rescanDelay = (!isset($params['rescanDelay'])) ? 10 : $params['rescanDelay'];
|
|
$params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region'];
|
|
$params['hostname'] = empty($params['hostname']) ? 's3.amazonaws.com' : $params['hostname'];
|
|
if (!isset($params['port']) || $params['port'] === '') {
|
|
$params['port'] = ($params['use_ssl'] === 'false') ? 80 : 443;
|
|
}
|
|
$base_url = $scheme . '://' . $params['hostname'] . ':' . $params['port'] . '/';
|
|
|
|
$this->connection = S3Client::factory(array(
|
|
'key' => $params['key'],
|
|
'secret' => $params['secret'],
|
|
'base_url' => $base_url,
|
|
'region' => $params['region']
|
|
));
|
|
|
|
if (!$this->connection->isValidBucketName($this->bucket)) {
|
|
throw new \Exception("The configured bucket name is invalid.");
|
|
}
|
|
|
|
if (!$this->connection->doesBucketExist($this->bucket)) {
|
|
try {
|
|
$this->connection->createBucket(array(
|
|
'Bucket' => $this->bucket
|
|
));
|
|
$this->connection->waitUntilBucketExists(array(
|
|
'Bucket' => $this->bucket,
|
|
'waiter.interval' => 1,
|
|
'waiter.max_attempts' => 15
|
|
));
|
|
$this->testTimeout();
|
|
} catch (S3Exception $e) {
|
|
\OCP\Util::logException('files_external', $e);
|
|
throw new \Exception('Creation of bucket failed. '.$e->getMessage());
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Updates old storage ids (v0.2.1 and older) that are based on key and secret to new ones based on the bucket name.
|
|
* TODO Do this in an update.php. requires iterating over all users and loading the mount.json from their home
|
|
*
|
|
* @param array $params
|
|
*/
|
|
public function updateLegacyId (array $params) {
|
|
$stmt = \OC::$server->getDatabaseConnection()->prepare(
|
|
'UPDATE `*PREFIX*storages` SET `id` = ? WHERE `id` = ?'
|
|
);
|
|
$oldId = 'amazon::' . $params['key'] . md5($params['secret']);
|
|
$stmt->execute(array($this->id, $oldId));
|
|
}
|
|
|
|
/**
|
|
* Remove a file or folder
|
|
*
|
|
* @param string $path
|
|
* @return bool
|
|
*/
|
|
protected function remove($path) {
|
|
// remember fileType to reduce http calls
|
|
$fileType = $this->filetype($path);
|
|
if ($fileType === 'dir') {
|
|
return $this->rmdir($path);
|
|
} else if ($fileType === 'file') {
|
|
return $this->unlink($path);
|
|
} else {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
public function mkdir($path) {
|
|
$path = $this->normalizePath($path);
|
|
|
|
if ($this->is_dir($path)) {
|
|
return false;
|
|
}
|
|
|
|
try {
|
|
$this->connection->putObject(array(
|
|
'Bucket' => $this->bucket,
|
|
'Key' => $path . '/',
|
|
'Body' => '',
|
|
'ContentType' => 'httpd/unix-directory'
|
|
));
|
|
$this->testTimeout();
|
|
} catch (S3Exception $e) {
|
|
\OCP\Util::logException('files_external', $e);
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
public function file_exists($path) {
|
|
return $this->filetype($path) !== false;
|
|
}
|
|
|
|
|
|
public function rmdir($path) {
|
|
$path = $this->normalizePath($path);
|
|
|
|
if ($this->isRoot($path)) {
|
|
return $this->clearBucket();
|
|
}
|
|
|
|
if (!$this->file_exists($path)) {
|
|
return false;
|
|
}
|
|
|
|
return $this->batchDelete($path);
|
|
}
|
|
|
|
protected function clearBucket() {
|
|
try {
|
|
$this->connection->clearBucket($this->bucket);
|
|
return true;
|
|
// clearBucket() is not working with Ceph, so if it fails we try the slower approach
|
|
} catch (\Exception $e) {
|
|
return $this->batchDelete();
|
|
}
|
|
return false;
|
|
}
|
|
|
|
private function batchDelete ($path = null) {
|
|
$params = array(
|
|
'Bucket' => $this->bucket
|
|
);
|
|
if ($path !== null) {
|
|
$params['Prefix'] = $path . '/';
|
|
}
|
|
try {
|
|
// Since there are no real directories on S3, we need
|
|
// to delete all objects prefixed with the path.
|
|
do {
|
|
// instead of the iterator, manually loop over the list ...
|
|
$objects = $this->connection->listObjects($params);
|
|
// ... so we can delete the files in batches
|
|
$this->connection->deleteObjects(array(
|
|
'Bucket' => $this->bucket,
|
|
'Objects' => $objects['Contents']
|
|
));
|
|
$this->testTimeout();
|
|
// we reached the end when the list is no longer truncated
|
|
} while ($objects['IsTruncated']);
|
|
} catch (S3Exception $e) {
|
|
\OCP\Util::logException('files_external', $e);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
public function opendir($path) {
|
|
$path = $this->normalizePath($path);
|
|
|
|
if ($this->isRoot($path)) {
|
|
$path = '';
|
|
} else {
|
|
$path .= '/';
|
|
}
|
|
|
|
try {
|
|
$files = array();
|
|
$result = $this->connection->getIterator('ListObjects', array(
|
|
'Bucket' => $this->bucket,
|
|
'Delimiter' => '/',
|
|
'Prefix' => $path
|
|
), array('return_prefixes' => true));
|
|
|
|
foreach ($result as $object) {
|
|
$file = basename(
|
|
isset($object['Key']) ? $object['Key'] : $object['Prefix']
|
|
);
|
|
|
|
if ($file != basename($path)) {
|
|
$files[] = $file;
|
|
}
|
|
}
|
|
|
|
\OC\Files\Stream\Dir::register('amazons3' . $path, $files);
|
|
|
|
return opendir('fakedir://amazons3' . $path);
|
|
} catch (S3Exception $e) {
|
|
\OCP\Util::logException('files_external', $e);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
public function stat($path) {
|
|
$path = $this->normalizePath($path);
|
|
|
|
try {
|
|
$stat = array();
|
|
if ($this->is_dir($path)) {
|
|
//folders don't really exist
|
|
$stat['size'] = -1; //unknown
|
|
$stat['mtime'] = time() - $this->rescanDelay * 1000;
|
|
} else {
|
|
$result = $this->connection->headObject(array(
|
|
'Bucket' => $this->bucket,
|
|
'Key' => $path
|
|
));
|
|
|
|
$stat['size'] = $result['ContentLength'] ? $result['ContentLength'] : 0;
|
|
if ($result['Metadata']['lastmodified']) {
|
|
$stat['mtime'] = strtotime($result['Metadata']['lastmodified']);
|
|
} else {
|
|
$stat['mtime'] = strtotime($result['LastModified']);
|
|
}
|
|
}
|
|
$stat['atime'] = time();
|
|
|
|
return $stat;
|
|
} catch(S3Exception $e) {
|
|
\OCP\Util::logException('files_external', $e);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
public function filetype($path) {
|
|
$path = $this->normalizePath($path);
|
|
|
|
if ($this->isRoot($path)) {
|
|
return 'dir';
|
|
}
|
|
|
|
try {
|
|
if ($this->connection->doesObjectExist($this->bucket, $path)) {
|
|
return 'file';
|
|
}
|
|
if ($this->connection->doesObjectExist($this->bucket, $path.'/')) {
|
|
return 'dir';
|
|
}
|
|
} catch (S3Exception $e) {
|
|
\OCP\Util::logException('files_external', $e);
|
|
return false;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
public function unlink($path) {
|
|
$path = $this->normalizePath($path);
|
|
|
|
if ($this->is_dir($path)) {
|
|
return $this->rmdir($path);
|
|
}
|
|
|
|
try {
|
|
$this->connection->deleteObject(array(
|
|
'Bucket' => $this->bucket,
|
|
'Key' => $path
|
|
));
|
|
$this->testTimeout();
|
|
} catch (S3Exception $e) {
|
|
\OCP\Util::logException('files_external', $e);
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
public function fopen($path, $mode) {
|
|
$path = $this->normalizePath($path);
|
|
|
|
switch ($mode) {
|
|
case 'r':
|
|
case 'rb':
|
|
$tmpFile = \OC_Helper::tmpFile();
|
|
self::$tmpFiles[$tmpFile] = $path;
|
|
|
|
try {
|
|
$this->connection->getObject(array(
|
|
'Bucket' => $this->bucket,
|
|
'Key' => $path,
|
|
'SaveAs' => $tmpFile
|
|
));
|
|
} catch (S3Exception $e) {
|
|
\OCP\Util::logException('files_external', $e);
|
|
return false;
|
|
}
|
|
|
|
return fopen($tmpFile, 'r');
|
|
case 'w':
|
|
case 'wb':
|
|
case 'a':
|
|
case 'ab':
|
|
case 'r+':
|
|
case 'w+':
|
|
case 'wb+':
|
|
case 'a+':
|
|
case 'x':
|
|
case 'x+':
|
|
case 'c':
|
|
case 'c+':
|
|
if (strrpos($path, '.') !== false) {
|
|
$ext = substr($path, strrpos($path, '.'));
|
|
} else {
|
|
$ext = '';
|
|
}
|
|
$tmpFile = \OC_Helper::tmpFile($ext);
|
|
\OC\Files\Stream\Close::registerCallback($tmpFile, array($this, 'writeBack'));
|
|
if ($this->file_exists($path)) {
|
|
$source = $this->fopen($path, 'r');
|
|
file_put_contents($tmpFile, $source);
|
|
}
|
|
self::$tmpFiles[$tmpFile] = $path;
|
|
|
|
return fopen('close://' . $tmpFile, $mode);
|
|
}
|
|
return false;
|
|
}
|
|
|
|
public function getMimeType($path) {
|
|
$path = $this->normalizePath($path);
|
|
|
|
if ($this->is_dir($path)) {
|
|
return 'httpd/unix-directory';
|
|
} else if ($this->file_exists($path)) {
|
|
try {
|
|
$result = $this->connection->headObject(array(
|
|
'Bucket' => $this->bucket,
|
|
'Key' => $path
|
|
));
|
|
} catch (S3Exception $e) {
|
|
\OCP\Util::logException('files_external', $e);
|
|
return false;
|
|
}
|
|
|
|
return $result['ContentType'];
|
|
}
|
|
return false;
|
|
}
|
|
|
|
public function touch($path, $mtime = null) {
|
|
$path = $this->normalizePath($path);
|
|
|
|
$metadata = array();
|
|
if (!is_null($mtime)) {
|
|
$metadata = array('lastmodified' => $mtime);
|
|
}
|
|
|
|
$fileType = $this->filetype($path);
|
|
try {
|
|
if ($fileType !== false) {
|
|
if ($fileType === 'dir' && ! $this->isRoot($path)) {
|
|
$path .= '/';
|
|
}
|
|
$this->connection->copyObject(array(
|
|
'Bucket' => $this->bucket,
|
|
'Key' => $this->cleanKey($path),
|
|
'Metadata' => $metadata,
|
|
'CopySource' => $this->bucket . '/' . $path
|
|
));
|
|
$this->testTimeout();
|
|
} else {
|
|
$mimeType = \OC_Helper::getMimetypeDetector()->detectPath($path);
|
|
$this->connection->putObject(array(
|
|
'Bucket' => $this->bucket,
|
|
'Key' => $this->cleanKey($path),
|
|
'Metadata' => $metadata,
|
|
'Body' => '',
|
|
'ContentType' => $mimeType
|
|
));
|
|
$this->testTimeout();
|
|
}
|
|
} catch (S3Exception $e) {
|
|
\OCP\Util::logException('files_external', $e);
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
public function copy($path1, $path2) {
|
|
$path1 = $this->normalizePath($path1);
|
|
$path2 = $this->normalizePath($path2);
|
|
|
|
if ($this->is_file($path1)) {
|
|
try {
|
|
$this->connection->copyObject(array(
|
|
'Bucket' => $this->bucket,
|
|
'Key' => $this->cleanKey($path2),
|
|
'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1)
|
|
));
|
|
$this->testTimeout();
|
|
} catch (S3Exception $e) {
|
|
\OCP\Util::logException('files_external', $e);
|
|
return false;
|
|
}
|
|
} else {
|
|
$this->remove($path2);
|
|
|
|
try {
|
|
$this->connection->copyObject(array(
|
|
'Bucket' => $this->bucket,
|
|
'Key' => $path2 . '/',
|
|
'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1 . '/')
|
|
));
|
|
$this->testTimeout();
|
|
} catch (S3Exception $e) {
|
|
\OCP\Util::logException('files_external', $e);
|
|
return false;
|
|
}
|
|
|
|
$dh = $this->opendir($path1);
|
|
if (is_resource($dh)) {
|
|
while (($file = readdir($dh)) !== false) {
|
|
if ($file === '.' || $file === '..') {
|
|
continue;
|
|
}
|
|
|
|
$source = $path1 . '/' . $file;
|
|
$target = $path2 . '/' . $file;
|
|
$this->copy($source, $target);
|
|
}
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
public function rename($path1, $path2) {
|
|
$path1 = $this->normalizePath($path1);
|
|
$path2 = $this->normalizePath($path2);
|
|
|
|
if ($this->is_file($path1)) {
|
|
|
|
if ($this->copy($path1, $path2) === false) {
|
|
return false;
|
|
}
|
|
|
|
if ($this->unlink($path1) === false) {
|
|
$this->unlink($path2);
|
|
return false;
|
|
}
|
|
} else {
|
|
|
|
if ($this->copy($path1, $path2) === false) {
|
|
return false;
|
|
}
|
|
|
|
if ($this->rmdir($path1) === false) {
|
|
$this->rmdir($path2);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
public function test() {
|
|
$test = $this->connection->getBucketAcl(array(
|
|
'Bucket' => $this->bucket,
|
|
));
|
|
if (isset($test) && !is_null($test->getPath('Owner/ID'))) {
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
public function getId() {
|
|
return $this->id;
|
|
}
|
|
|
|
public function getConnection() {
|
|
return $this->connection;
|
|
}
|
|
|
|
public function writeBack($tmpFile) {
|
|
if (!isset(self::$tmpFiles[$tmpFile])) {
|
|
return false;
|
|
}
|
|
|
|
try {
|
|
$this->connection->putObject(array(
|
|
'Bucket' => $this->bucket,
|
|
'Key' => $this->cleanKey(self::$tmpFiles[$tmpFile]),
|
|
'SourceFile' => $tmpFile,
|
|
'ContentType' => \OC_Helper::getMimeType($tmpFile),
|
|
'ContentLength' => filesize($tmpFile)
|
|
));
|
|
$this->testTimeout();
|
|
|
|
unlink($tmpFile);
|
|
} catch (S3Exception $e) {
|
|
\OCP\Util::logException('files_external', $e);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* check if curl is installed
|
|
*/
|
|
public static function checkDependencies() {
|
|
if (function_exists('curl_init')) {
|
|
return true;
|
|
} else {
|
|
return array('curl');
|
|
}
|
|
}
|
|
|
|
}
|