reworking and extending \OC\Files\Storage\AmazonS3
This commit is contained in:
Christian Berendt
parent
32597c380a
commit
8e5474394e
1 changed files with 389 additions and 143 deletions
|
|
@ -4,7 +4,9 @@
|
|||
* ownCloud
|
||||
*
|
||||
* @author Michael Gapczynski
|
||||
* @author Christian Berendt
|
||||
* @copyright 2012 Michael Gapczynski mtgap@owncloud.com
|
||||
* @copyright 2013 Christian Berendt berendt@b1-systems.de
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
|
||||
|
|
@ -22,164 +24,286 @@
|
|||
|
||||
namespace OC\Files\Storage;
|
||||
|
||||
require_once 'aws-sdk/sdk.class.php';
|
||||
set_include_path(get_include_path() . PATH_SEPARATOR .
|
||||
\OC_App::getAppPath('files_external') . '/3rdparty/aws-sdk-php');
|
||||
require 'aws.phar';
|
||||
|
||||
use Aws\S3\S3Client;
|
||||
use Aws\S3\Exception\S3Exception;
|
||||
|
||||
class AmazonS3 extends \OC\Files\Storage\Common {
|
||||
|
||||
private $s3;
|
||||
private $connection;
|
||||
private $bucket;
|
||||
private $objects = array();
|
||||
private $id;
|
||||
private static $tmpFiles = array();
|
||||
|
||||
private static $tempFiles = array();
|
||||
private function normalizePath($path) {
|
||||
if (substr($path, 0, 1) == '/') {
|
||||
$path = substr($path, 1);
|
||||
}
|
||||
|
||||
// TODO Update to new AWS SDK
|
||||
if (substr($path, -1) == '/') {
|
||||
$path = substr($path, 0, -1);
|
||||
}
|
||||
|
||||
if ( ! $path) {
|
||||
$path = '.';
|
||||
}
|
||||
|
||||
return $path;
|
||||
}
|
||||
|
||||
public function __construct($params) {
|
||||
if (isset($params['key']) && isset($params['secret']) && isset($params['bucket'])) {
|
||||
$this->id = 'amazon::' . $params['key'] . md5($params['secret']);
|
||||
$this->s3 = new \AmazonS3(array('key' => $params['key'], 'secret' => $params['secret']));
|
||||
$this->bucket = $params['bucket'];
|
||||
} else {
|
||||
if ( ! isset($params['key']) || ! isset($params['secret']) || ! isset($params['bucket'])) {
|
||||
throw new \Exception();
|
||||
}
|
||||
}
|
||||
|
||||
private function getObject($path) {
|
||||
if (array_key_exists($path, $this->objects)) {
|
||||
return $this->objects[$path];
|
||||
$this->id = 'amazon::' . $params['key'] . md5($params['secret']);
|
||||
$this->bucket = $params['bucket'];
|
||||
$scheme = ($params['use_ssl'] === 'false') ? 'http' : 'https';
|
||||
|
||||
if (isset($params['hostname']) && isset($params['port'])) {
|
||||
$base_url = $scheme.'://'.$params['hostname'].':'.$params['port'].'/';
|
||||
$this->connection = S3Client::factory(array(
|
||||
'key' => $params['key'],
|
||||
'secret' => $params['secret'],
|
||||
'base_url' => $base_url
|
||||
));
|
||||
} else {
|
||||
$response = $this->s3->get_object_metadata($this->bucket, $path);
|
||||
if ($response) {
|
||||
$this->objects[$path] = $response;
|
||||
return $response;
|
||||
// This object could be a folder, a '/' must be at the end of the path
|
||||
} else if (substr($path, -1) != '/') {
|
||||
$response = $this->s3->get_object_metadata($this->bucket, $path . '/');
|
||||
if ($response) {
|
||||
$this->objects[$path] = $response;
|
||||
return $response;
|
||||
}
|
||||
}
|
||||
$this->connection = S3Client::factory(array(
|
||||
'key' => $params['key'],
|
||||
'secret' => $params['secret'],
|
||||
'scheme' => $scheme,
|
||||
'region' => $region
|
||||
));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public function getId() {
|
||||
return $this->id;
|
||||
if ( ! $this->connection->doesBucketExist($this->bucket)) {
|
||||
$result = $this->connection->createBucket(array(
|
||||
'Bucket' => $this->bucket
|
||||
));
|
||||
}
|
||||
|
||||
if ( ! $this->file_exists('.')) {
|
||||
$result = $this->connection->putObject(array(
|
||||
'Bucket' => $this->bucket,
|
||||
'Key' => '.',
|
||||
'Body' => '',
|
||||
'ContentType' => 'httpd/unix-directory',
|
||||
'ContentLength' => 0
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
public function mkdir($path) {
|
||||
// Folders in Amazon S3 are 0 byte objects with a '/' at the end of the name
|
||||
if (substr($path, -1) != '/') {
|
||||
$path .= '/';
|
||||
}
|
||||
$response = $this->s3->create_object($this->bucket, $path, array('body' => ''));
|
||||
return $response->isOK();
|
||||
}
|
||||
$path = $this->normalizePath($path);
|
||||
|
||||
public function rmdir($path) {
|
||||
if (substr($path, -1) != '/') {
|
||||
$path .= '/';
|
||||
if($this->is_dir($path)) {
|
||||
return false;
|
||||
}
|
||||
return $this->unlink($path);
|
||||
}
|
||||
|
||||
public function opendir($path) {
|
||||
if ($path == '' || $path == '/') {
|
||||
// Use the '/' delimiter to only fetch objects inside the folder
|
||||
$opt = array('delimiter' => '/');
|
||||
} else {
|
||||
if (substr($path, -1) != '/') {
|
||||
$path .= '/';
|
||||
}
|
||||
$opt = array('delimiter' => '/', 'prefix' => $path);
|
||||
try {
|
||||
$result = $this->connection->putObject(array(
|
||||
'Bucket' => $this->bucket,
|
||||
'Key' => $path . '/',
|
||||
'Body' => '',
|
||||
'ContentType' => 'httpd/unix-directory',
|
||||
'ContentLength' => 0
|
||||
));
|
||||
} catch (S3Exception $e) {
|
||||
return false;
|
||||
}
|
||||
$response = $this->s3->list_objects($this->bucket, $opt);
|
||||
if ($response->isOK()) {
|
||||
$files = array();
|
||||
foreach ($response->body->Contents as $object) {
|
||||
// The folder being opened also shows up in the list of objects, don't add it to the files
|
||||
if ($object->Key != $path) {
|
||||
$files[] = basename($object->Key);
|
||||
}
|
||||
}
|
||||
// Sub folders show up as CommonPrefixes
|
||||
foreach ($response->body->CommonPrefixes as $object) {
|
||||
$files[] = basename($object->Prefix);
|
||||
}
|
||||
\OC\Files\Stream\Dir::register('amazons3' . $path, $files);
|
||||
return opendir('fakedir://amazons3' . $path);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public function stat($path) {
|
||||
if ($path == '' || $path == '/') {
|
||||
$stat['size'] = $this->s3->get_bucket_filesize($this->bucket);
|
||||
$stat['atime'] = time();
|
||||
$stat['mtime'] = $stat['atime'];
|
||||
} else if ($object = $this->getObject($path)) {
|
||||
$stat['size'] = $object['Size'];
|
||||
$stat['atime'] = time();
|
||||
$stat['mtime'] = strtotime($object['LastModified']);
|
||||
}
|
||||
if (isset($stat)) {
|
||||
return $stat;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public function filetype($path) {
|
||||
if ($path == '' || $path == '/') {
|
||||
return 'dir';
|
||||
} else {
|
||||
$object = $this->getObject($path);
|
||||
if ($object) {
|
||||
// Amazon S3 doesn't have typical folders, this is an alternative method to detect a folder
|
||||
if (substr($object['Key'], -1) == '/' && $object['Size'] == 0) {
|
||||
return 'dir';
|
||||
} else {
|
||||
return 'file';
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public function isReadable($path) {
|
||||
// TODO Check acl and determine who grantee is
|
||||
return true;
|
||||
}
|
||||
|
||||
public function isUpdatable($path) {
|
||||
// TODO Check acl and determine who grantee is
|
||||
return true;
|
||||
}
|
||||
|
||||
public function file_exists($path) {
|
||||
if ($this->filetype($path) == 'dir' && substr($path, -1) != '/') {
|
||||
$path = $this->normalizePath($path);
|
||||
|
||||
if ( ! $path) {
|
||||
$path = '.';
|
||||
} else if ($path != '.' && $this->is_dir($path)) {
|
||||
$path .= '/';
|
||||
}
|
||||
return $this->s3->if_object_exists($this->bucket, $path);
|
||||
|
||||
try {
|
||||
$result = $this->connection->doesObjectExist(
|
||||
$this->bucket,
|
||||
$path
|
||||
);
|
||||
} catch (S3Exception $e) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
|
||||
public function rmdir($path) {
|
||||
$path = $this->normalizePath($path);
|
||||
|
||||
if ( ! $this->file_exists($path)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$dh = $this->opendir($path);
|
||||
while ($file = readdir($dh)) {
|
||||
if ($file == '.' || $file != '..') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($this->is_dir(stripcslashes($file))) {
|
||||
$this->rmdir(stripcslashes($file));
|
||||
} else {
|
||||
$this->unlink(stripcslashes($file));
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
$result = $this->connection->deleteObject(array(
|
||||
'Bucket' => $this->bucket,
|
||||
'Key' => $path . '/'
|
||||
));
|
||||
} catch (S3Exception $e) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public function opendir($path) {
|
||||
$path = $this->normalizePath($path);
|
||||
|
||||
if ($path == '.') {
|
||||
$path = '';
|
||||
} else if ($path) {
|
||||
$path .= '/';
|
||||
}
|
||||
|
||||
try {
|
||||
$files = array();
|
||||
$result = $this->connection->getIterator('ListObjects', array(
|
||||
'Bucket' => $this->bucket,
|
||||
'Delimiter' => '/',
|
||||
'Prefix' => $path
|
||||
), array('return_prefixes' => true));
|
||||
|
||||
foreach ($result as $object) {
|
||||
$file = basename(
|
||||
isset($object['Key']) ? $object['Key'] : $object['Prefix']
|
||||
);
|
||||
|
||||
if ( $file != basename($path)) {
|
||||
$files[] = $file;
|
||||
}
|
||||
}
|
||||
|
||||
\OC\Files\Stream\Dir::register('amazons3' . $path, $files);
|
||||
|
||||
return opendir('fakedir://amazons3' . $path);
|
||||
} catch (S3Exception $e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public function stat($path) {
|
||||
$path = $this->normalizePath($path);
|
||||
|
||||
try {
|
||||
if ($this->is_dir($path) && $path != '.') {
|
||||
$path .= '/';
|
||||
}
|
||||
|
||||
$result = $this->connection->headObject(array(
|
||||
'Bucket' => $this->bucket,
|
||||
'Key' => $path
|
||||
));
|
||||
|
||||
$stat = array();
|
||||
$stat['size'] = $result['ContentLength'] ? $result['ContentLength'] : 0;
|
||||
if ($result['Metadata']['lastmodified']) {
|
||||
$stat['mtime'] = strtotime($result['Metadata']['lastmodified']);
|
||||
} else {
|
||||
$stat['mtime'] = strtotime($result['LastModified']);
|
||||
}
|
||||
$stat['atime'] = time();
|
||||
|
||||
return $stat;
|
||||
} catch(S3Exception $e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public function filetype($path) {
|
||||
$path = $this->normalizePath($path);
|
||||
|
||||
try {
|
||||
if ($path != '.' && $this->connection->doesObjectExist($this->bucket, $path)) {
|
||||
return 'file';
|
||||
}
|
||||
|
||||
if ($path != '.') {
|
||||
$path .= '/';
|
||||
}
|
||||
|
||||
if ($this->connection->doesObjectExist($this->bucket, $path)) {
|
||||
return 'dir';
|
||||
}
|
||||
} catch (S3Exception $e) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public function isReadable($path) {
|
||||
return true;
|
||||
}
|
||||
|
||||
public function isUpdatable($path) {
|
||||
return true;
|
||||
}
|
||||
|
||||
public function unlink($path) {
|
||||
$response = $this->s3->delete_object($this->bucket, $path);
|
||||
return $response->isOK();
|
||||
$path = $this->normalizePath($path);
|
||||
|
||||
if ( ! $this->file_exists($path)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
$result = $this->connection->deleteObject(array(
|
||||
'Bucket' => $this->bucket,
|
||||
'Key' => $path
|
||||
));
|
||||
} catch (S3Exception $e) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->touch(dirname($path));
|
||||
return true;
|
||||
}
|
||||
|
||||
public function fopen($path, $mode) {
|
||||
$path = $this->normalizePath($path);
|
||||
|
||||
switch ($mode) {
|
||||
case 'r':
|
||||
case 'rb':
|
||||
$tmpFile = \OC_Helper::tmpFile();
|
||||
$handle = fopen($tmpFile, 'w');
|
||||
$response = $this->s3->get_object($this->bucket, $path, array('fileDownload' => $handle));
|
||||
if ($response->isOK()) {
|
||||
return fopen($tmpFile, 'r');
|
||||
self::$tmpFiles[$tmpFile] = $path;
|
||||
|
||||
try {
|
||||
$result = $this->connection->getObject(array(
|
||||
'Bucket' => $this->bucket,
|
||||
'Key' => $path,
|
||||
'SaveAs' => $tmpFile
|
||||
));
|
||||
} catch (S3Exception $e) {
|
||||
return false;
|
||||
}
|
||||
break;
|
||||
|
||||
return fopen($tmpFile, 'r');
|
||||
case 'w':
|
||||
case 'wb':
|
||||
case 'a':
|
||||
|
|
@ -203,45 +327,139 @@ class AmazonS3 extends \OC\Files\Storage\Common {
|
|||
$source = $this->fopen($path, 'r');
|
||||
file_put_contents($tmpFile, $source);
|
||||
}
|
||||
self::$tempFiles[$tmpFile] = $path;
|
||||
self::$tmpFiles[$tmpFile] = $path;
|
||||
|
||||
return fopen('close://' . $tmpFile, $mode);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public function writeBack($tmpFile) {
|
||||
if (isset(self::$tempFiles[$tmpFile])) {
|
||||
$handle = fopen($tmpFile, 'r');
|
||||
$response = $this->s3->create_object($this->bucket,
|
||||
self::$tempFiles[$tmpFile],
|
||||
array('fileUpload' => $handle));
|
||||
if ($response->isOK()) {
|
||||
unlink($tmpFile);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function getMimeType($path) {
|
||||
if ($this->filetype($path) == 'dir') {
|
||||
$path = $this->normalizePath($path);
|
||||
|
||||
if ($this->is_dir($path)) {
|
||||
return 'httpd/unix-directory';
|
||||
} else {
|
||||
$object = $this->getObject($path);
|
||||
if ($object) {
|
||||
return $object['ContentType'];
|
||||
} else if ($this->file_exists($path)) {
|
||||
try {
|
||||
$result = $this->connection->headObject(array(
|
||||
'Bucket' => $this->bucket,
|
||||
'Key' => $path
|
||||
));
|
||||
} catch (S3Exception $e) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return $result['ContentType'];
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public function touch($path, $mtime = null) {
|
||||
if (is_null($mtime)) {
|
||||
$mtime = time();
|
||||
$path = $this->normalizePath($path);
|
||||
|
||||
$metadata = array();
|
||||
if ( ! is_null($mtime)) {
|
||||
$metadata = array('lastmodified' => $mtime);
|
||||
}
|
||||
if ($this->filetype($path) == 'dir' && substr($path, -1) != '/') {
|
||||
$path .= '/';
|
||||
|
||||
try {
|
||||
if ($this->file_exists($path)) {
|
||||
if ($this->is_dir($path) && $path != '.') {
|
||||
$path .= '/';
|
||||
}
|
||||
$result = $this->connection->copyObject(array(
|
||||
'Bucket' => $this->bucket,
|
||||
'Key' => $path,
|
||||
'Metadata' => $metadata,
|
||||
'CopySource' => $this->bucket . '/' . $path
|
||||
));
|
||||
} else {
|
||||
$result = $this->connection->putObject(array(
|
||||
'Bucket' => $this->bucket,
|
||||
'Key' => $path,
|
||||
'Metadata' => $metadata
|
||||
));
|
||||
}
|
||||
} catch (S3Exception $e) {
|
||||
return false;
|
||||
}
|
||||
$response = $this->s3->update_object($this->bucket, $path, array('meta' => array('LastModified' => $mtime)));
|
||||
return $response->isOK();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public function copy($path1, $path2) {
|
||||
$path1 = $this->normalizePath($path1);
|
||||
$path2 = $this->normalizePath($path2);
|
||||
|
||||
if ($this->is_file($path1)) {
|
||||
try {
|
||||
$result = $this->connection->copyObject(array(
|
||||
'Bucket' => $this->bucket,
|
||||
'Key' => $path2,
|
||||
'CopySource' => $this->bucket . '/' . $path1
|
||||
));
|
||||
} catch (S3Exception $e) {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
if ($this->file_exists($path2)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
$result = $this->connection->copyObject(array(
|
||||
'Bucket' => $this->bucket,
|
||||
'Key' => $path2 . '/',
|
||||
'CopySource' => $this->bucket . '/' . $path1 . '/'
|
||||
));
|
||||
} catch (S3Exception $e) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$dh = $this->opendir($path1);
|
||||
while ($file = readdir($dh)) {
|
||||
if ($file == '.' || $file == '..') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$source = $path1 . '/' . $file;
|
||||
$target = $path2 . '/' . $file;
|
||||
$this->copy($source, $target);
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public function rename($path1, $path2) {
|
||||
$path1 = $this->normalizePath($path1);
|
||||
$path2 = $this->normalizePath($path2);
|
||||
|
||||
if ($this->is_file($path1)) {
|
||||
if ($this->copy($path1, $path2) == false) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if ($this->unlink($path1) == false) {
|
||||
$this->unlink($path2);
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
if ($this->file_exists($path2)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if ($this->copy($path1, $path2) == false) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if($this->rmdir($path1) == false) {
|
||||
$this->rmdir($path2);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public function test() {
|
||||
|
|
@ -252,4 +470,32 @@ class AmazonS3 extends \OC\Files\Storage\Common {
|
|||
return false;
|
||||
}
|
||||
|
||||
public function getId() {
|
||||
return $this->id;
|
||||
}
|
||||
|
||||
public function getConnection() {
|
||||
return $this->connection;
|
||||
}
|
||||
|
||||
public function writeBack($tmpFile) {
|
||||
if ( ! isset(self::$tmpFiles[$tmpFile])) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
$result= $this->connection->putObject(array(
|
||||
'Bucket' => $this->bucket,
|
||||
'Key' => self::$tmpFiles[$tmpFile],
|
||||
'SourceFile' => $tmpFile,
|
||||
'ContentType' => \OC_Helper::getMimeType($tmpFile),
|
||||
'ContentLength' => filesize($tmpFile)
|
||||
));
|
||||
|
||||
unlink($tmpFile);
|
||||
$this->touch(dirname(self::$tmpFiles[$tmpFile]));
|
||||
} catch (S3Exception $e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue