wbrawner/server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

adjust swift external storage to new openstack sdk

Browse source 
Signed-off-by: Robin Appelman <robin@icewind.nl>
This commit is contained in:
Robin Appelman 2018-02-12 15:08:27 +01:00 committed by Roeland Jago Douma
parent 3192efc415
commit 817f2f8f49
No known key found for this signature in database
GPG key ID: F941078878347C0C
4 changed files with 150 additions and 198 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

247
apps/files_external/lib/Lib/Storage/Swift.php
View file

@ -1,4 +1,5 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2016, ownCloud, Inc.
*
@ -39,31 +40,21 @@
namespace OCA\Files_External\Lib\Storage;
use Guzzle\Http\Url;
use Guzzle\Http\Exception\ClientErrorResponseException;
use GuzzleHttp\Psr7\Uri;
use Icewind\Streams\CallbackWrapper;
use Icewind\Streams\IteratorDirectory;
use Icewind\Streams\RetryWrapper;
use OpenCloud;
use OpenCloud\Common\Exceptions;
use OpenCloud\OpenStack;
use OpenCloud\Rackspace;
use OpenCloud\ObjectStore\Resource\DataObject;
use OC\Files\ObjectStore\SwiftFactory;
use OCP\Files\StorageBadConfigException;
use OpenStack\Common\Error\BadResponseError;
use OpenStack\ObjectStore\v1\Models\StorageObject;
class Swift extends \OC\Files\Storage\Common {
/** @var SwiftFactory */
private $connectionFactory;
/**
* @var \OpenCloud\ObjectStore\Service
*/
private $connection;
/**
* @var \OpenCloud\ObjectStore\Resource\Container
* @var \OpenStack\ObjectStore\v1\Models\Container
*/
private $container;
/**
* @var \OpenCloud\OpenStack
*/
private $anchor;
/**
* @var string
*/
@ -75,21 +66,26 @@ class Swift extends \OC\Files\Storage\Common {
*/
private $params;
/** @var string */
/** @var string */
private $id;
/** @var \OC\Files\ObjectStore\Swift */
private $objectStore;
/**
* Key value cache mapping path to data object. Maps path to
* \OpenCloud\OpenStack\ObjectStorage\Resource\DataObject for existing
* paths and path to false for not existing paths.
*
* @var \OCP\ICache
*/
private $objectCache;
/**
* @param string $path
* @return mixed|string
*/
private function normalizePath($path) {
private function normalizePath(string $path) {
$path = trim($path, '/');
if (!$path) {
@ -117,24 +113,22 @@ class Swift extends \OC\Files\Storage\Common {
* that one will be returned.
*
* @param string $path
* @return \OpenCloud\ObjectStore\Resource\DataObject|bool object
* @return StorageObject|bool object
* or false if the object did not exist
* @throws \OCP\Files\StorageAuthException
* @throws \OCP\Files\StorageNotAvailableException
*/
private function fetchObject($path) {
private function fetchObject(string $path) {
if ($this->objectCache->hasKey($path)) {
// might be "false" if object did not exist from last check
return $this->objectCache->get($path);
}
try {
$object = $this->getContainer()->getPartialObject($path);
$object = $this->getContainer()->getObject($path);
$object->retrieve();
$this->objectCache->set($path, $object);
return $object;
} catch (ClientErrorResponseException $e) {
// this exception happens when the object does not exist, which
// is expected in most cases
$this->objectCache->set($path, false);
return false;
} catch (ClientErrorResponseException $e) {
} catch (BadResponseError $e) {
// Expected response is "404 Not Found", so only log if it isn't
if ($e->getResponse()->getStatusCode() !== 404) {
\OC::$server->getLogger()->logException($e, [
@ -142,6 +136,7 @@ class Swift extends \OC\Files\Storage\Common {
'app' => 'files_external',
]);
}
$this->objectCache->set($path, false);
return false;
}
}
@ -152,6 +147,8 @@ class Swift extends \OC\Files\Storage\Common {
* @param string $path
*
* @return bool true if the object exist, false otherwise
* @throws \OCP\Files\StorageAuthException
* @throws \OCP\Files\StorageNotAvailableException
*/
private function doesObjectExist($path) {
return $this->fetchObject($path) !== false;
@ -162,17 +159,15 @@ class Swift extends \OC\Files\Storage\Common {
or empty($params['user']) or empty($params['bucket'])
or empty($params['region'])
) {
throw new \Exception("API Key or password, Username, Bucket and Region have to be configured.");
throw new StorageBadConfigException("API Key or password, Username, Bucket and Region have to be configured.");
}
$this->id = 'swift::' . $params['user'] . md5($params['bucket']);
$bucketUrl = Url::factory($params['bucket']);
if ($bucketUrl->isAbsolute()) {
$this->bucket = end($bucketUrl->getPathSegments());
$params['endpoint_url'] = $bucketUrl->addPath('..')->normalizePath();
} else {
$this->bucket = $params['bucket'];
$bucketUrl = new Uri($params['bucket']);
if ($bucketUrl->getHost()) {
$params['bucket'] = basename($bucketUrl->getPath());
$params['endpoint_url'] = (string)$bucketUrl->withPath(dirname($bucketUrl->getPath()));
}
if (empty($params['url'])) {
@ -183,9 +178,14 @@ class Swift extends \OC\Files\Storage\Common {
$params['service_name'] = 'cloudFiles';
}
$params['autocreate'] = true;
$this->params = $params;
// FIXME: private class...
$this->objectCache = new \OC\Cache\CappedMemoryCache();
$this->connectionFactory = new SwiftFactory(\OC::$server->getMemCacheFactory()->createDistributed('swift/'), $this->params);
$this->objectStore = new \OC\Files\ObjectStore\Swift($this->params, $this->connectionFactory);
$this->bucket = $params['bucket'];
}
public function mkdir($path) {
@ -200,14 +200,15 @@ class Swift extends \OC\Files\Storage\Common {
}
try {
$customHeaders = array('content-type' => 'httpd/unix-directory');
$metadataHeaders = DataObject::stockHeaders(array());
$allHeaders = $customHeaders + $metadataHeaders;
$this->getContainer()->uploadObject($path, '', $allHeaders);
$this->getContainer()->createObject([
'name' => $path,
'content' => '',
'headers' => ['content-type' => 'httpd/unix-directory']
]);
// invalidate so that the next access gets the real object
// with all properties
$this->objectCache->remove($path);
} catch (Exceptions\CreateUpdateError $e) {
} catch (BadResponseError $e) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
@ -249,9 +250,9 @@ class Swift extends \OC\Files\Storage\Common {
}
try {
$this->getContainer()->dataObject()->setName($path . '/')->delete();
$this->objectStore->deleteObject($path . '/');
$this->objectCache->remove($path . '/');
} catch (Exceptions\DeleteError $e) {
} catch (BadResponseError $e) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
@ -271,19 +272,18 @@ class Swift extends \OC\Files\Storage\Common {
$path .= '/';
}
$path = str_replace('%23', '#', $path); // the prefix is sent as a query param, so revert the encoding of #
// $path = str_replace('%23', '#', $path); // the prefix is sent as a query param, so revert the encoding of #
try {
$files = array();
/** @var OpenCloud\Common\Collection $objects */
$objects = $this->getContainer()->objectList(array(
$files = [];
$objects = $this->getContainer()->listObjects([
'prefix' => $path,
'delimiter' => '/'
));
]);
/** @var OpenCloud\ObjectStore\Resource\DataObject $object */
/** @var StorageObject $object */
foreach ($objects as $object) {
$file = basename($object->getName());
$file = basename($object->name);
if ($file !== basename($path) && $file !== '.') {
$files[] = $file;
}
@ -310,12 +310,11 @@ class Swift extends \OC\Files\Storage\Common {
}
try {
/** @var DataObject $object */
$object = $this->fetchObject($path);
if (!$object) {
return false;
}
} catch (ClientErrorResponseException $e) {
} catch (BadResponseError $e) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
@ -323,16 +322,11 @@ class Swift extends \OC\Files\Storage\Common {
return false;
}
$dateTime = \DateTime::createFromFormat(\DateTime::RFC1123, $object->getLastModified());
if ($dateTime !== false) {
$mtime = $dateTime->getTimestamp();
} else {
$mtime = null;
}
$dateTime = $object->lastModified ? \DateTime::createFromFormat(\DateTime::RFC1123, $object->lastModified) : false;
$mtime = $dateTime ? $dateTime->getTimestamp() : null;
$objectMetadata = $object->getMetadata();
$metaTimestamp = $objectMetadata->getProperty('timestamp');
if (isset($metaTimestamp)) {
$mtime = $metaTimestamp;
if (isset($objectMetadata['timestamp'])) {
$mtime = $objectMetadata['timestamp'];
}
if (!empty($mtime)) {
@ -340,7 +334,7 @@ class Swift extends \OC\Files\Storage\Common {
}
$stat = array();
$stat['size'] = (int)$object->getContentLength();
$stat['size'] = (int)$object->contentLength;
$stat['mtime'] = $mtime;
$stat['atime'] = time();
return $stat;
@ -370,17 +364,17 @@ class Swift extends \OC\Files\Storage\Common {
}
try {
$this->getContainer()->dataObject()->setName($path)->delete();
$this->objectStore->deleteObject($path);
$this->objectCache->remove($path);
$this->objectCache->remove($path . '/');
} catch (ClientErrorResponseException $e) {
} catch (BadResponseError $e) {
if ($e->getResponse()->getStatusCode() !== 404) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
]);
throw $e;
}
return false;
}
return true;
@ -397,20 +391,8 @@ class Swift extends \OC\Files\Storage\Common {
case 'r':
case 'rb':
try {
$c = $this->getContainer();
$streamFactory = new \Guzzle\Stream\PhpStreamRequestFactory();
/** @var \OpenCloud\Common\Http\Client $client */
$client = $c->getClient();
$streamInterface = $streamFactory->fromRequest($client->get($c->getUrl($path)));
$streamInterface->rewind();
$stream = $streamInterface->getStream();
stream_context_set_option($stream, 'swift','content', $streamInterface);
if(!strrpos($streamInterface
->getMetaData('wrapper_data')[0], '404 Not Found')) {
return RetryWrapper::wrap($stream);
}
return false;
} catch (\Guzzle\Http\Exception\BadResponseException $e) {
return $this->objectStore->readObject($path);
} catch (BadResponseError $e) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
@ -453,24 +435,25 @@ class Swift extends \OC\Files\Storage\Common {
if (is_null($mtime)) {
$mtime = time();
}
$metadata = array('timestamp' => $mtime);
$metadata = ['timestamp' => $mtime];
if ($this->file_exists($path)) {
if ($this->is_dir($path) && $path !== '.') {
$path .= '/';
}
$object = $this->fetchObject($path);
if ($object->saveMetadata($metadata)) {
if ($object->mergeMetadata($metadata)) {
// invalidate target object to force repopulation on fetch
$this->objectCache->remove($path);
}
return true;
} else {
$mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
$customHeaders = array('content-type' => $mimeType);
$metadataHeaders = DataObject::stockHeaders($metadata);
$allHeaders = $customHeaders + $metadataHeaders;
$this->getContainer()->uploadObject($path, '', $allHeaders);
$this->getContainer()->createObject([
'name' => $path,
'content' => '',
'headers' => ['content-type' => 'httpd/unix-directory']
]);
// invalidate target object to force repopulation on fetch
$this->objectCache->remove($path);
return true;
@ -482,18 +465,21 @@ class Swift extends \OC\Files\Storage\Common {
$path2 = $this->normalizePath($path2);
$fileType = $this->filetype($path1);
if ($fileType === 'file') {
if ($fileType) {
// make way
$this->unlink($path2);
}
if ($fileType === 'file') {
try {
$source = $this->fetchObject($path1);
$source->copy($this->bucket . '/' . $path2);
$source->copy([
'destination' => $this->bucket . '/' . $path2
]);
// invalidate target object to force repopulation on fetch
$this->objectCache->remove($path2);
$this->objectCache->remove($path2 . '/');
} catch (ClientErrorResponseException $e) {
} catch (BadResponseError $e) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
@ -502,17 +488,15 @@ class Swift extends \OC\Files\Storage\Common {
}
} else if ($fileType === 'dir') {
// make way
$this->unlink($path2);
try {
$source = $this->fetchObject($path1 . '/');
$source->copy($this->bucket . '/' . $path2 . '/');
$source->copy([
'destination' => $this->bucket . '/' . $path2 . '/'
]);
// invalidate target object to force repopulation on fetch
$this->objectCache->remove($path2);
$this->objectCache->remove($path2 . '/');
} catch (ClientErrorResponseException $e) {
} catch (BadResponseError $e) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
@ -553,6 +537,7 @@ class Swift extends \OC\Files\Storage\Common {
// cleanup
if ($this->unlink($path1) === false) {
throw new \Exception('failed to remove original');
$this->unlink($path2);
return false;
}
@ -567,81 +552,27 @@ class Swift extends \OC\Files\Storage\Common {
return $this->id;
}
/**
* Returns the connection
*
* @return OpenCloud\ObjectStore\Service connected client
* @throws \Exception if connection could not be made
*/
public function getConnection() {
if (!is_null($this->connection)) {
return $this->connection;
}
$settings = array(
'username' => $this->params['user'],
);
if (!empty($this->params['password'])) {
$settings['password'] = $this->params['password'];
} else if (!empty($this->params['key'])) {
$settings['apiKey'] = $this->params['key'];
}
if (!empty($this->params['tenant'])) {
$settings['tenantName'] = $this->params['tenant'];
}
if (!empty($this->params['timeout'])) {
$settings['timeout'] = $this->params['timeout'];
}
if (isset($settings['apiKey'])) {
$this->anchor = new Rackspace($this->params['url'], $settings);
} else {
$this->anchor = new OpenStack($this->params['url'], $settings);
}
$connection = $this->anchor->objectStoreService($this->params['service_name'], $this->params['region']);
if (!empty($this->params['endpoint_url'])) {
$endpoint = $connection->getEndpoint();
$endpoint->setPublicUrl($this->params['endpoint_url']);
$endpoint->setPrivateUrl($this->params['endpoint_url']);
$connection->setEndpoint($endpoint);
}
$this->connection = $connection;
return $this->connection;
}
/**
* Returns the initialized object store container.
*
* @return OpenCloud\ObjectStore\Resource\Container
* @return \OpenStack\ObjectStore\v1\Models\Container
* @throws \OCP\Files\StorageAuthException
* @throws \OCP\Files\StorageNotAvailableException
*/
public function getContainer() {
if (!is_null($this->container)) {
return $this->container;
}
if (is_null($this->container)) {
$this->container = $this->connectionFactory->getContainer();
try {
$this->container = $this->getConnection()->getContainer($this->bucket);
} catch (ClientErrorResponseException $e) {
$this->container = $this->getConnection()->createContainer($this->bucket);
if (!$this->file_exists('.')) {
$this->mkdir('.');
}
}
if (!$this->file_exists('.')) {
$this->mkdir('.');
}
return $this->container;
}
public function writeBack($tmpFile, $path) {
$fileData = fopen($tmpFile, 'r');
$this->getContainer()->uploadObject($path, $fileData);
$this->objectStore->writeObject($path, $fileData);
// invalidate target object to force repopulation on fetch
$this->objectCache->remove($path);
unlink($tmpFile);

16
apps/files_external/tests/Storage/SwiftTest.php
View file

@ -27,6 +27,7 @@
namespace OCA\Files_External\Tests\Storage;
use GuzzleHttp\Exception\ClientException;
use \OCA\Files_External\Lib\Storage\Swift;
/**
@ -40,6 +41,11 @@ class SwiftTest extends \Test\Files\Storage\Storage {
private $config;
/**
* @var Swift instance
*/
protected $instance;
protected function setUp() {
parent::setUp();
@ -53,17 +59,15 @@ class SwiftTest extends \Test\Files\Storage\Storage {
protected function tearDown() {
if ($this->instance) {
try {
$connection = $this->instance->getConnection();
$container = $connection->getContainer($this->config['bucket']);
$container = $this->instance->getContainer();
$objects = $container->objectList();
while($object = $objects->next()) {
$object->setName(str_replace('#','%23',$object->getName()));
$objects = $container->listObjects();
foreach ($objects as $object) {
$object->delete();
}
$container->delete();
} catch (\Guzzle\Http\Exception\ClientErrorResponseException $e) {
} catch (ClientException $e) {
// container didn't exist, so we don't need to delete it
}
}

13
lib/private/Files/ObjectStore/Swift.php
View file

@ -41,24 +41,21 @@ class Swift implements IObjectStore {
*/
private $container = null;
/** @var SwiftConnectionFactory */
/** @var SwiftFactory */
private $swiftFactory;
public function __construct($params) {
$this->swiftFactory = new SwiftConnectionFactory(\OC::$server->getMemCacheFactory()->createDistributed('swift::'));
public function __construct($params, SwiftFactory $connectionFactory = null) {
$this->swiftFactory = $connectionFactory ?: new SwiftFactory(\OC::$server->getMemCacheFactory()->createDistributed('swift::'), $params);
$this->params = $params;
}
/**
* @return \OpenStack\ObjectStore\v1\Models\Container
* @throws StorageAuthException
* @throws \Exception
* @throws \OCP\Files\StorageNotAvailableException
*/
private function getContainer() {
if (is_null($this->container)) {
$this->container = $this->swiftFactory->getContainer($this->params);
}
return $this->container;
return $this->swiftFactory->getContainer();
}
/**

72
lib/private/Files/ObjectStore/SwiftConnectionFactory.php → lib/private/Files/ObjectStore/SwiftFactory.php
View file

@ -36,12 +36,17 @@ use OpenStack\Identity\v2\Service;
use OpenStack\OpenStack;
use OpenStack\Common\Transport\Utils as TransportUtils;
use Psr\Http\Message\RequestInterface;
use OpenStack\ObjectStore\v1\Models\Container;
class SwiftConnectionFactory {
class SwiftFactory {
private $cache;
private $params;
/** @var Container|null */
private $container = null;
public function __construct(ICache $cache) {
public function __construct(ICache $cache, array $params) {
$this->cache = $cache;
$this->params = $params;
}
private function getCachedToken(string $cacheKey) {
@ -58,42 +63,46 @@ class SwiftConnectionFactory {
}
/**
* @param array $params
* @return OpenStack
* @throws StorageAuthException
* @throws \Exception
*/
private function getClient(array &$params) {
if (isset($params['bucket'])) {
$params['container'] = $params['bucket'];
private function getClient() {
if (isset($this->params['bucket'])) {
$this->params['container'] = $this->params['bucket'];
}
if (!isset($params['container'])) {
$params['container'] = 'owncloud';
if (!isset($this->params['container'])) {
$this->params['container'] = 'owncloud';
}
if (!isset($params['autocreate'])) {
if (!isset($this->params['autocreate'])) {
// should only be true for tests
$params['autocreate'] = false;
$this->params['autocreate'] = false;
}
if (!isset($this->params['username']) && isset($this->params['user'])) {
$this->params['username'] = $this->params['user'];
}
if (!isset($this->params['tenantName']) && isset($this->params['tenant'])) {
$this->params['tenantName'] = $this->params['tenant'];
}
$cacheKey = $params['username'] . '@' . $params['url'] . '/' . $params['bucket'];
$cacheKey = $this->params['username'] . '@' . $this->params['url'] . '/' . $this->params['bucket'];
$token = $this->getCachedToken($cacheKey);
$hasToken = is_array($token) && (new \DateTimeImmutable($token['expires_at'])) > (new \DateTimeImmutable('now'));
if ($hasToken) {
$params['cachedToken'] = $token;
$this->params['cachedToken'] = $token;
}
$httpClient = new Client([
'base_uri' => TransportUtils::normalizeUrl($params['url']),
'base_uri' => TransportUtils::normalizeUrl($this->params['url']),
'handler' => HandlerStack::create()
]);
$authService = Service::factory($httpClient);
$params['identityService'] = $authService;
$params['authUrl'] = $params['url'];
$client = new OpenStack($params);
$this->params['identityService'] = $authService;
$this->params['authUrl'] = $this->params['url'];
$client = new OpenStack($this->params);
if (!$hasToken) {
try {
$token = $authService->generateToken($params);
$token = $authService->generateToken($this->params);
$this->cacheToken($token, $cacheKey);
} catch (ConnectException $e) {
throw new StorageAuthException('Failed to connect to keystone, verify the keystone url', $e);
@ -117,19 +126,30 @@ class SwiftConnectionFactory {
}
/**
* @param array $params
* @return \OpenStack\ObjectStore\v1\Models\Container
* @throws StorageAuthException
* @throws \Exception
* @throws StorageNotAvailableException
*/
public function getContainer(array $params) {
public function getContainer() {
if (is_null($this->container)) {
$this->container = $this->createContainer();
}
$client = $this->getClient($params);
return $this->container;
}
/**
* @return \OpenStack\ObjectStore\v1\Models\Container
* @throws StorageAuthException
* @throws StorageNotAvailableException
*/
private function createContainer() {
$client = $this->getClient();
$objectStoreService = $client->objectStoreV1();
$autoCreate = isset($params['autocreate']) && $params['autocreate'] === true;
$autoCreate = isset($this->params['autocreate']) && $this->params['autocreate'] === true;
try {
$container = $objectStoreService->getContainer($params['container']);
$container = $objectStoreService->getContainer($this->params['container']);
if ($autoCreate) {
$container->getMetadata();
}
@ -138,10 +158,10 @@ class SwiftConnectionFactory {
// if the container does not exist and autocreate is true try to create the container on the fly
if ($ex->getResponse()->getStatusCode() === 404 && $autoCreate) {
return $objectStoreService->createContainer([
'name' => $params['container']
'name' => $this->params['container']
]);
} else {
throw $ex;
throw new StorageNotAvailableException('Invalid response while trying to get container info', StorageNotAvailableException::STATUS_ERROR, $e);
}
} catch (ConnectException $e) {
/** @var RequestInterface $request */