wbrawner/server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #8314 from nextcloud/openstack

Browse source 
update openstack sdk
This commit is contained in:
Roeland Jago Douma 2018-02-19 22:22:55 +01:00 committed by GitHub
commit 980c78048b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 363 additions and 387 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.drone.yml
View file

@ -851,9 +851,8 @@ services:
matrix:
OBJECT_STORE: s3
dockswift:
image: icewind1991/dockswift
image: icewind1991/dockswift:nextcloud-ci
environment:
- INITIALIZE=yes
- IPADDRESS=dockswift
when:
matrix:

2
3rdparty

@ -1 +1 @@
Subproject commit ffb6a3a10985ae6dca121200760a78b5a75aa5d5
Subproject commit 600b0a7ab4c8f3c5e0fb48c68e171190f1c9006a

247
apps/files_external/lib/Lib/Storage/Swift.php
View file

@ -1,4 +1,5 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2016, ownCloud, Inc.
*
@ -39,31 +40,21 @@
namespace OCA\Files_External\Lib\Storage;
use Guzzle\Http\Url;
use Guzzle\Http\Exception\ClientErrorResponseException;
use GuzzleHttp\Psr7\Uri;
use Icewind\Streams\CallbackWrapper;
use Icewind\Streams\IteratorDirectory;
use Icewind\Streams\RetryWrapper;
use OpenCloud;
use OpenCloud\Common\Exceptions;
use OpenCloud\OpenStack;
use OpenCloud\Rackspace;
use OpenCloud\ObjectStore\Resource\DataObject;
use OC\Files\ObjectStore\SwiftFactory;
use OCP\Files\StorageBadConfigException;
use OpenStack\Common\Error\BadResponseError;
use OpenStack\ObjectStore\v1\Models\StorageObject;
class Swift extends \OC\Files\Storage\Common {
/** @var SwiftFactory */
private $connectionFactory;
/**
* @var \OpenCloud\ObjectStore\Service
*/
private $connection;
/**
* @var \OpenCloud\ObjectStore\Resource\Container
* @var \OpenStack\ObjectStore\v1\Models\Container
*/
private $container;
/**
* @var \OpenCloud\OpenStack
*/
private $anchor;
/**
* @var string
*/
@ -75,21 +66,26 @@ class Swift extends \OC\Files\Storage\Common {
*/
private $params;
/** @var string */
/** @var string */
private $id;
/** @var \OC\Files\ObjectStore\Swift */
private $objectStore;
/**
* Key value cache mapping path to data object. Maps path to
* \OpenCloud\OpenStack\ObjectStorage\Resource\DataObject for existing
* paths and path to false for not existing paths.
*
* @var \OCP\ICache
*/
private $objectCache;
/**
* @param string $path
* @return mixed|string
*/
private function normalizePath($path) {
private function normalizePath(string $path) {
$path = trim($path, '/');
if (!$path) {
@ -117,24 +113,22 @@ class Swift extends \OC\Files\Storage\Common {
* that one will be returned.
*
* @param string $path
* @return \OpenCloud\ObjectStore\Resource\DataObject|bool object
* @return StorageObject|bool object
* or false if the object did not exist
* @throws \OCP\Files\StorageAuthException
* @throws \OCP\Files\StorageNotAvailableException
*/
private function fetchObject($path) {
private function fetchObject(string $path) {
if ($this->objectCache->hasKey($path)) {
// might be "false" if object did not exist from last check
return $this->objectCache->get($path);
}
try {
$object = $this->getContainer()->getPartialObject($path);
$object = $this->getContainer()->getObject($path);
$object->retrieve();
$this->objectCache->set($path, $object);
return $object;
} catch (ClientErrorResponseException $e) {
// this exception happens when the object does not exist, which
// is expected in most cases
$this->objectCache->set($path, false);
return false;
} catch (ClientErrorResponseException $e) {
} catch (BadResponseError $e) {
// Expected response is "404 Not Found", so only log if it isn't
if ($e->getResponse()->getStatusCode() !== 404) {
\OC::$server->getLogger()->logException($e, [
@ -142,6 +136,7 @@ class Swift extends \OC\Files\Storage\Common {
'app' => 'files_external',
]);
}
$this->objectCache->set($path, false);
return false;
}
}
@ -152,6 +147,8 @@ class Swift extends \OC\Files\Storage\Common {
* @param string $path
*
* @return bool true if the object exist, false otherwise
* @throws \OCP\Files\StorageAuthException
* @throws \OCP\Files\StorageNotAvailableException
*/
private function doesObjectExist($path) {
return $this->fetchObject($path) !== false;
@ -162,17 +159,15 @@ class Swift extends \OC\Files\Storage\Common {
or empty($params['user']) or empty($params['bucket'])
or empty($params['region'])
) {
throw new \Exception("API Key or password, Username, Bucket and Region have to be configured.");
throw new StorageBadConfigException("API Key or password, Username, Bucket and Region have to be configured.");
}
$this->id = 'swift::' . $params['user'] . md5($params['bucket']);
$bucketUrl = Url::factory($params['bucket']);
if ($bucketUrl->isAbsolute()) {
$this->bucket = end($bucketUrl->getPathSegments());
$params['endpoint_url'] = $bucketUrl->addPath('..')->normalizePath();
} else {
$this->bucket = $params['bucket'];
$bucketUrl = new Uri($params['bucket']);
if ($bucketUrl->getHost()) {
$params['bucket'] = basename($bucketUrl->getPath());
$params['endpoint_url'] = (string)$bucketUrl->withPath(dirname($bucketUrl->getPath()));
}
if (empty($params['url'])) {
@ -183,9 +178,14 @@ class Swift extends \OC\Files\Storage\Common {
$params['service_name'] = 'cloudFiles';
}
$params['autocreate'] = true;
$this->params = $params;
// FIXME: private class...
$this->objectCache = new \OC\Cache\CappedMemoryCache();
$this->connectionFactory = new SwiftFactory(\OC::$server->getMemCacheFactory()->createDistributed('swift/'), $this->params);
$this->objectStore = new \OC\Files\ObjectStore\Swift($this->params, $this->connectionFactory);
$this->bucket = $params['bucket'];
}
public function mkdir($path) {
@ -200,14 +200,15 @@ class Swift extends \OC\Files\Storage\Common {
}
try {
$customHeaders = array('content-type' => 'httpd/unix-directory');
$metadataHeaders = DataObject::stockHeaders(array());
$allHeaders = $customHeaders + $metadataHeaders;
$this->getContainer()->uploadObject($path, '', $allHeaders);
$this->getContainer()->createObject([
'name' => $path,
'content' => '',
'headers' => ['content-type' => 'httpd/unix-directory']
]);
// invalidate so that the next access gets the real object
// with all properties
$this->objectCache->remove($path);
} catch (Exceptions\CreateUpdateError $e) {
} catch (BadResponseError $e) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
@ -249,9 +250,9 @@ class Swift extends \OC\Files\Storage\Common {
}
try {
$this->getContainer()->dataObject()->setName($path . '/')->delete();
$this->objectStore->deleteObject($path . '/');
$this->objectCache->remove($path . '/');
} catch (Exceptions\DeleteError $e) {
} catch (BadResponseError $e) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
@ -271,19 +272,18 @@ class Swift extends \OC\Files\Storage\Common {
$path .= '/';
}
$path = str_replace('%23', '#', $path); // the prefix is sent as a query param, so revert the encoding of #
// $path = str_replace('%23', '#', $path); // the prefix is sent as a query param, so revert the encoding of #
try {
$files = array();
/** @var OpenCloud\Common\Collection $objects */
$objects = $this->getContainer()->objectList(array(
$files = [];
$objects = $this->getContainer()->listObjects([
'prefix' => $path,
'delimiter' => '/'
));
]);
/** @var OpenCloud\ObjectStore\Resource\DataObject $object */
/** @var StorageObject $object */
foreach ($objects as $object) {
$file = basename($object->getName());
$file = basename($object->name);
if ($file !== basename($path) && $file !== '.') {
$files[] = $file;
}
@ -310,12 +310,11 @@ class Swift extends \OC\Files\Storage\Common {
}
try {
/** @var DataObject $object */
$object = $this->fetchObject($path);
if (!$object) {
return false;
}
} catch (ClientErrorResponseException $e) {
} catch (BadResponseError $e) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
@ -323,16 +322,11 @@ class Swift extends \OC\Files\Storage\Common {
return false;
}
$dateTime = \DateTime::createFromFormat(\DateTime::RFC1123, $object->getLastModified());
if ($dateTime !== false) {
$mtime = $dateTime->getTimestamp();
} else {
$mtime = null;
}
$dateTime = $object->lastModified ? \DateTime::createFromFormat(\DateTime::RFC1123, $object->lastModified) : false;
$mtime = $dateTime ? $dateTime->getTimestamp() : null;
$objectMetadata = $object->getMetadata();
$metaTimestamp = $objectMetadata->getProperty('timestamp');
if (isset($metaTimestamp)) {
$mtime = $metaTimestamp;
if (isset($objectMetadata['timestamp'])) {
$mtime = $objectMetadata['timestamp'];
}
if (!empty($mtime)) {
@ -340,7 +334,7 @@ class Swift extends \OC\Files\Storage\Common {
}
$stat = array();
$stat['size'] = (int)$object->getContentLength();
$stat['size'] = (int)$object->contentLength;
$stat['mtime'] = $mtime;
$stat['atime'] = time();
return $stat;
@ -370,17 +364,17 @@ class Swift extends \OC\Files\Storage\Common {
}
try {
$this->getContainer()->dataObject()->setName($path)->delete();
$this->objectStore->deleteObject($path);
$this->objectCache->remove($path);
$this->objectCache->remove($path . '/');
} catch (ClientErrorResponseException $e) {
} catch (BadResponseError $e) {
if ($e->getResponse()->getStatusCode() !== 404) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
]);
throw $e;
}
return false;
}
return true;
@ -397,20 +391,8 @@ class Swift extends \OC\Files\Storage\Common {
case 'r':
case 'rb':
try {
$c = $this->getContainer();
$streamFactory = new \Guzzle\Stream\PhpStreamRequestFactory();
/** @var \OpenCloud\Common\Http\Client $client */
$client = $c->getClient();
$streamInterface = $streamFactory->fromRequest($client->get($c->getUrl($path)));
$streamInterface->rewind();
$stream = $streamInterface->getStream();
stream_context_set_option($stream, 'swift','content', $streamInterface);
if(!strrpos($streamInterface
->getMetaData('wrapper_data')[0], '404 Not Found')) {
return RetryWrapper::wrap($stream);
}
return false;
} catch (\Guzzle\Http\Exception\BadResponseException $e) {
return $this->objectStore->readObject($path);
} catch (BadResponseError $e) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
@ -453,24 +435,25 @@ class Swift extends \OC\Files\Storage\Common {
if (is_null($mtime)) {
$mtime = time();
}
$metadata = array('timestamp' => $mtime);
$metadata = ['timestamp' => $mtime];
if ($this->file_exists($path)) {
if ($this->is_dir($path) && $path !== '.') {
$path .= '/';
}
$object = $this->fetchObject($path);
if ($object->saveMetadata($metadata)) {
if ($object->mergeMetadata($metadata)) {
// invalidate target object to force repopulation on fetch
$this->objectCache->remove($path);
}
return true;
} else {
$mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
$customHeaders = array('content-type' => $mimeType);
$metadataHeaders = DataObject::stockHeaders($metadata);
$allHeaders = $customHeaders + $metadataHeaders;
$this->getContainer()->uploadObject($path, '', $allHeaders);
$this->getContainer()->createObject([
'name' => $path,
'content' => '',
'headers' => ['content-type' => 'httpd/unix-directory']
]);
// invalidate target object to force repopulation on fetch
$this->objectCache->remove($path);
return true;
@ -482,18 +465,21 @@ class Swift extends \OC\Files\Storage\Common {
$path2 = $this->normalizePath($path2);
$fileType = $this->filetype($path1);
if ($fileType === 'file') {
if ($fileType) {
// make way
$this->unlink($path2);
}
if ($fileType === 'file') {
try {
$source = $this->fetchObject($path1);
$source->copy($this->bucket . '/' . $path2);
$source->copy([
'destination' => $this->bucket . '/' . $path2
]);
// invalidate target object to force repopulation on fetch
$this->objectCache->remove($path2);
$this->objectCache->remove($path2 . '/');
} catch (ClientErrorResponseException $e) {
} catch (BadResponseError $e) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
@ -502,17 +488,15 @@ class Swift extends \OC\Files\Storage\Common {
}
} else if ($fileType === 'dir') {
// make way
$this->unlink($path2);
try {
$source = $this->fetchObject($path1 . '/');
$source->copy($this->bucket . '/' . $path2 . '/');
$source->copy([
'destination' => $this->bucket . '/' . $path2 . '/'
]);
// invalidate target object to force repopulation on fetch
$this->objectCache->remove($path2);
$this->objectCache->remove($path2 . '/');
} catch (ClientErrorResponseException $e) {
} catch (BadResponseError $e) {
\OC::$server->getLogger()->logException($e, [
'level' => \OCP\Util::ERROR,
'app' => 'files_external',
@ -553,6 +537,7 @@ class Swift extends \OC\Files\Storage\Common {
// cleanup
if ($this->unlink($path1) === false) {
throw new \Exception('failed to remove original');
$this->unlink($path2);
return false;
}
@ -567,81 +552,27 @@ class Swift extends \OC\Files\Storage\Common {
return $this->id;
}
/**
* Returns the connection
*
* @return OpenCloud\ObjectStore\Service connected client
* @throws \Exception if connection could not be made
*/
public function getConnection() {
if (!is_null($this->connection)) {
return $this->connection;
}
$settings = array(
'username' => $this->params['user'],
);
if (!empty($this->params['password'])) {
$settings['password'] = $this->params['password'];
} else if (!empty($this->params['key'])) {
$settings['apiKey'] = $this->params['key'];
}
if (!empty($this->params['tenant'])) {
$settings['tenantName'] = $this->params['tenant'];
}
if (!empty($this->params['timeout'])) {
$settings['timeout'] = $this->params['timeout'];
}
if (isset($settings['apiKey'])) {
$this->anchor = new Rackspace($this->params['url'], $settings);
} else {
$this->anchor = new OpenStack($this->params['url'], $settings);
}
$connection = $this->anchor->objectStoreService($this->params['service_name'], $this->params['region']);
if (!empty($this->params['endpoint_url'])) {
$endpoint = $connection->getEndpoint();
$endpoint->setPublicUrl($this->params['endpoint_url']);
$endpoint->setPrivateUrl($this->params['endpoint_url']);
$connection->setEndpoint($endpoint);
}
$this->connection = $connection;
return $this->connection;
}
/**
* Returns the initialized object store container.
*
* @return OpenCloud\ObjectStore\Resource\Container
* @return \OpenStack\ObjectStore\v1\Models\Container
* @throws \OCP\Files\StorageAuthException
* @throws \OCP\Files\StorageNotAvailableException
*/
public function getContainer() {
if (!is_null($this->container)) {
return $this->container;
}
if (is_null($this->container)) {
$this->container = $this->connectionFactory->getContainer();
try {
$this->container = $this->getConnection()->getContainer($this->bucket);
} catch (ClientErrorResponseException $e) {
$this->container = $this->getConnection()->createContainer($this->bucket);
if (!$this->file_exists('.')) {
$this->mkdir('.');
}
}
if (!$this->file_exists('.')) {
$this->mkdir('.');
}
return $this->container;
}
public function writeBack($tmpFile, $path) {
$fileData = fopen($tmpFile, 'r');
$this->getContainer()->uploadObject($path, $fileData);
$this->objectStore->writeObject($path, $fileData);
// invalidate target object to force repopulation on fetch
$this->objectCache->remove($path);
unlink($tmpFile);

16
apps/files_external/tests/Storage/SwiftTest.php
View file

@ -27,6 +27,7 @@
namespace OCA\Files_External\Tests\Storage;
use GuzzleHttp\Exception\ClientException;
use \OCA\Files_External\Lib\Storage\Swift;
/**
@ -40,6 +41,11 @@ class SwiftTest extends \Test\Files\Storage\Storage {
private $config;
/**
* @var Swift instance
*/
protected $instance;
protected function setUp() {
parent::setUp();
@ -53,17 +59,15 @@ class SwiftTest extends \Test\Files\Storage\Storage {
protected function tearDown() {
if ($this->instance) {
try {
$connection = $this->instance->getConnection();
$container = $connection->getContainer($this->config['bucket']);
$container = $this->instance->getContainer();
$objects = $container->objectList();
while($object = $objects->next()) {
$object->setName(str_replace('#','%23',$object->getName()));
$objects = $container->listObjects();
foreach ($objects as $object) {
$object->delete();
}
$container->delete();
} catch (\Guzzle\Http\Exception\ClientErrorResponseException $e) {
} catch (ClientException $e) {
// container didn't exist, so we don't need to delete it
}
}

1
lib/composer/composer/autoload_classmap.php
View file

@ -653,6 +653,7 @@ return array(
'OC\\Files\\ObjectStore\\S3Signature' => $baseDir . '/lib/private/Files/ObjectStore/S3Signature.php',
'OC\\Files\\ObjectStore\\StorageObjectStore' => $baseDir . '/lib/private/Files/ObjectStore/StorageObjectStore.php',
'OC\\Files\\ObjectStore\\Swift' => $baseDir . '/lib/private/Files/ObjectStore/Swift.php',
'OC\\Files\\ObjectStore\\SwiftFactory' => $baseDir . '/lib/private/Files/ObjectStore/SwiftFactory.php',
'OC\\Files\\Search\\SearchBinaryOperator' => $baseDir . '/lib/private/Files/Search/SearchBinaryOperator.php',
'OC\\Files\\Search\\SearchComparison' => $baseDir . '/lib/private/Files/Search/SearchComparison.php',
'OC\\Files\\Search\\SearchOrder' => $baseDir . '/lib/private/Files/Search/SearchOrder.php',

1
lib/composer/composer/autoload_static.php
View file

@ -683,6 +683,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
'OC\\Files\\ObjectStore\\S3Signature' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/S3Signature.php',
'OC\\Files\\ObjectStore\\StorageObjectStore' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/StorageObjectStore.php',
'OC\\Files\\ObjectStore\\Swift' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Swift.php',
'OC\\Files\\ObjectStore\\SwiftFactory' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/SwiftFactory.php',
'OC\\Files\\Search\\SearchBinaryOperator' => __DIR__ . '/../../..' . '/lib/private/Files/Search/SearchBinaryOperator.php',
'OC\\Files\\Search\\SearchComparison' => __DIR__ . '/../../..' . '/lib/private/Files/Search/SearchComparison.php',
'OC\\Files\\Search\\SearchOrder' => __DIR__ . '/../../..' . '/lib/private/Files/Search/SearchOrder.php',

243
lib/private/Files/ObjectStore/Swift.php
View file

@ -25,223 +25,37 @@
namespace OC\Files\ObjectStore;
use Guzzle\Http\Exception\ClientErrorResponseException;
use Guzzle\Http\Exception\CurlException;
use function GuzzleHttp\Psr7\stream_for;
use Icewind\Streams\RetryWrapper;
use OCP\Files\ObjectStore\IObjectStore;
use OCP\Files\StorageAuthException;
use OCP\Files\StorageNotAvailableException;
use OpenCloud\Common\Service\Catalog;
use OpenCloud\Common\Service\CatalogItem;
use OpenCloud\Identity\Resource\Token;
use OpenCloud\ObjectStore\Service;
use OpenCloud\OpenStack;
use OpenCloud\Rackspace;
class Swift implements IObjectStore {
/**
* @var \OpenCloud\OpenStack
*/
private $client;
/**
* @var array
*/
private $params;
/**
* @var \OpenCloud\ObjectStore\Service
* @var \OpenStack\ObjectStore\v1\Models\Container|null
*/
private $objectStoreService;
private $container = null;
/**
* @var \OpenCloud\ObjectStore\Resource\Container
*/
private $container;
private $memcache;
public function __construct($params) {
if (isset($params['bucket'])) {
$params['container'] = $params['bucket'];
}
if (!isset($params['container'])) {
$params['container'] = 'owncloud';
}
if (!isset($params['autocreate'])) {
// should only be true for tests
$params['autocreate'] = false;
}
if (isset($params['apiKey'])) {
$this->client = new Rackspace($params['url'], $params);
$cacheKey = $params['username'] . '@' . $params['url'] . '/' . $params['bucket'];
} else {
$this->client = new OpenStack($params['url'], $params);
$cacheKey = $params['username'] . '@' . $params['url'] . '/' . $params['bucket'];
}
$cacheFactory = \OC::$server->getMemCacheFactory();
$this->memcache = $cacheFactory->createDistributed('swift::' . $cacheKey);
/** @var SwiftFactory */
private $swiftFactory;
public function __construct($params, SwiftFactory $connectionFactory = null) {
$this->swiftFactory = $connectionFactory ?: new SwiftFactory(\OC::$server->getMemCacheFactory()->createDistributed('swift::'), $params);
$this->params = $params;
}
/**
* @suppress PhanNonClassMethodCall
* @return \OpenStack\ObjectStore\v1\Models\Container
* @throws StorageAuthException
* @throws \OCP\Files\StorageNotAvailableException
*/
protected function init() {
if ($this->container) {
return;
}
$this->importToken();
/** @var Token $token */
$token = $this->client->getTokenObject();
if (!$token || $token->hasExpired()) {
try {
$this->client->authenticate();
$this->exportToken();
} catch (ClientErrorResponseException $e) {
$statusCode = $e->getResponse()->getStatusCode();
if ($statusCode == 412) {
throw new StorageAuthException('Precondition failed, verify the keystone url', $e);
} else if ($statusCode === 401) {
throw new StorageAuthException('Authentication failed, verify the username, password and possibly tenant', $e);
} else {
throw new StorageAuthException('Unknown error', $e);
}
}
}
/** @var Catalog $catalog */
$catalog = $this->client->getCatalog();
if (count($catalog->getItems()) === 0) {
throw new StorageAuthException('Keystone did not provide a valid catalog, verify the credentials');
}
if (isset($this->params['serviceName'])) {
$serviceName = $this->params['serviceName'];
} else {
$serviceName = Service::DEFAULT_NAME;
}
if (isset($this->params['urlType'])) {
$urlType = $this->params['urlType'];
if ($urlType !== 'internalURL' && $urlType !== 'publicURL') {
throw new StorageNotAvailableException('Invalid url type');
}
} else {
$urlType = Service::DEFAULT_URL_TYPE;
}
$catalogItem = $this->getCatalogForService($catalog, $serviceName);
if (!$catalogItem) {
$available = implode(', ', $this->getAvailableServiceNames($catalog));
throw new StorageNotAvailableException(
"Service $serviceName not found in service catalog, available services: $available"
);
} else if (isset($this->params['region'])) {
$this->validateRegion($catalogItem, $this->params['region']);
}
$this->objectStoreService = $this->client->objectStoreService($serviceName, $this->params['region'], $urlType);
try {
$this->container = $this->objectStoreService->getContainer($this->params['container']);
} catch (ClientErrorResponseException $ex) {
// if the container does not exist and autocreate is true try to create the container on the fly
if (isset($this->params['autocreate']) && $this->params['autocreate'] === true) {
$this->container = $this->objectStoreService->createContainer($this->params['container']);
} else {
throw $ex;
}
} catch (CurlException $e) {
if ($e->getErrorNo() === 7) {
$host = $e->getCurlHandle()->getUrl()->getHost() . ':' . $e->getCurlHandle()->getUrl()->getPort();
\OC::$server->getLogger()->error("Can't connect to object storage server at $host");
throw new StorageNotAvailableException("Can't connect to object storage server at $host", StorageNotAvailableException::STATUS_ERROR, $e);
}
throw $e;
}
}
private function exportToken() {
$export = $this->client->exportCredentials();
$export['catalog'] = array_map(function (CatalogItem $item) {
return [
'name' => $item->getName(),
'endpoints' => $item->getEndpoints(),
'type' => $item->getType()
];
}, $export['catalog']->getItems());
$this->memcache->set('token', json_encode($export));
}
private function importToken() {
$cachedTokenString = $this->memcache->get('token');
if ($cachedTokenString) {
$cachedToken = json_decode($cachedTokenString, true);
$cachedToken['catalog'] = array_map(function (array $item) {
$itemClass = new \stdClass();
$itemClass->name = $item['name'];
$itemClass->endpoints = array_map(function (array $endpoint) {
return (object)$endpoint;
}, $item['endpoints']);
$itemClass->type = $item['type'];
return $itemClass;
}, $cachedToken['catalog']);
try {
$this->client->importCredentials($cachedToken);
} catch (\Exception $e) {
$this->client->setTokenObject(new Token());
}
}
}
/**
* @param Catalog $catalog
* @param $name
* @return null|CatalogItem
*/
private function getCatalogForService(Catalog $catalog, $name) {
foreach ($catalog->getItems() as $item) {
/** @var CatalogItem $item */
if ($item->hasType(Service::DEFAULT_TYPE) && $item->hasName($name)) {
return $item;
}
}
return null;
}
private function validateRegion(CatalogItem $item, $region) {
$endPoints = $item->getEndpoints();
foreach ($endPoints as $endPoint) {
if ($endPoint->region === $region) {
return;
}
}
$availableRegions = implode(', ', array_map(function ($endpoint) {
return $endpoint->region;
}, $endPoints));
throw new StorageNotAvailableException("Invalid region '$region', available regions: $availableRegions");
}
private function getAvailableServiceNames(Catalog $catalog) {
return array_map(function (CatalogItem $item) {
return $item->getName();
}, array_filter($catalog->getItems(), function (CatalogItem $item) {
return $item->hasType(Service::DEFAULT_TYPE);
}));
private function getContainer() {
return $this->swiftFactory->getContainer();
}
/**
@ -254,29 +68,29 @@ class Swift implements IObjectStore {
/**
* @param string $urn the unified resource name used to identify the object
* @param resource $stream stream with the data to write
* @throws Exception from openstack lib when something goes wrong
* @throws \Exception from openstack lib when something goes wrong
*/
public function writeObject($urn, $stream) {
$this->init();
$this->container->uploadObject($urn, $stream);
$this->getContainer()->createObject([
'name' => $urn,
'stream' => stream_for($stream)
]);
}
/**
* @param string $urn the unified resource name used to identify the object
* @return resource stream with the read data
* @throws Exception from openstack lib when something goes wrong
* @throws \Exception from openstack lib when something goes wrong
*/
public function readObject($urn) {
$this->init();
$object = $this->container->getObject($urn);
$object = $this->getContainer()->getObject($urn);
// we need to keep a reference to objectContent or
// the stream will be closed before we can do anything with it
/** @var $objectContent \Guzzle\Http\EntityBody * */
$objectContent = $object->getContent();
$objectContent = $object->download();
$objectContent->rewind();
$stream = $objectContent->getStream();
$stream = $objectContent->detach();
// save the object content in the context of the stream to prevent it being gc'd until the stream is closed
stream_context_set_option($stream, 'swift', 'content', $objectContent);
@ -286,17 +100,18 @@ class Swift implements IObjectStore {
/**
* @param string $urn Unified Resource Name
* @return void
* @throws Exception from openstack lib when something goes wrong
* @throws \Exception from openstack lib when something goes wrong
*/
public function deleteObject($urn) {
$this->init();
// see https://github.com/rackspace/php-opencloud/issues/243#issuecomment-30032242
$this->container->dataObject()->setName($urn)->delete();
$this->getContainer()->getObject($urn)->delete();
}
public function deleteContainer($recursive = false) {
$this->init();
$this->container->delete($recursive);
/**
* @return void
* @throws \Exception from openstack lib when something goes wrong
*/
public function deleteContainer() {
$this->getContainer()->delete();
}
}

174
lib/private/Files/ObjectStore/SwiftFactory.php Normal file
View file

@ -0,0 +1,174 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2018 Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OC\Files\ObjectStore;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\ClientException;
use GuzzleHttp\Exception\ConnectException;
use GuzzleHttp\Exception\RequestException;
use GuzzleHttp\HandlerStack;
use OCP\Files\StorageAuthException;
use OCP\Files\StorageNotAvailableException;
use OCP\ICache;
use OpenStack\Common\Error\BadResponseError;
use OpenStack\Identity\v2\Models\Token;
use OpenStack\Identity\v2\Service;
use OpenStack\OpenStack;
use OpenStack\Common\Transport\Utils as TransportUtils;
use Psr\Http\Message\RequestInterface;
use OpenStack\ObjectStore\v1\Models\Container;
class SwiftFactory {
private $cache;
private $params;
/** @var Container|null */
private $container = null;
public function __construct(ICache $cache, array $params) {
$this->cache = $cache;
$this->params = $params;
}
private function getCachedToken(string $cacheKey) {
$cachedTokenString = $this->cache->get($cacheKey . '/token');
if ($cachedTokenString) {
return json_decode($cachedTokenString);
} else {
return null;
}
}
private function cacheToken(Token $token, string $cacheKey) {
$this->cache->set($cacheKey . '/token', json_encode($token));
}
/**
* @return OpenStack
* @throws StorageAuthException
*/
private function getClient() {
if (isset($this->params['bucket'])) {
$this->params['container'] = $this->params['bucket'];
}
if (!isset($this->params['container'])) {
$this->params['container'] = 'owncloud';
}
if (!isset($this->params['autocreate'])) {
// should only be true for tests
$this->params['autocreate'] = false;
}
if (!isset($this->params['username']) && isset($this->params['user'])) {
$this->params['username'] = $this->params['user'];
}
if (!isset($this->params['tenantName']) && isset($this->params['tenant'])) {
$this->params['tenantName'] = $this->params['tenant'];
}
$cacheKey = $this->params['username'] . '@' . $this->params['url'] . '/' . $this->params['bucket'];
$token = $this->getCachedToken($cacheKey);
$hasToken = is_array($token) && (new \DateTimeImmutable($token['expires_at'])) > (new \DateTimeImmutable('now'));
if ($hasToken) {
$this->params['cachedToken'] = $token;
}
$httpClient = new Client([
'base_uri' => TransportUtils::normalizeUrl($this->params['url']),
'handler' => HandlerStack::create()
]);
$authService = Service::factory($httpClient);
$this->params['identityService'] = $authService;
$this->params['authUrl'] = $this->params['url'];
$client = new OpenStack($this->params);
if (!$hasToken) {
try {
$token = $authService->generateToken($this->params);
$this->cacheToken($token, $cacheKey);
} catch (ConnectException $e) {
throw new StorageAuthException('Failed to connect to keystone, verify the keystone url', $e);
} catch (ClientException $e) {
$statusCode = $e->getResponse()->getStatusCode();
if ($statusCode === 404) {
throw new StorageAuthException('Keystone not found, verify the keystone url', $e);
} else if ($statusCode === 412) {
throw new StorageAuthException('Precondition failed, verify the keystone url', $e);
} else if ($statusCode === 401) {
throw new StorageAuthException('Authentication failed, verify the username, password and possibly tenant', $e);
} else {
throw new StorageAuthException('Unknown error', $e);
}
} catch (RequestException $e) {
throw new StorageAuthException('Connection reset while connecting to keystone, verify the keystone url', $e);
}
}
return $client;
}
/**
* @return \OpenStack\ObjectStore\v1\Models\Container
* @throws StorageAuthException
* @throws StorageNotAvailableException
*/
public function getContainer() {
if (is_null($this->container)) {
$this->container = $this->createContainer();
}
return $this->container;
}
/**
* @return \OpenStack\ObjectStore\v1\Models\Container
* @throws StorageAuthException
* @throws StorageNotAvailableException
*/
private function createContainer() {
$client = $this->getClient();
$objectStoreService = $client->objectStoreV1();
$autoCreate = isset($this->params['autocreate']) && $this->params['autocreate'] === true;
try {
$container = $objectStoreService->getContainer($this->params['container']);
if ($autoCreate) {
$container->getMetadata();
}
return $container;
} catch (BadResponseError $ex) {
// if the container does not exist and autocreate is true try to create the container on the fly
if ($ex->getResponse()->getStatusCode() === 404 && $autoCreate) {
return $objectStoreService->createContainer([
'name' => $this->params['container']
]);
} else {
throw new StorageNotAvailableException('Invalid response while trying to get container info', StorageNotAvailableException::STATUS_ERROR, $e);
}
} catch (ConnectException $e) {
/** @var RequestInterface $request */
$request = $e->getRequest();
$host = $request->getUri()->getHost() . ':' . $request->getUri()->getPort();
\OC::$server->getLogger()->error("Can't connect to object storage server at $host");
throw new StorageNotAvailableException("Can't connect to object storage server at $host", StorageNotAvailableException::STATUS_ERROR, $e);
}
}
}

6
tests/Settings/Controller/CheckSetupControllerTest.php
View file

@ -21,7 +21,6 @@
namespace Tests\Settings\Controller;
use Guzzle\Http\Message\Response;
use OC\Settings\Controller\CheckSetupController;
use OCP\AppFramework\Http;
use OCP\AppFramework\Http\DataDisplayResponse;
@ -34,6 +33,7 @@ use OCP\ILogger;
use OCP\IRequest;
use OCP\IURLGenerator;
use OC_Util;
use Psr\Http\Message\ResponseInterface;
use Test\TestCase;
use OC\IntegrityCheck\Checker;
@ -461,7 +461,7 @@ class CheckSetupControllerTest extends TestCase {
->disableOriginalConstructor()->getMock();
$exception = $this->getMockBuilder('\GuzzleHttp\Exception\ClientException')
->disableOriginalConstructor()->getMock();
$response = $this->getMockBuilder(Response::class)
$response = $this->getMockBuilder(ResponseInterface::class)
->disableOriginalConstructor()->getMock();
$response->expects($this->once())
->method('getStatusCode')
@ -495,7 +495,7 @@ class CheckSetupControllerTest extends TestCase {
->disableOriginalConstructor()->getMock();
$exception = $this->getMockBuilder('\GuzzleHttp\Exception\ClientException')
->disableOriginalConstructor()->getMock();
$response = $this->getMockBuilder(Response::class)
$response = $this->getMockBuilder(ResponseInterface::class)
->disableOriginalConstructor()->getMock();
$response->expects($this->once())
->method('getStatusCode')

57
tests/drone-wait-objectstore.sh
View file

@ -1,10 +1,61 @@
#!/bin/bash
function get_swift_token() {
KEYSTONE_OUT=$(curl -s 'http://dockswift:5000/v2.0/tokens' -H 'Content-Type: application/json' -d '{"auth":{"passwordCredentials":{"username":"swift","password":"swift"},"tenantName":"service"}}')
if (echo "$KEYSTONE_OUT" | grep -q 'object-store')
then
SWIFT_ENDPOINT=$(echo "$KEYSTONE_OUT" | php -r "echo array_values(array_filter(json_decode(file_get_contents('php://stdin'),true)['access']['serviceCatalog'], function(\$endpoint){return \$endpoint['type']==='object-store';}))[0]['endpoints'][0]['publicURL'];")
SWIFT_TOKEN=$(echo "$KEYSTONE_OUT" | php -r "echo json_decode(file_get_contents('php://stdin'),true)['access']['token']['id'];")
return 0
else
return -1
fi
}
if [ "$OBJECT_STORE" == "swift" ]; then
echo "waiting for swift"
until curl -I http://dockswift:5000/v3
echo "waiting for keystone"
until get_swift_token
do
sleep 2
done
sleep 60
echo "waiting for object store at $SWIFT_ENDPOINT"
until curl -s -H "X-Auth-Token: $SWIFT_TOKEN" "$SWIFT_ENDPOINT"
do
sleep 2
done
echo "creating container"
sleep 2
while [ 1 ]
do
sleep 2
respCode=$(curl -s -o /dev/null -w "%{http_code}" -X PUT -H "X-Auth-Token: $SWIFT_TOKEN" "$SWIFT_ENDPOINT/nextcloud")
if [ "$respCode" == "201" ]
then
break
fi
done
echo "creating test file"
while [ 1 ]
do
sleep 2
respCode=$(curl -s -o /dev/null -w "%{http_code}" -X PUT -H "X-Auth-Token: $SWIFT_TOKEN" -H "Content-Type: text/html; charset=UTF-8" -d "Hello world" "$SWIFT_ENDPOINT/nextcloud/helloworld.txt")
if [ "$respCode" == "201" ]
then
break
fi
done
echo "deleting test file"
curl -s -o /dev/null -w "%{http_code}\n" -X DELETE -H "X-Auth-Token: $SWIFT_TOKEN" "$SWIFT_ENDPOINT/nextcloud/helloworld.txt"
fi