Merge branch 'master' into exclude-custom-data-dir

This commit is contained in:
C. Montero Luque 2016-03-01 13:35:14 -05:00
commit c51a01b43f
30 changed files with 510 additions and 333 deletions

View file

@ -20,6 +20,7 @@
*/
use OCA\Dav\AppInfo\Application;
use Symfony\Component\EventDispatcher\GenericEvent;
$app = new Application();
$app->registerHooks();
@ -28,6 +29,20 @@ $app->registerHooks();
return $app->getSyncService();
});
$eventDispatcher = \OC::$server->getEventDispatcher();
$eventDispatcher->addListener('OCP\Federation\TrustedServerEvent::remove',
function(GenericEvent $event) use ($app) {
/** @var \OCA\DAV\CardDAV\CardDavBackend $cardDavBackend */
$cardDavBackend = $app->getContainer()->query('CardDavBackend');
$addressBookUri = $event->getSubject();
$addressBook = $cardDavBackend->getAddressBooksByUri('principals/system/system', $addressBookUri);
if (!is_null($addressBook)) {
$cardDavBackend->deleteAddressBook($addressBook['id']);
}
}
);
$cm = \OC::$server->getContactsManager();
$cm->register(function() use ($cm, $app) {
$userId = \OC::$server->getUserSession()->getUser()->getUID();

View file

@ -69,7 +69,8 @@ class Application extends App {
/** @var IAppContainer $c */
return new SyncService(
$c->query('CardDavBackend'),
$c->getServer()->getUserManager()
$c->getServer()->getUserManager(),
$c->getServer()->getLogger()
);
});

View file

@ -21,11 +21,14 @@
namespace OCA\DAV\CardDAV;
use OCP\AppFramework\Http;
use OCP\ILogger;
use OCP\IUser;
use OCP\IUserManager;
use Sabre\DAV\Client;
use Sabre\DAV\Xml\Response\MultiStatus;
use Sabre\DAV\Xml\Service;
use Sabre\HTTP\ClientHttpException;
use Sabre\VObject\Reader;
class SyncService {
@ -36,12 +39,16 @@ class SyncService {
/** @var IUserManager */
private $userManager;
/** @var ILogger */
private $logger;
/** @var array */
private $localSystemAddressBook;
public function __construct(CardDavBackend $backend, IUserManager $userManager) {
public function __construct(CardDavBackend $backend, IUserManager $userManager, ILogger $logger) {
$this->backend = $backend;
$this->userManager = $userManager;
$this->logger = $logger;
}
/**
@ -53,6 +60,7 @@ class SyncService {
* @param string $targetPrincipal
* @param array $targetProperties
* @return string
* @throws \Exception
*/
public function syncRemoteAddressBook($url, $userName, $sharedSecret, $syncToken, $targetBookId, $targetPrincipal, $targetProperties) {
// 1. create addressbook
@ -60,7 +68,16 @@ class SyncService {
$addressBookId = $book['id'];
// 2. query changes
$response = $this->requestSyncReport($url, $userName, $sharedSecret, $syncToken);
try {
$response = $this->requestSyncReport($url, $userName, $sharedSecret, $syncToken);
} catch (ClientHttpException $ex) {
if ($ex->getCode() === Http::STATUS_UNAUTHORIZED) {
// remote server revoked access to the address book, remove it
$this->backend->deleteAddressBook($addressBookId);
$this->logger->info('Authorization failed, remove address book: ' . $url, ['app' => 'dav']);
throw $ex;
}
}
// 3. apply changes
// TODO: use multi-get for download

View file

@ -215,11 +215,16 @@ class File extends Node implements IFile {
}
}
$this->refreshInfo();
if (isset($request->server['HTTP_OC_CHECKSUM'])) {
$checksum = trim($request->server['HTTP_OC_CHECKSUM']);
$this->fileView->putFileInfo($this->path, ['checksum' => $checksum]);
$this->refreshInfo();
} else if ($this->getChecksum() !== null && $this->getChecksum() !== '') {
$this->fileView->putFileInfo($this->path, ['checksum' => '']);
$this->refreshInfo();
}
$this->refreshInfo();
} catch (StorageNotAvailableException $e) {
throw new ServiceUnavailable("Failed to check file size: " . $e->getMessage());
@ -457,8 +462,16 @@ class File extends Node implements IFile {
$this->emitPostHooks($exists, $targetPath);
// FIXME: should call refreshInfo but can't because $this->path is not the of the final file
$info = $this->fileView->getFileInfo($targetPath);
if (isset($request->server['HTTP_OC_CHECKSUM'])) {
$checksum = trim($request->server['HTTP_OC_CHECKSUM']);
$this->fileView->putFileInfo($targetPath, ['checksum' => $checksum]);
} else if ($info->getChecksum() !== null && $info->getChecksum() !== '') {
$this->fileView->putFileInfo($this->path, ['checksum' => '']);
}
$this->fileView->unlockFile($targetPath, ILockingProvider::LOCK_SHARED);
return $info->getEtag();

View file

@ -27,6 +27,7 @@
namespace OCA\DAV\Connector\Sabre;
use Sabre\DAV\Exception\NotFound;
use Sabre\DAV\IFile;
use \Sabre\DAV\PropFind;
use \Sabre\DAV\PropPatch;
@ -197,7 +198,7 @@ class FilesPlugin extends \Sabre\DAV\ServerPlugin {
//Add OC-Checksum header
/** @var $node File */
$checksum = $node->getChecksum();
if ($checksum !== null) {
if ($checksum !== null && $checksum !== '') {
$response->addHeader('OC-Checksum', $checksum);
}
}
@ -252,6 +253,10 @@ class FilesPlugin extends \Sabre\DAV\ServerPlugin {
$propFind->handle(self::CHECKSUMS_PROPERTYNAME, function() use ($node) {
$checksum = $node->getChecksum();
if ($checksum === NULL || $checksum === '') {
return null;
}
return new ChecksumList($checksum);
});

View file

@ -68,13 +68,15 @@ class SyncServiceTest extends TestCase {
/** @var IUserManager $userManager */
$userManager = $this->getMockBuilder('OCP\IUserManager')->disableOriginalConstructor()->getMock();
$ss = new SyncService($backend, $userManager);
$logger = $this->getMockBuilder('OCP\ILogger')->disableOriginalConstructor()->getMock();
$ss = new SyncService($backend, $userManager, $logger);
$book = $ss->ensureSystemAddressBookExists('principals/users/adam', 'contacts', []);
}
public function testUpdateAndDeleteUser() {
/** @var CardDavBackend | \PHPUnit_Framework_MockObject_MockObject $backend */
$backend = $this->getMockBuilder('OCA\DAV\CardDAV\CardDAVBackend')->disableOriginalConstructor()->getMock();
$logger = $this->getMockBuilder('OCP\ILogger')->disableOriginalConstructor()->getMock();
$backend->expects($this->once())->method('createCard');
$backend->expects($this->once())->method('updateCard');
@ -92,7 +94,7 @@ class SyncServiceTest extends TestCase {
$user->method('getBackendClassName')->willReturn('unittest');
$user->method('getUID')->willReturn('test-user');
$ss = new SyncService($backend, $userManager);
$ss = new SyncService($backend, $userManager, $logger);
$ss->updateUser($user);
$user->method('getDisplayName')->willReturn('A test user for unit testing');
@ -123,8 +125,9 @@ class SyncServiceTest extends TestCase {
*/
private function getSyncServiceMock($backend, $response) {
$userManager = $this->getMockBuilder('OCP\IUserManager')->disableOriginalConstructor()->getMock();
$logger = $this->getMockBuilder('OCP\ILogger')->disableOriginalConstructor()->getMock();
/** @var SyncService | \PHPUnit_Framework_MockObject_MockObject $ss */
$ss = $this->getMock('OCA\DAV\CardDAV\SyncService', ['ensureSystemAddressBookExists', 'requestSyncReport', 'download'], [$backend, $userManager]);
$ss = $this->getMock('OCA\DAV\CardDAV\SyncService', ['ensureSystemAddressBookExists', 'requestSyncReport', 'download'], [$backend, $userManager, $logger]);
$ss->method('requestSyncReport')->withAnyParameters()->willReturn(['response' => $response, 'token' => 'sync-token-1']);
$ss->method('ensureSystemAddressBookExists')->willReturn(['id' => 1]);
$ss->method('download')->willReturn([

View file

@ -75,13 +75,15 @@ class Application extends \OCP\AppFramework\App {
});
$container->registerService('TrustedServers', function(IAppContainer $c) {
$server = $c->getServer();
return new TrustedServers(
$c->query('DbHandler'),
\OC::$server->getHTTPClientService(),
\OC::$server->getLogger(),
\OC::$server->getJobList(),
\OC::$server->getSecureRandom(),
\OC::$server->getConfig()
$server->getHTTPClientService(),
$server->getLogger(),
$server->getJobList(),
$server->getSecureRandom(),
$server->getConfig(),
$server->getEventDispatcher()
);
});
@ -94,6 +96,7 @@ class Application extends \OCP\AppFramework\App {
$c->query('TrustedServers')
);
});
}
private function registerMiddleware() {

View file

@ -27,8 +27,7 @@
<type>text</type>
<default></default>
<notnull>true</notnull>
<length>32</length>
<comments>md5 hash of the url without the protocol</comments>
<comments>sha1 hash of the url without the protocol</comments>
</field>
<field>
<name>token</name>

View file

@ -5,7 +5,7 @@
<description>ownCloud Federation allows you to connect with other trusted ownClouds to exchange the user directory. For example this will be used to auto-complete external users for federated sharing.</description>
<licence>AGPL</licence>
<author>Bjoern Schiessle</author>
<version>0.0.3</version>
<version>0.0.4</version>
<namespace>Federation</namespace>
<category>other</category>
<dependencies>

View file

@ -91,12 +91,13 @@ class GetSharedSecret extends QueuedJob{
$this->trustedServers = $trustedServers;
} else {
$this->trustedServers = new TrustedServers(
$this->dbHandler,
\OC::$server->getHTTPClientService(),
$this->logger,
$this->jobList,
\OC::$server->getSecureRandom(),
\OC::$server->getConfig()
$this->dbHandler,
\OC::$server->getHTTPClientService(),
$this->logger,
$this->jobList,
\OC::$server->getSecureRandom(),
\OC::$server->getConfig(),
\OC::$server->getEventDispatcher()
);
}
}

View file

@ -95,7 +95,8 @@ class RequestSharedSecret extends QueuedJob {
$this->logger,
$this->jobList,
\OC::$server->getSecureRandom(),
\OC::$server->getConfig()
\OC::$server->getConfig(),
\OC::$server->getEventDispatcher()
);
}
}

View file

@ -40,6 +40,7 @@ class SyncFederationAddressBooks extends Command {
$this->syncService->syncThemAll(function($url, $ex) use ($progress, $output) {
if ($ex instanceof \Exception) {
$output->writeln("Error while syncing $url : " . $ex->getMessage());
} else {
$progress->advance();
}

View file

@ -105,6 +105,28 @@ class DbHandler {
$query->execute();
}
/**
* get trusted server with given ID
*
* @param int $id
* @return array
* @throws \Exception
*/
public function getServerById($id) {
$query = $this->connection->getQueryBuilder();
$query->select('*')->from($this->dbTable)
->where($query->expr()->eq('id', $query->createParameter('id')))
->setParameter('id', $id);
$query->execute();
$result = $query->execute()->fetchAll();
if (empty($result)) {
throw new \Exception('No Server found with ID: ' . $id);
}
return $result[0];
}
/**
* get all trusted servers
*
@ -112,7 +134,7 @@ class DbHandler {
*/
public function getAllServer() {
$query = $this->connection->getQueryBuilder();
$query->select(['url', 'id', 'status', 'shared_secret', 'sync_token'])->from($this->dbTable);
$query->select(['url', 'url_hash', 'id', 'status', 'shared_secret', 'sync_token'])->from($this->dbTable);
$result = $query->execute()->fetchAll();
return $result;
}
@ -252,11 +274,11 @@ class DbHandler {
*/
protected function hash($url) {
$normalized = $this->normalizeUrl($url);
return md5($normalized);
return sha1($normalized);
}
/**
* normalize URL, used to create the md5 hash
* normalize URL, used to create the sha1 hash
*
* @param string $url
* @return string

View file

@ -3,6 +3,7 @@
namespace OCA\Federation;
use OCA\DAV\CardDAV\SyncService;
use OCP\AppFramework\Http;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Input\InputInterface;
@ -40,7 +41,7 @@ class SyncFederationAddressBooks {
if (is_null($sharedSecret)) {
continue;
}
$targetBookId = sha1($url);
$targetBookId = $trustedServer['url_hash'];
$targetPrincipal = "principals/system/system";
$targetBookProperties = [
'{DAV:}displayname' => $url
@ -51,6 +52,9 @@ class SyncFederationAddressBooks {
$this->dbHandler->setServerStatus($url, TrustedServers::STATUS_OK, $newToken);
}
} catch (\Exception $ex) {
if ($ex->getCode() === Http::STATUS_UNAUTHORIZED) {
$this->dbHandler->setServerStatus($url, TrustedServers::STATUS_ACCESS_REVOKED);
}
$callback($url, $ex);
}
}

View file

@ -30,6 +30,8 @@ use OCP\Http\Client\IClientService;
use OCP\IConfig;
use OCP\ILogger;
use OCP\Security\ISecureRandom;
use Symfony\Component\EventDispatcher\EventDispatcherInterface;
use Symfony\Component\EventDispatcher\GenericEvent;
class TrustedServers {
@ -39,6 +41,8 @@ class TrustedServers {
const STATUS_PENDING = 2;
/** something went wrong, misconfigured server, software bug,... user interaction needed */
const STATUS_FAILURE = 3;
/** remote server revoked access */
const STATUS_ACCESS_REVOKED = 4;
/** @var dbHandler */
private $dbHandler;
@ -58,6 +62,9 @@ class TrustedServers {
/** @var IConfig */
private $config;
/** @var EventDispatcherInterface */
private $dispatcher;
/**
* @param DbHandler $dbHandler
* @param IClientService $httpClientService
@ -65,6 +72,7 @@ class TrustedServers {
* @param IJobList $jobList
* @param ISecureRandom $secureRandom
* @param IConfig $config
* @param EventDispatcherInterface $dispatcher
*/
public function __construct(
DbHandler $dbHandler,
@ -72,7 +80,8 @@ class TrustedServers {
ILogger $logger,
IJobList $jobList,
ISecureRandom $secureRandom,
IConfig $config
IConfig $config,
EventDispatcherInterface $dispatcher
) {
$this->dbHandler = $dbHandler;
$this->httpClientService = $httpClientService;
@ -80,6 +89,7 @@ class TrustedServers {
$this->jobList = $jobList;
$this->secureRandom = $secureRandom;
$this->config = $config;
$this->dispatcher = $dispatcher;
}
/**
@ -154,7 +164,10 @@ class TrustedServers {
* @param int $id
*/
public function removeServer($id) {
$server = $this->dbHandler->getServerById($id);
$this->dbHandler->removeServer($id);
$event = new GenericEvent($server['url_hash']);
$this->dispatcher->dispatch('OCP\Federation\TrustedServerEvent::remove', $event);
}
/**
@ -222,6 +235,7 @@ class TrustedServers {
*
* @param $status
* @return bool
* @throws HintException
*/
protected function checkOwnCloudVersion($status) {
$decoded = json_decode($status, true);

View file

@ -34,7 +34,8 @@ $trustedServers = new \OCA\Federation\TrustedServers(
\OC::$server->getLogger(),
\OC::$server->getJobList(),
\OC::$server->getSecureRandom(),
\OC::$server->getConfig()
\OC::$server->getConfig(),
\OC::$server->getEventDispatcher()
);
$template->assign('trustedServers', $trustedServers->getServers());

View file

@ -26,7 +26,11 @@ style('federation', 'settings-admin')
<li id="<?php p($trustedServer['id']); ?>" class="icon-delete">
<?php if((int)$trustedServer['status'] === TrustedServers::STATUS_OK) { ?>
<span class="status success"></span>
<?php } elseif((int)$trustedServer['status'] === TrustedServers::STATUS_PENDING) { ?>
<?php
} elseif(
(int)$trustedServer['status'] === TrustedServers::STATUS_PENDING ||
(int)$trustedServer['status'] === TrustedServers::STATUS_ACCESS_REVOKED
) { ?>
<span class="status indeterminate"></span>
<?php } else {?>
<span class="status error"></span>

View file

@ -89,9 +89,9 @@ class DbHandlerTest extends TestCase {
public function dataTestAddServer() {
return [
['http://owncloud.org', 'http://owncloud.org', md5('owncloud.org')],
['https://owncloud.org', 'https://owncloud.org', md5('owncloud.org')],
['http://owncloud.org/', 'http://owncloud.org', md5('owncloud.org')],
['http://owncloud.org', 'http://owncloud.org', sha1('owncloud.org')],
['https://owncloud.org', 'https://owncloud.org', sha1('owncloud.org')],
['http://owncloud.org/', 'http://owncloud.org', sha1('owncloud.org')],
];
}
@ -115,6 +115,15 @@ class DbHandlerTest extends TestCase {
$this->assertSame($id1, (int)$result[0]['id']);
}
public function testGetServerById() {
$this->dbHandler->addServer('server1');
$id = $this->dbHandler->addServer('server2');
$result = $this->dbHandler->getServerById($id);
$this->assertSame('server2', $result['url']);
}
public function testGetAll() {
$id1 = $this->dbHandler->addServer('server1');
$id2 = $this->dbHandler->addServer('server2');
@ -233,10 +242,10 @@ class DbHandlerTest extends TestCase {
public function dataTestHash() {
return [
['server1', md5('server1')],
['http://server1', md5('server1')],
['https://server1', md5('server1')],
['http://server1/', md5('server1')],
['server1', sha1('server1')],
['http://server1', sha1('server1')],
['https://server1', sha1('server1')],
['http://server1/', sha1('server1')],
];
}

View file

@ -19,6 +19,7 @@ class SyncFederationAddressbooksTest extends \Test\TestCase {
willReturn([
[
'url' => 'https://cloud.drop.box',
'url_hash' => 'sha1',
'shared_secret' => 'iloveowncloud',
'sync_token' => '0'
]
@ -47,6 +48,7 @@ class SyncFederationAddressbooksTest extends \Test\TestCase {
willReturn([
[
'url' => 'https://cloud.drop.box',
'url_hash' => 'sha1',
'shared_secret' => 'iloveowncloud',
'sync_token' => '0'
]

View file

@ -23,7 +23,6 @@
namespace OCA\Federation\Tests\lib;
use OC\HintException;
use OCA\Federation\DbHandler;
use OCA\Federation\TrustedServers;
use OCP\BackgroundJob\IJobList;
@ -33,6 +32,7 @@ use OCP\Http\Client\IResponse;
use OCP\IConfig;
use OCP\ILogger;
use OCP\Security\ISecureRandom;
use Symfony\Component\EventDispatcher\EventDispatcherInterface;
use Test\TestCase;
class TrustedServersTest extends TestCase {
@ -64,11 +64,16 @@ class TrustedServersTest extends TestCase {
/** @var \PHPUnit_Framework_MockObject_MockObject | IConfig */
private $config;
/** @var \PHPUnit_Framework_MockObject_MockObject | EventDispatcherInterface */
private $dispatcher;
public function setUp() {
parent::setUp();
$this->dbHandler = $this->getMockBuilder('\OCA\Federation\DbHandler')
->disableOriginalConstructor()->getMock();
$this->dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcherInterface')
->disableOriginalConstructor()->getMock();
$this->httpClientService = $this->getMock('OCP\Http\Client\IClientService');
$this->httpClient = $this->getMock('OCP\Http\Client\IClient');
$this->response = $this->getMock('OCP\Http\Client\IResponse');
@ -83,7 +88,8 @@ class TrustedServersTest extends TestCase {
$this->logger,
$this->jobList,
$this->secureRandom,
$this->config
$this->config,
$this->dispatcher
);
}
@ -103,7 +109,8 @@ class TrustedServersTest extends TestCase {
$this->logger,
$this->jobList,
$this->secureRandom,
$this->config
$this->config,
$this->dispatcher
]
)
->setMethods(['normalizeUrl', 'updateProtocol'])
@ -191,7 +198,18 @@ class TrustedServersTest extends TestCase {
public function testRemoveServer() {
$id = 42;
$server = ['url_hash' => 'url_hash'];
$this->dbHandler->expects($this->once())->method('removeServer')->with($id);
$this->dbHandler->expects($this->once())->method('getServerById')->with($id)
->willReturn($server);
$this->dispatcher->expects($this->once())->method('dispatch')
->willReturnCallback(
function($eventId, $event) {
$this->assertSame($eventId, 'OCP\Federation\TrustedServerEvent::remove');
$this->assertInstanceOf('Symfony\Component\EventDispatcher\GenericEvent', $event);
$this->assertSame('url_hash', $event->getSubject());
}
);
$this->trustedServers->removeServer($id);
}
@ -247,7 +265,8 @@ class TrustedServersTest extends TestCase {
$this->logger,
$this->jobList,
$this->secureRandom,
$this->config
$this->config,
$this->dispatcher
]
)
->setMethods(['checkOwnCloudVersion'])

View file

@ -31,6 +31,10 @@ if [ -z "$thisFolder" ]; then
thisFolder="."
fi;
# create readiness notification socket
notify_sock=$(readlink -f "$thisFolder"/dockerContainerCeph.$EXECUTOR_NUMBER.amazons3.sock)
mkfifo "$notify_sock"
user=test
accesskey=aaabbbccc
secretkey=cccbbbaaa
@ -39,6 +43,7 @@ port=80
container=`docker run -d \
-e RGW_CIVETWEB_PORT=$port \
-v "$notify_sock":/run/notifyme.sock \
${docker_image}`
host=`docker inspect --format="{{.NetworkSettings.IPAddress}}" $container`
@ -50,7 +55,8 @@ echo "${docker_image} container: $container"
echo $container >> $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.amazons3
echo -n "Waiting for ceph initialization"
if ! "$thisFolder"/env/wait-for-connection ${host} ${port} 60; then
ready=$(timeout 60 cat "$notify_sock")
if [[ $ready != 'READY=1' ]]; then
echo "[ERROR] Waited 60 seconds, no response" >&2
exit 1
fi

View file

@ -31,6 +31,10 @@ if [ -z "$thisFolder" ]; then
thisFolder="."
fi;
# create readiness notification socket
notify_sock=$(readlink -f "$thisFolder"/dockerContainerCeph.$EXECUTOR_NUMBER.swift.sock)
mkfifo "$notify_sock"
port=5001
user=test
@ -48,6 +52,7 @@ container=`docker run -d \
-e KEYSTONE_SERVICE=${service} \
-e OSD_SIZE=300 \
--privileged \
-v "$notify_sock":/run/notifyme.sock \
${docker_image}`
host=`docker inspect --format="{{.NetworkSettings.IPAddress}}" $container`
@ -59,7 +64,8 @@ echo "${docker_image} container: $container"
echo $container >> $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.swift
echo -n "Waiting for ceph initialization"
if ! "$thisFolder"/env/wait-for-connection ${host} 80 600; then
ready=$(timeout 600 cat "$notify_sock")
if [[ $ready != 'READY=1' ]]; then
echo "[ERROR] Waited 600 seconds, no response" >&2
docker logs $container
exit 1

View file

@ -33,4 +33,5 @@ done;
# cleanup
rm $thisFolder/config.amazons3.php
rm $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.amazons3
rm $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.amazons3.sock

View file

@ -35,4 +35,5 @@ done;
# cleanup
rm $thisFolder/config.swift.php
rm $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.swift
rm $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.swift.sock

View file

@ -20,6 +20,8 @@ default:
baseUrl: http://localhost:8080
- CalDavContext:
baseUrl: http://localhost:8080
- ChecksumsContext:
baseUrl: http://localhost:8080
federation:
paths:
- %paths.base%/../federation_features

View file

@ -0,0 +1,227 @@
<?php
require __DIR__ . '/../../vendor/autoload.php';
use Behat\Gherkin\Node\TableNode;
use GuzzleHttp\Client;
use GuzzleHttp\Message\ResponseInterface;
class ChecksumsContext implements \Behat\Behat\Context\Context {
/** @var string */
private $baseUrl;
/** @var Client */
private $client;
/** @var ResponseInterface */
private $response;
/**
* @param string $baseUrl
*/
public function __construct($baseUrl) {
$this->baseUrl = $baseUrl;
// in case of ci deployment we take the server url from the environment
$testServerUrl = getenv('TEST_SERVER_URL');
if ($testServerUrl !== false) {
$this->baseUrl = substr($testServerUrl, 0, -5);
}
}
/** @BeforeScenario */
public function tearUpScenario() {
$this->client = new Client();
}
/** @AfterScenario */
public function tearDownScenario() {
}
/**
* @param string $userName
* @return string
*/
private function getPasswordForUser($userName) {
if($userName === 'admin') {
return 'admin';
}
return '123456';
}
/**
* @When user :user uploads file :source to :destination with checksum :checksum
*/
public function userUploadsFileToWithChecksum($user, $source, $destination, $checksum)
{
$file = \GuzzleHttp\Stream\Stream::factory(fopen($source, 'r'));
try {
$this->response = $this->client->put(
$this->baseUrl . '/remote.php/webdav' . $destination,
[
'auth' => [
$user,
$this->getPasswordForUser($user)
],
'body' => $file,
'headers' => [
'OC-Checksum' => $checksum
]
]
);
} catch (\GuzzleHttp\Exception\ServerException $e) {
// 4xx and 5xx responses cause an exception
$this->response = $e->getResponse();
}
}
/**
* @Then The webdav response should have a status code :statusCode
*/
public function theWebdavResponseShouldHaveAStatusCode($statusCode) {
if((int)$statusCode !== $this->response->getStatusCode()) {
throw new \Exception("Expected $statusCode, got ".$this->response->getStatusCode());
}
}
/**
* @When user :user request the checksum of :path via propfind
*/
public function userRequestTheChecksumOfViaPropfind($user, $path)
{
$request = $this->client->createRequest(
'PROPFIND',
$this->baseUrl . '/remote.php/webdav' . $path,
[
'body' => '<?xml version="1.0"?>
<d:propfind xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns">
<d:prop>
<oc:checksums />
</d:prop>
</d:propfind>',
'auth' => [
$user,
$this->getPasswordForUser($user),
]
]
);
$this->response = $this->client->send($request);
}
/**
* @Then The webdav checksum should match :checksum
*/
public function theWebdavChecksumShouldMatch($checksum)
{
$service = new Sabre\Xml\Service();
$parsed = $service->parse($this->response->getBody()->getContents());
/*
* Fetch the checksum array
* Maybe we want to do this a bit cleaner ;)
*/
$checksums = $parsed[0]['value'][1]['value'][0]['value'][0];
if ($checksums['value'][0]['value'] !== $checksum) {
throw new \Exception("Expected $checksum, got ".$checksums['value'][0]['value']);
}
}
/**
* @When user :user downloads the file :path
*/
public function userDownloadsTheFile($user, $path)
{
$this->response = $this->client->get(
$this->baseUrl . '/remote.php/webdav' . $path,
[
'auth' => [
$user,
$this->getPasswordForUser($user),
]
]
);
}
/**
* @Then The header checksum should match :checksum
*/
public function theHeaderChecksumShouldMatch($checksum)
{
if ($this->response->getHeader('OC-Checksum') !== $checksum) {
throw new \Exception("Expected $checksum, got ".$this->response->getHeader('OC-Checksum'));
}
}
/**
* @Given User :user copied file :source to :destination
*/
public function userCopiedFileTo($user, $source, $destination)
{
$request = $this->client->createRequest(
'MOVE',
$this->baseUrl . '/remote.php/webdav' . $source,
[
'auth' => [
$user,
$this->getPasswordForUser($user),
],
'headers' => [
'Destination' => $this->baseUrl . '/remote.php/webdav' . $destination,
],
]
);
$this->response = $this->client->send($request);
}
/**
* @Then The webdav checksum should be empty
*/
public function theWebdavChecksumShouldBeEmpty()
{
$service = new Sabre\Xml\Service();
$parsed = $service->parse($this->response->getBody()->getContents());
/*
* Fetch the checksum array
* Maybe we want to do this a bit cleaner ;)
*/
$status = $parsed[0]['value'][1]['value'][1]['value'];
if ($status !== 'HTTP/1.1 404 Not Found') {
throw new \Exception("Expected 'HTTP/1.1 404 Not Found', got ".$status);
}
}
/**
* @Then The OC-Checksum header should not be there
*/
public function theOcChecksumHeaderShouldNotBeThere()
{
if ($this->response->hasHeader('OC-Checksum')) {
throw new \Exception("Expected no checksum header but got ".$this->response->getHeader('OC-Checksum'));
}
}
/**
* @Given user :user uploads chunk file :num of :total with :data to :destination with checksum :checksum
*/
public function userUploadsChunkFileOfWithToWithChecksum($user, $num, $total, $data, $destination, $checksum)
{
$num -= 1;
$this->response = $this->client->put(
$this->baseUrl . '/remote.php/webdav' . $destination . '-chunking-42-'.$total.'-'.$num,
[
'auth' => [
$user,
$this->getPasswordForUser($user)
],
'body' => $data,
'headers' => [
'OC-Checksum' => $checksum,
'OC-Chunked' => '1',
]
]
);
}
}

View file

@ -0,0 +1,76 @@
Feature: checksums
Scenario: Uploading a file with checksum should work
Given user "user0" exists
When user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a"
Then The webdav response should have a status code "201"
Scenario: Uploading a file with checksum should return the checksum in the propfind
Given user "user0" exists
And user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a"
When user "user0" request the checksum of "/myChecksumFile.txt" via propfind
Then The webdav checksum should match "MD5:d70b40f177b14b470d1756a3c12b963a"
Scenario: Uploading a file with checksum should return the checksum in the download header
Given user "user0" exists
And user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a"
When user "user0" downloads the file "/myChecksumFile.txt"
Then The header checksum should match "MD5:d70b40f177b14b470d1756a3c12b963a"
Scenario: Moving a file with checksum should return the checksum in the propfind
Given user "user0" exists
And user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a"
When User "user0" moved file "/myChecksumFile.txt" to "/myMovedChecksumFile.txt"
And user "user0" request the checksum of "/myMovedChecksumFile.txt" via propfind
Then The webdav checksum should match "MD5:d70b40f177b14b470d1756a3c12b963a"
Scenario: Moving file with checksum should return the checksum in the download header
Given user "user0" exists
And user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a"
When User "user0" moved file "/myChecksumFile.txt" to "/myMovedChecksumFile.txt"
And user "user0" downloads the file "/myMovedChecksumFile.txt"
Then The header checksum should match "MD5:d70b40f177b14b470d1756a3c12b963a"
Scenario: Copying a file with checksum should return the checksum in the propfind
Given user "user0" exists
And user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a"
When User "user0" copied file "/myChecksumFile.txt" to "/myChecksumFileCopy.txt"
And user "user0" request the checksum of "/myChecksumFileCopy.txt" via propfind
Then The webdav checksum should match "MD5:d70b40f177b14b470d1756a3c12b963a"
Scenario: Copying file with checksum should return the checksum in the download header
Given user "user0" exists
And user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a"
When User "user0" copied file "/myChecksumFile.txt" to "/myChecksumFileCopy.txt"
And user "user0" downloads the file "/myChecksumFileCopy.txt"
Then The header checksum should match "MD5:d70b40f177b14b470d1756a3c12b963a"
Scenario: Overwriting a file with checksum should remove the checksum and not return it in the propfind
Given user "user0" exists
And user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a"
When user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt"
And user "user0" request the checksum of "/myChecksumFile.txt" via propfind
Then The webdav checksum should be empty
Scenario: Overwriting a file with checksum should remove the checksum and not return it in the download header
Given user "user0" exists
And user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a"
When user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt"
And user "user0" downloads the file "/myChecksumFile.txt"
Then The OC-Checksum header should not be there
Scenario: Uploading a chunked file with checksum should return the checksum in the propfind
Given user "user0" exists
And user "user0" uploads chunk file "1" of "3" with "AAAAA" to "/myChecksumFile.txt" with checksum "MD5:e892fdd61a74bc89cd05673cc2e22f88"
And user "user0" uploads chunk file "2" of "3" with "BBBBB" to "/myChecksumFile.txt" with checksum "MD5:e892fdd61a74bc89cd05673cc2e22f88"
And user "user0" uploads chunk file "3" of "3" with "CCCCC" to "/myChecksumFile.txt" with checksum "MD5:e892fdd61a74bc89cd05673cc2e22f88"
When user "user0" request the checksum of "/myChecksumFile.txt" via propfind
Then The webdav checksum should match "MD5:e892fdd61a74bc89cd05673cc2e22f88"
Scenario: Uploading a chunked file with checksum should return the checksum in the download header
Given user "user0" exists
And user "user0" uploads chunk file "1" of "3" with "AAAAA" to "/myChecksumFile.txt" with checksum "MD5:e892fdd61a74bc89cd05673cc2e22f88"
And user "user0" uploads chunk file "2" of "3" with "BBBBB" to "/myChecksumFile.txt" with checksum "MD5:e892fdd61a74bc89cd05673cc2e22f88"
And user "user0" uploads chunk file "3" of "3" with "CCCCC" to "/myChecksumFile.txt" with checksum "MD5:e892fdd61a74bc89cd05673cc2e22f88"
When user "user0" downloads the file "/myChecksumFile.txt"
Then The header checksum should match "MD5:e892fdd61a74bc89cd05673cc2e22f88"

View file

@ -1,274 +0,0 @@
#!/bin/bash
set -e
: ${CLUSTER:=ceph}
: ${RGW_NAME:=$(hostname -s)}
: ${MON_NAME:=$(hostname -s)}
: ${RGW_CIVETWEB_PORT:=80}
: ${OSD_SIZE:=100}
: ${KEYSTONE_ADMIN_TOKEN:=admin}
: ${KEYSTONE_ADMIN_PORT:=35357}
: ${KEYSTONE_PUBLIC_PORT:=5001}
: ${KEYSTONE_SERVICE:=${CLUSTER}}
: ${KEYSTONE_ENDPOINT_REGION:=region}
: ${KEYSTONE_ADMIN_USER:=admin}
: ${KEYSTONE_ADMIN_TENANT:=admin}
: ${KEYSTONE_ADMIN_PASS:=admin}
ip_address=$(head -n1 /etc/hosts | cut -d" " -f1)
: ${MON_IP:=${ip_address}}
subnet=$(ip route | grep "src ${ip_address}" | cut -d" " -f1)
: ${CEPH_NETWORK:=${subnet}}
#######
# MON #
#######
if [ ! -n "$CEPH_NETWORK" ]; then
echo "ERROR- CEPH_NETWORK must be defined as the name of the network for the OSDs"
exit 1
fi
if [ ! -n "$MON_IP" ]; then
echo "ERROR- MON_IP must be defined as the IP address of the monitor"
exit 1
fi
# bootstrap MON
if [ ! -e /etc/ceph/ceph.conf ]; then
fsid=$(uuidgen)
cat <<ENDHERE >/etc/ceph/${CLUSTER}.conf
[global]
fsid = $fsid
mon initial members = ${MON_NAME}
mon host = ${MON_IP}
auth cluster required = cephx
auth service required = cephx
auth client required = cephx
osd crush chooseleaf type = 0
osd journal size = 100
osd pool default pg num = 8
osd pool default pgp num = 8
osd pool default size = 1
public network = ${CEPH_NETWORK}
cluster network = ${CEPH_NETWORK}
debug ms = 1
[mon]
debug mon = 20
debug paxos = 20
debug auth = 20
[osd]
debug osd = 20
debug filestore = 20
debug journal = 20
debug monc = 20
[mds]
debug mds = 20
debug mds balancer = 20
debug mds log = 20
debug mds migrator = 20
[client.radosgw.gateway]
rgw keystone url = http://${MON_IP}:${KEYSTONE_ADMIN_PORT}
rgw keystone admin token = ${KEYSTONE_ADMIN_TOKEN}
rgw keystone accepted roles = _member_
ENDHERE
# Generate administrator key
ceph-authtool /etc/ceph/${CLUSTER}.client.admin.keyring --create-keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow'
# Generate the mon. key
ceph-authtool /etc/ceph/${CLUSTER}.mon.keyring --create-keyring --gen-key -n mon. --cap mon 'allow *'
# Generate initial monitor map
monmaptool --create --add ${MON_NAME} ${MON_IP} --fsid ${fsid} /etc/ceph/monmap
fi
# If we don't have a monitor keyring, this is a new monitor
if [ ! -e /var/lib/ceph/mon/${CLUSTER}-${MON_NAME}/keyring ]; then
if [ ! -e /etc/ceph/${CLUSTER}.client.admin.keyring ]; then
echo "ERROR- /etc/ceph/${CLUSTER}.client.admin.keyring must exist; get it from your existing mon"
exit 2
fi
if [ ! -e /etc/ceph/${CLUSTER}.mon.keyring ]; then
echo "ERROR- /etc/ceph/${CLUSTER}.mon.keyring must exist. You can extract it from your current monitor by running 'ceph auth get mon. -o /tmp/${CLUSTER}.mon.keyring'"
exit 3
fi
if [ ! -e /etc/ceph/monmap ]; then
echo "ERROR- /etc/ceph/monmap must exist. You can extract it from your current monitor by running 'ceph mon getmap -o /tmp/monmap'"
exit 4
fi
# Import the client.admin keyring and the monitor keyring into a new, temporary one
ceph-authtool /tmp/${CLUSTER}.mon.keyring --create-keyring --import-keyring /etc/ceph/${CLUSTER}.client.admin.keyring
ceph-authtool /tmp/${CLUSTER}.mon.keyring --import-keyring /etc/ceph/${CLUSTER}.mon.keyring
# Make the monitor directory
mkdir -p /var/lib/ceph/mon/${CLUSTER}-${MON_NAME}
# Prepare the monitor daemon's directory with the map and keyring
ceph-mon --mkfs -i ${MON_NAME} --monmap /etc/ceph/monmap --keyring /tmp/${CLUSTER}.mon.keyring
# Clean up the temporary key
rm /tmp/${CLUSTER}.mon.keyring
fi
# start MON
ceph-mon -i ${MON_NAME} --public-addr ${MON_IP}:6789
# change replica size
ceph osd pool set rbd size 1
#######
# OSD #
#######
if [ ! -e /var/lib/ceph/osd/${CLUSTER}-0/keyring ]; then
# bootstrap OSD
mkdir -p /var/lib/ceph/osd/${CLUSTER}-0
# skip btrfs HACK if btrfs is already in place
if [ "$(stat -f /var/lib/ceph/osd/${CLUSTER}-0 2>/dev/null | grep btrfs | wc -l)" == "0" ]; then
# HACK create btrfs loopback device
echo "creating osd storage image"
dd if=/dev/zero of=/tmp/osddata bs=1M count=${OSD_SIZE}
mkfs.btrfs /tmp/osddata
echo "mounting via loopback"
mount -o loop /tmp/osddata /var/lib/ceph/osd/${CLUSTER}-0
echo "now mounted:"
mount
# end HACK
fi
echo "creating osd"
ceph osd create
echo "creating osd filesystem"
ceph-osd -i 0 --mkfs
echo "creating osd keyring"
ceph auth get-or-create osd.0 osd 'allow *' mon 'allow profile osd' -o /var/lib/ceph/osd/${CLUSTER}-0/keyring
echo "configuring osd crush"
ceph osd crush add 0 1 root=default host=$(hostname -s)
echo "adding osd keyring"
ceph-osd -i 0 -k /var/lib/ceph/osd/${CLUSTER}-0/keyring
fi
# start OSD
echo "starting osd"
ceph-osd --cluster=${CLUSTER} -i 0
#sleep 10
#######
# MDS #
#######
if [ ! -e /var/lib/ceph/mds/${CLUSTER}-0/keyring ]; then
# create ceph filesystem
echo "creating osd pool"
ceph osd pool create cephfs_data 8
echo "creating osd pool metadata"
ceph osd pool create cephfs_metadata 8
echo "creating cephfs"
ceph fs new cephfs cephfs_metadata cephfs_data
# bootstrap MDS
mkdir -p /var/lib/ceph/mds/${CLUSTER}-0
echo "creating mds auth"
ceph auth get-or-create mds.0 mds 'allow' osd 'allow *' mon 'allow profile mds' > /var/lib/ceph/mds/${CLUSTER}-0/keyring
fi
# start MDS
echo "starting mds"
ceph-mds --cluster=${CLUSTER} -i 0
#sleep 10
#######
# RGW #
#######
if [ ! -e /var/lib/ceph/radosgw/${RGW_NAME}/keyring ]; then
# bootstrap RGW
mkdir -p /var/lib/ceph/radosgw/${RGW_NAME}
echo "creating rgw auth"
ceph auth get-or-create client.radosgw.gateway osd 'allow rwx' mon 'allow rw' -o /var/lib/ceph/radosgw/${RGW_NAME}/keyring
fi
# start RGW
echo "starting rgw"
radosgw -c /etc/ceph/ceph.conf -n client.radosgw.gateway -k /var/lib/ceph/radosgw/${RGW_NAME}/keyring --rgw-socket-path="" --rgw-frontends="civetweb port=${RGW_CIVETWEB_PORT}"
#######
# API #
#######
# start ceph-rest-api
echo "starting rest api"
ceph-rest-api -n client.admin &
############
# Keystone #
############
if [ ! -e /etc/keystone/${CLUSTER}.conf ]; then
cat <<ENDHERE > /etc/keystone/${CLUSTER}.conf
[DEFAULT]
admin_token=${KEYSTONE_ADMIN_TOKEN}
admin_port=${KEYSTONE_ADMIN_PORT}
public_port=${KEYSTONE_PUBLIC_PORT}
[database]
connection = sqlite:////var/lib/keystone/keystone.db
ENDHERE
# start Keystone
echo "starting keystone"
keystone-all --config-file /etc/keystone/${CLUSTER}.conf &
# wait until up
while ! nc ${MON_IP} ${KEYSTONE_ADMIN_PORT} </dev/null; do
sleep 1
done
export OS_SERVICE_TOKEN=${KEYSTONE_ADMIN_TOKEN}
export OS_SERVICE_ENDPOINT=http://${MON_IP}:${KEYSTONE_ADMIN_PORT}/v2.0
echo "creating keystone service ${KEYSTONE_SERVICE}"
keystone service-create --name ${KEYSTONE_SERVICE} --type object-store
echo "creating keystone endpoint ${KEYSTONE_SERVICE}"
keystone endpoint-create --service ${KEYSTONE_SERVICE} \
--region ${KEYSTONE_ENDPOINT_REGION} \
--publicurl http://${MON_IP}:${RGW_CIVETWEB_PORT}/swift/v1 \
--internalurl http://${MON_IP}:${RGW_CIVETWEB_PORT}/swift/v1 \
--adminurl http://${MON_IP}:${RGW_CIVETWEB_PORT}/swift/v1
echo "creating keystone user ${KEYSTONE_ADMIN_USER}"
keystone user-create --name=${KEYSTONE_ADMIN_USER} --pass=${KEYSTONE_ADMIN_PASS} --email=dev@null.com
echo "creating keystone tenant ${KEYSTONE_ADMIN_TENANT}"
keystone tenant-create --name=${KEYSTONE_ADMIN_TENANT} --description=admin
echo "adding keystone role _member_"
keystone user-role-add --user=${KEYSTONE_ADMIN_USER} --tenant=${KEYSTONE_ADMIN_TENANT} --role=_member_
echo "creating keystone role admin"
keystone role-create --name=admin
echo "adding keystone role admin"
keystone user-role-add --user=${KEYSTONE_ADMIN_USER} --tenant=${KEYSTONE_ADMIN_TENANT} --role=admin
fi
#########
# WATCH #
#########
echo "watching ceph"
exec ceph -w

View file

@ -28,6 +28,10 @@ docker pull ${docker_image}
# retrieve current folder to place the config in the parent folder
thisFolder="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# create readiness notification socket
notify_sock=$(readlink -f "$thisFolder"/dockerContainerCeph.$EXECUTOR_NUMBER.swift.sock)
mkfifo "$notify_sock"
port=5034
user=test
@ -44,9 +48,9 @@ container=`docker run -d \
-e KEYSTONE_ENDPOINT_REGION=${region} \
-e KEYSTONE_SERVICE=${service} \
-e OSD_SIZE=300 \
-v ${thisFolder}/entrypoint.sh:/entrypoint.sh \
-v "$notify_sock":/run/notifyme.sock \
--privileged \
--entrypoint /entrypoint.sh ${docker_image}`
${docker_image}`
host=$(docker inspect --format="{{.NetworkSettings.IPAddress}}" "$container")
@ -57,20 +61,13 @@ echo "${docker_image} container: $container"
echo $container >> $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.swift
echo -n "Waiting for ceph initialization"
starttime=$(date +%s)
# support for GNU netcat and BSD netcat
while ! (nc -c -w 1 ${host} 80 </dev/null >&/dev/null \
|| nc -w 1 ${host} 80 </dev/null >&/dev/null); do
sleep 1
echo -n '.'
if (( $(date +%s) > starttime + 160 )); then
echo
echo "[ERROR] Waited 120 seconds, no response" >&2
exit 1
fi
done
echo
sleep 20 # the keystone server also needs some time to fully initialize
ready=$(timeout 600 cat "$notify_sock")
if [[ $ready != 'READY=1' ]]; then
echo "[ERROR] Waited 600 seconds, no response" >&2
docker logs $container
exit 1
fi
sleep 1
cat > $thisFolder/swift.config.php <<DELIM
<?php

View file

@ -38,3 +38,4 @@ fi;
# cleanup
rm -rf $thisFolder/swift.config.php
rm -rf $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.swift
rm -rf $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.swift.sock