wbrawner/server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Move files/ajax/scan.php to background job

Browse source 
The background job will now be executed in chunks of 500 users all 10 minutes.
This commit is contained in:
Lukas Reschke 2015-11-16 22:23:16 +01:00
parent d305412a35
commit 391bc49dab
9 changed files with 280 additions and 146 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

94
apps/files/ajax/scan.php
View file

@ -1,94 +0,0 @@
<?php
/**
* @author Bart Visscher <bartv@thisnet.nl>
* @author Jörn Friedrich Dreyer <jfd@butonic.de>
* @author Lukas Reschke <lukas@owncloud.com>
* @author Robin Appelman <icewind@owncloud.com>
* @author Vincent Petry <pvince81@owncloud.com>
*
* @copyright Copyright (c) 2015, ownCloud, Inc.
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
set_time_limit(0); //scanning can take ages
\OCP\JSON::checkLoggedIn();
\OCP\JSON::callCheck();
\OC::$server->getSession()->close();
$force = (isset($_GET['force']) and ($_GET['force'] === 'true'));
$dir = isset($_GET['dir']) ? (string)$_GET['dir'] : '';
if (isset($_GET['users'])) {
\OCP\JSON::checkAdminUser();
if ($_GET['users'] === 'all') {
$users = OC_User::getUsers();
} else {
$users = json_decode($_GET['users']);
}
} else {
$users = array(OC_User::getUser());
}
$eventSource = \OC::$server->createEventSource();
$listener = new ScanListener($eventSource);
foreach ($users as $user) {
$eventSource->send('user', $user);
$scanner = new \OC\Files\Utils\Scanner($user, \OC::$server->getDatabaseConnection(), \OC::$server->getLogger());
$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', array($listener, 'file'));
try {
if ($force) {
$scanner->scan($dir);
} else {
$scanner->backgroundScan($dir);
}
} catch (\Exception $e) {
$eventSource->send('error', get_class($e) . ': ' . $e->getMessage());
}
}
$eventSource->send('done', $listener->getCount());
$eventSource->close();
class ScanListener {
private $fileCount = 0;
private $lastCount = 0;
/**
* @var \OCP\IEventSource event source to pass events to
*/
private $eventSource;
/**
* @param \OCP\IEventSource $eventSource
*/
public function __construct($eventSource) {
$this->eventSource = $eventSource;
}
public function file() {
$this->fileCount++;
if ($this->fileCount > $this->lastCount + 20) { //send a count update every 20 files
$this->lastCount = $this->fileCount;
$this->eventSource->send('count', $this->fileCount);
}
}
public function getCount() {
return $this->fileCount;
}
}

2
apps/files/appinfo/info.xml
View file

@ -8,7 +8,7 @@
<shipped>true</shipped>
<standalone/>
<default_enable/>
<version>1.4.0</version>
<version>1.4.1</version>
<types>
<filesystem/>
</types>

26
apps/files/appinfo/install.php Normal file
View file

@ -0,0 +1,26 @@
<?php
/**
* @author Lukas Reschke <lukas@owncloud.com>
*
* @copyright Copyright (c) 2015, ownCloud, Inc.
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
// Cron job for scanning user storages
$jobList = \OC::$server->getJobList();
$job = 'OCA\Files\BackgroundJob\ScanFiles';
\OC::$server->getJobList()->add($job);

2
apps/files/appinfo/routes.php
View file

@ -75,8 +75,6 @@ $this->create('files_ajax_newfolder', 'ajax/newfolder.php')
->actionInclude('files/ajax/newfolder.php');
$this->create('files_ajax_rename', 'ajax/rename.php')
->actionInclude('files/ajax/rename.php');
$this->create('files_ajax_scan', 'ajax/scan.php')
->actionInclude('files/ajax/scan.php');
$this->create('files_ajax_upload', 'ajax/upload.php')
->actionInclude('files/ajax/upload.php');

5
apps/files/appinfo/update.php
View file

@ -96,6 +96,11 @@ if ($installedVersion === '1.1.9' && (
}
}
// Add cron job for scanning user storages
$jobList = \OC::$server->getJobList();
$job = 'OCA\Files\BackgroundJob\ScanFiles';
\OC::$server->getJobList()->add($job);
/**
* migrate old constant DEBUG to new config value 'debug'
*

48
apps/files/js/files.js
View file

@ -243,9 +243,6 @@
e.preventDefault(); // prevent browser from doing anything, if file isn't dropped in dropZone
});
//do a background scan if needed
scanFiles();
// display storage warnings
setTimeout(Files.displayStorageWarnings, 100);
@ -323,51 +320,6 @@
OCA.Files.Files = Files;
})();
function scanFiles(force, dir, users) {
if (!OC.currentUser) {
return;
}
if (!dir) {
dir = '';
}
force = !!force; //cast to bool
scanFiles.scanning = true;
var scannerEventSource;
if (users) {
var usersString;
if (users === 'all') {
usersString = users;
} else {
usersString = JSON.stringify(users);
}
scannerEventSource = new OC.EventSource(OC.filePath('files','ajax','scan.php'),{force: force,dir: dir, users: usersString});
} else {
scannerEventSource = new OC.EventSource(OC.filePath('files','ajax','scan.php'),{force: force,dir: dir});
}
scanFiles.cancel = scannerEventSource.close.bind(scannerEventSource);
scannerEventSource.listen('count',function(count) {
console.log(count + ' files scanned');
});
scannerEventSource.listen('folder',function(path) {
console.log('now scanning ' + path);
});
scannerEventSource.listen('error',function(message) {
console.error('Scanner error: ', message);
});
scannerEventSource.listen('done',function(count) {
scanFiles.scanning=false;
console.log('done after ' + count + ' files');
if (OCA.Files.App) {
OCA.Files.App.fileList.updateStorageStatistics(true);
}
});
scannerEventSource.listen('user',function(user) {
console.log('scanning files for ' + user);
});
}
scanFiles.scanning=false;
// TODO: move to FileList
var createDragShadow = function(event) {
// FIXME: inject file list instance somehow

114
apps/files/lib/backgroundjob/scanfiles.php Normal file
View file

@ -0,0 +1,114 @@
<?php
/**
* @author Lukas Reschke <lukas@owncloud.com>
*
* @copyright Copyright (c) 2015, ownCloud, Inc.
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OCA\Files\BackgroundJob;
use OC\Files\Utils\Scanner;
use OCP\IConfig;
use OCP\IDBConnection;
use OCP\ILogger;
use OCP\IUser;
use OCP\IUserManager;
/**
* Class ScanFiles is a background job used to run the file scanner over the user
* accounts to ensure integrity of the file cache.
*
* @package OCA\Files\BackgroundJob
*/
class ScanFiles extends \OC\BackgroundJob\TimedJob {
/** @var IConfig */
private $config;
/** @var IUserManager */
private $userManager;
/** @var IDBConnection */
private $dbConnection;
/** @var ILogger */
private $logger;
/** Amount of users that should get scanned per execution */
const USERS_PER_SESSION = 500;
/**
* @param IConfig|null $config
* @param IUserManager|null $userManager
* @param IDBConnection|null $dbConnection
* @param ILogger|null $logger
*/
public function __construct(IConfig $config = null,
IUserManager $userManager = null,
IDBConnection $dbConnection = null,
ILogger $logger = null) {
// Run once per 10 minutes
$this->setInterval(60 * 10);
if (is_null($userManager) || is_null($config)) {
$this->fixDIForJobs();
} else {
$this->config = $config;
$this->userManager = $userManager;
$this->logger = $logger;
}
}
protected function fixDIForJobs() {
$this->config = \OC::$server->getConfig();
$this->userManager = \OC::$server->getUserManager();
$this->logger = \OC::$server->getLogger();
}
/**
* @param IUser $user
*/
protected function runScanner(IUser $user) {
try {
$scanner = new Scanner(
$user->getUID(),
$this->dbConnection,
$this->logger
);
$scanner->backgroundScan('');
} catch (\Exception $e) {
$this->logger->logException($e, ['app' => 'files']);
}
\OC_Util::tearDownFS();
}
/**
* @param $argument
* @throws \Exception
*/
protected function run($argument) {
$offset = $this->config->getAppValue('files', 'cronjob_scan_files', 0);
$users = $this->userManager->search('', self::USERS_PER_SESSION, $offset);
if (!count($users)) {
// No users found, reset offset and retry
$offset = 0;
$users = $this->userManager->search('', self::USERS_PER_SESSION);
}
$offset += self::USERS_PER_SESSION;
$this->config->setAppValue('files', 'cronjob_scan_files', $offset);
foreach ($users as $user) {
$this->runScanner($user);
}
}
}

134
apps/files/tests/backgroundjob/ScanFilesTest.php Normal file
View file

@ -0,0 +1,134 @@
<?php
/**
* @author Lukas Reschke <lukas@owncloud.com>
*
* @copyright Copyright (c) 2015, ownCloud, Inc.
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OCA\Files\Tests\BackgroundJob;
use Test\TestCase;
use OCP\IConfig;
use OCP\IUserManager;
use OCA\Files\BackgroundJob\ScanFiles;
use OCP\ILogger;
/**
* Class ScanFilesTest
*
* @package OCA\Files\Tests\BackgroundJob
*/
class ScanFilesTest extends TestCase {
/** @var IConfig */
private $config;
/** @var IUserManager */
private $userManager;
/** @var ScanFiles */
private $scanFiles;
public function setUp() {
parent::setUp();
$this->config = $this->getMock('\OCP\IConfig');
$this->userManager = $this->getMock('\OCP\IUserManager');
$this->scanFiles = $this->getMockBuilder('\OCA\Files\BackgroundJob\ScanFiles')
->setConstructorArgs([
$this->config,
$this->userManager,
])
->setMethods(['runScanner'])
->getMock();
}
public function testRunWithoutUsers() {
$this->config
->expects($this->at(0))
->method('getAppValue')
->with('files', 'cronjob_scan_files', 0)
->will($this->returnValue(50));
$this->userManager
->expects($this->at(0))
->method('search')
->with('', 500, 50)
->will($this->returnValue([]));
$this->userManager
->expects($this->at(1))
->method('search')
->with('', 500)
->will($this->returnValue([]));
$this->config
->expects($this->at(1))
->method('setAppValue')
->with('files', 'cronjob_scan_files', 500);
$this->invokePrivate($this->scanFiles, 'run', [[]]);
}
public function testRunWithUsers() {
$fakeUser = $this->getMock('\OCP\IUser');
$this->config
->expects($this->at(0))
->method('getAppValue')
->with('files', 'cronjob_scan_files', 0)
->will($this->returnValue(50));
$this->userManager
->expects($this->at(0))
->method('search')
->with('', 500, 50)
->will($this->returnValue([
$fakeUser
]));
$this->config
->expects($this->at(1))
->method('setAppValue')
->with('files', 'cronjob_scan_files', 550);
$this->scanFiles
->expects($this->once())
->method('runScanner')
->with($fakeUser);
$this->invokePrivate($this->scanFiles, 'run', [[]]);
}
public function testRunWithUsersAndOffsetAtEndOfUserList() {
$this->config
->expects($this->at(0))
->method('getAppValue')
->with('files', 'cronjob_scan_files', 0)
->will($this->returnValue(50));
$this->userManager
->expects($this->at(0))
->method('search')
->with('', 500, 50)
->will($this->returnValue([]));
$this->userManager
->expects($this->at(1))
->method('search')
->with('', 500)
->will($this->returnValue([]));
$this->config
->expects($this->at(1))
->method('setAppValue')
->with('files', 'cronjob_scan_files', 500);
$this->scanFiles
->expects($this->never())
->method('runScanner');
$this->invokePrivate($this->scanFiles, 'run', [[]]);
}
}

1
lib/public/backgroundjob/ijoblist.php
View file

@ -36,7 +36,6 @@ interface IJobList {
*
* @param \OCP\BackgroundJob\IJob|string $job
* @param mixed $argument The argument to be passed to $job->run() when the job is exectured
* @param string $job
* @return void
* @since 7.0.0
*/