wbrawner/server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

move LDAP user attributes "sync" to background (except for ajax jobs)

Browse source 
Signed-off-by: Arthur Schiwon <blizzz@arthur-schiwon.de>
This commit is contained in:
Arthur Schiwon 2017-10-31 19:42:17 +01:00
parent ef3cd32916
commit 59c05d5447
No known key found for this signature in database
GPG key ID: 7424F1874854DF23
11 changed files with 418 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
apps/user_ldap/appinfo/info.xml
View file

@ -10,7 +10,7 @@ A user logs into Nextcloud with their LDAP or AD credentials, and is granted acc
<licence>AGPL</licence>
<author>Dominik Schmidt</author>
<author>Arthur Schiwon</author>
<version>1.3.0</version>
<version>1.3.1</version>
<types>
<authentication/>
</types>
@ -27,6 +27,7 @@ A user logs into Nextcloud with their LDAP or AD credentials, and is granted acc
<background-jobs>
<job>OCA\User_LDAP\Jobs\UpdateGroups</job>
<job>OCA\User_LDAP\Jobs\CleanUp</job>
<job>OCA\User_LDAP\Jobs\Sync</job>
</background-jobs>
<settings>

1
apps/user_ldap/composer/composer/autoload_classmap.php
View file

@ -34,6 +34,7 @@ return array(
'OCA\\User_LDAP\\ILDAPWrapper' => $baseDir . '/../lib/ILDAPWrapper.php',
'OCA\\User_LDAP\\IUserLDAP' => $baseDir . '/../lib/IUserLDAP.php',
'OCA\\User_LDAP\\Jobs\\CleanUp' => $baseDir . '/../lib/Jobs/CleanUp.php',
'OCA\\User_LDAP\\Jobs\\Sync' => $baseDir . '/../lib/Jobs/Sync.php',
'OCA\\User_LDAP\\Jobs\\UpdateGroups' => $baseDir . '/../lib/Jobs/UpdateGroups.php',
'OCA\\User_LDAP\\LDAP' => $baseDir . '/../lib/LDAP.php',
'OCA\\User_LDAP\\LDAPProvider' => $baseDir . '/../lib/LDAPProvider.php',

1
apps/user_ldap/composer/composer/autoload_static.php
View file

@ -49,6 +49,7 @@ class ComposerStaticInitUser_LDAP
'OCA\\User_LDAP\\ILDAPWrapper' => __DIR__ . '/..' . '/../lib/ILDAPWrapper.php',
'OCA\\User_LDAP\\IUserLDAP' => __DIR__ . '/..' . '/../lib/IUserLDAP.php',
'OCA\\User_LDAP\\Jobs\\CleanUp' => __DIR__ . '/..' . '/../lib/Jobs/CleanUp.php',
'OCA\\User_LDAP\\Jobs\\Sync' => __DIR__ . '/..' . '/../lib/Jobs/Sync.php',
'OCA\\User_LDAP\\Jobs\\UpdateGroups' => __DIR__ . '/..' . '/../lib/Jobs/UpdateGroups.php',
'OCA\\User_LDAP\\LDAP' => __DIR__ . '/..' . '/../lib/LDAP.php',
'OCA\\User_LDAP\\LDAPProvider' => __DIR__ . '/..' . '/../lib/LDAPProvider.php',

72
apps/user_ldap/lib/Access.php
View file

@ -58,6 +58,8 @@ use OCP\IServerContainer;
* @package OCA\User_LDAP
*/
class Access extends LDAPUtility implements IUserTools {
const UUID_ATTRIBUTES = ['entryuuid', 'nsuniqueid', 'objectguid', 'guid', 'ipauniqueid'];
/** @var \OCA\User_LDAP\Connection */
public $connection;
/** @var Manager */
@ -518,13 +520,15 @@ class Access extends LDAPUtility implements IUserTools {
/**
* returns an internal Nextcloud name for the given LDAP DN, false on DN outside of search DN
*
* @param string $fdn the dn of the user object
* @param string $ldapName optional, the display name of the object
* @param bool $isUser optional, whether it is a user object (otherwise group assumed)
* @param bool|null $newlyMapped
* @return string|false with with the name to use in Nextcloud
* @param array|null $record
* @return false|string with with the name to use in Nextcloud
*/
public function dn2ocname($fdn, $ldapName = null, $isUser = true, &$newlyMapped = null) {
public function dn2ocname($fdn, $ldapName = null, $isUser = true, &$newlyMapped = null, array $record = null) {
$newlyMapped = false;
if($isUser) {
$mapper = $this->getUserMapper();
@ -541,7 +545,7 @@ class Access extends LDAPUtility implements IUserTools {
}
//second try: get the UUID and check if it is known. Then, update the DN and return the name.
$uuid = $this->getUUID($fdn, $isUser);
$uuid = $this->getUUID($fdn, $isUser, $record);
if(is_string($uuid)) {
$ncName = $mapper->getNameByUUID($uuid);
if(is_string($ncName)) {
@ -800,7 +804,7 @@ class Access extends LDAPUtility implements IUserTools {
* utilizing the login filter.
*
* @param string $loginName
* @return array
* @return int
*/
public function countUsersByLoginName($loginName) {
$loginName = $this->escapeFilterPart($loginName);
@ -814,11 +818,22 @@ class Access extends LDAPUtility implements IUserTools {
* @param string|string[] $attr
* @param int $limit
* @param int $offset
* @param bool $forceApplyAttributes
* @return array
*/
public function fetchListOfUsers($filter, $attr, $limit = null, $offset = null) {
public function fetchListOfUsers($filter, $attr, $limit = null, $offset = null, $forceApplyAttributes = false) {
$ldapRecords = $this->searchUsers($filter, $attr, $limit, $offset);
$this->batchApplyUserAttributes($ldapRecords);
$recordsToUpdate = $ldapRecords;
if(!$forceApplyAttributes) {
$isBackgroundJobModeAjax = $this->c->getConfig()
->getAppValue('core', 'backgroundjobs_mode', 'ajax') === 'ajax';
$recordsToUpdate = array_filter($ldapRecords, function($record) use ($isBackgroundJobModeAjax) {
$newlyMapped = false;
$uid = $this->dn2ocname($record['dn'][0], null, true, $newlyMapped, $record);
return ($uid !== false) && ($newlyMapped || $isBackgroundJobModeAjax);
});
}
$this->batchApplyUserAttributes($recordsToUpdate);
return $this->fetchList($ldapRecords, (count($attr) > 1));
}
@ -830,16 +845,13 @@ class Access extends LDAPUtility implements IUserTools {
*/
public function batchApplyUserAttributes(array $ldapRecords){
$displayNameAttribute = strtolower($this->connection->ldapUserDisplayName);
$config = $this->c->getConfig();
$isBackgroundJobModeAjax = $config->getAppValue('core', 'backgroundjobs_mode', 'ajax') === 'ajax';
foreach($ldapRecords as $userRecord) {
if(!isset($userRecord[$displayNameAttribute])) {
// displayName is obligatory
continue;
}
$newlyMapped = false;
$ocName = $this->dn2ocname($userRecord['dn'][0], null, true, $newlyMapped);
if($ocName === false || ($newlyMapped === false && !$isBackgroundJobModeAjax)) {
$ocName = $this->dn2ocname($userRecord['dn'][0], null, true);
if($ocName === false) {
continue;
}
$this->cacheUserExists($ocName);
@ -1235,7 +1247,9 @@ class Access extends LDAPUtility implements IUserTools {
if($key !== 'dn') {
$selection[$i][$key] = $this->resemblesDN($key) ?
$this->helper->sanitizeDN($item[$key])
: $item[$key];
: $key === 'objectguid' || $key === 'guid' ?
$selection[$i][$key] = $this->convertObjectGUID2Str($item[$key])
: $item[$key];
} else {
$selection[$i][$key] = [$this->helper->sanitizeDN($item[$key])];
}
@ -1527,12 +1541,14 @@ class Access extends LDAPUtility implements IUserTools {
/**
* auto-detects the directory's UUID attribute
*
* @param string $dn a known DN used to check against
* @param bool $isUser
* @param bool $force the detection should be run, even if it is not set to auto
* @param array|null $ldapRecord
* @return bool true on success, false otherwise
*/
private function detectUuidAttribute($dn, $isUser = true, $force = false) {
private function detectUuidAttribute($dn, $isUser = true, $force = false, array $ldapRecord = null) {
if($isUser) {
$uuidAttr = 'ldapUuidUserAttribute';
$uuidOverride = $this->connection->ldapExpertUUIDUserAttr;
@ -1550,10 +1566,17 @@ class Access extends LDAPUtility implements IUserTools {
return true;
}
// for now, supported attributes are entryUUID, nsuniqueid, objectGUID, ipaUniqueID
$testAttributes = array('entryuuid', 'nsuniqueid', 'objectguid', 'guid', 'ipauniqueid');
foreach(self::UUID_ATTRIBUTES as $attribute) {
if($ldapRecord !== null) {
// we have the info from LDAP already, we don't need to talk to the server again
if(isset($ldapRecord[$attribute])) {
$this->connection->$uuidAttr = $attribute;
return true;
} else {
continue;
}
}
foreach($testAttributes as $attribute) {
$value = $this->readAttribute($dn, $attribute);
if(is_array($value) && isset($value[0]) && !empty($value[0])) {
\OCP\Util::writeLog('user_ldap',
@ -1573,9 +1596,10 @@ class Access extends LDAPUtility implements IUserTools {
/**
* @param string $dn
* @param bool $isUser
* @return string|bool
* @param null $ldapRecord
* @return bool|string
*/
public function getUUID($dn, $isUser = true) {
public function getUUID($dn, $isUser = true, $ldapRecord = null) {
if($isUser) {
$uuidAttr = 'ldapUuidUserAttribute';
$uuidOverride = $this->connection->ldapExpertUUIDUserAttr;
@ -1585,14 +1609,16 @@ class Access extends LDAPUtility implements IUserTools {
}
$uuid = false;
if($this->detectUuidAttribute($dn, $isUser)) {
if($this->detectUuidAttribute($dn, $isUser, false, $ldapRecord)) {
$attr = $this->connection->$uuidAttr;
$uuid = $this->readAttribute($dn, $attr);
$uuid = isset($ldapRecord[$attr]) ? $ldapRecord[$attr] : $this->readAttribute($dn, $attr);
if( !is_array($uuid)
&& $uuidOverride !== ''
&& $this->detectUuidAttribute($dn, $isUser, true)) {
$uuid = $this->readAttribute($dn,
$this->connection->$uuidAttr);
&& $this->detectUuidAttribute($dn, $isUser, true, $ldapRecord))
{
$uuid = isset($ldapRecord[$this->connection->$uuidAttr])
? $ldapRecord[$this->connection->$uuidAttr]
: $this->readAttribute($dn, $this->connection->$uuidAttr);
}
if(is_array($uuid) && isset($uuid[0]) && !empty($uuid[0])) {
$uuid = $uuid[0];

1
apps/user_ldap/lib/Configuration.php
View file

@ -282,6 +282,7 @@ class Configuration {
}
$this->saveValue($cta[$key], $value);
}
$this->saveValue('_lastChange', time());
$this->unsavedChanges = [];
}

6
apps/user_ldap/lib/Connection.php
View file

@ -54,6 +54,8 @@ use OC\ServerNotAvailableException;
* @property bool|mixed|void ldapGroupMemberAssocAttr
* @property string ldapUuidUserAttribute
* @property string ldapUuidGroupAttribute
* @property string ldapExpertUUIDUserAttr
* @property string ldapExpertUUIDGroupAttr
*/
class Connection extends LDAPUtility {
private $ldapConnectionRes = null;
@ -350,8 +352,8 @@ class Connection extends LDAPUtility {
if(!empty($uuidOverride)) {
$this->configuration->$effectiveSetting = $uuidOverride;
} else {
$uuidAttributes = array('auto', 'entryuuid', 'nsuniqueid',
'objectguid', 'guid', 'ipauniqueid');
$uuidAttributes = Access::UUID_ATTRIBUTES;
array_unshift($uuidAttributes, 'auto');
if(!in_array($this->configuration->$effectiveSetting,
$uuidAttributes)
&& (!is_null($this->configID))) {

332
apps/user_ldap/lib/Jobs/Sync.php Normal file
View file

@ -0,0 +1,332 @@
<?php
/**
* @copyright Copyright (c) 2017 Arthur Schiwon <blizzz@arthur-schiwon.de>
*
* @author Arthur Schiwon <blizzz@arthur-schiwon.de>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\User_LDAP\Jobs;
use OC\BackgroundJob\TimedJob;
use OC\ServerNotAvailableException;
use OCA\User_LDAP\Access;
use OCA\User_LDAP\Configuration;
use OCA\User_LDAP\Connection;
use OCA\User_LDAP\FilesystemHelper;
use OCA\User_LDAP\Helper;
use OCA\User_LDAP\LDAP;
use OCA\User_LDAP\LogWrapper;
use OCA\User_LDAP\Mapping\UserMapping;
use OCA\User_LDAP\User\Manager;
use OCA\User_LDAP\User_LDAP;
use OCP\Image;
use OCP\IServerContainer;
class Sync extends TimedJob {
/** @var IServerContainer */
protected $c;
/** @var Helper */
protected $ldapHelper;
/** @var LDAP */
protected $ldap;
/** @var Manager */
protected $userManager;
/** @var UserMapping */
protected $mapper;
/** @var int */
protected $maxInterval = 12 * 60 * 60; // 12h
/** @var int */
protected $minInterval = 30 * 60; // 30min
public function __construct() {
$this->setInterval(
\OC::$server->getConfig()->getAppValue(
'user_ldap',
'background_sync_interval',
$this->minInterval
)
);
}
/**
* updates the interval
*
* the idea is to adjust the interval depending on the amount of known users
* and the attempt to update each user one day. At most it would run every
* 30 minutes, and at least every 12 hours.
*/
public function updateInterval() {
$minPagingSize = $this->getMinPagingSize();
$mappedUsers = $this->mapper->count();
$runsPerDay = ($minPagingSize === 0) ? $this->maxInterval : $mappedUsers / $minPagingSize;
$interval = floor(24 * 60 * 60 / $runsPerDay);
$interval = min(max($interval, $this->minInterval), $this->maxInterval);
$this->c->getConfig()->setAppValue('user_ldap', 'background_sync_interval', $interval);
}
/**
* returns the smallest configured paging size
* @return int
*/
protected function getMinPagingSize() {
$config = $this->c->getConfig();
$configKeys = $config->getAppKeys('user_ldap');
$configKeys = array_filter($configKeys, function($key) {
return strpos($key, 'ldap_paging_size') !== false;
});
$minPagingSize = null;
foreach ($configKeys as $configKey) {
$pagingSize = $config->getAppValue('user_ldap', $configKey, $minPagingSize);
$minPagingSize = $minPagingSize === null ? $pagingSize : min($minPagingSize, $pagingSize);
}
return (int)$minPagingSize;
}
/**
* @param array $argument
*/
protected function run($argument) {
$this->setArgument($argument);
$isBackgroundJobModeAjax = $this->c->getConfig()
->getAppValue('core', 'backgroundjobs_mode', 'ajax') === 'ajax';
if($isBackgroundJobModeAjax) {
return;
}
$cycleData = $this->getCycle();
if($cycleData === null) {
$cycleData = $this->determineNextCycle();
if($cycleData === null) {
$this->updateInterval();
return;
}
}
if(!$this->qualifiesToRun($cycleData)) {
$this->updateInterval();
return;
}
try {
$expectMoreResults = $this->runCycle($cycleData);
if ($expectMoreResults) {
$this->increaseOffset($cycleData);
} else {
$this->determineNextCycle();
}
$this->updateInterval();
} catch (ServerNotAvailableException $e) {
$this->determineNextCycle();
}
}
/**
* @param array $cycleData
* @return bool whether more results are expected from the same configuration
*/
public function runCycle($cycleData) {
$connection = new Connection($this->ldap, $cycleData['prefix']);
$access = new Access($connection, $this->ldap, $this->userManager, $this->ldapHelper, $this->c);
$access->setUserMapper($this->mapper);
$filter = $access->combineFilterWithAnd(array(
$access->connection->ldapUserFilter,
$access->connection->ldapUserDisplayName . '=*',
$access->getFilterPartForUserSearch('')
));
$results = $access->fetchListOfUsers(
$filter,
$access->userManager->getAttributes(),
$connection->ldapPagingSize,
$cycleData['offset'],
true
);
if($connection->ldapPagingSize === 0) {
return true;
}
return count($results) !== $connection->ldapPagingSize;
}
/**
* returns the info about the current cycle that should be run, if any,
* otherwise null
*
* @return array|null
*/
public function getCycle() {
$prefixes = $this->ldapHelper->getServerConfigurationPrefixes(true);
if(count($prefixes) === 0) {
return null;
}
$config = $this->c->getConfig();
$cycleData = [
'prefix' => $config->getAppValue('user_ldap', 'background_sync_prefix', null),
'offset' => (int)$config->getAppValue('user_ldap', 'background_sync_offset', 0),
];
if(
$cycleData['prefix'] !== null
&& in_array($cycleData['prefix'], $prefixes)
) {
return $cycleData;
}
return null;
}
/**
* Save the provided cycle information in the DB
*
* @param array $cycleData
*/
public function setCycle(array $cycleData) {
$config = $this->c->getConfig();
$config->setAppValue('user_ldap', 'background_sync_prefix', $cycleData['prefix']);
$config->setAppValue('user_ldap', 'background_sync_offset', $cycleData['offset']);
}
/**
* returns data about the next cycle that should run, if any, otherwise
* null. It also always goes for the next LDAP configuration!
*
* @param array|null $cycleData the old cycle
* @return array|null
*/
public function determineNextCycle(array $cycleData = null) {
$prefixes = $this->ldapHelper->getServerConfigurationPrefixes(true);
if(count($prefixes) === 0) {
return null;
}
// get the next prefix in line and remember it
$oldPrefix = $cycleData === null ? null : $cycleData['prefix'];
$prefix = $this->getNextPrefix($oldPrefix);
if($prefix === null) {
return null;
}
$cycleData['prefix'] = $prefix;
$cycleData['offset'] = 0;
$this->setCycle(['prefix' => $prefix, 'offset' => 0]);
return $cycleData;
}
/**
* Checks whether the provided cycle should be run. Currently only the
* last configuration change goes into account (at least one hour).
*
* @param $cycleData
* @return bool
*/
protected function qualifiesToRun($cycleData) {
$config = $this->c->getConfig();
$lastChange = $config->getAppValue('user_ldap', $cycleData['prefix'] . '_lastChange', 0);
if((time() - $lastChange) > 60 * 30) {
return true;
}
return false;
}
/**
* increases the offset of the current cycle for the next run
*
* @param $cycleData
*/
protected function increaseOffset($cycleData) {
$ldapConfig = new Configuration($cycleData['prefix']);
$cycleData['offset'] += (int)$ldapConfig->ldapPagingSize;
$this->setCycle($cycleData);
}
/**
* determines the next configuration prefix based on the last one (if any)
*
* @param string|null $lastPrefix
* @return string|null
*/
protected function getNextPrefix($lastPrefix) {
$prefixes = $this->ldapHelper->getServerConfigurationPrefixes(true);
$noOfPrefixes = count($prefixes);
if($noOfPrefixes === 0) {
return null;
}
$i = $lastPrefix === null ? false : array_search($lastPrefix, $prefixes, true);
if($i === false) {
$i = -1;
} else {
$i++;
}
if(!isset($prefixes[$i])) {
$i = 0;
}
return $prefixes[$i];
}
/**
* "fixes" DI
*
* @param array $argument
*/
public function setArgument($argument) {
if(isset($argument['c'])) {
$this->c = $argument['c'];
} else {
$this->c = \OC::$server;
}
if(isset($argument['helper'])) {
$this->ldapHelper = $argument['helper'];
} else {
$this->ldapHelper = new Helper($this->c->getConfig());
}
if(isset($argument['ldapWrapper'])) {
$this->ldap = $argument['ldapWrapper'];
} else {
$this->ldap = new LDAP();
}
if(isset($argument['userManager'])) {
$this->userManager = $argument['userManager'];
} else {
$this->userManager = new Manager(
$this->c->getConfig(),
new FilesystemHelper(),
new LogWrapper(),
$this->c->getAvatarManager(),
new Image(),
$this->c->getDatabaseConnection(),
$this->c->getUserManager(),
$this->c->getNotificationManager()
);
}
if(isset($argument['mapper'])) {
$this->mapper = $argument['mapper'];
} else {
$this->mapper = new UserMapping($this->c->getDatabaseConnection());
}
}
}

15
apps/user_ldap/lib/Mapping/AbstractMapping.php
View file

@ -277,4 +277,19 @@ abstract class AbstractMapping {
->getTruncateTableSQL('`' . $this->getTableName() . '`');
return $this->dbc->prepare($sql)->execute();
}
/**
* returns the number of entries in the mappings table
*
* @return int
*/
public function count() {
$qb = $this->dbc->getQueryBuilder();
$query = $qb->select($qb->createFunction('COUNT(`ldap_dn`)'))
->from($this->getTableName());
$res = $query->execute();
$count = $res->fetchColumn();
$res->closeCursor();
return (int)$count;
}
}

4
apps/user_ldap/lib/User/Manager.php
View file

@ -28,6 +28,7 @@
namespace OCA\User_LDAP\User;
use OC\Cache\CappedMemoryCache;
use OCA\User_LDAP\Access;
use OCA\User_LDAP\LogWrapper;
use OCA\User_LDAP\FilesystemHelper;
use OCP\IAvatarManager;
@ -167,8 +168,9 @@ class Manager {
* @return string[]
*/
public function getAttributes($minimal = false) {
$attributes = array('dn', 'uid', 'samaccountname', 'memberof');
$attributes = array_merge(Access::UUID_ATTRIBUTES, ['dn', 'uid', 'samaccountname', 'memberof']);
$possible = array(
$this->access->getConnection()->ldapExpertUUIDUserAttr,
$this->access->getConnection()->ldapQuotaAttribute,
$this->access->getConnection()->ldapEmailAttribute,
$this->access->getConnection()->ldapUserDisplayName,

16
apps/user_ldap/lib/User/User.php
View file

@ -190,13 +190,6 @@ class User {
}
unset($attr);
//Email
$attr = strtolower($this->connection->ldapEmailAttribute);
if(isset($ldapEntry[$attr])) {
$this->updateEmail($ldapEntry[$attr][0]);
}
unset($attr);
//displayName
$displayName = $displayName2 = '';
$attr = strtolower($this->connection->ldapUserDisplayName);
@ -217,6 +210,15 @@ class User {
}
unset($attr);
//Email
//email must be stored after displayname, because it would cause a user
//change event that will trigger fetching the display name again
$attr = strtolower($this->connection->ldapEmailAttribute);
if(isset($ldapEntry[$attr])) {
$this->updateEmail($ldapEntry[$attr][0]);
}
unset($attr);
// LDAP Username, needed for s2s sharing
if(isset($ldapEntry['uid'])) {
$this->storeLDAPUserName($ldapEntry['uid'][0]);

2
apps/user_ldap/lib/User_LDAP.php
View file

@ -278,7 +278,7 @@ class User_LDAP extends BackendUtility implements \OCP\IUserBackend, \OCP\UserIn
Util::writeLog('user_ldap',
'getUsers: Options: search '.$search.' limit '.$limit.' offset '.$offset.' Filter: '.$filter,
Util::DEBUG);
//do the search and translate results to owncloud names
//do the search and translate results to Nextcloud names
$ldap_users = $this->access->fetchListOfUsers(
$filter,
$this->access->userManager->getAttributes(true),