From 286b61d167d1ca0b67b09c71511871e64768d36a Mon Sep 17 00:00:00 2001 From: Angel Fernando Quiroz Campos <1697880+AngelFQC@users.noreply.github.com> Date: Mon, 18 Nov 2024 17:37:30 -0500 Subject: [PATCH 1/2] Plugin: Azure: Add options to user delta queries when syncing - refs BT#21930 Requires DB changes: ```sql CREATE TABLE azure_ad_sync_state (id INT AUTO_INCREMENT NOT NULL, title VARCHAR(255) NOT NULL, value LONGTEXT NOT NULL, created_at DATETIME NOT NULL, updated_at DATETIME NOT NULL, PRIMARY KEY(id)) DEFAULT CHARACTER SET utf8 COLLATE `utf8_unicode_ci` ENGINE = InnoDB; ``` --- plugin/azure_active_directory/lang/dutch.php | 4 + .../azure_active_directory/lang/english.php | 4 + plugin/azure_active_directory/lang/french.php | 4 + .../azure_active_directory/lang/spanish.php | 4 + .../src/AzureActiveDirectory.php | 61 +++++++++++++++ .../src/AzureCommand.php | 67 ++++++++++++++--- .../src/AzureSyncUsersCommand.php | 16 ++-- .../src/Entity/AzureSyncState.php | 74 +++++++++++++++++++ plugin/azure_active_directory/uninstall.php | 9 +++ 9 files changed, 224 insertions(+), 19 deletions(-) create mode 100644 plugin/azure_active_directory/src/Entity/AzureSyncState.php create mode 100644 plugin/azure_active_directory/uninstall.php diff --git a/plugin/azure_active_directory/lang/dutch.php b/plugin/azure_active_directory/lang/dutch.php index 32885f3f68c..48a8049ec37 100644 --- a/plugin/azure_active_directory/lang/dutch.php +++ b/plugin/azure_active_directory/lang/dutch.php @@ -46,3 +46,7 @@ $strings['tenant_id_help'] = 'Required to run scripts.'; $strings['deactivate_nonexisting_users'] = 'Deactivate non-existing users'; $strings['deactivate_nonexisting_users_help'] = 'Compare registered users in Chamilo with those in Azure and deactivate accounts in Chamilo that do not exist in Azure.'; +$strings['script_users_delta'] = 'Delta query for users'; +$strings['script_users_delta_help'] = 'Get newly created, updated, or deleted users without having to perform a full read of the entire user collection. By default, is No.'; +$strings['script_usergroups_delta'] = 'Delta query for usergroups'; +$strings['script_usergroups_delta_help'] = 'Get newly created, updated, or deleted groups, including group membership changes, without having to perform a full read of the entire group collection. By default, is No.'; diff --git a/plugin/azure_active_directory/lang/english.php b/plugin/azure_active_directory/lang/english.php index 0d000a7d797..defb392d1e0 100644 --- a/plugin/azure_active_directory/lang/english.php +++ b/plugin/azure_active_directory/lang/english.php @@ -46,3 +46,7 @@ $strings['tenant_id_help'] = 'Required to run scripts.'; $strings['deactivate_nonexisting_users'] = 'Deactivate non-existing users'; $strings['deactivate_nonexisting_users_help'] = 'Compare registered users in Chamilo with those in Azure and deactivate accounts in Chamilo that do not exist in Azure.'; +$strings['script_users_delta'] = 'Delta query for users'; +$strings['script_users_delta_help'] = 'Get newly created, updated, or deleted users without having to perform a full read of the entire user collection. By default, is No.'; +$strings['script_usergroups_delta'] = 'Delta query for usergroups'; +$strings['script_usergroups_delta_help'] = 'Get newly created, updated, or deleted groups, including group membership changes, without having to perform a full read of the entire group collection. By default, is No.'; diff --git a/plugin/azure_active_directory/lang/french.php b/plugin/azure_active_directory/lang/french.php index e699c6d91d2..3707e64de8b 100644 --- a/plugin/azure_active_directory/lang/french.php +++ b/plugin/azure_active_directory/lang/french.php @@ -46,3 +46,7 @@ $strings['tenant_id_help'] = 'Nécessaire pour exécuter des scripts.'; $strings['deactivate_nonexisting_users'] = 'Deactivate non-existing users'; $strings['deactivate_nonexisting_users_help'] = 'Compare registered users in Chamilo with those in Azure and deactivate accounts in Chamilo that do not exist in Azure.'; +$strings['script_users_delta'] = 'Requête delta pour les utilisateurs'; +$strings['script_users_delta_help'] = 'Get newly created, updated, or deleted users without having to perform a full read of the entire user collection. By default, is No.'; +$strings['script_usergroups_delta'] = 'Requête delta pour les groupes d\'utilisateurs'; +$strings['script_usergroups_delta_help'] = 'Get newly created, updated, or deleted groups, including group membership changes, without having to perform a full read of the entire group collection. By default, is No.'; diff --git a/plugin/azure_active_directory/lang/spanish.php b/plugin/azure_active_directory/lang/spanish.php index 389a4912e49..4885c5df063 100644 --- a/plugin/azure_active_directory/lang/spanish.php +++ b/plugin/azure_active_directory/lang/spanish.php @@ -46,3 +46,7 @@ $strings['tenant_id_help'] = 'Necesario para ejecutar scripts.'; $strings['deactivate_nonexisting_users'] = 'Desactivar usuarios no existentes'; $strings['deactivate_nonexisting_users_help'] = 'Compara los usuarios registrados en Chamilo con los de Azure y desactiva las cuentas en Chamilo que no existan en Azure.'; +$strings['script_users_delta'] = 'Consula delta para usuarios'; +$strings['script_users_delta_help'] = 'Obtiene usuarios recién creados, actualizados o eliminados sin tener que realizar una lectura completa de toda la colección de usuarios. De forma predeterminada, es No.'; +$strings['script_usergroups_delta'] = 'Consulta delta para grupos de usuarios'; +$strings['script_usergroups_delta_help'] = 'Obtiene grupos recién creados, actualizados o eliminados, incluidos los cambios de membresía del grupo, sin tener que realizar una lectura completa de toda la colección de grupos. De forma predeterminada, es No'; diff --git a/plugin/azure_active_directory/src/AzureActiveDirectory.php b/plugin/azure_active_directory/src/AzureActiveDirectory.php index fce9cf9b689..d2493f11be4 100644 --- a/plugin/azure_active_directory/src/AzureActiveDirectory.php +++ b/plugin/azure_active_directory/src/AzureActiveDirectory.php @@ -1,7 +1,10 @@ 'text', self::SETTING_TENANT_ID => 'text', self::SETTING_DEACTIVATE_NONEXISTING_USERS => 'boolean', + self::SETTING_GET_USERS_DELTA => 'boolean', + self::SETTING_GET_USERGROUPS_DELTA => 'boolean', ]; parent::__construct('2.4', 'Angel Fernando Quiroz Campos, Yannick Warnier', $settings); @@ -131,6 +138,8 @@ public function getUrl($urlType) /** * Create extra fields for user when installing. + * + * @throws ToolsException */ public function install() { @@ -152,6 +161,35 @@ public function install() $this->get_lang('AzureUid'), '' ); + + $em = Database::getManager(); + + if ($em->getConnection()->getSchemaManager()->tablesExist(['course_home_notify_notification'])) { + return; + } + + $schemaTool = new SchemaTool($em); + $schemaTool->createSchema( + [ + $em->getClassMetadata(AzureSyncState::class), + ] + ); + } + + public function uninstall() + { + $em = Database::getManager(); + + if (!$em->getConnection()->getSchemaManager()->tablesExist(['course_home_notify_notification'])) { + return; + } + + $schemaTool = new SchemaTool($em); + $schemaTool->dropSchema( + [ + $em->getClassMetadata(AzureSyncState::class), + ] + ); } public function getExistingUserVerificationOrder(): array @@ -385,4 +423,27 @@ private function formatUserData( $extra, ]; } + + public function getSyncState(string $title): ?AzureSyncState + { + $stateRepo = Database::getManager()->getRepository(AzureSyncState::class); + + return $stateRepo->findOneBy(['title' => $title]); + } + + public function saveSyncState(string $title, $value) + { + $state = $this->getSyncState($title); + + if (!$state) { + $state = new AzureSyncState(); + $state->setTitle($title); + + Database::getManager()->persist($state); + } + + $state->setValue($value); + + Database::getManager()->flush(); + } } diff --git a/plugin/azure_active_directory/src/AzureCommand.php b/plugin/azure_active_directory/src/AzureCommand.php index ce79c45ca2f..b07dd79a7fb 100644 --- a/plugin/azure_active_directory/src/AzureCommand.php +++ b/plugin/azure_active_directory/src/AzureCommand.php @@ -2,6 +2,7 @@ /* For license terms, see /license.txt */ +use Chamilo\PluginBundle\Entity\AzureActiveDirectory\AzureSyncState; use League\OAuth2\Client\Provider\Exception\IdentityProviderException; use League\OAuth2\Client\Token\AccessTokenInterface; use TheNetworg\OAuth2\Client\Provider\Azure; @@ -56,11 +57,21 @@ protected function getAzureUsers(): Generator 'id', ]; - $query = sprintf( - '$top=%d&$select=%s', - AzureActiveDirectory::API_PAGE_SIZE, - implode(',', $userFields) - ); + $getUsersDelta = 'true' === $this->plugin->get(AzureActiveDirectory::SETTING_GET_USERS_DELTA); + + if ($getUsersDelta) { + $usersDeltaLink = $this->plugin->getSyncState(AzureSyncState::USERS_DATALINK); + + $query = $usersDeltaLink + ? $usersDeltaLink->getValue() + : sprintf('$select=%s', implode(',', $userFields)); + } else { + $query = sprintf( + '$top=%d&$select=%s', + AzureActiveDirectory::API_PAGE_SIZE, + implode(',', $userFields) + ); + } $token = null; @@ -70,7 +81,7 @@ protected function getAzureUsers(): Generator try { $azureUsersRequest = $this->provider->request( 'get', - "users?$query", + $getUsersDelta ? "users/delta?$query" : "users?$query", $token ); } catch (Exception $e) { @@ -80,6 +91,10 @@ protected function getAzureUsers(): Generator $azureUsersInfo = $azureUsersRequest['value'] ?? []; foreach ($azureUsersInfo as $azureUserInfo) { + $azureUserInfo['mail'] = $azureUserInfo['mail'] ?? null; + $azureUserInfo['surname'] = $azureUserInfo['surname'] ?? null; + $azureUserInfo['givenName'] = $azureUserInfo['givenName'] ?? null; + yield $azureUserInfo; } @@ -89,6 +104,13 @@ protected function getAzureUsers(): Generator $hasNextLink = true; $query = parse_url($azureUsersRequest['@odata.nextLink'], PHP_URL_QUERY); } + + if ($getUsersDelta && !empty($azureUsersRequest['@odata.deltaLink'])) { + $this->plugin->saveSyncState( + AzureSyncState::USERS_DATALINK, + parse_url($azureUsersRequest['@odata.deltaLink'], PHP_URL_QUERY), + ); + } } while ($hasNextLink); } @@ -105,11 +127,21 @@ protected function getAzureGroups(): Generator 'description', ]; - $query = sprintf( - '$top=%d&$select=%s', - AzureActiveDirectory::API_PAGE_SIZE, - implode(',', $groupFields) - ); + $getUsergroupsDelta = 'true' === $this->plugin->get(AzureActiveDirectory::SETTING_GET_USERGROUPS_DELTA); + + if ($getUsergroupsDelta) { + $usergroupsDeltaLink = $this->plugin->getSyncState(AzureSyncState::USERGROUPS_DATALINK); + + $query = $usergroupsDeltaLink + ? $usergroupsDeltaLink->getValue() + : sprintf('$select=%s', implode(',', $groupFields)); + } else { + $query = sprintf( + '$top=%d&$select=%s', + AzureActiveDirectory::API_PAGE_SIZE, + implode(',', $groupFields) + ); + } $token = null; @@ -117,7 +149,11 @@ protected function getAzureGroups(): Generator $this->generateOrRefreshToken($token); try { - $azureGroupsRequest = $this->provider->request('get', "groups?$query", $token); + $azureGroupsRequest = $this->provider->request( + 'get', + $getUsergroupsDelta ? "groups/delta?$query" : "groups?$query", + $token + ); } catch (Exception $e) { throw new Exception('Exception when requesting groups from Azure: '.$e->getMessage()); } @@ -134,6 +170,13 @@ protected function getAzureGroups(): Generator $hasNextLink = true; $query = parse_url($azureGroupsRequest['@odata.nextLink'], PHP_URL_QUERY); } + + if ($getUsergroupsDelta && !empty($azureGroupsRequest['@odata.deltaLink'])) { + $this->plugin->saveSyncState( + AzureSyncState::USERGROUPS_DATALINK, + parse_url($azureGroupsRequest['@odata.deltaLink'], PHP_URL_QUERY), + ); + } } while ($hasNextLink); } diff --git a/plugin/azure_active_directory/src/AzureSyncUsersCommand.php b/plugin/azure_active_directory/src/AzureSyncUsersCommand.php index 36b60f9a2f7..72c198ee779 100644 --- a/plugin/azure_active_directory/src/AzureSyncUsersCommand.php +++ b/plugin/azure_active_directory/src/AzureSyncUsersCommand.php @@ -15,8 +15,8 @@ public function __invoke(): Generator { yield 'Synchronizing users from Azure.'; - /** @var array $existingUsers */ - $existingUsers = []; + /** @var array $azureCreatedUserIdList */ + $azureCreatedUserIdList = []; foreach ($this->getAzureUsers() as $azureUserInfo) { try { @@ -27,7 +27,7 @@ public function __invoke(): Generator continue; } - $existingUsers[$azureUserInfo['id']] = $userId; + $azureCreatedUserIdList[$azureUserInfo['id']] = $userId; yield sprintf('User (ID %d) with received info: %s ', $userId, serialize($azureUserInfo)); } @@ -53,7 +53,7 @@ public function __invoke(): Generator $azureGroupMembersUids = array_column($azureGroupMembersInfo, 'id'); foreach ($azureGroupMembersUids as $azureGroupMembersUid) { - $userId = $existingUsers[$azureGroupMembersUid] ?? null; + $userId = $azureCreatedUserIdList[$azureGroupMembersUid] ?? null; if (!$userId) { continue; @@ -72,20 +72,22 @@ public function __invoke(): Generator $em->flush(); } - if ('true' === $this->plugin->get(AzureActiveDirectory::SETTING_DEACTIVATE_NONEXISTING_USERS)) { + if ('true' === $this->plugin->get(AzureActiveDirectory::SETTING_DEACTIVATE_NONEXISTING_USERS) + && 'true' !== $this->plugin->get(AzureActiveDirectory::SETTING_GET_USERS_DELTA) + ) { yield '----------------'; yield 'Trying deactivate non-existing users in Azure'; $users = UserManager::getRepository()->findByAuthSource('azure'); - $userIdList = array_map( + $chamiloUserIdList = array_map( function ($user) { return $user->getId(); }, $users ); - $nonExistingUsers = array_diff($userIdList, $existingUsers); + $nonExistingUsers = array_diff($chamiloUserIdList, $azureCreatedUserIdList); UserManager::deactivate_users($nonExistingUsers); diff --git a/plugin/azure_active_directory/src/Entity/AzureSyncState.php b/plugin/azure_active_directory/src/Entity/AzureSyncState.php new file mode 100644 index 00000000000..a61016770b1 --- /dev/null +++ b/plugin/azure_active_directory/src/Entity/AzureSyncState.php @@ -0,0 +1,74 @@ +id; + } + + public function getTitle(): string + { + return $this->title; + } + + public function setTitle(string $title): AzureSyncState + { + $this->title = $title; + + return $this; + } + + public function getValue(): string + { + return $this->value; + } + + public function setValue(string $value): AzureSyncState + { + $this->value = $value; + + return $this; + } +} diff --git a/plugin/azure_active_directory/uninstall.php b/plugin/azure_active_directory/uninstall.php new file mode 100644 index 00000000000..43a1f4fa731 --- /dev/null +++ b/plugin/azure_active_directory/uninstall.php @@ -0,0 +1,9 @@ +uninstall(); From 5e1031678798f1229acb4315e07df261cbbbcab8 Mon Sep 17 00:00:00 2001 From: Angel Fernando Quiroz Campos <1697880+AngelFQC@users.noreply.github.com> Date: Mon, 18 Nov 2024 17:37:58 -0500 Subject: [PATCH 2/2] Plugin: Azure: Bump version to v2.5 - refs BT#21930 --- plugin/azure_active_directory/CHANGELOG.md | 8 ++++++++ .../azure_active_directory/src/AzureActiveDirectory.php | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/plugin/azure_active_directory/CHANGELOG.md b/plugin/azure_active_directory/CHANGELOG.md index cb19ecdeb9a..f2c618ee346 100644 --- a/plugin/azure_active_directory/CHANGELOG.md +++ b/plugin/azure_active_directory/CHANGELOG.md @@ -1,5 +1,13 @@ # Azure Active Directory Changelog +## 2.5 - 2024-11-18 + +* Added new options to get the user and groups with delta query (or change tracking) when syncing with scripts. +this requires manually doing the following changes to your database if you are upgrading from v2.4 +```sql +CREATE TABLE azure_ad_sync_state (id INT AUTO_INCREMENT NOT NULL, title VARCHAR(255) NOT NULL, value LONGTEXT NOT NULL, created_at DATETIME NOT NULL, updated_at DATETIME NOT NULL, PRIMARY KEY(id)) DEFAULT CHARACTER SET utf8 COLLATE `utf8_unicode_ci` ENGINE = InnoDB; +``` + ## 2.4 - 2024-08-28 * Added a new user extra field to save the unique Azure ID (internal UID). diff --git a/plugin/azure_active_directory/src/AzureActiveDirectory.php b/plugin/azure_active_directory/src/AzureActiveDirectory.php index d2493f11be4..4b1e3cc63a5 100644 --- a/plugin/azure_active_directory/src/AzureActiveDirectory.php +++ b/plugin/azure_active_directory/src/AzureActiveDirectory.php @@ -68,7 +68,7 @@ protected function __construct() self::SETTING_GET_USERGROUPS_DELTA => 'boolean', ]; - parent::__construct('2.4', 'Angel Fernando Quiroz Campos, Yannick Warnier', $settings); + parent::__construct('2.5', 'Angel Fernando Quiroz Campos, Yannick Warnier', $settings); } /**