chamilo · AngelFQC · Nov 18, 2024 · Nov 18, 2024
diff --git a/plugin/azure_active_directory/CHANGELOG.md b/plugin/azure_active_directory/CHANGELOG.md
@@ -1,5 +1,13 @@
 # Azure Active Directory Changelog
 
+## 2.5 - 2024-11-18
+
+* Added new options to get the user and groups with delta query (or change tracking) when syncing with scripts.
+this requires manually doing the following changes to  your database if you are upgrading from v2.4
+```sql
+CREATE TABLE azure_ad_sync_state (id INT AUTO_INCREMENT NOT NULL, title VARCHAR(255) NOT NULL, value LONGTEXT NOT NULL, created_at DATETIME NOT NULL, updated_at DATETIME NOT NULL, PRIMARY KEY(id)) DEFAULT CHARACTER SET utf8 COLLATE `utf8_unicode_ci` ENGINE = InnoDB;
+```
+
 ## 2.4 - 2024-08-28
 
 * Added a new user extra field to save the unique Azure ID (internal UID).

diff --git a/plugin/azure_active_directory/lang/dutch.php b/plugin/azure_active_directory/lang/dutch.php
@@ -46,3 +46,7 @@
 $strings['tenant_id_help'] = 'Required to run scripts.';
 $strings['deactivate_nonexisting_users'] = 'Deactivate non-existing users';
 $strings['deactivate_nonexisting_users_help'] = 'Compare registered users in Chamilo with those in Azure and deactivate accounts in Chamilo that do not exist in Azure.';
+$strings['script_users_delta'] = 'Delta query for users';
+$strings['script_users_delta_help'] = 'Get newly created, updated, or deleted users without having to perform a full read of the entire user collection. By default, is <code>No</code>.';
+$strings['script_usergroups_delta'] = 'Delta query for usergroups';
+$strings['script_usergroups_delta_help'] = 'Get newly created, updated, or deleted groups, including group membership changes, without having to perform a full read of the entire group collection. By default, is <code>No</code>.';
diff --git a/plugin/azure_active_directory/lang/english.php b/plugin/azure_active_directory/lang/english.php
@@ -46,3 +46,7 @@
 $strings['tenant_id_help'] = 'Required to run scripts.';
 $strings['deactivate_nonexisting_users'] = 'Deactivate non-existing users';
 $strings['deactivate_nonexisting_users_help'] = 'Compare registered users in Chamilo with those in Azure and deactivate accounts in Chamilo that do not exist in Azure.';
+$strings['script_users_delta'] = 'Delta query for users';
+$strings['script_users_delta_help'] = 'Get newly created, updated, or deleted users without having to perform a full read of the entire user collection. By default, is <code>No</code>.';
+$strings['script_usergroups_delta'] = 'Delta query for usergroups';
+$strings['script_usergroups_delta_help'] = 'Get newly created, updated, or deleted groups, including group membership changes, without having to perform a full read of the entire group collection. By default, is <code>No</code>.';
diff --git a/plugin/azure_active_directory/lang/french.php b/plugin/azure_active_directory/lang/french.php
@@ -46,3 +46,7 @@
 $strings['tenant_id_help'] = 'Nécessaire pour exécuter des scripts.';
 $strings['deactivate_nonexisting_users'] = 'Deactivate non-existing users';
 $strings['deactivate_nonexisting_users_help'] = 'Compare registered users in Chamilo with those in Azure and deactivate accounts in Chamilo that do not exist in Azure.';
+$strings['script_users_delta'] = 'Requête delta pour les utilisateurs';
+$strings['script_users_delta_help'] = 'Get newly created, updated, or deleted users without having to perform a full read of the entire user collection. By default, is <code>No</code>.';
+$strings['script_usergroups_delta'] = 'Requête delta pour les groupes d\'utilisateurs';
+$strings['script_usergroups_delta_help'] = 'Get newly created, updated, or deleted groups, including group membership changes, without having to perform a full read of the entire group collection. By default, is <code>No</code>.';
diff --git a/plugin/azure_active_directory/lang/spanish.php b/plugin/azure_active_directory/lang/spanish.php
@@ -46,3 +46,7 @@
 $strings['tenant_id_help'] = 'Necesario para ejecutar scripts.';
 $strings['deactivate_nonexisting_users'] = 'Desactivar usuarios no existentes';
 $strings['deactivate_nonexisting_users_help'] = 'Compara los usuarios registrados en Chamilo con los de Azure y desactiva las cuentas en Chamilo que no existan en Azure.';
+$strings['script_users_delta'] = 'Consula delta para usuarios';
+$strings['script_users_delta_help'] = 'Obtiene usuarios recién creados, actualizados o eliminados sin tener que realizar una lectura completa de toda la colección de usuarios. De forma predeterminada, es <code>No</code>.';
+$strings['script_usergroups_delta'] = 'Consulta delta para grupos de usuarios';
+$strings['script_usergroups_delta_help'] = 'Obtiene grupos recién creados, actualizados o eliminados, incluidos los cambios de membresía del grupo, sin tener que realizar una lectura completa de toda la colección de grupos. De forma predeterminada, es <code>No</code>';
diff --git a/plugin/azure_active_directory/src/AzureActiveDirectory.php b/plugin/azure_active_directory/src/AzureActiveDirectory.php
@@ -1,7 +1,10 @@
 <?php
 /* For license terms, see /license.txt */
 
+use Chamilo\PluginBundle\Entity\AzureActiveDirectory\AzureSyncState;
 use Chamilo\UserBundle\Entity\User;
+use Doctrine\ORM\Tools\SchemaTool;
+use Doctrine\ORM\Tools\ToolsException;
 use TheNetworg\OAuth2\Client\Provider\Azure;
 
 /**
@@ -28,6 +31,8 @@ class AzureActiveDirectory extends Plugin
     public const SETTING_EXISTING_USER_VERIFICATION_ORDER = 'existing_user_verification_order';
     public const SETTING_TENANT_ID = 'tenant_id';
     public const SETTING_DEACTIVATE_NONEXISTING_USERS = 'deactivate_nonexisting_users';
+    public const SETTING_GET_USERS_DELTA = 'script_users_delta';
+    public const SETTING_GET_USERGROUPS_DELTA = 'script_usergroups_delta';
 
     public const URL_TYPE_AUTHORIZE = 'login';
     public const URL_TYPE_LOGOUT = 'logout';
@@ -59,9 +64,11 @@ protected function __construct()
             self::SETTING_EXISTING_USER_VERIFICATION_ORDER => 'text',
             self::SETTING_TENANT_ID => 'text',
             self::SETTING_DEACTIVATE_NONEXISTING_USERS => 'boolean',
+            self::SETTING_GET_USERS_DELTA => 'boolean',
+            self::SETTING_GET_USERGROUPS_DELTA => 'boolean',
         ];
 
-        parent::__construct('2.4', 'Angel Fernando Quiroz Campos, Yannick Warnier', $settings);
+        parent::__construct('2.5', 'Angel Fernando Quiroz Campos, Yannick Warnier', $settings);
     }
 
     /**
@@ -131,6 +138,8 @@ public function getUrl($urlType)
 
     /**
      * Create extra fields for user when installing.
+     *
+     * @throws ToolsException
      */
     public function install()
     {
@@ -152,6 +161,35 @@ public function install()
             $this->get_lang('AzureUid'),
             ''
         );
+
+        $em = Database::getManager();
+
+        if ($em->getConnection()->getSchemaManager()->tablesExist(['course_home_notify_notification'])) {
+            return;
+        }
+
+        $schemaTool = new SchemaTool($em);
+        $schemaTool->createSchema(
+            [
+                $em->getClassMetadata(AzureSyncState::class),
+            ]
+        );
+    }
+
+    public function uninstall()
+    {
+        $em = Database::getManager();
+
+        if (!$em->getConnection()->getSchemaManager()->tablesExist(['course_home_notify_notification'])) {
+            return;
+        }
+
+        $schemaTool = new SchemaTool($em);
+        $schemaTool->dropSchema(
+            [
+                $em->getClassMetadata(AzureSyncState::class),
+            ]
+        );
     }
 
     public function getExistingUserVerificationOrder(): array
@@ -385,4 +423,27 @@ private function formatUserData(
             $extra,
         ];
     }
+
+    public function getSyncState(string $title): ?AzureSyncState
+    {
+        $stateRepo = Database::getManager()->getRepository(AzureSyncState::class);
+
+        return $stateRepo->findOneBy(['title' => $title]);
+    }
+
+    public function saveSyncState(string $title, $value)
+    {
+        $state = $this->getSyncState($title);
+
+        if (!$state) {
+            $state = new AzureSyncState();
+            $state->setTitle($title);
+
+            Database::getManager()->persist($state);
+        }
+
+        $state->setValue($value);
+
+        Database::getManager()->flush();
+    }
 }
diff --git a/plugin/azure_active_directory/src/AzureCommand.php b/plugin/azure_active_directory/src/AzureCommand.php
@@ -2,6 +2,7 @@
 
 /* For license terms, see /license.txt */
 
+use Chamilo\PluginBundle\Entity\AzureActiveDirectory\AzureSyncState;
 use League\OAuth2\Client\Provider\Exception\IdentityProviderException;
 use League\OAuth2\Client\Token\AccessTokenInterface;
 use TheNetworg\OAuth2\Client\Provider\Azure;
@@ -56,11 +57,21 @@ protected function getAzureUsers(): Generator
             'id',
         ];
 
-        $query = sprintf(
-            '$top=%d&$select=%s',
-            AzureActiveDirectory::API_PAGE_SIZE,
-            implode(',', $userFields)
-        );
+        $getUsersDelta = 'true' === $this->plugin->get(AzureActiveDirectory::SETTING_GET_USERS_DELTA);
+
+        if ($getUsersDelta) {
+            $usersDeltaLink = $this->plugin->getSyncState(AzureSyncState::USERS_DATALINK);
+
+            $query = $usersDeltaLink
+                ? $usersDeltaLink->getValue()
+                : sprintf('$select=%s', implode(',', $userFields));
+        } else {
+            $query = sprintf(
+                '$top=%d&$select=%s',
+                AzureActiveDirectory::API_PAGE_SIZE,
+                implode(',', $userFields)
+            );
+        }
 
         $token = null;
 
@@ -70,7 +81,7 @@ protected function getAzureUsers(): Generator
             try {
                 $azureUsersRequest = $this->provider->request(
                     'get',
-                    "users?$query",
+                    $getUsersDelta ? "users/delta?$query" : "users?$query",
                     $token
                 );
             } catch (Exception $e) {
@@ -80,6 +91,10 @@ protected function getAzureUsers(): Generator
             $azureUsersInfo = $azureUsersRequest['value'] ?? [];
 
             foreach ($azureUsersInfo as $azureUserInfo) {
+                $azureUserInfo['mail'] = $azureUserInfo['mail'] ?? null;
+                $azureUserInfo['surname'] = $azureUserInfo['surname'] ?? null;
+                $azureUserInfo['givenName'] = $azureUserInfo['givenName'] ?? null;
+
                 yield $azureUserInfo;
             }
 
@@ -89,6 +104,13 @@ protected function getAzureUsers(): Generator
                 $hasNextLink = true;
                 $query = parse_url($azureUsersRequest['@odata.nextLink'], PHP_URL_QUERY);
             }
+
+            if ($getUsersDelta && !empty($azureUsersRequest['@odata.deltaLink'])) {
+                $this->plugin->saveSyncState(
+                    AzureSyncState::USERS_DATALINK,
+                    parse_url($azureUsersRequest['@odata.deltaLink'], PHP_URL_QUERY),
+                );
+            }
         } while ($hasNextLink);
     }
 
@@ -105,19 +127,33 @@ protected function getAzureGroups(): Generator
             'description',
         ];
 
-        $query = sprintf(
-            '$top=%d&$select=%s',
-            AzureActiveDirectory::API_PAGE_SIZE,
-            implode(',', $groupFields)
-        );
+        $getUsergroupsDelta = 'true' === $this->plugin->get(AzureActiveDirectory::SETTING_GET_USERGROUPS_DELTA);
+
+        if ($getUsergroupsDelta) {
+            $usergroupsDeltaLink = $this->plugin->getSyncState(AzureSyncState::USERGROUPS_DATALINK);
+
+            $query = $usergroupsDeltaLink
+                ? $usergroupsDeltaLink->getValue()
+                : sprintf('$select=%s', implode(',', $groupFields));
+        } else {
+            $query = sprintf(
+                '$top=%d&$select=%s',
+                AzureActiveDirectory::API_PAGE_SIZE,
+                implode(',', $groupFields)
+            );
+        }
 
         $token = null;
 
         do {
             $this->generateOrRefreshToken($token);
 
             try {
-                $azureGroupsRequest = $this->provider->request('get', "groups?$query", $token);
+                $azureGroupsRequest = $this->provider->request(
+                    'get',
+                    $getUsergroupsDelta ? "groups/delta?$query" : "groups?$query",
+                    $token
+                );
             } catch (Exception $e) {
                 throw new Exception('Exception when requesting groups from Azure: '.$e->getMessage());
             }
@@ -134,6 +170,13 @@ protected function getAzureGroups(): Generator
                 $hasNextLink = true;
                 $query = parse_url($azureGroupsRequest['@odata.nextLink'], PHP_URL_QUERY);
             }
+
+            if ($getUsergroupsDelta && !empty($azureGroupsRequest['@odata.deltaLink'])) {
+                $this->plugin->saveSyncState(
+                    AzureSyncState::USERGROUPS_DATALINK,
+                    parse_url($azureGroupsRequest['@odata.deltaLink'], PHP_URL_QUERY),
+                );
+            }
         } while ($hasNextLink);
     }
 

diff --git a/plugin/azure_active_directory/src/AzureSyncUsersCommand.php b/plugin/azure_active_directory/src/AzureSyncUsersCommand.php
@@ -15,8 +15,8 @@ public function __invoke(): Generator
     {
         yield 'Synchronizing users from Azure.';
 
-        /** @var array<string, int> $existingUsers */
-        $existingUsers = [];
+        /** @var array<string, int> $azureCreatedUserIdList */
+        $azureCreatedUserIdList = [];
 
         foreach ($this->getAzureUsers() as $azureUserInfo) {
             try {
@@ -27,7 +27,7 @@ public function __invoke(): Generator
                 continue;
             }
 
-            $existingUsers[$azureUserInfo['id']] = $userId;
+            $azureCreatedUserIdList[$azureUserInfo['id']] = $userId;
 
             yield sprintf('User (ID %d) with received info: %s ', $userId, serialize($azureUserInfo));
         }
@@ -53,7 +53,7 @@ public function __invoke(): Generator
             $azureGroupMembersUids = array_column($azureGroupMembersInfo, 'id');
 
             foreach ($azureGroupMembersUids as $azureGroupMembersUid) {
-                $userId = $existingUsers[$azureGroupMembersUid] ?? null;
+                $userId = $azureCreatedUserIdList[$azureGroupMembersUid] ?? null;
 
                 if (!$userId) {
                     continue;
@@ -72,20 +72,22 @@ public function __invoke(): Generator
             $em->flush();
         }
 
-        if ('true' === $this->plugin->get(AzureActiveDirectory::SETTING_DEACTIVATE_NONEXISTING_USERS)) {
+        if ('true' === $this->plugin->get(AzureActiveDirectory::SETTING_DEACTIVATE_NONEXISTING_USERS)
+            && 'true' !== $this->plugin->get(AzureActiveDirectory::SETTING_GET_USERS_DELTA)
+        ) {
             yield '----------------';
 
             yield 'Trying deactivate non-existing users in Azure';
 
             $users = UserManager::getRepository()->findByAuthSource('azure');
-            $userIdList = array_map(
+            $chamiloUserIdList = array_map(
                 function ($user) {
                     return $user->getId();
                 },
                 $users
             );
 
-            $nonExistingUsers = array_diff($userIdList, $existingUsers);
+            $nonExistingUsers = array_diff($chamiloUserIdList, $azureCreatedUserIdList);
 
             UserManager::deactivate_users($nonExistingUsers);