filecheck uses limit and offset to perform on large tables

aurelien-riv · May 16, 2018 · d1a217c · d1a217c
1 parent 354c28e
commit d1a217c
Show file tree

Hide file tree

Showing 7 changed files with 34 additions and 25 deletions.
diff --git a/main.php b/main.php
@@ -9,5 +9,5 @@
 
 foreach ($dbChecker->run() as $error)
 {
-    echo get_class($error).' '.$error;
+    echo $error;
 }
diff --git a/src/DBAL/AbstractDBAL.php b/src/DBAL/AbstractDBAL.php
@@ -61,10 +61,10 @@ protected function getInnerJoinString(string $table, array $innerJoinColumns)
         return $joins;
     }
 
-    public function getDistinctValuesWithJoinColumnsWithoutNulls(string $table, array $columns, array $innerJoins) : array
+    public function getDistinctValuesWithJoinColumnsWithoutNulls(string $table, array $columns, array $innerJoins, int $limit, int $offset) : array
     {
         $innerJoins = $this->getInnerJoinString($table, $innerJoins);
-        return $this->queries->getDistinctValuesWithJoinColumnsWithoutNulls($table, $columns, $innerJoins)->fetchAll(\PDO::FETCH_ASSOC);
+        return $this->queries->getDistinctValuesWithJoinColumnsWithoutNulls($table, $columns, $innerJoins, $limit, $offset)->fetchAll(\PDO::FETCH_ASSOC);
     }
 
     public function getTableNames() : array

diff --git a/src/DBQueries/AbstractDbQueries.php b/src/DBQueries/AbstractDbQueries.php
@@ -46,9 +46,7 @@ public function getDuplicateForColumnsWithCount(string $table, string $columns)
         {
             $query .= " AND $columns IS NOT NULL";
         }
-        $stmt = $this->pdo->prepare($query);
-        $stmt->execute();
-        return $stmt;
+        return $this->pdo->query($query);
     }
 
     public function getDistinctValuesWithoutNulls($table, $columns)
@@ -59,10 +57,7 @@ public function getDistinctValuesWithoutNulls($table, $columns)
             $selectColumns = implode(',', $columns);
             $whereColumns = implode(' IS NOT NULL AND ', $columns);
         }
-        $query = "SELECT DISTINCT $selectColumns FROM $table WHERE $whereColumns IS NOT NULL;";
-        $stmt = $this->pdo->prepare($query);
-        $stmt->execute();
-        return $stmt;
+        return $this->pdo->query("SELECT DISTINCT $selectColumns FROM $table WHERE $whereColumns IS NOT NULL;");
     }
 
     public abstract function getDistantTableAndColumnFromTableAndColumnFK(string $table, string $column) : \PDOStatement;
@@ -71,12 +66,14 @@ public abstract function getDistantTableAndColumnFromTableAndColumnFK(string $ta
      * @param string   $table
      * @param string[] $columns
      * @param          $innerJoinString
+     * @param int      $limit
+     * @param int      $offset
      * @return bool|\PDOStatement
      * Warning, composed key are not supported yet
      */
-    public function getDistinctValuesWithJoinColumnsWithoutNulls($table, $columns, $innerJoinString) : \PDOStatement
+    public function getDistinctValuesWithJoinColumnsWithoutNulls($table, $columns, $innerJoinString, int $limit, int $offset) : \PDOStatement
     {
-        $columns          = array_unique($columns);
+        $columns = array_unique($columns);
 
         $selectColumns = '';
         foreach ($columns as $column)
@@ -85,7 +82,10 @@ public function getDistinctValuesWithJoinColumnsWithoutNulls($table, $columns, $
         }
         $stmt = $this->pdo->prepare("SELECT DISTINCT " . rtrim($selectColumns, ',')
                                     . " FROM $table $innerJoinString"
-                                    . " WHERE " . implode(' IS NOT NULL AND ', $columns) . " IS NOT NULL;");
+                                    . " WHERE " . implode(' IS NOT NULL AND ', $columns) . " IS NOT NULL;"
+                                    . " LIMIT :limit OFFSET :offset");
+        $stmt->bindParam(':limit',  $limit,  \PDO::PARAM_INT);
+        $stmt->bindParam(':offset', $offset, \PDO::PARAM_INT);
         $stmt->execute();
         return $stmt;
     }

diff --git a/src/modules/FileCheck/DBQueriesInterface.php b/src/modules/FileCheck/DBQueriesInterface.php
@@ -6,5 +6,5 @@ interface DBQueriesInterface
 {
     public function getName() : string;
 
-    public function getDistinctValuesWithJoinColumnsWithoutNulls(string $table, array $columns, array $innerJoinColumns) : array;
+    public function getDistinctValuesWithJoinColumnsWithoutNulls(string $table, array $columns, array $innerJoinColumns, int $limit, int $offset) : array;
 }
diff --git a/src/modules/FileCheck/FileCheck.php b/src/modules/FileCheck/FileCheck.php
@@ -68,6 +68,7 @@ private function replaceVariablesFromPath($path, $value, &$columns)
 
     public function run(AbstractDBAL $dbal)
     {
+        $limit = 1000;
         foreach ($this->config['mapping'] as $mapping)
         {
             $table = key($mapping);
@@ -76,13 +77,21 @@ public function run(AbstractDBAL $dbal)
             $columns = $innerJoins = [];
             $this->extractVariablesFromPath($path, $columns, $innerJoins);
 
-            $values = $dbal->getDistinctValuesWithJoinColumnsWithoutNulls($table, array_keys($columns), $innerJoins);
-            foreach ($values as $value)
-            {
-                $tmpColumns = $columns;
-                $tmpPath = $this->replaceVariablesFromPath($path, $value, $tmpColumns);
-                yield from $this->testFile($dbal->getName(), $table, $tmpColumns, $tmpPath);
-            }
+            $offset = 0;
+            do {
+                $values = $dbal->getDistinctValuesWithJoinColumnsWithoutNulls($table, array_keys($columns), $innerJoins, $limit, $offset);
+                yield from $this->processData($dbal, $path, $table, $columns, $values);
+                $offset += $limit;
+            } while (count($values) === $limit);
+        }
+    }
+
+    private function processData(AbstractDBAL $dbal, $path, $table, $columns, $values)
+    {
+        foreach ($values as $value)
+        {
+            $tmpPath = $this->replaceVariablesFromPath($path, $value, $columns);
+            yield from $this->testFile($dbal->getName(), $table, $columns, $tmpPath);
         }
     }
 

diff --git a/tests/modules/RelCheckTest/RelCheckTest.php b/tests/modules/RelCheckTest/RelCheckTest.php
@@ -74,7 +74,7 @@ private function initDb(AbstractDBAL $dbal, \PDO $pdo)
         $pdo->exec("CREATE TABLE t4 (t4_id INTEGER PRIMARY KEY, FOREIGN KEY (t4_id) REFERENCES t3(t3_id));");
         if ($dbal instanceof SQLiteDBAL)
         {
-            $pdo->exec("PRAGMA foreign_keys = OFF");
+            $pdo->exec("PRAGMA foreign_keys = OFF"); // Valeur par défaut
         }
         else if ($dbal instanceof MySQLDBAL)
         {

diff --git a/tests/modules/UniqueIntegrityCheck/UniqueIntegrityCheckTest.php b/tests/modules/UniqueIntegrityCheck/UniqueIntegrityCheckTest.php
@@ -1,7 +1,8 @@
 <?php
 
-namespace DBCheckerTests\modules\RelCheckTest;
+namespace DBCheckerTests\modules\UniqueIntegrityCheckTest;
 
+use DBChecker\DBAL\MySQLDBAL;
 use DBChecker\modules\DataBase\DatabasesModule;
 use DBChecker\modules\ModuleManager;
 use DBChecker\modules\UniqueIntegrityCheck\UniqueIntegrityCheck;
@@ -55,10 +56,9 @@ private function init($dbIndex)
     private function initDb(\PDO $pdo)
     {
         $pdo->exec("CREATE TABLE t1 (id INTEGER PRIMARY KEY, data1 CHAR(2), data2 CHAR(2));");
-        $pdo->exec("CREATE UNIQUE INDEX t2_data_unique ON t1 (data1, data2);");
+        $pdo->exec("CREATE UNIQUE INDEX t1_data_unique ON t1 (data1, data2);");
         $pdo->exec("INSERT INTO t1 VALUES (1, 'v1', NULL);");
         $pdo->exec("INSERT INTO t1 VALUES (2, 'v1', NULL);");
-
     }
 
     #region SQLite