Skip to content

Commit

Permalink
Merge pull request #854 from nextcloud/feature/detect-data-type-durin…
Browse files Browse the repository at this point in the history 
…g-import

feat: autodetect data type during import
  • Loading branch information
@juliusknorr
juliusknorr authored Feb 28, 2024
2 parents 9558791 + 262d093 commit 3a3a246
Show file tree
Hide file tree
Showing 2 changed files with 129 additions and 20 deletions.
27 changes: 25 additions & 2 deletions lib/Service/ColumnService.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -478,6 +478,7 @@ public function delete(int $id, bool $skipRowCleanup = false, ?string $userId =
* @param int|null $tableId
* @param int|null $viewId
* @param array $titles example ['Test column 1', 'And so on', '3rd column title']
* @param array $dataTypes example ['datetime', 'number', 'text']
* @param string|null $userId
* @param bool $createUnknownColumns
* @param int $countCreatedColumns
Expand All @@ -488,7 +489,7 @@ public function delete(int $id, bool $skipRowCleanup = false, ?string $userId =
* @throws NotFoundError
* @throws PermissionError
*/
public function findOrCreateColumnsByTitleForTableAsArray(?int $tableId, ?int $viewId, array $titles, ?string $userId, bool $createUnknownColumns, int &$countCreatedColumns, int &$countMatchingColumns): array {
public function findOrCreateColumnsByTitleForTableAsArray(?int $tableId, ?int $viewId, array $titles, array $dataTypes, ?string $userId, bool $createUnknownColumns, int &$countCreatedColumns, int &$countMatchingColumns): array {
$result = [];

if($userId === null) {
Expand Down Expand Up @@ -522,7 +523,29 @@ public function findOrCreateColumnsByTitleForTableAsArray(?int $tableId, ?int $v
// if column was not found
if($result[$i] === '' && $createUnknownColumns) {
$description = $this->l->t('This column was automatically created by the import service.');
$result[$i] = $this->create($userId, $tableId, $viewId, 'text', 'line', $title, false, $description, null, null, null, null, null, null, null, null, null, null, null, null, []);
$result[$i] = $this->create(
$userId,
$tableId,
$viewId,
$dataTypes[$i]['type'],
$dataTypes[$i]['subtype'] ?? '',
$title,
false,
$description,
null,
null,
null,
$dataTypes[$i]['number_prefix'] ?? null,
$dataTypes[$i]['number_suffix'] ?? null,
null,
null,
null,
$dataTypes[$i]['number_decimals'] ?? null,
null,
$dataTypes[$i]['selection_default'] ?? null,
null,
[]
);
$countCreatedColumns++;
}
}
Expand Down
122 changes: 104 additions & 18 deletions lib/Service/ImportService.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@
use OCP\Files\NotPermittedException;
use OCP\IUserManager;
use OCP\Server;
use PhpOffice\PhpSpreadsheet\Cell\Cell;
use PhpOffice\PhpSpreadsheet\Cell\DataType;
use PhpOffice\PhpSpreadsheet\IOFactory;
use PhpOffice\PhpSpreadsheet\Shared\Date;
use PhpOffice\PhpSpreadsheet\Worksheet\Row;
use PhpOffice\PhpSpreadsheet\Worksheet\Worksheet;
use Psr\Container\ContainerExceptionInterface;
Expand Down Expand Up @@ -145,18 +148,17 @@ public function import(?int $tableId, ?int $viewId, string $path, bool $createMi
* @throws PermissionError
*/
private function loop(Worksheet $worksheet): void {
$firstRow = true;
foreach ($worksheet->getRowIterator() as $row) {
if ($firstRow) {
$this->getColumns($row);
if (empty(array_filter($this->columns))) {
return;
}
$firstRow = false;
} else {
// parse row data
$this->createRow($row);
}
$firstRow = $worksheet->getRowIterator()->current();
$secondRow = $worksheet->getRowIterator()->seek(2)->current();
$this->getColumns($firstRow, $secondRow);

if (empty(array_filter($this->columns))) {
return;
}

foreach ($worksheet->getRowIterator(2) as $row) {
// parse row data
$this->createRow($row);
}
}

Expand Down Expand Up @@ -204,20 +206,32 @@ private function createRow(Row $row): void {
continue;
}

/** @var Column $column */
$column = $this->columns[$i];

// if cell is empty
if(!$cell || $cell->getValue() === null) {
$this->logger->info('Cell is empty while fetching rows data for importing.');
if($this->columns[$i]->getMandatory()) {
if($column->getMandatory()) {
$this->logger->warning('Mandatory column was not set');
$this->countErrors++;
return;
}
continue;
}

$value = $cell->getValue();
if ($column->getType() === 'datetime') {
$value = Date::excelToDateTimeObject($value)->format('Y-m-d H:i');
} elseif ($column->getType() === 'number' && $column->getNumberSuffix() === '%') {
$value = $value * 100;
} elseif ($column->getType() === 'selection' && $column->getSubtype() === 'check') {
$value = $cell->getFormattedValue() === 'TRUE' ? 'true' : 'false';
}

$data[] = [
'columnId' => (int) $this->columns[$i]->getId(),
'value' => json_decode($this->parseValueByColumnType($cell->getValue(), $this->columns[$i])),
'value' => json_decode($this->parseValueByColumnType($value, $this->columns[$i])),
];
}
try {
Expand All @@ -237,29 +251,101 @@ private function createRow(Row $row): void {
}

/**
* @param Row $row
* @param Row $firstRow
* @param Row $secondRow
* @throws InternalError
* @throws NotFoundError
* @throws PermissionError
* @throws DoesNotExistException
* @throws MultipleObjectsReturnedException
*/
private function getColumns(Row $row): void {
$cellIterator = $row->getCellIterator();
private function getColumns(Row $firstRow, Row $secondRow): void {
$cellIterator = $firstRow->getCellIterator();
$secondRowCellIterator = $secondRow->getCellIterator();
$titles = [];
$dataTypes = [];
foreach ($cellIterator as $cell) {
if ($cell && $cell->getValue() !== null && $cell->getValue() !== '') {
$titles[] = $cell->getValue();

// Convert data type to our data type
$dataTypes[] = $this->parseColumnDataType($secondRowCellIterator->current());
} else {
$this->logger->debug('No cell given or cellValue is empty while loading columns for importing');
$this->countErrors++;
}
$secondRowCellIterator->next();
}
try {
$this->columns = $this->columnService->findOrCreateColumnsByTitleForTableAsArray($this->tableId, $this->viewId, $titles, $this->userId, $this->createUnknownColumns, $this->countCreatedColumns, $this->countMatchingColumns);
$this->columns = $this->columnService->findOrCreateColumnsByTitleForTableAsArray($this->tableId, $this->viewId, $titles, $dataTypes, $this->userId, $this->createUnknownColumns, $this->countCreatedColumns, $this->countMatchingColumns);
} catch (Exception $e) {
throw new InternalError($e->getMessage());
}
}

private function parseColumnDataType(Cell $cell): array {
$originDataType = $cell->getDataType();
$value = $cell->getValue();
$formattedValue = $cell->getFormattedValue();
$dataType = [
'type' => 'text',
'subtype' => 'line',
];

if (Date::isDateTime($cell) || $originDataType === DataType::TYPE_ISO_DATE) {
$dataType = [
'type' => 'datetime',
];
} elseif ($originDataType === DataType::TYPE_NUMERIC) {
if (str_contains($formattedValue, '%')) {
$dataType = [
'type' => 'number',
'number_decimals' => 2,
'number_suffix' => '%',
];
} elseif (str_contains($formattedValue, '€')) {
$dataType = [
'type' => 'number',
'number_decimals' => 2,
'number_suffix' => '€',
];
} elseif (str_contains($formattedValue, 'EUR')) {
$dataType = [
'type' => 'number',
'number_decimals' => 2,
'number_suffix' => 'EUR',
];
} elseif (str_contains($formattedValue, '$')) {
$dataType = [
'type' => 'number',
'number_decimals' => 2,
'number_prefix' => '$',
];
} elseif (str_contains($formattedValue, 'USD')) {
$dataType = [
'type' => 'number',
'number_decimals' => 2,
'number_suffix' => 'USD',
];
} elseif (is_float($value)) {
$decimals = strlen(substr(strrchr((string)$value, "."), 1));
$dataType = [
'type' => 'number',
'number_decimals' => $decimals,
];
} else {
$dataType = [
'type' => 'number',
];
}
} elseif ($originDataType === DataType::TYPE_BOOL) {
$dataType = [
'type' => 'selection',
'subtype' => 'check',
'selection_default' => 'false',
];
}

return $dataType;
}
}

0 comments on commit 3a3a246

Please sign in to comment.