datacube: support for extended columns (#3335)

finos · Jul 10, 2024 · 845d16b · 845d16b
1 parent d442265
commit 845d16b
Show file tree

Hide file tree

Showing 57 changed files with 673 additions and 486 deletions.
diff --git a/.changeset/unlucky-boats-remain.md b/.changeset/unlucky-boats-remain.md
@@ -0,0 +1,40 @@
+---
+'@finos/legend-application-pure-ide-deployment': patch
+'@finos/legend-extension-dsl-data-space-studio': patch
+'@finos/legend-application-studio-deployment': patch
+'@finos/legend-extension-store-service-store': patch
+'@finos/legend-vscode-extension-dependencies': patch
+'@finos/legend-application-query-deployment': patch
+'@finos/legend-application-studio-bootstrap': patch
+'@finos/legend-application-query-bootstrap': patch
+'@finos/legend-application-repl-deployment': patch
+'@finos/legend-extension-dsl-data-quality': patch
+'@finos/legend-extension-store-relational': patch
+'@finos/legend-server-showcase-deployment': patch
+'@finos/legend-extension-dsl-persistence': patch
+'@finos/legend-extension-store-flat-data': patch
+'@finos/legend-extension-dsl-data-space': patch
+'@finos/legend-extension-tracer-zipkin': patch
+'@finos/legend-extension-dsl-diagram': patch
+'@finos/legend-extension-dsl-service': patch
+'@finos/legend-application-pure-ide': patch
+'@finos/legend-extension-assortment': patch
+'@finos/legend-application-studio': patch
+'@finos/legend-extension-dsl-text': patch
+'@finos/legend-application-query': patch
+'@finos/legend-application-repl': patch
+'@finos/legend-server-showcase': patch
+'@finos/legend-query-builder': patch
+'@finos/legend-server-depot': patch
+'@finos/legend-application': patch
+'@finos/legend-server-sdlc': patch
+'@finos/legend-dev-utils': patch
+'@finos/stylelint-config-legend-studio': patch
+'@finos/legend-storage': patch
+'@finos/eslint-plugin-legend-studio': patch
+'@finos/legend-shared': patch
+'@finos/babel-preset-legend-studio': patch
+'@finos/legend-graph': patch
+'@finos/legend-lego': patch
+'@finos/legend-art': patch
+---
diff --git a/fixtures/legend-mock-server/package.json b/fixtures/legend-mock-server/package.json
@@ -35,7 +35,7 @@
     "eslint": "8.57.0",
     "nodemon": "3.1.4",
     "npm-run-all": "4.1.5",
-    "rimraf": "5.0.9",
+    "rimraf": "6.0.0",
     "typescript": "5.5.3"
   }
 }
diff --git a/package.json b/package.json
@@ -114,12 +114,12 @@
     "npm-run-all": "4.1.5",
     "prettier": "3.3.2",
     "prettier-plugin-tailwindcss": "0.6.5",
-    "rimraf": "5.0.9",
-    "sass": "1.77.6",
+    "rimraf": "6.0.0",
+    "sass": "1.77.7",
     "semver": "7.6.2",
     "sort-package-json": "2.10.0",
     "stylelint": "16.6.1",
-    "typedoc": "0.26.3",
+    "typedoc": "0.26.4",
     "typescript": "5.5.3",
     "yargs": "17.7.2"
   },

diff --git a/packages/babel-preset/package.json b/packages/babel-preset/package.json
@@ -41,7 +41,7 @@
   "devDependencies": {
     "cross-env": "7.0.3",
     "eslint": "8.57.0",
-    "rimraf": "5.0.9",
+    "rimraf": "6.0.0",
     "typescript": "5.5.3"
   },
   "peerDependencies": {

diff --git a/packages/eslint-plugin/package.json b/packages/eslint-plugin/package.json
@@ -44,7 +44,7 @@
   },
   "devDependencies": {
     "cross-env": "7.0.3",
-    "rimraf": "5.0.9"
+    "rimraf": "6.0.0"
   },
   "peerDependencies": {
     "eslint": "^8.0.0",

diff --git a/packages/legend-application-pure-ide-deployment/package.json b/packages/legend-application-pure-ide-deployment/package.json
@@ -49,7 +49,7 @@
     "cross-env": "7.0.3",
     "eslint": "8.57.0",
     "npm-run-all": "4.1.5",
-    "rimraf": "5.0.9",
+    "rimraf": "6.0.0",
     "typescript": "5.5.3",
     "webpack": "5.92.1",
     "webpack-bundle-analyzer": "4.10.2",

diff --git a/packages/legend-application-pure-ide/package.json b/packages/legend-application-pure-ide/package.json
@@ -66,8 +66,8 @@
     "eslint": "8.57.0",
     "jest": "29.7.0",
     "npm-run-all": "4.1.5",
-    "rimraf": "5.0.9",
-    "sass": "1.77.6",
+    "rimraf": "6.0.0",
+    "sass": "1.77.7",
     "typescript": "5.5.3"
   },
   "peerDependencies": {

diff --git a/packages/legend-application-query-bootstrap/package.json b/packages/legend-application-query-bootstrap/package.json
@@ -66,8 +66,8 @@
     "cross-env": "7.0.3",
     "eslint": "8.57.0",
     "npm-run-all": "4.1.5",
-    "rimraf": "5.0.9",
-    "sass": "1.77.6",
+    "rimraf": "6.0.0",
+    "sass": "1.77.7",
     "typescript": "5.5.3"
   },
   "publishConfig": {

diff --git a/packages/legend-application-query-deployment/package.json b/packages/legend-application-query-deployment/package.json
@@ -42,7 +42,7 @@
     "cross-env": "7.0.3",
     "eslint": "8.57.0",
     "npm-run-all": "4.1.5",
-    "rimraf": "5.0.9",
+    "rimraf": "6.0.0",
     "typescript": "5.5.3",
     "webpack": "5.92.1",
     "webpack-bundle-analyzer": "4.10.2",

diff --git a/packages/legend-application-query/package.json b/packages/legend-application-query/package.json
@@ -69,8 +69,8 @@
     "eslint": "8.57.0",
     "jest": "29.7.0",
     "npm-run-all": "4.1.5",
-    "rimraf": "5.0.9",
-    "sass": "1.77.6",
+    "rimraf": "6.0.0",
+    "sass": "1.77.7",
     "typescript": "5.5.3"
   },
   "peerDependencies": {

diff --git a/packages/legend-application-repl-deployment/package.json b/packages/legend-application-repl-deployment/package.json
@@ -50,7 +50,7 @@
     "cross-env": "7.0.3",
     "eslint": "8.57.0",
     "npm-run-all": "4.1.5",
-    "rimraf": "5.0.9",
+    "rimraf": "6.0.0",
     "tailwindcss": "3.4.4",
     "typescript": "5.5.3",
     "webpack": "5.92.1",

diff --git a/packages/legend-application-repl/package.json b/packages/legend-application-repl/package.json
@@ -47,11 +47,13 @@
   "dependencies": {
     "@ag-grid-community/client-side-row-model": "32.0.1",
     "@ag-grid-community/core": "32.0.1",
+    "@ag-grid-community/csv-export": "32.0.1",
     "@ag-grid-community/react": "32.0.1",
     "@ag-grid-community/styles": "32.0.1",
     "@ag-grid-enterprise/clipboard": "32.0.1",
     "@ag-grid-enterprise/column-tool-panel": "32.0.1",
     "@ag-grid-enterprise/core": "32.0.1",
+    "@ag-grid-enterprise/excel-export": "32.0.1",
     "@ag-grid-enterprise/menu": "32.0.1",
     "@ag-grid-enterprise/range-selection": "32.0.1",
     "@ag-grid-enterprise/row-grouping": "32.0.1",
@@ -80,8 +82,8 @@
     "eslint": "8.57.0",
     "jest": "29.7.0",
     "npm-run-all": "4.1.5",
-    "rimraf": "5.0.9",
-    "sass": "1.77.6",
+    "rimraf": "6.0.0",
+    "sass": "1.77.7",
     "typescript": "5.5.3"
   },
   "peerDependencies": {

diff --git a/.../legend-application-repl/src/components/dataCube/editor/DataCubeEditorColumnsSelector.tsx b/.../legend-application-repl/src/components/dataCube/editor/DataCubeEditorColumnsSelector.tsx
@@ -32,9 +32,10 @@ import {
   type CustomNoRowsOverlayProps,
 } from '@ag-grid-community/react';
 import { ClientSideRowModelModule } from '@ag-grid-community/client-side-row-model';
-import type {
-  DataCubeEditorColumnsSelectorColumnState,
-  DataCubeEditorColumnsSelectorState,
+import {
+  DataCubeEditorColumnsSelectorColumnsVisibility,
+  type DataCubeEditorColumnsSelectorColumnState,
+  type DataCubeEditorColumnsSelectorState,
 } from '../../../stores/dataCube/editor/DataCubeEditorColumnsSelectorState.js';
 import { isNonNullable } from '@finos/legend-shared';
 import {
@@ -128,7 +129,7 @@ const ColumnsSearchResultCountBadge = observer(
     const { selector, gridApi, scope } = props;
     return (
       <div className="flex items-center justify-center rounded-lg bg-neutral-500 px-1 py-0.5 font-mono text-xs font-bold text-white">
-        {`${getDataForAllFilteredNodes(gridApi).length}/${scope === 'available' ? selector.availableColumns.length : selector.selectedColumns.length}`}
+        {`${getDataForAllFilteredNodes(gridApi).length}/${scope === 'available' ? selector.availableColumnsForDisplay.length : selector.selectedColumnsForDisplay.length}`}
         <span className="hidden">
           {scope === 'available'
             ? // subscribing to the search text to trigger re-render as it changes
@@ -177,13 +178,7 @@ export const DataCubeEditorColumnsSelector = observer(
      */
     const onSelectedColumnsDragStop = useCallback(
       (params: RowDragEndEvent<T>) => {
-        const newData = getDataForAllNodes(params.api);
-        selector.setAllSelectedColumns(newData);
-        selector.setAllAvailableColumns(
-          selector.availableColumns.filter(
-            (column) => !newData.includes(column),
-          ),
-        );
+        selector.setSelectedColumns(getDataForAllNodes(params.api));
       },
       [selector],
     );
@@ -194,12 +189,7 @@ export const DataCubeEditorColumnsSelector = observer(
         const columnsToMove = nodes
           .map((node) => node.data)
           .filter(isNonNullable);
-
-        selector.setAllAvailableColumns([
-          ...selector.availableColumns,
-          ...columnsToMove,
-        ]);
-        selector.setAllSelectedColumns(
+        selector.setSelectedColumns(
           selector.selectedColumns.filter(
             (column) => !columnsToMove.includes(column),
           ),
@@ -219,8 +209,7 @@ export const DataCubeEditorColumnsSelector = observer(
         if (event.overIndex === -1) {
           return;
         }
-        const newData = getDataForAllNodes(event.api);
-        selector.setAllSelectedColumns(newData);
+        selector.setSelectedColumns(getDataForAllNodes(event.api));
       },
       [selector],
     );
@@ -315,7 +304,7 @@ export const DataCubeEditorColumnsSelector = observer(
                 }}
                 // Using ag-grid quick filter is a cheap way to implement search
                 quickFilterText={selector.availableColumnsSearchText}
-                rowData={selector.availableColumns}
+                rowData={selector.availableColumnsForDisplay}
                 columnDefs={[
                   {
                     ...getBaseColumnDef<T>(),
@@ -331,15 +320,10 @@ export const DataCubeEditorColumnsSelector = observer(
                           const filteredData = getDataForAllFilteredNodes(
                             params.api,
                           );
-                          selector.setAllSelectedColumns([
+                          selector.setSelectedColumns([
                             ...selector.selectedColumns,
                             ...filteredData,
                           ]);
-                          selector.setAllAvailableColumns(
-                            selector.availableColumns.filter(
-                              (column) => !filteredData.includes(column),
-                            ),
-                          );
                           params.api.clearFocusedCell();
                         }}
                       >
@@ -356,25 +340,24 @@ export const DataCubeEditorColumnsSelector = observer(
                       if (!data) {
                         return null;
                       }
-                      const isHidden =
-                        selector.hiddenAvailableColumns.includes(data);
+                      const showHiddenIndicator =
+                        selector.columnsVisibility ===
+                          DataCubeEditorColumnsSelectorColumnsVisibility.VISIBLE_WITH_WARNING &&
+                        selector.editor.columnProperties.columns.find(
+                          (col) => col.name === data.name,
+                        )?.hideFromView;
 
                       return (
                         <div
                           className={cn('flex h-full w-full cursor-pointer', {
-                            'text-neutral-400': isHidden,
+                            'text-neutral-400': showHiddenIndicator,
                           })}
-                          title={`[${data.name}]${isHidden ? ' - Hidden' : ''}\nDouble-click to add column`}
+                          title={`[${data.name}]${showHiddenIndicator ? ' - Hidden' : ''}\nDouble-click to add column`}
                           onDoubleClick={() => {
-                            selector.setAllSelectedColumns([
+                            selector.setSelectedColumns([
                               ...selector.selectedColumns,
                               data,
                             ]);
-                            selector.setAllAvailableColumns(
-                              selector.availableColumns.filter(
-                                (column) => column !== data,
-                              ),
-                            );
                             params.api.clearFocusedCell();
                           }}
                         >
@@ -416,15 +399,10 @@ export const DataCubeEditorColumnsSelector = observer(
                 const columnsToMove = selectedAvailableColumns.filter(
                   (column) => filteredData.includes(column),
                 );
-                selector.setAllSelectedColumns([
+                selector.setSelectedColumns([
                   ...selector.selectedColumns,
                   ...columnsToMove,
                 ]);
-                selector.setAllAvailableColumns(
-                  selector.availableColumns.filter(
-                    (column) => !columnsToMove.includes(column),
-                  ),
-                );
                 availableColumnsGridApi.clearFocusedCell();
               }}
               disabled={selectedAvailableColumns.length === 0}
@@ -449,11 +427,7 @@ export const DataCubeEditorColumnsSelector = observer(
                 const columnsToMove = selectedSelectedColumns.filter((column) =>
                   filteredData.includes(column),
                 );
-                selector.setAllAvailableColumns([
-                  ...selector.availableColumns,
-                  ...columnsToMove,
-                ]);
-                selector.setAllSelectedColumns(
+                selector.setSelectedColumns(
                   selector.selectedColumns.filter(
                     (column) => !columnsToMove.includes(column),
                   ),
@@ -523,7 +497,7 @@ export const DataCubeEditorColumnsSelector = observer(
                 noRowsOverlayComponentParams={{
                   noColumnsSelectedRenderer,
                 }}
-                rowData={selector.selectedColumns}
+                rowData={selector.selectedColumnsForDisplay}
                 columnDefs={[
                   {
                     ...getBaseColumnDef<T>(),
@@ -539,11 +513,7 @@ export const DataCubeEditorColumnsSelector = observer(
                           const filteredData = getDataForAllFilteredNodes(
                             params.api,
                           );
-                          selector.setAllAvailableColumns([
-                            ...selector.availableColumns,
-                            ...filteredData,
-                          ]);
-                          selector.setAllSelectedColumns(
+                          selector.setSelectedColumns(
                             selector.selectedColumns.filter(
                               (column) => !filteredData.includes(column),
                             ),
@@ -564,21 +534,21 @@ export const DataCubeEditorColumnsSelector = observer(
                       if (!data) {
                         return null;
                       }
-                      const isHidden =
-                        selector.hiddenSelectedColumns.includes(data);
+                      const showHiddenIndicator =
+                        selector.columnsVisibility ===
+                          DataCubeEditorColumnsSelectorColumnsVisibility.VISIBLE_WITH_WARNING &&
+                        selector.editor.columnProperties.columns.find(
+                          (col) => col.name === data.name,
+                        )?.hideFromView;
 
                       return (
                         <div
                           className={cn('flex h-full w-full cursor-pointer', {
-                            'text-neutral-400': isHidden,
+                            'text-neutral-400': showHiddenIndicator,
                           })}
-                          title={`[${data.name}]${isHidden ? ' - Hidden' : ''}\nDouble-click to remove column`}
+                          title={`[${data.name}]${showHiddenIndicator ? ' - Hidden' : ''}\nDouble-click to remove column`}
                           onDoubleClick={() => {
-                            selector.setAllAvailableColumns([
-                              ...selector.availableColumns,
-                              data,
-                            ]);
-                            selector.setAllSelectedColumns(
+                            selector.setSelectedColumns(
                               selector.selectedColumns.filter(
                                 (column) => column !== data,
                               ),

diff --git a/packages/legend-application-repl/src/components/dataCube/grid/DataCubeGrid.tsx b/packages/legend-application-repl/src/components/dataCube/grid/DataCubeGrid.tsx
@@ -24,6 +24,8 @@ import { MenuModule } from '@ag-grid-enterprise/menu';
 import { SideBarModule } from '@ag-grid-enterprise/side-bar';
 import { ColumnsToolPanelModule } from '@ag-grid-enterprise/column-tool-panel';
 import { AgGridReact } from '@ag-grid-community/react';
+import { CsvExportModule } from '@ag-grid-community/csv-export';
+import { ExcelExportModule } from '@ag-grid-enterprise/excel-export';
 import { useEffect } from 'react';
 import { useREPLStore } from '../../REPLStoreProvider.js';
 import { DataCubeIcon, Switch, cn, Global, css } from '@finos/legend-art';
@@ -351,6 +353,7 @@ const DataCubeGridClient = observer(() => {
         modules={[
           // community
           ClientSideRowModelModule,
+          CsvExportModule,
           // enterprise
           ServerSideRowModelModule,
           RowGroupingModule,
@@ -359,6 +362,7 @@ const DataCubeGridClient = observer(() => {
           RangeSelectionModule,
           SideBarModule,
           ColumnsToolPanelModule,
+          ExcelExportModule,
         ]}
         {...generateBaseGridOptions(dataCube)}
       />