diff --git a/web/src/core/usecases/dataExplorer/selectors.ts b/web/src/core/usecases/dataExplorer/selectors.ts index d33c78db2..3895f487e 100644 --- a/web/src/core/usecases/dataExplorer/selectors.ts +++ b/web/src/core/usecases/dataExplorer/selectors.ts @@ -10,7 +10,7 @@ const columns = createSelector( createSelector( createSelector(state, state => state.data), data => { - if (data === undefined) { + if (data.state !== "loaded") { return undefined; } @@ -45,12 +45,21 @@ const main = createSelector(state, columns, (state, columns) => { return { isQuerying, errorMessage: errorMessage }; } - if (data === undefined) { - return { isQuerying, rows: undefined }; + if (data.state === "empty") { + return { + isQuerying, + rows: undefined + }; + } + + if (data.state === "unknownFileType") { + return { isQuerying, queryParams, shouldAskFileType: true }; } assert(columns !== undefined); assert(queryParams !== undefined); + assert(queryParams.rowsPerPage !== undefined); + assert(queryParams.page !== undefined); assert(extraRestorableStates !== undefined); const { rowsPerPage, page } = queryParams; diff --git a/web/src/core/usecases/dataExplorer/state.ts b/web/src/core/usecases/dataExplorer/state.ts index 428b9d651..d520fedbb 100644 --- a/web/src/core/usecases/dataExplorer/state.ts +++ b/web/src/core/usecases/dataExplorer/state.ts @@ -9,8 +9,8 @@ export type State = { queryParams: | { sourceUrl: string; - rowsPerPage: number; - page: number; + rowsPerPage: number | undefined; + page: number | undefined; } | undefined; extraRestorableStates: @@ -22,12 +22,14 @@ export type State = { errorMessage: string | undefined; data: | { + state: "loaded"; rows: any[]; rowCount: number | undefined; fileDownloadUrl: string; - // fileType: "parquet" | "csv" | "json"; + fileType: "parquet" | "csv" | "json"; } - | undefined; + | { state: "unknownFileType"; fileType: undefined; fileDownloadUrl: string } + | { state: "empty" }; }; export const { actions, reducer } = createUsecaseActions({ @@ -37,7 +39,7 @@ export const { actions, reducer } = createUsecaseActions({ queryParams: undefined, extraRestorableStates: undefined, errorMessage: undefined, - data: undefined + data: { state: "empty" } }), reducers: { queryStarted: ( @@ -96,10 +98,42 @@ export const { actions, reducer } = createUsecaseActions({ assert(state.extraRestorableStates !== undefined); state.extraRestorableStates.columnVisibility = columnVisibility; }, - querySucceeded: (state, { payload }: { payload: NonNullable }) => { - const { rowCount, rows, fileDownloadUrl } = payload; + + querySucceeded: ( + state, + { + payload + }: { + payload: { + rows: any[]; + rowCount: number | undefined; + fileDownloadUrl: string; + fileType: "parquet" | "csv" | "json"; + }; + } + ) => { + const { rowCount, rows, fileDownloadUrl, fileType } = payload; + state.isQuerying = false; + state.data = { state: "loaded", rowCount, rows, fileDownloadUrl, fileType }; + }, + //Rename this, i want to end query because not able to auto detect fileType + terminateQueryDueToUnknownFileType: ( + state, + { + payload + }: { + payload: { + fileDownloadUrl: string; + }; + } + ) => { + const { fileDownloadUrl } = payload; state.isQuerying = false; - state.data = { rowCount, rows, fileDownloadUrl }; + state.data = { + state: "unknownFileType", + fileDownloadUrl, + fileType: undefined + }; }, queryCanceled: state => { state.isQuerying = false; @@ -114,7 +148,7 @@ export const { actions, reducer } = createUsecaseActions({ restoreState: state => { state.queryParams = undefined; state.extraRestorableStates = undefined; - state.data = undefined; + state.data = { state: "empty" }; } } }); diff --git a/web/src/core/usecases/dataExplorer/thunks.ts b/web/src/core/usecases/dataExplorer/thunks.ts index 7144ffff1..41880e023 100644 --- a/web/src/core/usecases/dataExplorer/thunks.ts +++ b/web/src/core/usecases/dataExplorer/thunks.ts @@ -54,10 +54,9 @@ const privateThunks = { rowsPerPage: number; page: number; }; - shouldVerifyUrl: boolean; }) => async (...args) => { - const { queryParams, shouldVerifyUrl } = params; + const { queryParams } = params; const [dispatch, getState, rootContext] = args; const coreQueryParams = getState()[name].queryParams; @@ -74,32 +73,47 @@ const privateThunks = { dispatch(actions.queryStarted({ queryParams })); - if (shouldVerifyUrl && !thunks.getIsValidSourceUrl({ sourceUrl })()) { - dispatch(actions.queryFailed({ errorMessage: "Invalid sourceUrl" })); + const data = getState()[name].data; + + const { fileType, fileDownloadUrl: fileDownloadUrlOrUndefined } = + await (async () => { + if (!isSourceUrlChanged && data.state !== "empty") { + return { + fileType: data.fileType, + fileDownloadUrl: data.fileDownloadUrl + }; + } + return dispatch(privateThunks.detectFileType({ sourceUrl })); + })(); + + if (fileType === undefined) { + dispatch( + actions.terminateQueryDueToUnknownFileType({ + fileDownloadUrl: fileDownloadUrlOrUndefined + }) + ); return; } - const { fileDownloadUrl, rowCountOrErrorMessage } = await (async () => { - if (!isSourceUrlChanged) { - const data = getState()[name].data; - assert(data !== undefined); - return { - fileDownloadUrl: data.fileDownloadUrl, - rowCountOrErrorMessage: data.rowCount - }; - } - - const fileDownloadUrl = await dispatch( + const fileDownloadUrl = + fileDownloadUrlOrUndefined ?? + (await dispatch( privateThunks.getFileDonwloadUrl({ sourceUrl }) - ); + )); + + const rowCountOrErrorMessage = await (async () => { + if (!isSourceUrlChanged) { + assert(data.state === "loaded"); + return data.rowCount; + } const rowCountOrErrorMessage = await sqlOlap .getRowCount(sourceUrl) .catch(error => String(error)); - return { fileDownloadUrl, rowCountOrErrorMessage }; + return rowCountOrErrorMessage; })(); if (typeof rowCountOrErrorMessage === "string") { @@ -144,53 +158,101 @@ const privateThunks = { ? undefined : queryParams.rowsPerPage * (queryParams.page - 1) + rows.length, - fileDownloadUrl + fileDownloadUrl, + fileType }) ); }, - detectFileType: (params: { sourceUrl: string }) => async () => { - const { sourceUrl } = params; - //const [dispatch] = args; + detectFileType: + (params: { sourceUrl: string }) => + async (...args) => { + const { sourceUrl } = params; + const [dispatch] = args; - const extension = (() => { - const validExtensions = ["parquet", "csv", "json"] as const; - type ValidExtension = (typeof validExtensions)[number]; + const validFileType = ["parquet", "csv", "json"] as const; + type ValidFileType = (typeof validFileType)[number]; - const isValidExtension = (ext: string): ext is ValidExtension => - validExtensions.includes(ext as ValidExtension); + const isValidFileType = (ext: string): ext is ValidFileType => + validFileType.includes(ext as ValidFileType); - let pathname: string; + const extension = (() => { + let pathname: string; - try { - pathname = new URL(sourceUrl).pathname; - } catch { - return undefined; - } - const match = pathname.match(/\.(\w+)$/); + try { + pathname = new URL(sourceUrl).pathname; + } catch { + return undefined; + } + const match = pathname.match(/\.(\w+)$/); - if (match === null) { - return undefined; - } + if (match === null) { + return undefined; + } - const [, extension] = match; + const [, extension] = match; - return isValidExtension(extension) ? extension : undefined; - })(); + return isValidFileType(extension) ? extension : undefined; + })(); - if (extension) { - return extension; - } + if (extension) { + return { fileType: extension, fileDownloadUrl: undefined }; + } + + const fileDownloadUrl = await dispatch( + privateThunks.getFileDonwloadUrl({ sourceUrl }) + ); - /* const contentType = await (async () => { - const fileDownloadUrl = await dispatch( - privateThunks.getFileDonwloadUrl({ - sourceUrl - }) - ); + try { + const response = await fetch(fileDownloadUrl, { method: "HEAD" }); + + if (!response.ok) { + return undefined; + } + + return response.headers.get("Content-Type") ?? undefined; + } catch (error) { + return undefined; + } })(); - */ - }, + + const contentTypeToExtension = [ + { + keyword: "application/parquet" as const, + extension: "parquet" as const + }, + { + keyword: "application/x-parquet" as const, + extension: "parquet" as const + }, + { keyword: "text/csv" as const, extension: "csv" as const }, + { keyword: "application/csv" as const, extension: "csv" as const }, + { keyword: "application/json" as const, extension: "json" as const }, + { keyword: "text/json" as const, extension: "json" as const } + ]; + + const getExtensionFromContentType = ( + contentType?: string + ): ValidFileType | undefined => { + if (!contentType) { + return undefined; + } + + const match = contentTypeToExtension.find( + ({ keyword }) => contentType === keyword + ); + return match ? match.extension : undefined; + }; + + return { + fileType: getExtensionFromContentType(contentType), + fileDownloadUrl + }; + }, + /* + getParquetMetadata: (params: { sourceUrl: string }) => async () => {}, + + */ updateDataSource: (params: { queryParams: { @@ -198,12 +260,10 @@ const privateThunks = { rowsPerPage: number | undefined; page: number | undefined; }; - shouldVerifyUrl: boolean; }) => async (...args) => { const { - queryParams: { sourceUrl, rowsPerPage = 25, page = 1 }, - shouldVerifyUrl + queryParams: { sourceUrl, rowsPerPage = 25, page = 1 } } = params; const [dispatch, getState, rootContext] = args; @@ -232,8 +292,7 @@ const privateThunks = { sourceUrl, rowsPerPage, page - }, - shouldVerifyUrl + } }) ); } @@ -282,11 +341,11 @@ export const thunks = { await dispatch( privateThunks.updateDataSource({ - queryParams: { sourceUrl, rowsPerPage, page }, - shouldVerifyUrl: true + queryParams: { sourceUrl, rowsPerPage, page } }) ); }, + /* getIsValidSourceUrl: (params: { sourceUrl: string }) => () => { const { sourceUrl } = params; @@ -322,6 +381,7 @@ export const thunks = { return true; }, + */ updateDataSource: (params: { sourceUrl: string }) => async (...args) => { @@ -335,8 +395,7 @@ export const thunks = { sourceUrl, rowsPerPage: undefined, page: undefined - }, - shouldVerifyUrl: false + } }) ); }, @@ -350,8 +409,7 @@ export const thunks = { dispatch( privateThunks.performQuery({ - queryParams: { ...stateQueryParams, page, rowsPerPage }, - shouldVerifyUrl: false + queryParams: { ...stateQueryParams, page, rowsPerPage } }) ); }, diff --git a/web/src/ui/pages/dataExplorer/DataExplorer.tsx b/web/src/ui/pages/dataExplorer/DataExplorer.tsx index 7ff7e4768..3505da6d7 100644 --- a/web/src/ui/pages/dataExplorer/DataExplorer.tsx +++ b/web/src/ui/pages/dataExplorer/DataExplorer.tsx @@ -58,7 +58,8 @@ export default function DataExplorer(props: Props) { columns, rowCount, errorMessage, - isQuerying + isQuerying, + shouldAskFileType } = useCoreState("dataExplorer", "main"); useEffect(() => { @@ -73,18 +74,22 @@ export default function DataExplorer(props: Props) { return; } + const { selectedRowIndex: selectedRow, columnVisibility } = + extraRestorableStates || {}; + routes[route.name]({ ...route.params, source: queryParams.sourceUrl, page: queryParams.page, rowsPerPage: queryParams.rowsPerPage, - selectedRow: extraRestorableStates.selectedRowIndex, - columnVisibility: extraRestorableStates.columnVisibility + selectedRow, + columnVisibility }).replace(); }, [queryParams, extraRestorableStates]); const { classes, cx } = useStyles(); + console.log("core props", { rows, queryParams, errorMessage, shouldAskFileType }); // Theres a bug in MUI classes.panel does not apply so have to apply the class manually const { childrenClassName: dataGridPanelWrapperRefClassName } = useApplyClassNameToParent({ @@ -118,11 +123,6 @@ export default function DataExplorer(props: Props) { /> - dataExplorer.getIsValidSourceUrl({ - sourceUrl: url - }) - } onUrlChange={value => { dataExplorer.updateDataSource({ sourceUrl: value }); }} @@ -155,6 +155,9 @@ export default function DataExplorer(props: Props) { ); } + assert(queryParams.page !== undefined); + assert(queryParams.rowsPerPage !== undefined); + return (
; export const Default: Story = { args: { url: "https://example.com", - onUrlChange: action("URL changed"), - getIsValidUrl: (url: string) => url.startsWith("https://") + onUrlChange: action("URL changed") } }; diff --git a/web/src/ui/pages/dataExplorer/UrlInput.tsx b/web/src/ui/pages/dataExplorer/UrlInput.tsx index 08e52b8f1..430a6fbe7 100644 --- a/web/src/ui/pages/dataExplorer/UrlInput.tsx +++ b/web/src/ui/pages/dataExplorer/UrlInput.tsx @@ -10,16 +10,15 @@ type Props = { className?: string; url: string; onUrlChange: (value: string) => void; - getIsValidUrl: (url: string) => boolean; }; export function UrlInput(props: Props) { - const { className, url, onUrlChange, getIsValidUrl } = props; + const { className, url, onUrlChange } = props; const { t } = useTranslation({ UrlInput }); const [urlBeingTyped, setUrlBeingTyped] = useState(url); - const isLoadable = urlBeingTyped !== url && getIsValidUrl(urlBeingTyped); + const isLoadable = urlBeingTyped !== url; const { classes, cx } = useStyles({ isLoadable });