Skip to content

Commit

Permalink
wip too
Browse files Browse the repository at this point in the history
  • Loading branch information
@ddecrulle
ddecrulle committed Dec 20, 2024
1 parent 28619d4 commit ce076e9
Show file tree
Hide file tree
Showing 4 changed files with 136 additions and 90 deletions.
59 changes: 29 additions & 30 deletions web/src/core/usecases/dataExplorer/selectors.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,40 +42,39 @@ const main = createSelector(state, columns, (state, columns) => {
const { isQuerying, queryParams, errorMessage, data, extraRestorableStates } = state;

if (errorMessage !== undefined) {
return { isQuerying, errorMessage: errorMessage };
return { isQuerying, errorMessage: errorMessage, queryParams };
}

if (data.state === "empty") {
return {
isQuerying,
rows: undefined
};
}
switch (data.state) {
case "empty":
return {
isQuerying,
rows: undefined
};
case "unknownFileType":
return { isQuerying, queryParams, shouldAskFileType: true };
case "loaded": {
assert(columns !== undefined);
assert(queryParams !== undefined);
assert(queryParams.rowsPerPage !== undefined);
assert(queryParams.page !== undefined);
assert(extraRestorableStates !== undefined);

if (data.state === "unknownFileType") {
return { isQuerying, queryParams, shouldAskFileType: true };
const { rowsPerPage, page } = queryParams;
return {
isQuerying,
rows: data.rows.map((row, i) => ({
id: i + rowsPerPage * (page - 1),
...row
})),
rowCount: data.rowCount,
queryParams,
extraRestorableStates,
fileDownloadUrl: data.fileDownloadUrl,
columns
};
}
}

assert(columns !== undefined);
assert(queryParams !== undefined);
assert(queryParams.rowsPerPage !== undefined);
assert(queryParams.page !== undefined);
assert(extraRestorableStates !== undefined);

const { rowsPerPage, page } = queryParams;

return {
isQuerying,
rows: data.rows.map((row, i) => ({
id: i + rowsPerPage * (page - 1),
...row
})),
rowCount: data.rowCount,
queryParams,
extraRestorableStates,
fileDownloadUrl: data.fileDownloadUrl,
columns
};
});

export const selectors = { main };
36 changes: 19 additions & 17 deletions web/src/core/usecases/dataExplorer/state.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,21 +42,6 @@ export const { actions, reducer } = createUsecaseActions({
data: { state: "empty" }
}),
reducers: {
queryStarted: (
state,
{
payload
}: {
payload: {
queryParams: NonNullable<State["queryParams"]>;
};
}
) => {
const { queryParams } = payload;
state.errorMessage = undefined;
state.isQuerying = true;
state.queryParams = queryParams;
},
extraRestorableStateSet: (
state,
{
Expand Down Expand Up @@ -98,7 +83,21 @@ export const { actions, reducer } = createUsecaseActions({
assert(state.extraRestorableStates !== undefined);
state.extraRestorableStates.columnVisibility = columnVisibility;
},

queryStarted: (
state,
{
payload
}: {
payload: {
queryParams: NonNullable<State["queryParams"]>;
};
}
) => {
const { queryParams } = payload;
state.errorMessage = undefined;
state.isQuerying = true;
state.queryParams = queryParams;
},
querySucceeded: (
state,
{
Expand All @@ -115,6 +114,10 @@ export const { actions, reducer } = createUsecaseActions({
const { rowCount, rows, fileDownloadUrl, fileType } = payload;
state.isQuerying = false;
state.data = { state: "loaded", rowCount, rows, fileDownloadUrl, fileType };
state.extraRestorableStates = {
selectedRowIndex: undefined,
columnVisibility: {}
};
},
//Rename this, i want to end query because not able to auto detect fileType
terminateQueryDueToUnknownFileType: (
Expand Down Expand Up @@ -143,7 +146,6 @@ export const { actions, reducer } = createUsecaseActions({
const { errorMessage } = payload;
state.isQuerying = false;
state.errorMessage = errorMessage;
state.queryParams = undefined;
},
restoreState: state => {
state.queryParams = undefined;
Expand Down
130 changes: 87 additions & 43 deletions web/src/core/usecases/dataExplorer/thunks.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,11 @@ const privateThunks = {
fileDownloadUrl: data.fileDownloadUrl
};
}
return dispatch(privateThunks.detectFileType({ sourceUrl }));
const toto = await dispatch(
privateThunks.detectFileType({ sourceUrl })
);
console.log(toto);
return toto;
})();

if (fileType === undefined) {
Expand Down Expand Up @@ -202,57 +206,97 @@ const privateThunks = {
privateThunks.getFileDonwloadUrl({ sourceUrl })
);

const contentType = await (async () => {
try {
const response = await fetch(fileDownloadUrl, { method: "HEAD" });
try {
const response = await fetch(fileDownloadUrl, {
method: "GET",
headers: { Range: "bytes=0-15" } // Fetch the first 16 bytes
});

if (!response.ok) {
return { fileType: undefined, fileDownloadUrl };
}

const contentType = response.headers.get("Content-Type");

if (!response.ok) {
const filTypeExtractdByContentType = (() => {
if (!contentType) {
return undefined;
}

return response.headers.get("Content-Type") ?? undefined;
} catch (error) {
return undefined;
}
})();
const contentTypeToExtension = [
{
keyword: "application/parquet" as const,
extension: "parquet" as const
},
{
keyword: "application/x-parquet" as const,
extension: "parquet" as const
},
{ keyword: "text/csv" as const, extension: "csv" as const },
{
keyword: "application/csv" as const,
extension: "csv" as const
},
{
keyword: "application/json" as const,
extension: "json" as const
},
{ keyword: "text/json" as const, extension: "json" as const }
];

const match = contentTypeToExtension.find(
({ keyword }) => contentType === keyword
);
return match ? match.extension : undefined;
})();

const contentTypeToExtension = [
{
keyword: "application/parquet" as const,
extension: "parquet" as const
},
{
keyword: "application/x-parquet" as const,
extension: "parquet" as const
},
{ keyword: "text/csv" as const, extension: "csv" as const },
{ keyword: "application/csv" as const, extension: "csv" as const },
{ keyword: "application/json" as const, extension: "json" as const },
{ keyword: "text/json" as const, extension: "json" as const }
];

const getExtensionFromContentType = (
contentType?: string
): ValidFileType | undefined => {
if (!contentType) {
return undefined;
if (filTypeExtractdByContentType !== undefined) {
return { fileType: filTypeExtractdByContentType, fileDownloadUrl };
}

const match = contentTypeToExtension.find(
({ keyword }) => contentType === keyword
);
return match ? match.extension : undefined;
};
const fileSignatures = [
{
condition: (bytes: Uint8Array) =>
bytes[0] === 80 &&
bytes[1] === 65 &&
bytes[2] === 82 &&
bytes[3] === 49, // "PAR1"
extension: "parquet" as const
},
{
condition: (bytes: Uint8Array) => [91, 123].includes(bytes[0]), // "[" or "{"
extension: "json" as const // JSON
},
{
condition: (bytes: Uint8Array) => {
const fileContent = new TextDecoder().decode(bytes);
return (
fileContent.includes(",") ||
fileContent.includes("\n") ||
fileContent.includes(";")
); // CSV heuristic
},
extension: "csv" as const
}
];

return {
fileType: getExtensionFromContentType(contentType),
fileDownloadUrl
};
},
/*
getParquetMetadata: (params: { sourceUrl: string }) => async () => {},
const arrayBuffer = await response.arrayBuffer();
const bytes = new Uint8Array(arrayBuffer);

*/
const match = fileSignatures.find(({ condition }) => condition(bytes));

if (match) {
return { fileType: match.extension, fileDownloadUrl };
}
} catch (error) {
console.error("Failed to fetch file for type detection:", error);
//TODO: reject an error
return { fileType: undefined, fileDownloadUrl };
}

//Ask user to manualy specify the file type
return { fileType: undefined, fileDownloadUrl };
},
updateDataSource:
(params: {
queryParams: {
Expand Down
1 change: 1 addition & 0 deletions web/src/ui/pages/dataExplorer/DataExplorer.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ export default function DataExplorer(props: Props) {
<div className={classes.mainArea}>
{(() => {
if (errorMessage !== undefined) {
console.log(queryParams);
return (
<Alert className={classes.errorAlert} severity="error">
{errorMessage}
Expand Down

0 comments on commit ce076e9

Please sign in to comment.