diff --git a/package.json b/package.json
index 5f15f1815..323f0ebc7 100644
--- a/package.json
+++ b/package.json
@@ -41,6 +41,7 @@
"query-string": "^6.12.1",
"react": "^16.8.3",
"react-app-rewire-postcss": "^3.0.2",
+ "react-charts": "^2.0.0-beta.7",
"react-copy-to-clipboard": "^5.0.1",
"react-datasheet": "^1.4.1",
"react-dom": "^16.8.3",
@@ -49,6 +50,7 @@
"react-editor-js": "^1.9.0",
"react-helmet": "^5.2.1",
"react-hot-loader": "^4.3.12",
+ "react-icons": "^4.3.1",
"react-markdown": "^4.2.2",
"react-modal": "^3.8.1",
"react-router-dom": "^5.2.0",
diff --git a/src/App.css b/src/App.css
index 8808b45ac..71124787d 100755
--- a/src/App.css
+++ b/src/App.css
@@ -44,6 +44,10 @@ body {
margin-top: 0px !important;
}
+.mt--10 {
+ margin-top: -10px !important;
+}
+
.mr-0 {
margin-right: 0px !important;
}
@@ -342,6 +346,10 @@ body {
color: #676767 !important;
}
+.text-dark-grey {
+ color: #5e5e5e !important;
+}
+
.text-black {
color: #000000 !important;
}
@@ -542,6 +550,14 @@ body {
height: 38px !important;
}
+.h-24 {
+ height: 24px !important;
+}
+
+.h-220 {
+ height: 220px !important;
+}
+
.small-inline {
display: inline-block;
padding-top: 10px;
diff --git a/src/assets/s3.png b/src/assets/s3.png
new file mode 100644
index 000000000..f89de6f6c
Binary files /dev/null and b/src/assets/s3.png differ
diff --git a/src/common/dateUtils.js b/src/common/dateUtils.js
index 9e80462c9..c3838e479 100644
--- a/src/common/dateUtils.js
+++ b/src/common/dateUtils.js
@@ -1,5 +1,5 @@
// Reformat time to the "DD MMM YYYY" format like "12 Aug 2019"
-export const longDate = (date) => {
+export const longDate = date => {
var mydate = new Date(date);
var month = [
'Jan',
@@ -19,7 +19,7 @@ export const longDate = (date) => {
return str;
};
-export const toYearMonthDay = (isoDate) => {
+export const toYearMonthDay = isoDate => {
/*
* Convert ISO8601 UTC datetime string to local datetime and format as
* yyyy-mm-dd date string.
@@ -39,3 +39,34 @@ export const toYearMonthDay = (isoDate) => {
}
return date;
};
+
+// Reformat time to the date and time format
+export const dateTime = date => {
+ var mydate = new Date(date);
+ var month = [
+ 'Jan',
+ 'Feb',
+ 'Mar',
+ 'Apr',
+ 'May',
+ 'Jun',
+ 'Jul',
+ 'Aug',
+ 'Sept',
+ 'Oct',
+ 'Nov',
+ 'Dec',
+ ][mydate.getMonth()];
+ const weekday = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'];
+ var str =
+ weekday[mydate.getDay()] +
+ ', ' +
+ month +
+ ' ' +
+ mydate.getUTCDate() +
+ ', ' +
+ mydate.getFullYear() +
+ ', ' +
+ mydate.toLocaleTimeString([], {hour: '2-digit', minute: '2-digit'});
+ return str;
+};
diff --git a/src/common/enums.js b/src/common/enums.js
index 80163a4d7..1123fe587 100644
--- a/src/common/enums.js
+++ b/src/common/enums.js
@@ -667,6 +667,7 @@ export const permissionIcon = {
validationresultset: 'code',
datatemplate: 'file excel',
organization: 'group',
+ storageanalysis: 'box',
};
// Icon color for different permissions
diff --git a/src/components/Header/Header.js b/src/components/Header/Header.js
index a66948ebe..395f37b7c 100644
--- a/src/components/Header/Header.js
+++ b/src/components/Header/Header.js
@@ -277,6 +277,12 @@ const Header = ({location}) => {
)}
+ {hasPermission(profile, 'view_settings') && (
+
+ S3 Accounting
+
+
+ )}
{hasPermission(profile, 'add_referraltoken') && (
diff --git a/src/documents/utilities.js b/src/documents/utilities.js
index f23eb7bfb..b2f3241e6 100644
--- a/src/documents/utilities.js
+++ b/src/documents/utilities.js
@@ -1,6 +1,7 @@
-import {KF_STUDY_API} from '../common/globals';
import * as stringSimilarity from 'string-similarity';
+import {KF_STUDY_API} from '../common/globals';
+
// Compare date of file versions based on their createdAt time. (Latest first)
export const dateCompare = (version1, version2) => {
return new Date(version2.node.createdAt) - new Date(version1.node.createdAt);
diff --git a/src/routes/index.js b/src/routes/index.js
index 8f55bfe2b..a64a70237 100644
--- a/src/routes/index.js
+++ b/src/routes/index.js
@@ -51,6 +51,7 @@ import {
TemplatesView,
} from '../admin/views';
import ReleaseRoutes from '../releases/routes';
+import S3AccountingRoutes from '../s3Accounting/routes';
import TrackedRoute from './TrackedRoute';
const Routes = () => (
@@ -74,6 +75,7 @@ const Routes = () => (
+
{
+ const [tab, setTab] = useState('matched');
+ const [dropdown, setDropdown] = useState('count_by_data_type');
+ const [pages, setPages] = useState([{first: 10}]);
+
+ const storageAnalyses =
+ data && data.allStorageAnalyses.edges.length > 0
+ ? data.allStorageAnalyses.edges[0].node
+ : {};
+
+ const {
+ loading: fileAuditsLoading,
+ error: fileAuditsError,
+ data: fileAuditsData,
+ refetch: fileAuditsRefetch,
+ fetchMore,
+ } = useQuery(ALL_FILE_AUDITS, {
+ variables: {
+ storageAnalysis: storageAnalyses.id,
+ result: tab,
+ ...pages[0],
+ },
+ });
+
+ const audit = storageAnalyses.stats
+ ? JSON.parse(storageAnalyses.stats).audit
+ : {};
+ console.log(storageAnalyses.stats && JSON.parse(storageAnalyses.stats));
+ const tabContent = {
+ differ: {
+ title: ' S3 locations that have a different file than expected',
+ description:
+ 'This metric is computed by counting the urls in the upload manifest(s) that match urls in the S3 inventory, but have a different file hash in the upload manfiest(s) vs the S3 inventory. These represent locations in S3 where file content changed or got overwritten.',
+ color: '#db2828',
+ },
+ matched: {
+ title: ' files that were expected but not found in S3',
+ description:
+ 'This metric is computed by counting the file hashes in the upload manifest(s) that were not found in the S3 inventory. These represent file blobs that do not exist anywhere in S3 at the time of scan.',
+ color: '#21ba45',
+ },
+ missing: {
+ title: ' files in S3 with different content than expected',
+ description:
+ 'This metric is computed by counting the urls in the upload manifest(s) whose hash value is different than the hash value in the S3 inventory. These items could indicate corrupted uploads or S3 objects that have been overwritten with different content since the time of upload.',
+ color: '#db2828',
+ },
+ moved: {
+ title:
+ ' files that exist in the S3 but at a different location than expected',
+ description:
+ 'This metric is computed by counting the file hashes in the upload manifest(s) whose url is different than the url in the S3 inventory. These represent file blobs that exist in S3 but at different locations than specified by a user in the upload manifest(s).',
+ color: '#fbbd08',
+ },
+ unexpected: {
+ title: ' files that were found in S3 but not expected',
+ description:
+ 'This metric is computed by counting the file hashes in the S3 inventory that were not in any of the upload manifests. These could be files that are unaccounted for or were uploaded by someone besides the intended submitter',
+ color: '#fbbd08',
+ },
+ };
+
+ const dropdownText = {
+ count_by_bucket: 'Bucket',
+ count_by_data_type: 'Data Type',
+ count_by_ext: 'Extension',
+ count_by_size: 'Size',
+ };
+
+ const dropdownOptions = () => {
+ var options = {};
+ Object.keys(audit).length > 0 &&
+ Object.keys(audit).map(t => {
+ options[t] = Object.keys(audit[t])
+ .filter(k => k.includes('count_by_'))
+ .map(v => ({
+ key: v,
+ text: dropdownText[v],
+ value: v,
+ }));
+ return true;
+ });
+ return options;
+ };
+
+ const CountStat = ({number, title, icon, color}) => (
+
+
+
+ {title}
+
+
+ );
+
+ const DateStat = ({date, title}) => (
+ <>
+
+ {dateTime(date)}
+ >
+ );
+
+ const series = React.useMemo(
+ () => ({
+ type: 'bar',
+ }),
+ [],
+ );
+
+ const axes = React.useMemo(
+ () => [
+ {primary: true, type: 'ordinal', position: 'left'},
+ {position: 'bottom', type: 'linear', stacked: true},
+ ],
+ [],
+ );
+
+ const tooltip = React.useMemo(
+ () => ({
+ align: 'auto',
+ anchor: 'closest',
+ }),
+ [],
+ );
+
+ const dataChart = React.useMemo(() => {
+ var rawData = audit && audit[tab] ? audit[tab][dropdown] : {};
+ const col = Object.keys(rawData);
+ const formatData = col.map(c => ({x: c, y: rawData[c]}));
+ return [
+ {
+ datums: formatData,
+ },
+ ];
+ }, [audit, dropdown, tab]);
+
+ const getSeriesStyle = React.useCallback(
+ series => ({
+ color: tabContent[tab].color || '#919090',
+ }),
+ [tab, tabContent],
+ );
+
+ if (loading)
+ return (
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ );
+
+ if (error)
+ return (
+
+
+
+ );
+
+ return (
+
+ {Object.keys(audit).length > 0 && (
+
+
+ {
+ setTab('matched');
+ setDropdown('count_by_data_type');
+ fileAuditsRefetch({
+ result: tab,
+ });
+ }}
+ >
+ }
+ title="files matched"
+ color={tab === 'matched' && 'text-green'}
+ />
+ {tab === 'matched' && (
+
+ )}
+
+ {
+ setTab('missing');
+ setDropdown('count_by_data_type');
+ fileAuditsRefetch({
+ result: tab,
+ });
+ }}
+ >
+ }
+ title="files missing"
+ color={tab === 'missing' && 'text-red'}
+ />
+ {tab === 'missing' && (
+
+ )}
+
+ {
+ setTab('moved');
+ setDropdown('count_by_data_type');
+ fileAuditsRefetch({
+ result: tab,
+ });
+ }}
+ >
+ }
+ title="files moved"
+ color={tab === 'moved' && 'text-yellow'}
+ />
+ {tab === 'moved' && (
+
+ )}
+
+ {
+ setTab('unexpected');
+ setDropdown('count_by_data_type');
+ fileAuditsRefetch({
+ result: tab,
+ });
+ }}
+ >
+ }
+ title="files unexpected"
+ color={tab === 'unexpected' && 'text-yellow'}
+ />
+ {tab === 'unexpected' && (
+
+ )}
+
+
+
+
+
+
+
+
+
+
+
+ Found{' '}
+ {audit[tab] && audit[tab].total_count
+ ? audit[tab].total_count
+ : '-'}
+ {tabContent[tab].title}
+
+ {tabContent[tab].description}
+
+
+
+ )}
+
+
+ Files by {dropdownText[dropdown]}
+
+ {
+ setDropdown(data.value);
+ }}
+ />
+ {dataChart[0].datums.length > 0 ? (
+
+
+
+ ) : (
+
+
+
+ )}
+
+
+
+ );
+};
+
+export default ComparisonTab;
diff --git a/src/s3Accounting/components/FileUploadTable.js b/src/s3Accounting/components/FileUploadTable.js
new file mode 100644
index 000000000..46deed40c
--- /dev/null
+++ b/src/s3Accounting/components/FileUploadTable.js
@@ -0,0 +1,200 @@
+import {
+ Container,
+ Header,
+ Icon,
+ Menu,
+ Message,
+ Placeholder,
+ Popup,
+ Segment,
+ Table,
+} from 'semantic-ui-react';
+import React, {useState} from 'react';
+
+import TimeAgo from 'react-timeago';
+import {formatFileSize} from '../../documents/utilities';
+import {longDate} from '../../common/dateUtils';
+
+const FileUploadTable = ({
+ loading,
+ error,
+ data,
+ title,
+ fetchMore,
+ pages,
+ setPages,
+}) => {
+ const pageSize = 4;
+ const [currPage, setCurrPage] = useState(0);
+
+ const pageInfo = data && data.pageInfo;
+ const pagesLoaded = data && data.edges.length / pageSize;
+ const loadingPage = data && pagesLoaded < currPage + 1;
+ const content =
+ data &&
+ (!loadingPage
+ ? data.edges.slice(currPage * pageSize, (currPage + 1) * pageSize)
+ : data.edges.slice((currPage - 1) * pageSize, currPage * pageSize));
+ // console.log(data);
+ // console.log(pageInfo);
+
+ if (loading)
+ return (
+
+
+
+
+
+
+
+
+
+
+ );
+
+ if (error)
+ return (
+
+
+
+
+ );
+
+ const CutoffCell = ({content}) => (
+ <>
+ {content ? (
+
+ {content.substring(0, 20)}
+ {content.length > 20 && '...'}
+
+ }
+ />
+ ) : (
+ -
+ )}
+ >
+ );
+
+ return (
+
+
+ {data && data.edges.length > 0 ? (
+
+
+
+
+ Expected Url
+
+
+ Expected Hash
+
+
+ Actual Hash
+
+
+ Expected Size
+
+
+ Actual Size
+
+
+ Hash Algorithm
+
+ Result
+
+ Source Filename
+
+ Created At
+
+
+
+ {content.map(({node}) => (
+
+
+
+
+
+ {node.expectedSize ? formatFileSize(node.expectedSize) : '-'}
+
+
+ {node.actualSize ? formatFileSize(node.actualSize) : '-'}
+
+
+ {node.hashAlgorithm ? node.hashAlgorithm : '-'}
+
+ {node.result}
+
+
+
+
+
+ ))}
+
+
+
+
+ {loadingPage && }
+
+ {
+ setCurrPage(currPage - 1);
+ setPages([...pages.splice(-1, 1)]);
+ fetchMore({
+ variables: pages[currPage - 1],
+ });
+ }}
+ disabled={currPage === 0 || loading}
+ />
+ {
+ setCurrPage(currPage + 1);
+ const variables = {
+ after: data.pageInfo.endCursor,
+ first: 10,
+ };
+ setPages([...pages, variables]);
+ fetchMore({
+ variables,
+ });
+ }}
+ />
+
+
+
+
+
+ ) : (
+
+
+
+ )}
+
+ );
+};
+
+export default FileUploadTable;
diff --git a/src/s3Accounting/components/FileUploadsTab.js b/src/s3Accounting/components/FileUploadsTab.js
new file mode 100644
index 000000000..53db807dd
--- /dev/null
+++ b/src/s3Accounting/components/FileUploadsTab.js
@@ -0,0 +1,216 @@
+import {
+ Container,
+ Dropdown,
+ Grid,
+ Header,
+ Icon,
+ Message,
+ Placeholder,
+ Segment,
+} from 'semantic-ui-react';
+import {FaAmbulance, FaFile, FaPhotoVideo} from 'react-icons/fa';
+import React, {useState} from 'react';
+
+import {ALL_UPLOAD_MANIFEST_FILES} from '../queries';
+import {Chart} from 'react-charts';
+import FileUploadTable from './FileUploadTable';
+import {dateTime} from '../../common/dateUtils';
+import {useQuery} from '@apollo/client';
+
+const FileUploadsTab = ({match, loading, error, data}) => {
+ const [dropdown, setDropdown] = useState('count_by_data_type');
+
+ const storageAnalyses =
+ data && data.allStorageAnalyses.edges.length > 0
+ ? data.allStorageAnalyses.edges[0].node
+ : {};
+
+ const {
+ loading: uploadManifestLoading,
+ error: uploadManifestError,
+ data: uploadManifestData,
+ } = useQuery(ALL_UPLOAD_MANIFEST_FILES, {
+ variables: {
+ storageAnalysis: storageAnalyses.id,
+ },
+ });
+
+ const uploads = storageAnalyses.stats
+ ? JSON.parse(storageAnalyses.stats).uploads
+ : {};
+
+ const dropdownText = {
+ count_by_bucket: 'Bucket',
+ count_by_data_type: 'Data Type',
+ count_by_ext: 'Extension',
+ count_by_size: 'Size',
+ };
+
+ const CountStat = ({number, title, icon, color}) => (
+
+
+
+ {title}
+
+
+ );
+
+ const DateStat = ({date, title}) => (
+ <>
+
+ {dateTime(date)}
+ >
+ );
+
+ const series = React.useMemo(
+ () => ({
+ type: 'bar',
+ }),
+ [],
+ );
+ const axes = React.useMemo(
+ () => [
+ {primary: true, type: 'ordinal', position: 'left'},
+ {position: 'bottom', type: 'linear', stacked: true},
+ ],
+ [],
+ );
+
+ const dataChart = React.useMemo(() => {
+ const rawData = uploads ? uploads[dropdown] : {};
+ const col = Object.keys(rawData);
+ const formatData = col.map(c => ({x: c, y: rawData[c]}));
+ return [
+ {
+ datums: formatData,
+ },
+ ];
+ }, [dropdown, uploads]);
+
+ const getSeriesStyle = React.useCallback(
+ series => ({
+ color: '#4183c4',
+ }),
+ [],
+ );
+
+ if (loading)
+ return (
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ );
+
+ if (error)
+ return (
+
+
+
+ );
+ console.log(uploadManifestData);
+ return (
+
+ {uploads && (
+
+
+
+ }
+ title="files found"
+ color="text-blue"
+ />
+
+
+ }
+ title="file extensions detected"
+ color="text-blue"
+ />
+
+
+ }
+ title="file types detected"
+ color="text-blue"
+ />
+
+
+
+
+
+
+ )}
+
+
+ Files by {dropdownText[dropdown]}
+
+ k.includes('count_by_'))
+ .map(v => ({
+ key: v,
+ text: dropdownText[v],
+ value: v,
+ }))}
+ value={dropdown}
+ onChange={(ev, data) => {
+ setDropdown(data.value);
+ }}
+ />
+ {dataChart[0].datums.length > 0 ? (
+
+
+
+ ) : (
+
+
+
+ )}
+
+
+
+ );
+};
+
+export default FileUploadsTab;
diff --git a/src/s3Accounting/components/S3InventoryTab.js b/src/s3Accounting/components/S3InventoryTab.js
new file mode 100644
index 000000000..3de21824c
--- /dev/null
+++ b/src/s3Accounting/components/S3InventoryTab.js
@@ -0,0 +1,215 @@
+import {
+ Container,
+ Dropdown,
+ Grid,
+ Header,
+ Icon,
+ Message,
+ Placeholder,
+ Segment,
+} from 'semantic-ui-react';
+import {FaFile, FaFill, FaPhotoVideo} from 'react-icons/fa';
+import React, {useState} from 'react';
+
+import {ALL_CLOUD_INVENTORY_FILES} from '../queries';
+import {Chart} from 'react-charts';
+import FileUploadTable from './FileUploadTable';
+import {dateTime} from '../../common/dateUtils';
+import {useQuery} from '@apollo/client';
+
+const S3InventoryTab = ({match, loading, error, data}) => {
+ const [dropdown, setDropdown] = useState('count_by_data_type');
+
+ const storageAnalyses =
+ data && data.allStorageAnalyses.edges.length > 0
+ ? data.allStorageAnalyses.edges[0].node
+ : {};
+
+ const {
+ loading: cloudInventoryLoading,
+ error: cloudInventoryError,
+ data: cloudInventoryData,
+ } = useQuery(ALL_CLOUD_INVENTORY_FILES, {
+ variables: {
+ storageAnalysis: storageAnalyses.id,
+ },
+ });
+
+ const inventory = storageAnalyses.stats
+ ? JSON.parse(storageAnalyses.stats).inventory
+ : {};
+
+ const dropdownText = {
+ count_by_bucket: 'Bucket',
+ count_by_data_type: 'Data Type',
+ count_by_ext: 'Extension',
+ count_by_size: 'Size',
+ };
+
+ const CountStat = ({number, title, icon, color}) => (
+
+
+
+ {title}
+
+
+ );
+ const DateStat = ({date, title}) => (
+ <>
+
+ {dateTime(date)}
+ >
+ );
+
+ const series = React.useMemo(
+ () => ({
+ type: 'bar',
+ }),
+ [],
+ );
+ const axes = React.useMemo(
+ () => [
+ {primary: true, type: 'ordinal', position: 'left'},
+ {position: 'bottom', type: 'linear', stacked: true},
+ ],
+ [],
+ );
+
+ const dataChart = React.useMemo(() => {
+ const rawData = inventory ? inventory[dropdown] : {};
+ const col = Object.keys(rawData);
+ const formatData = col.map(c => ({x: c, y: rawData[c]}));
+ return [
+ {
+ datums: formatData,
+ },
+ ];
+ }, [dropdown, inventory]);
+
+ const getSeriesStyle = React.useCallback(
+ series => ({
+ color: '#4183c4',
+ }),
+ [],
+ );
+
+ if (loading)
+ return (
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ );
+
+ if (error)
+ return (
+
+
+
+ );
+
+ return (
+
+ {inventory && (
+
+
+
+ }
+ title="bucket found"
+ color="text-blue"
+ />
+
+
+ }
+ title="files found"
+ color="text-blue"
+ />
+
+
+ }
+ title="file extensions detected"
+ color="text-blue"
+ />
+
+
+
+
+
+
+ )}
+
+
+ Files by {dropdownText[dropdown]}
+
+ k.includes('count_by_'))
+ .map(v => ({
+ key: v,
+ text: dropdownText[v],
+ value: v,
+ }))}
+ value={dropdown}
+ onChange={(ev, data) => {
+ setDropdown(data.value);
+ }}
+ />
+ {dataChart[0].datums.length > 0 ? (
+
+
+
+ ) : (
+
+
+
+ )}
+
+
+
+ );
+};
+
+export default S3InventoryTab;
diff --git a/src/s3Accounting/fragments.js b/src/s3Accounting/fragments.js
new file mode 100644
index 000000000..00115fc8a
--- /dev/null
+++ b/src/s3Accounting/fragments.js
@@ -0,0 +1,17 @@
+import gql from 'graphql-tag';
+
+export const FILE_AUDI_FIELDS = gql`
+ fragment FileAudiFields on FileAuditNode {
+ id
+ result
+ sourceFilename
+ expectedUrl
+ actualHash
+ expectedHash
+ expectedSize
+ actualSize
+ hashAlgorithm
+ customFields
+ createdAt
+ }
+`;
diff --git a/src/s3Accounting/queries.js b/src/s3Accounting/queries.js
new file mode 100644
index 000000000..cf8fc4337
--- /dev/null
+++ b/src/s3Accounting/queries.js
@@ -0,0 +1,104 @@
+import {FILE_AUDI_FIELDS} from './fragments';
+import gql from 'graphql-tag';
+
+export const ALL_STORAGE_ANALYSES = gql`
+ query allStorageAnalyses($studyKfId: String) {
+ allStorageAnalyses(studyKfId: $studyKfId) {
+ edges {
+ node {
+ id
+ uuid
+ createdAt
+ refreshedAt
+ scannedStorageAt
+ stats
+ study {
+ id
+ kfId
+ name
+ }
+ fileAudits {
+ edges {
+ node {
+ ...FileAudiFields
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ ${FILE_AUDI_FIELDS}
+`;
+
+export const ALL_FILE_AUDITS = gql`
+ query allFileAudits(
+ $storageAnalysis: ID
+ $result: ResultEnum
+ $first: Int
+ $last: Int
+ $after: String
+ $before: String
+ ) {
+ allFileAudits(
+ storageAnalysis: $storageAnalysis
+ result: $result
+ first: $first
+ last: $last
+ after: $after
+ before: $before
+ orderBy: "-created_at"
+ ) {
+ pageInfo {
+ hasNextPage
+ hasPreviousPage
+ startCursor
+ endCursor
+ }
+ edges {
+ node {
+ ...FileAudiFields
+ }
+ }
+ }
+ }
+ ${FILE_AUDI_FIELDS}
+`;
+
+export const ALL_CLOUD_INVENTORY_FILES = gql`
+ query allCloudInventoryFiles($storageAnalysis: ID, $result: ResultEnum) {
+ allCloudInventoryFiles(storageAnalysis: $storageAnalysis, result: $result) {
+ edges {
+ node {
+ ...FileAudiFields
+ }
+ }
+ pageInfo {
+ hasNextPage
+ hasPreviousPage
+ startCursor
+ endCursor
+ }
+ }
+ }
+ ${FILE_AUDI_FIELDS}
+`;
+
+export const ALL_UPLOAD_MANIFEST_FILES = gql`
+ query allUploadManifestFiles($storageAnalysis: ID, $result: ResultEnum) {
+ allUploadManifestFiles(storageAnalysis: $storageAnalysis, result: $result) {
+ edges {
+ node {
+ ...FileAudiFields
+ }
+ }
+ pageInfo {
+ hasNextPage
+ hasPreviousPage
+ startCursor
+ endCursor
+ }
+ }
+ }
+ ${FILE_AUDI_FIELDS}
+`;
diff --git a/src/s3Accounting/routes.js b/src/s3Accounting/routes.js
new file mode 100644
index 000000000..cdb2b76ae
--- /dev/null
+++ b/src/s3Accounting/routes.js
@@ -0,0 +1,15 @@
+import {DetailView, ListView} from './views';
+import {Route, Switch} from 'react-router-dom';
+
+import React from 'react';
+
+const Routes = () => (
+ <>
+
+
+ } />
+
+ >
+);
+
+export default Routes;
diff --git a/src/s3Accounting/views/DetailView.js b/src/s3Accounting/views/DetailView.js
new file mode 100644
index 000000000..30635c513
--- /dev/null
+++ b/src/s3Accounting/views/DetailView.js
@@ -0,0 +1,151 @@
+import {
+ Button,
+ Container,
+ Header,
+ Icon,
+ Menu,
+ Segment,
+} from 'semantic-ui-react';
+import React, {useState} from 'react';
+
+import {ALL_STORAGE_ANALYSES} from '../queries';
+import ComparisonTab from '../components/ComparisonTab';
+import FileUploadsTab from '../components/FileUploadsTab';
+import {Helmet} from 'react-helmet';
+import {ImageMessage} from '../../components/ImageMessage';
+import {MY_PROFILE} from '../../state/queries';
+import S3InventoryTab from '../components/S3InventoryTab';
+import {hasPermission} from '../../common/permissions';
+import s3 from '../../assets/s3.png';
+import {useQuery} from '@apollo/client';
+
+const DetailView = ({match, history}) => {
+ const [tab, setTab] = useState(0);
+ const {data: profileData, loading: myProfileLoading} = useQuery(MY_PROFILE);
+ const myProfile = profileData && profileData.myProfile;
+
+ const {loading, error, data} = useQuery(ALL_STORAGE_ANALYSES, {
+ variables: {
+ studyKfId: match.params.studyId,
+ },
+ });
+ const storageAnalyses =
+ data && data.allStorageAnalyses.edges.length > 0
+ ? data.allStorageAnalyses.edges[0].node
+ : {};
+ const study = storageAnalyses && storageAnalyses.study;
+ const studyName = study ? study.name || study.shortName : 'Uknown Study';
+
+ const navTab = [
+ {
+ tab: 'Comparison',
+ hash: '#comparison',
+ },
+ {
+ tab: 'S3 Inventory',
+ hash: '#s3-inventory',
+ },
+ {
+ tab: 'File Uploads',
+ hash: '#file-uploads',
+ },
+ ];
+
+ if (
+ !myProfileLoading &&
+ myProfile &&
+ !(
+ hasPermission(myProfile, 'list_all_storageanalysis') ||
+ hasPermission(myProfile, 'list_all_fileaudits')
+ )
+ )
+ return (
+
+ You may not have been granted permissions for analysis of files in
+ study's a cloud storage. Please contact us at{' '}
+
+ support@kidsfirstdrc.org
+
+ >
+ }
+ />
+ );
+
+ return (
+
+
+ {`KF Data Tracker - S3 Accounting`}
+
+
+ history.push(`/study/${match.params.studyId}/basic-info/info`)
+ }
+ />
+
+
+ View snapshots of your study's data files in cloud storage and compare
+ them against the file upload manifests that your users submitted. This
+ will help you understand the differential between what a user tried to
+ upload and what actually got uploaded to your cloud storage.
+
+
+ {navTab.map((item, i) => (
+ {
+ setTab(i);
+ window.history.pushState(
+ {},
+ '',
+ history.location.pathname + navTab[i].hash,
+ );
+ }}
+ >
+ {item.tab}
+
+ ))}
+
+ {tab === 0 && (
+
+ )}
+ {tab === 1 && (
+
+ )}
+ {tab === 2 && (
+
+ )}
+
+ );
+};
+export default DetailView;
diff --git a/src/s3Accounting/views/ListView.js b/src/s3Accounting/views/ListView.js
new file mode 100644
index 000000000..c9cb95dec
--- /dev/null
+++ b/src/s3Accounting/views/ListView.js
@@ -0,0 +1,200 @@
+import {
+ Container,
+ Header,
+ Icon,
+ Message,
+ Placeholder,
+ Segment,
+ Table,
+} from 'semantic-ui-react';
+
+import {ALL_STORAGE_ANALYSES} from '../queries';
+import {Helmet} from 'react-helmet';
+import {ImageMessage} from '../../components/ImageMessage';
+import KfId from '../../components/StudyList/KfId';
+import {MY_PROFILE} from '../../state/queries';
+import React from 'react';
+import TimeAgo from 'react-timeago';
+import {hasPermission} from '../../common/permissions';
+import {longDate} from '../../common/dateUtils';
+import s3 from '../../assets/s3.png';
+import {useHistory} from 'react-router-dom';
+import {useQuery} from '@apollo/client';
+
+const ListView = ({match}) => {
+ const history = useHistory();
+
+ const {data: profileData, loading: myProfileLoading} = useQuery(MY_PROFILE);
+ const myProfile = profileData && profileData.myProfile;
+
+ const {loading, error, data} = useQuery(ALL_STORAGE_ANALYSES);
+ const storageAnalyses =
+ data && data.allStorageAnalyses.edges.length > 0
+ ? data.allStorageAnalyses.edges
+ : [];
+ console.log(myProfile);
+ if (
+ !myProfileLoading &&
+ myProfile &&
+ !hasPermission(myProfile, 'list_all_storageanalysis')
+ )
+ return (
+
+ You may not have been granted permissions for analysis of files in
+ study's a cloud storage. Please contact us at{' '}
+
+ support@kidsfirstdrc.org
+
+ >
+ }
+ />
+ );
+
+ if (loading)
+ return (
+
+
+
+ S3 Accounting Reports
+
+
+ View snapshots of your study's data files in cloud storage and compare
+ them against the file upload manifests that your users submitted. This
+ will help you understand the differential between what a user tried to
+ upload and what actually got uploaded to your cloud storage.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ );
+
+ if (error)
+ return (
+
+
+ {`KF Data Tracker - S3 Accounting - Error`}
+
+
+
+ S3 Accounting Reports
+
+
+ View snapshots of your study's data files in cloud storage and compare
+ them against the file upload manifests that your users submitted. This
+ will help you understand the differential between what a user tried to
+ upload and what actually got uploaded to your cloud storage.
+
+
+
+ );
+
+ return (
+
+
+ {`KF Data Tracker - S3 Accounting`}
+
+
+
+ S3 Accounting Reports
+
+
+ View snapshots of your study's data files in cloud storage and compare
+ them against the file upload manifests that your users submitted. This
+ will help you understand the differential between what a user tried to
+ upload and what actually got uploaded to your cloud storage.
+
+ {storageAnalyses.length > 0 ? (
+
+
+
+ Study Name
+ Study ID
+ Updated At
+
+ Files Found
+
+
+ Files Missing
+
+
+ Files Moved
+
+
+
+
+ {storageAnalyses.map(({node}) => {
+ const audit = node.stats
+ ? JSON.parse(node.stats).audit
+ : {
+ matched: {total_count: '-'},
+ missing: {total_count: '-'},
+ moved: {total_count: '-'},
+ };
+ return (
+
+ history.push(`/s3-accounting/${node.study.kfId}#comparison`)
+ }
+ >
+ {node.study.shortName}
+
+
+
+
+
+ {audit.matched.total_count}
+
+
+ {audit.missing.total_count}
+
+
+ {audit.moved.total_count}
+
+
+ );
+ })}
+
+
+ ) : (
+
+
+
+ )}
+
+ );
+};
+
+export default ListView;
diff --git a/src/s3Accounting/views/index.js b/src/s3Accounting/views/index.js
new file mode 100644
index 000000000..8e9e68d07
--- /dev/null
+++ b/src/s3Accounting/views/index.js
@@ -0,0 +1,2 @@
+export {default as ListView} from './ListView';
+export {default as DetailView} from './DetailView';
diff --git a/yarn.lock b/yarn.lock
index 3cce1cf8f..5e1f44df2 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1909,6 +1909,11 @@
"@nodelib/fs.scandir" "2.1.4"
fastq "^1.6.0"
+"@reach/observe-rect@^1.1.0":
+ version "1.2.0"
+ resolved "https://registry.yarnpkg.com/@reach/observe-rect/-/observe-rect-1.2.0.tgz#d7a6013b8aafcc64c778a0ccb83355a11204d3b2"
+ integrity sha512-Ba7HmkFgfQxZqqaeIWWkNK0rEhpxVQHIoVyW1YDSkGsGIXzcaW4deC8B0pZrNSSyLTdIk7y+5olKt5+g0GmFIQ==
+
"@react-dnd/asap@^4.0.0":
version "4.0.0"
resolved "https://registry.yarnpkg.com/@react-dnd/asap/-/asap-4.0.0.tgz#b300eeed83e9801f51bd66b0337c9a6f04548651"
@@ -3929,9 +3934,9 @@ caniuse-api@^3.0.0:
lodash.uniq "^4.5.0"
caniuse-lite@^1.0.0, caniuse-lite@^1.0.30000939, caniuse-lite@^1.0.30000989, caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001173:
- version "1.0.30001180"
- resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001180.tgz#67abcd6d1edf48fa5e7d1e84091d1d65ab76e33b"
- integrity sha512-n8JVqXuZMVSPKiPiypjFtDTXc4jWIdjxull0f92WLo7e1MSi3uJ3NvveakSh/aCl1QKFAvIz3vIj0v+0K+FrXw==
+ version "1.0.30001278"
+ resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001278.tgz"
+ integrity sha512-mpF9KeH8u5cMoEmIic/cr7PNS+F5LWBk0t2ekGT60lFf0Wq+n9LspAj0g3P+o7DQhD3sUdlMln4YFAWhFYn9jg==
capture-exit@^2.0.0:
version "2.0.0"
@@ -4944,6 +4949,13 @@ cypress@^4.2.0:
url "^0.11.0"
yauzl "^2.10.0"
+d3-array@2:
+ version "2.12.1"
+ resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-2.12.1.tgz#e20b41aafcdffdf5d50928004ececf815a465e81"
+ integrity sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==
+ dependencies:
+ internmap "^1.0.0"
+
d3-array@^2.3.0:
version "2.11.0"
resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-2.11.0.tgz#5ed6a2869bc7d471aec8df9ff6ed9fef798facc4"
@@ -4956,6 +4968,13 @@ d3-array@^2.3.0:
resolved "https://registry.yarnpkg.com/d3-color/-/d3-color-2.0.0.tgz#8d625cab42ed9b8f601a1760a389f7ea9189d62e"
integrity sha512-SPXi0TSKPD4g9tw0NMZFnR95XVgUZiBH+uUTqQuDu1OsE2zomHU7ho0FISciaPvosimixwHFl3WHLGabv6dDgQ==
+d3-delaunay@^5.2.1:
+ version "5.3.0"
+ resolved "https://registry.yarnpkg.com/d3-delaunay/-/d3-delaunay-5.3.0.tgz#b47f05c38f854a4e7b3cea80e0bb12e57398772d"
+ integrity sha512-amALSrOllWVLaHTnDLHwMIiz0d1bBu9gZXd1FiLfXf8sHcX9jrcj81TVZOqD4UX7MgBZZ07c8GxzEgBpJqc74w==
+ dependencies:
+ delaunator "4"
+
"d3-format@1 - 2":
version "2.0.0"
resolved "https://registry.yarnpkg.com/d3-format/-/d3-format-2.0.0.tgz#a10bcc0f986c372b729ba447382413aabf5b0767"
@@ -5002,7 +5021,18 @@ d3-scale@^3.0.0:
d3-time "1 - 2"
d3-time-format "2 - 3"
-d3-shape@^1.2.2, d3-shape@^1.3.5:
+d3-scale@^3.2.1:
+ version "3.3.0"
+ resolved "https://registry.yarnpkg.com/d3-scale/-/d3-scale-3.3.0.tgz#28c600b29f47e5b9cd2df9749c206727966203f3"
+ integrity sha512-1JGp44NQCt5d1g+Yy+GeOnZP7xHo0ii8zsQp6PGzd+C1/dl0KGsp9A7Mxwp+1D1o4unbTTxVdU/ZOIEBoeZPbQ==
+ dependencies:
+ d3-array "^2.3.0"
+ d3-format "1 - 2"
+ d3-interpolate "1.2.0 - 2"
+ d3-time "^2.1.1"
+ d3-time-format "2 - 3"
+
+d3-shape@^1.2.2, d3-shape@^1.3.5, d3-shape@^1.3.7:
version "1.3.7"
resolved "https://registry.yarnpkg.com/d3-shape/-/d3-shape-1.3.7.tgz#df63801be07bc986bc54f63789b4fe502992b5d7"
integrity sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==
@@ -5033,6 +5063,18 @@ d3-time@1, d3-time@^1.0.10, d3-time@^1.0.11:
resolved "https://registry.yarnpkg.com/d3-time/-/d3-time-2.0.0.tgz#ad7c127d17c67bd57a4c61f3eaecb81108b1e0ab"
integrity sha512-2mvhstTFcMvwStWd9Tj3e6CEqtOivtD8AUiHT8ido/xmzrI9ijrUUihZ6nHuf/vsScRBonagOdj0Vv+SEL5G3Q==
+d3-time@^2.1.1:
+ version "2.1.1"
+ resolved "https://registry.yarnpkg.com/d3-time/-/d3-time-2.1.1.tgz#e9d8a8a88691f4548e68ca085e5ff956724a6682"
+ integrity sha512-/eIQe/eR4kCQwq7yxi7z4c6qEXf2IYGcjoWB5OOQy4Tq9Uv39/947qlDcN2TLkiTzQWzvnsuYPB9TrWaNfipKQ==
+ dependencies:
+ d3-array "2"
+
+d3-voronoi@^1.1.2:
+ version "1.1.4"
+ resolved "https://registry.yarnpkg.com/d3-voronoi/-/d3-voronoi-1.1.4.tgz#dd3c78d7653d2bb359284ae478645d95944c8297"
+ integrity sha512-dArJ32hchFsrQ8uMiTBLq256MpnZjeuBtdHpaDlYuQyjU0CVzCJl/BVW+SkszaAeH95D/8gxqAhgx0ouAWAfRg==
+
damerau-levenshtein@^1.0.4:
version "1.0.6"
resolved "https://registry.yarnpkg.com/damerau-levenshtein/-/damerau-levenshtein-1.0.6.tgz#143c1641cb3d85c60c32329e26899adea8701791"
@@ -5209,6 +5251,11 @@ del@^3.0.0:
pify "^3.0.0"
rimraf "^2.2.8"
+delaunator@4:
+ version "4.0.1"
+ resolved "https://registry.yarnpkg.com/delaunator/-/delaunator-4.0.1.tgz#3d779687f57919a7a418f8ab947d3bddb6846957"
+ integrity sha512-WNPWi1IRKZfCt/qIDMfERkDp93+iZEmOxN2yy4Jg+Xhv8SLk2UTqqbe1sfiipn0and9QrE914/ihdx82Y/Giag==
+
delayed-stream@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
@@ -11708,6 +11755,17 @@ react-app-rewired@^2.1.0:
dependencies:
semver "^5.6.0"
+react-charts@^2.0.0-beta.7:
+ version "2.0.0-beta.7"
+ resolved "https://registry.yarnpkg.com/react-charts/-/react-charts-2.0.0-beta.7.tgz#cc989d78f976dc70796af3af816874274f0c75be"
+ integrity sha512-iUspg9rnx7kD0H/wsK67HNUioOgKgJ8WRXr/Tk3EGP2qcFb9Vo7pjDk4oz1jH12TC+mqL+HFxNYraMkhWd6CUw==
+ dependencies:
+ "@reach/observe-rect" "^1.1.0"
+ d3-delaunay "^5.2.1"
+ d3-scale "^3.2.1"
+ d3-shape "^1.3.7"
+ d3-voronoi "^1.1.2"
+
react-copy-to-clipboard@^5.0.1:
version "5.0.3"
resolved "https://registry.yarnpkg.com/react-copy-to-clipboard/-/react-copy-to-clipboard-5.0.3.tgz#2a0623b1115a1d8c84144e9434d3342b5af41ab4"
@@ -11880,6 +11938,11 @@ react-hot-loader@^4.3.12:
shallowequal "^1.1.0"
source-map "^0.7.3"
+react-icons@^4.3.1:
+ version "4.3.1"
+ resolved "https://registry.yarnpkg.com/react-icons/-/react-icons-4.3.1.tgz#2fa92aebbbc71f43d2db2ed1aed07361124e91ca"
+ integrity sha512-cB10MXLTs3gVuXimblAdI71jrJx8njrJZmNMEMC+sQu5B/BIOmlsAjskdqpn81y8UBVEGuHODd7/ci5DvoSzTQ==
+
react-is@^16.12.0, react-is@^16.6.0, react-is@^16.6.3, react-is@^16.7.0, react-is@^16.8.1, react-is@^16.8.4, react-is@^16.8.6:
version "16.13.1"
resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.13.1.tgz#789729a4dc36de2999dc156dd6c1d9c18cea56a4"
@@ -14460,10 +14523,8 @@ watchpack@^1.5.0:
resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-1.7.5.tgz#1267e6c55e0b9b5be44c2023aed5437a2c26c453"
integrity sha512-9P3MWk6SrKjHsGkLT2KHXdQ/9SNkyoJbabxnKOoJepsvJjJG8uYTR3yTPxPQvNDI3w4Nz1xnE0TLHK4RIVe/MQ==
dependencies:
- chokidar "^3.4.1"
graceful-fs "^4.1.2"
neo-async "^2.5.0"
- watchpack-chokidar2 "^2.0.1"
optionalDependencies:
chokidar "^3.4.1"
watchpack-chokidar2 "^2.0.1"