diff --git a/package.json b/package.json
index 5f15f1815..00e1f0370 100644
--- a/package.json
+++ b/package.json
@@ -41,6 +41,7 @@
"query-string": "^6.12.1",
"react": "^16.8.3",
"react-app-rewire-postcss": "^3.0.2",
+ "react-charts": "^2.0.0-beta.7",
"react-copy-to-clipboard": "^5.0.1",
"react-datasheet": "^1.4.1",
"react-dom": "^16.8.3",
diff --git a/src/App.css b/src/App.css
index 6e24e137f..82025cd51 100755
--- a/src/App.css
+++ b/src/App.css
@@ -44,6 +44,10 @@ body {
margin-top: 0px !important;
}
+.mt--10 {
+ margin-top: -10px !important;
+}
+
.mr-0 {
margin-right: 0px !important;
}
@@ -338,6 +342,10 @@ body {
color: #676767 !important;
}
+.text-dark-grey {
+ color: #5e5e5e !important;
+}
+
.text-black {
color: #000000 !important;
}
@@ -538,6 +546,14 @@ body {
height: 38px !important;
}
+.h-24 {
+ height: 24px !important;
+}
+
+.h-220 {
+ height: 220px !important;
+}
+
.small-inline {
display: inline-block;
padding-top: 10px;
diff --git a/src/assets/s3.png b/src/assets/s3.png
new file mode 100644
index 000000000..f89de6f6c
Binary files /dev/null and b/src/assets/s3.png differ
diff --git a/src/common/dateUtils.js b/src/common/dateUtils.js
index 9e80462c9..c3838e479 100644
--- a/src/common/dateUtils.js
+++ b/src/common/dateUtils.js
@@ -1,5 +1,5 @@
// Reformat time to the "DD MMM YYYY" format like "12 Aug 2019"
-export const longDate = (date) => {
+export const longDate = date => {
var mydate = new Date(date);
var month = [
'Jan',
@@ -19,7 +19,7 @@ export const longDate = (date) => {
return str;
};
-export const toYearMonthDay = (isoDate) => {
+export const toYearMonthDay = isoDate => {
/*
* Convert ISO8601 UTC datetime string to local datetime and format as
* yyyy-mm-dd date string.
@@ -39,3 +39,34 @@ export const toYearMonthDay = (isoDate) => {
}
return date;
};
+
+// Reformat time to the date and time format
+export const dateTime = date => {
+ var mydate = new Date(date);
+ var month = [
+ 'Jan',
+ 'Feb',
+ 'Mar',
+ 'Apr',
+ 'May',
+ 'Jun',
+ 'Jul',
+ 'Aug',
+ 'Sept',
+ 'Oct',
+ 'Nov',
+ 'Dec',
+ ][mydate.getMonth()];
+ const weekday = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'];
+ var str =
+ weekday[mydate.getDay()] +
+ ', ' +
+ month +
+ ' ' +
+ mydate.getUTCDate() +
+ ', ' +
+ mydate.getFullYear() +
+ ', ' +
+ mydate.toLocaleTimeString([], {hour: '2-digit', minute: '2-digit'});
+ return str;
+};
diff --git a/src/components/Header/Header.js b/src/components/Header/Header.js
index a66948ebe..395f37b7c 100644
--- a/src/components/Header/Header.js
+++ b/src/components/Header/Header.js
@@ -277,6 +277,12 @@ const Header = ({location}) => {
)}
+ {hasPermission(profile, 'view_settings') && (
+
+ S3 Accounting
+
+
+ )}
{hasPermission(profile, 'add_referraltoken') && (
diff --git a/src/releases/views/ReleaseDetailView.js b/src/releases/views/ReleaseDetailView.js
index 1c7fd971e..749f2d39c 100644
--- a/src/releases/views/ReleaseDetailView.js
+++ b/src/releases/views/ReleaseDetailView.js
@@ -1,33 +1,33 @@
-import React from 'react';
-import {Helmet} from 'react-helmet';
-import {useQuery} from '@apollo/client';
-import {withRouter, Link} from 'react-router-dom';
+import {ALL_EVENTS, GET_RELEASE} from '../queries';
import {
Button,
Container,
Dimmer,
- Header,
Grid,
- Image,
+ Header,
Icon,
+ Image,
List,
+ Loader,
Message,
Segment,
- Loader,
} from 'semantic-ui-react';
-import {Progress} from '../components/Progress';
-import {ReleaseTaskList} from '../components/ReleaseTaskList';
+import {Link, withRouter} from 'react-router-dom';
import {
- ReleaseHeader,
+ LogViewer,
MarkdownEditor,
ReleaseActions,
- LogViewer,
+ ReleaseHeader,
} from '../components/ReleaseDetail';
-import paragraph from '../../assets/paragraph.png';
-import {GET_RELEASE, ALL_EVENTS} from '../queries';
+import {Helmet} from 'react-helmet';
import {MY_PROFILE} from '../../state/queries';
+import {Progress} from '../components/Progress';
+import React from 'react';
+import {ReleaseTaskList} from '../components/ReleaseTaskList';
import {hasPermission} from '../../common/permissions';
+import paragraph from '../../assets/paragraph.png';
+import {useQuery} from '@apollo/client';
const ReleaseDetailView = ({user, history, match}) => {
const relayId = Buffer.from('ReleaseNode:' + match.params.releaseId).toString(
@@ -40,7 +40,7 @@ const ReleaseDetailView = ({user, history, match}) => {
data: releaseData,
} = useQuery(GET_RELEASE, {
variables: {id: relayId},
- pollInterval: 5000,
+ pollInterval: 5001,
});
const {loading: eventsLoading, error: eventsError, data: events} = useQuery(
diff --git a/src/routes/index.js b/src/routes/index.js
index 8f55bfe2b..a64a70237 100644
--- a/src/routes/index.js
+++ b/src/routes/index.js
@@ -51,6 +51,7 @@ import {
TemplatesView,
} from '../admin/views';
import ReleaseRoutes from '../releases/routes';
+import S3AccountingRoutes from '../s3Accounting/routes';
import TrackedRoute from './TrackedRoute';
const Routes = () => (
@@ -74,6 +75,7 @@ const Routes = () => (
+
{
+ const [tab, setTab] = useState('matched');
+ const [dropdown, setDropdown] = useState('count_by_data_type');
+
+ const storageAnalyses =
+ data && data.allStorageAnalyses.edges.length > 0
+ ? data.allStorageAnalyses.edges[0].node
+ : {};
+
+ const {
+ loading: fileAuditsLoading,
+ error: fileAuditsError,
+ data: fileAuditsData,
+ refetch: fileAuditsRefetch,
+ } = useQuery(ALL_FILE_AUDITS, {
+ variables: {
+ storageAnalysis: storageAnalyses.id,
+ result: tab,
+ },
+ });
+
+ const audit = storageAnalyses.stats
+ ? JSON.parse(storageAnalyses.stats).audit
+ : {};
+
+ const tabContent = {
+ differ: {
+ title: ' files were successfully uploaded to S3',
+ description:
+ 'This metric is computed by counting the file hashes in the upload manifest(s) whose url and hash match a record in the S3 inventory. These represent files that were successfully uploaded to the expected location in S3.',
+ color: '#db2828',
+ },
+ matched: {
+ title: ' files that were expected but not found in S3',
+ description:
+ 'This metric is computed by counting the file hashes in the upload manifest(s) that were not found in the S3 inventory. These represent file blobs that do not exist anywhere in S3 at the time of scan.',
+ color: '#21ba45',
+ },
+ missing: {
+ title: ' files in S3 with different content than expected',
+ description:
+ 'This metric is computed by counting the urls in the upload manifest(s) whose hash value is different than the hash value in the S3 inventory. These items could indicate corrupted uploads or S3 objects that have been overwritten with different content since the time of upload.',
+ color: '#db2828',
+ },
+ moved: {
+ title:
+ ' files that exist in the S3 but at a different location than expected',
+ description:
+ 'This metric is computed by counting the file hashes in the upload manifest(s) whose url is different than the url in the S3 inventory. These represent file blobs that exist in S3 but at different locations than specified by a user in the upload manifest(s).',
+ color: '#fbbd08',
+ },
+ };
+
+ const dropdownText = {
+ count_by_bucket: 'Bucket',
+ count_by_data_type: 'Data Type',
+ count_by_ext: 'Extension',
+ count_by_size: 'Size',
+ };
+
+ const dropdownOptions = {
+ matched: [
+ {
+ key: 'count_by_data_type',
+ text: 'Data Type',
+ value: 'count_by_data_type',
+ },
+ {key: 'count_by_ext', text: 'Extension', value: 'count_by_ext'},
+ {key: 'count_by_size', text: 'Size', value: 'count_by_size'},
+ {key: 'count_by_bucket', text: 'Bucket', value: 'count_by_bucket'},
+ ],
+ missing: [
+ {
+ key: 'count_by_data_type',
+ text: 'Data Type',
+ value: 'count_by_data_type',
+ },
+ {key: 'count_by_ext', text: 'Extension', value: 'count_by_ext'},
+ {key: 'count_by_size', text: 'Size', value: 'count_by_size'},
+ ],
+ differ: [
+ {
+ key: 'count_by_data_type',
+ text: 'Data Type',
+ value: 'count_by_data_type',
+ },
+ {key: 'count_by_ext', text: 'Extension', value: 'count_by_ext'},
+ {key: 'count_by_size', text: 'Size', value: 'count_by_size'},
+ {key: 'count_by_bucket', text: 'Bucket', value: 'count_by_bucket'},
+ ],
+ moved: [
+ {
+ key: 'count_by_data_type',
+ text: 'Data Type',
+ value: 'count_by_data_type',
+ },
+ {key: 'count_by_ext', text: 'Extension', value: 'count_by_ext'},
+ {key: 'count_by_size', text: 'Size', value: 'count_by_size'},
+ ],
+ };
+
+ const CountStat = ({number, title, icon, color}) => (
+
+
+
+ {title}
+
+
+ );
+
+ const DateStat = ({date, title}) => (
+ <>
+
+ {dateTime(date)}
+ >
+ );
+
+ const series = React.useMemo(
+ () => ({
+ type: 'bar',
+ }),
+ [],
+ );
+ const axes = React.useMemo(
+ () => [
+ {primary: true, type: 'ordinal', position: 'left'},
+ {position: 'bottom', type: 'linear', stacked: true},
+ ],
+ [],
+ );
+
+ const dataChart = React.useMemo(() => {
+ var rawData = audit && audit[tab] ? audit[tab][dropdown] : {};
+ const col = Object.keys(rawData);
+ const formatData = col.map(c => ({x: c, y: rawData[c]}));
+ return [
+ {
+ datums: formatData,
+ },
+ ];
+ }, [audit, dropdown, tab]);
+
+ const getSeriesStyle = React.useCallback(
+ series => ({
+ color: tabContent[tab].color || '#919090',
+ }),
+ [tab, tabContent],
+ );
+
+ if (loading)
+ return (
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ );
+
+ if (error)
+ return (
+
+
+
+ );
+
+ return (
+
+ {Object.keys(audit).length > 0 && (
+
+
+ {
+ setTab('matched');
+ setDropdown('count_by_data_type');
+ fileAuditsRefetch({
+ result: tab,
+ });
+ }}
+ >
+
+ {tab === 'matched' && (
+
+ )}
+
+ {
+ setTab('missing');
+ setDropdown('count_by_data_type');
+ fileAuditsRefetch({
+ result: tab,
+ });
+ }}
+ >
+
+ {tab === 'missing' && (
+
+ )}
+
+ {
+ setTab('differ');
+ setDropdown('count_by_data_type');
+ fileAuditsRefetch({
+ result: tab,
+ });
+ }}
+ >
+
+ {tab === 'differ' && (
+
+ )}
+
+
+
+
+
+
+
+
+
+
+
+ Found{' '}
+ {audit[tab] && audit[tab].total_count
+ ? audit[tab].total_count
+ : '-'}
+ {tabContent[tab].title}
+
+ {tabContent[tab].description}
+
+
+
+ )}
+
+
+ Files by {dropdownText[dropdown]}
+
+ {
+ setDropdown(data.value);
+ }}
+ />
+ {dataChart[0].datums.length > 0 ? (
+
+
+
+ ) : (
+
+
+
+ )}
+
+
+
+ );
+};
+
+export default ComparisonTab;
diff --git a/src/s3Accounting/components/FileUploadTable.js b/src/s3Accounting/components/FileUploadTable.js
new file mode 100644
index 000000000..c7a68b188
--- /dev/null
+++ b/src/s3Accounting/components/FileUploadTable.js
@@ -0,0 +1,130 @@
+import {
+ Container,
+ Header,
+ Message,
+ Placeholder,
+ Popup,
+ Segment,
+ Table,
+} from 'semantic-ui-react';
+
+import React from 'react';
+import TimeAgo from 'react-timeago';
+import {longDate} from '../../common/dateUtils';
+
+const FileUploadTable = ({loading, error, data, title}) => {
+ if (loading)
+ return (
+
+
+
+
+
+
+
+
+
+
+ );
+
+ if (error)
+ return (
+
+
+
+
+ );
+
+ const CutoffCell = ({content}) => (
+ <>
+ {content ? (
+
+ {content.substring(0, 20)}
+ {content.length > 20 && '...'}
+
+ }
+ />
+ ) : (
+ -
+ )}
+ >
+ );
+
+ return (
+
+
+ {data && data.edges.length > 0 ? (
+
+
+
+
+ expectedUrl
+
+
+ expectedHash
+
+ actualHash
+
+ expectedSize
+
+ actualSize
+
+ hashAlgorithm
+
+ result
+
+ sourceFilename
+
+ createdAt
+
+
+
+ {data.edges.map(({node}) => (
+
+
+
+
+ {node.expectedSize}
+ {node.actualSize}
+ {node.hashAlgorithm}
+ {node.result}
+
+
+
+
+
+ ))}
+
+
+ ) : (
+
+
+
+ )}
+
+ );
+};
+
+export default FileUploadTable;
diff --git a/src/s3Accounting/components/FileUploadsTab.js b/src/s3Accounting/components/FileUploadsTab.js
new file mode 100644
index 000000000..145e1aea5
--- /dev/null
+++ b/src/s3Accounting/components/FileUploadsTab.js
@@ -0,0 +1,216 @@
+import {
+ Container,
+ Dropdown,
+ Grid,
+ Header,
+ Icon,
+ Message,
+ Placeholder,
+ Segment,
+} from 'semantic-ui-react';
+import React, {useState} from 'react';
+
+import {ALL_UPLOAD_MANIFEST_FILES} from '../queries';
+import {Chart} from 'react-charts';
+import FileUploadTable from './FileUploadTable';
+import {dateTime} from '../../common/dateUtils';
+import {useQuery} from '@apollo/client';
+
+const FileUploadsTab = ({match, loading, error, data}) => {
+ const [dropdown, setDropdown] = useState('count_by_data_type');
+
+ const storageAnalyses =
+ data && data.allStorageAnalyses.edges.length > 0
+ ? data.allStorageAnalyses.edges[0].node
+ : {};
+
+ const {
+ loading: uploadManifestLoading,
+ error: uploadManifestError,
+ data: uploadManifestData,
+ } = useQuery(ALL_UPLOAD_MANIFEST_FILES, {
+ variables: {
+ storageAnalysis: storageAnalyses.id,
+ },
+ });
+
+ const uploads = storageAnalyses.stats
+ ? JSON.parse(storageAnalyses.stats).uploads
+ : {};
+
+ const dropdownText = {
+ count_by_data_type: 'Data Type',
+ count_by_ext: 'Ext',
+ count_by_size: 'Size',
+ };
+
+ const CountStat = ({number, title, icon, color}) => (
+
+
+
+ {title}
+
+
+ );
+
+ const DateStat = ({date, title}) => (
+ <>
+
+ {dateTime(date)}
+ >
+ );
+
+ const series = React.useMemo(
+ () => ({
+ type: 'bar',
+ }),
+ [],
+ );
+ const axes = React.useMemo(
+ () => [
+ {primary: true, type: 'ordinal', position: 'left'},
+ {position: 'bottom', type: 'linear', stacked: true},
+ ],
+ [],
+ );
+
+ const dataChart = React.useMemo(() => {
+ const rawData = uploads ? uploads[dropdown] : {};
+ const col = Object.keys(rawData);
+ const formatData = col.map(c => ({x: c, y: rawData[c]}));
+ return [
+ {
+ datums: formatData,
+ },
+ ];
+ }, [dropdown, uploads]);
+
+ const getSeriesStyle = React.useCallback(
+ series => ({
+ color: '#4183c4',
+ }),
+ [],
+ );
+
+ if (loading)
+ return (
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ );
+
+ if (error)
+ return (
+
+
+
+ );
+
+ return (
+
+ {uploads && (
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ )}
+
+
+ Files by {dropdownText[dropdown]}
+
+ {
+ setDropdown(data.value);
+ }}
+ />
+ {dataChart[0].datums.length > 0 ? (
+
+
+
+ ) : (
+
+
+
+ )}
+
+
+
+ );
+};
+
+export default FileUploadsTab;
diff --git a/src/s3Accounting/components/S3InventoryTab.js b/src/s3Accounting/components/S3InventoryTab.js
new file mode 100644
index 000000000..71c991a1e
--- /dev/null
+++ b/src/s3Accounting/components/S3InventoryTab.js
@@ -0,0 +1,217 @@
+import {
+ Container,
+ Dropdown,
+ Grid,
+ Header,
+ Icon,
+ Message,
+ Placeholder,
+ Segment,
+} from 'semantic-ui-react';
+import React, {useState} from 'react';
+
+import {ALL_CLOUD_INVENTORY_FILES} from '../queries';
+import {Chart} from 'react-charts';
+import FileUploadTable from './FileUploadTable';
+import {dateTime} from '../../common/dateUtils';
+import {useQuery} from '@apollo/client';
+
+const S3InventoryTab = ({match, loading, error, data}) => {
+ const [dropdown, setDropdown] = useState('count_by_data_type');
+
+ const storageAnalyses =
+ data && data.allStorageAnalyses.edges.length > 0
+ ? data.allStorageAnalyses.edges[0].node
+ : {};
+
+ const {
+ loading: cloudInventoryLoading,
+ error: cloudInventoryError,
+ data: cloudInventoryData,
+ } = useQuery(ALL_CLOUD_INVENTORY_FILES, {
+ variables: {
+ storageAnalysis: storageAnalyses.id,
+ },
+ });
+
+ const inventory = storageAnalyses.stats
+ ? JSON.parse(storageAnalyses.stats).inventory
+ : {};
+
+ const dropdownText = {
+ count_by_bucket: 'Bucket',
+ count_by_data_type: 'Data Type',
+ count_by_ext: 'Ext',
+ count_by_size: 'Size',
+ };
+
+ const CountStat = ({number, title, icon, color}) => (
+
+
+
+ {title}
+
+
+ );
+ const DateStat = ({date, title}) => (
+ <>
+
+ {dateTime(date)}
+ >
+ );
+
+ const series = React.useMemo(
+ () => ({
+ type: 'bar',
+ }),
+ [],
+ );
+ const axes = React.useMemo(
+ () => [
+ {primary: true, type: 'ordinal', position: 'left'},
+ {position: 'bottom', type: 'linear', stacked: true},
+ ],
+ [],
+ );
+
+ const dataChart = React.useMemo(() => {
+ const rawData = inventory ? inventory[dropdown] : {};
+ const col = Object.keys(rawData);
+ const formatData = col.map(c => ({x: c, y: rawData[c]}));
+ return [
+ {
+ datums: formatData,
+ },
+ ];
+ }, [dropdown, inventory]);
+
+ const getSeriesStyle = React.useCallback(
+ series => ({
+ color: '#4183c4',
+ }),
+ [],
+ );
+
+ if (loading)
+ return (
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ );
+
+ if (error)
+ return (
+
+
+
+ );
+
+ return (
+
+ {inventory && (
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ )}
+
+
+ Files by {dropdownText[dropdown]}
+
+ {
+ setDropdown(data.value);
+ }}
+ />
+ {dataChart[0].datums.length > 0 ? (
+
+
+
+ ) : (
+
+
+
+ )}
+
+
+
+ );
+};
+
+export default S3InventoryTab;
diff --git a/src/s3Accounting/fragments.js b/src/s3Accounting/fragments.js
new file mode 100644
index 000000000..00115fc8a
--- /dev/null
+++ b/src/s3Accounting/fragments.js
@@ -0,0 +1,17 @@
+import gql from 'graphql-tag';
+
+export const FILE_AUDI_FIELDS = gql`
+ fragment FileAudiFields on FileAuditNode {
+ id
+ result
+ sourceFilename
+ expectedUrl
+ actualHash
+ expectedHash
+ expectedSize
+ actualSize
+ hashAlgorithm
+ customFields
+ createdAt
+ }
+`;
diff --git a/src/s3Accounting/queries.js b/src/s3Accounting/queries.js
new file mode 100644
index 000000000..cc12ba6a7
--- /dev/null
+++ b/src/s3Accounting/queries.js
@@ -0,0 +1,71 @@
+import {FILE_AUDI_FIELDS} from './fragments';
+import gql from 'graphql-tag';
+
+export const ALL_STORAGE_ANALYSES = gql`
+ query allStorageAnalyses($studyKfId: String) {
+ allStorageAnalyses(studyKfId: $studyKfId) {
+ edges {
+ node {
+ id
+ uuid
+ createdAt
+ refreshedAt
+ scannedStorageAt
+ stats
+ study {
+ id
+ kfId
+ name
+ }
+ fileAudits {
+ edges {
+ node {
+ ...FileAudiFields
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ ${FILE_AUDI_FIELDS}
+`;
+
+export const ALL_FILE_AUDITS = gql`
+ query allFileAudits($storageAnalysis: ID, $result: ResultEnum) {
+ allFileAudits(storageAnalysis: $storageAnalysis, result: $result) {
+ edges {
+ node {
+ ...FileAudiFields
+ }
+ }
+ }
+ }
+ ${FILE_AUDI_FIELDS}
+`;
+
+export const ALL_CLOUD_INVENTORY_FILES = gql`
+ query allCloudInventoryFiles($storageAnalysis: ID, $result: ResultEnum) {
+ allCloudInventoryFiles(storageAnalysis: $storageAnalysis, result: $result) {
+ edges {
+ node {
+ ...FileAudiFields
+ }
+ }
+ }
+ }
+ ${FILE_AUDI_FIELDS}
+`;
+
+export const ALL_UPLOAD_MANIFEST_FILES = gql`
+ query allUploadManifestFiles($storageAnalysis: ID, $result: ResultEnum) {
+ allUploadManifestFiles(storageAnalysis: $storageAnalysis, result: $result) {
+ edges {
+ node {
+ ...FileAudiFields
+ }
+ }
+ }
+ }
+ ${FILE_AUDI_FIELDS}
+`;
diff --git a/src/s3Accounting/routes.js b/src/s3Accounting/routes.js
new file mode 100644
index 000000000..cdb2b76ae
--- /dev/null
+++ b/src/s3Accounting/routes.js
@@ -0,0 +1,15 @@
+import {DetailView, ListView} from './views';
+import {Route, Switch} from 'react-router-dom';
+
+import React from 'react';
+
+const Routes = () => (
+ <>
+
+
+ } />
+
+ >
+);
+
+export default Routes;
diff --git a/src/s3Accounting/views/DetailView.js b/src/s3Accounting/views/DetailView.js
new file mode 100644
index 000000000..30635c513
--- /dev/null
+++ b/src/s3Accounting/views/DetailView.js
@@ -0,0 +1,151 @@
+import {
+ Button,
+ Container,
+ Header,
+ Icon,
+ Menu,
+ Segment,
+} from 'semantic-ui-react';
+import React, {useState} from 'react';
+
+import {ALL_STORAGE_ANALYSES} from '../queries';
+import ComparisonTab from '../components/ComparisonTab';
+import FileUploadsTab from '../components/FileUploadsTab';
+import {Helmet} from 'react-helmet';
+import {ImageMessage} from '../../components/ImageMessage';
+import {MY_PROFILE} from '../../state/queries';
+import S3InventoryTab from '../components/S3InventoryTab';
+import {hasPermission} from '../../common/permissions';
+import s3 from '../../assets/s3.png';
+import {useQuery} from '@apollo/client';
+
+const DetailView = ({match, history}) => {
+ const [tab, setTab] = useState(0);
+ const {data: profileData, loading: myProfileLoading} = useQuery(MY_PROFILE);
+ const myProfile = profileData && profileData.myProfile;
+
+ const {loading, error, data} = useQuery(ALL_STORAGE_ANALYSES, {
+ variables: {
+ studyKfId: match.params.studyId,
+ },
+ });
+ const storageAnalyses =
+ data && data.allStorageAnalyses.edges.length > 0
+ ? data.allStorageAnalyses.edges[0].node
+ : {};
+ const study = storageAnalyses && storageAnalyses.study;
+ const studyName = study ? study.name || study.shortName : 'Uknown Study';
+
+ const navTab = [
+ {
+ tab: 'Comparison',
+ hash: '#comparison',
+ },
+ {
+ tab: 'S3 Inventory',
+ hash: '#s3-inventory',
+ },
+ {
+ tab: 'File Uploads',
+ hash: '#file-uploads',
+ },
+ ];
+
+ if (
+ !myProfileLoading &&
+ myProfile &&
+ !(
+ hasPermission(myProfile, 'list_all_storageanalysis') ||
+ hasPermission(myProfile, 'list_all_fileaudits')
+ )
+ )
+ return (
+
+ You may not have been granted permissions for analysis of files in
+ study's a cloud storage. Please contact us at{' '}
+
+ support@kidsfirstdrc.org
+
+ >
+ }
+ />
+ );
+
+ return (
+
+
+ {`KF Data Tracker - S3 Accounting`}
+
+
+ history.push(`/study/${match.params.studyId}/basic-info/info`)
+ }
+ />
+
+
+ View snapshots of your study's data files in cloud storage and compare
+ them against the file upload manifests that your users submitted. This
+ will help you understand the differential between what a user tried to
+ upload and what actually got uploaded to your cloud storage.
+
+
+ {navTab.map((item, i) => (
+ {
+ setTab(i);
+ window.history.pushState(
+ {},
+ '',
+ history.location.pathname + navTab[i].hash,
+ );
+ }}
+ >
+ {item.tab}
+
+ ))}
+
+ {tab === 0 && (
+
+ )}
+ {tab === 1 && (
+
+ )}
+ {tab === 2 && (
+
+ )}
+
+ );
+};
+export default DetailView;
diff --git a/src/s3Accounting/views/ListView.js b/src/s3Accounting/views/ListView.js
new file mode 100644
index 000000000..a47cb7b16
--- /dev/null
+++ b/src/s3Accounting/views/ListView.js
@@ -0,0 +1,207 @@
+import {
+ Container,
+ Header,
+ Icon,
+ Message,
+ Placeholder,
+ Segment,
+ Table,
+} from 'semantic-ui-react';
+
+import {ALL_STORAGE_ANALYSES} from '../queries';
+import {Helmet} from 'react-helmet';
+import {ImageMessage} from '../../components/ImageMessage';
+import KfId from '../../components/StudyList/KfId';
+import {MY_PROFILE} from '../../state/queries';
+import React from 'react';
+import TimeAgo from 'react-timeago';
+import {hasPermission} from '../../common/permissions';
+import {longDate} from '../../common/dateUtils';
+import s3 from '../../assets/s3.png';
+import {useHistory} from 'react-router-dom';
+import {useQuery} from '@apollo/client';
+
+const ListView = ({match}) => {
+ const history = useHistory();
+
+ const {data: profileData, loading: myProfileLoading} = useQuery(MY_PROFILE);
+ const myProfile = profileData && profileData.myProfile;
+
+ const {loading, error, data} = useQuery(ALL_STORAGE_ANALYSES);
+ const storageAnalyses =
+ data && data.allStorageAnalyses.edges.length > 0
+ ? data.allStorageAnalyses.edges
+ : [];
+
+ if (
+ !myProfileLoading &&
+ myProfile &&
+ !hasPermission(myProfile, 'list_all_storageanalysis')
+ )
+ return (
+
+ You may not have been granted permissions for analysis of files in
+ study's a cloud storage. Please contact us at{' '}
+
+ support@kidsfirstdrc.org
+
+ >
+ }
+ />
+ );
+
+ if (loading)
+ return (
+
+
+
+ S3 Accounting Reports
+
+
+ View snapshots of your study's data files in cloud storage and compare
+ them against the file upload manifests that your users submitted. This
+ will help you understand the differential between what a user tried to
+ upload and what actually got uploaded to your cloud storage.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ );
+
+ if (error)
+ return (
+
+
+ {`KF Data Tracker - S3 Accounting - Error`}
+
+
+
+ S3 Accounting Reports
+
+
+ View snapshots of your study's data files in cloud storage and compare
+ them against the file upload manifests that your users submitted. This
+ will help you understand the differential between what a user tried to
+ upload and what actually got uploaded to your cloud storage.
+
+
+
+ );
+
+ return (
+
+
+ {`KF Data Tracker - S3 Accounting`}
+
+
+
+ S3 Accounting Reports
+
+
+ View snapshots of your study's data files in cloud storage and compare
+ them against the file upload manifests that your users submitted. This
+ will help you understand the differential between what a user tried to
+ upload and what actually got uploaded to your cloud storage.
+
+ {storageAnalyses.length > 0 ? (
+
+
+
+ Study Name
+ Study ID
+ Updated At
+
+ Files Found
+
+
+ Files Missing
+
+
+ Files Differ
+
+
+ Files Moved
+
+
+
+
+ {storageAnalyses.map(({node}) => {
+ const audit = node.stats
+ ? JSON.parse(node.stats).audit
+ : {
+ matched: {total_count: '-'},
+ missing: {total_count: '-'},
+ differ: {total_count: '-'},
+ moved: {total_count: '-'},
+ };
+ return (
+
+ history.push(`/s3-accounting/${node.study.kfId}#comparison`)
+ }
+ >
+ {node.study.name}
+
+
+
+
+
+ {audit.matched.total_count}
+
+
+ {audit.missing.total_count}
+
+
+ {audit.differ.total_count}
+
+
+ -
+
+
+ );
+ })}
+
+
+ ) : (
+
+
+
+ )}
+
+ );
+};
+
+export default ListView;
diff --git a/src/s3Accounting/views/index.js b/src/s3Accounting/views/index.js
new file mode 100644
index 000000000..8e9e68d07
--- /dev/null
+++ b/src/s3Accounting/views/index.js
@@ -0,0 +1,2 @@
+export {default as ListView} from './ListView';
+export {default as DetailView} from './DetailView';
diff --git a/yarn.lock b/yarn.lock
index 3cce1cf8f..2c88c406f 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1909,6 +1909,11 @@
"@nodelib/fs.scandir" "2.1.4"
fastq "^1.6.0"
+"@reach/observe-rect@^1.1.0":
+ version "1.2.0"
+ resolved "https://registry.yarnpkg.com/@reach/observe-rect/-/observe-rect-1.2.0.tgz#d7a6013b8aafcc64c778a0ccb83355a11204d3b2"
+ integrity sha512-Ba7HmkFgfQxZqqaeIWWkNK0rEhpxVQHIoVyW1YDSkGsGIXzcaW4deC8B0pZrNSSyLTdIk7y+5olKt5+g0GmFIQ==
+
"@react-dnd/asap@^4.0.0":
version "4.0.0"
resolved "https://registry.yarnpkg.com/@react-dnd/asap/-/asap-4.0.0.tgz#b300eeed83e9801f51bd66b0337c9a6f04548651"
@@ -3929,9 +3934,9 @@ caniuse-api@^3.0.0:
lodash.uniq "^4.5.0"
caniuse-lite@^1.0.0, caniuse-lite@^1.0.30000939, caniuse-lite@^1.0.30000989, caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001173:
- version "1.0.30001180"
- resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001180.tgz#67abcd6d1edf48fa5e7d1e84091d1d65ab76e33b"
- integrity sha512-n8JVqXuZMVSPKiPiypjFtDTXc4jWIdjxull0f92WLo7e1MSi3uJ3NvveakSh/aCl1QKFAvIz3vIj0v+0K+FrXw==
+ version "1.0.30001278"
+ resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001278.tgz"
+ integrity sha512-mpF9KeH8u5cMoEmIic/cr7PNS+F5LWBk0t2ekGT60lFf0Wq+n9LspAj0g3P+o7DQhD3sUdlMln4YFAWhFYn9jg==
capture-exit@^2.0.0:
version "2.0.0"
@@ -4944,6 +4949,13 @@ cypress@^4.2.0:
url "^0.11.0"
yauzl "^2.10.0"
+d3-array@2:
+ version "2.12.1"
+ resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-2.12.1.tgz#e20b41aafcdffdf5d50928004ececf815a465e81"
+ integrity sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==
+ dependencies:
+ internmap "^1.0.0"
+
d3-array@^2.3.0:
version "2.11.0"
resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-2.11.0.tgz#5ed6a2869bc7d471aec8df9ff6ed9fef798facc4"
@@ -4956,6 +4968,13 @@ d3-array@^2.3.0:
resolved "https://registry.yarnpkg.com/d3-color/-/d3-color-2.0.0.tgz#8d625cab42ed9b8f601a1760a389f7ea9189d62e"
integrity sha512-SPXi0TSKPD4g9tw0NMZFnR95XVgUZiBH+uUTqQuDu1OsE2zomHU7ho0FISciaPvosimixwHFl3WHLGabv6dDgQ==
+d3-delaunay@^5.2.1:
+ version "5.3.0"
+ resolved "https://registry.yarnpkg.com/d3-delaunay/-/d3-delaunay-5.3.0.tgz#b47f05c38f854a4e7b3cea80e0bb12e57398772d"
+ integrity sha512-amALSrOllWVLaHTnDLHwMIiz0d1bBu9gZXd1FiLfXf8sHcX9jrcj81TVZOqD4UX7MgBZZ07c8GxzEgBpJqc74w==
+ dependencies:
+ delaunator "4"
+
"d3-format@1 - 2":
version "2.0.0"
resolved "https://registry.yarnpkg.com/d3-format/-/d3-format-2.0.0.tgz#a10bcc0f986c372b729ba447382413aabf5b0767"
@@ -5002,7 +5021,18 @@ d3-scale@^3.0.0:
d3-time "1 - 2"
d3-time-format "2 - 3"
-d3-shape@^1.2.2, d3-shape@^1.3.5:
+d3-scale@^3.2.1:
+ version "3.3.0"
+ resolved "https://registry.yarnpkg.com/d3-scale/-/d3-scale-3.3.0.tgz#28c600b29f47e5b9cd2df9749c206727966203f3"
+ integrity sha512-1JGp44NQCt5d1g+Yy+GeOnZP7xHo0ii8zsQp6PGzd+C1/dl0KGsp9A7Mxwp+1D1o4unbTTxVdU/ZOIEBoeZPbQ==
+ dependencies:
+ d3-array "^2.3.0"
+ d3-format "1 - 2"
+ d3-interpolate "1.2.0 - 2"
+ d3-time "^2.1.1"
+ d3-time-format "2 - 3"
+
+d3-shape@^1.2.2, d3-shape@^1.3.5, d3-shape@^1.3.7:
version "1.3.7"
resolved "https://registry.yarnpkg.com/d3-shape/-/d3-shape-1.3.7.tgz#df63801be07bc986bc54f63789b4fe502992b5d7"
integrity sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==
@@ -5033,6 +5063,18 @@ d3-time@1, d3-time@^1.0.10, d3-time@^1.0.11:
resolved "https://registry.yarnpkg.com/d3-time/-/d3-time-2.0.0.tgz#ad7c127d17c67bd57a4c61f3eaecb81108b1e0ab"
integrity sha512-2mvhstTFcMvwStWd9Tj3e6CEqtOivtD8AUiHT8ido/xmzrI9ijrUUihZ6nHuf/vsScRBonagOdj0Vv+SEL5G3Q==
+d3-time@^2.1.1:
+ version "2.1.1"
+ resolved "https://registry.yarnpkg.com/d3-time/-/d3-time-2.1.1.tgz#e9d8a8a88691f4548e68ca085e5ff956724a6682"
+ integrity sha512-/eIQe/eR4kCQwq7yxi7z4c6qEXf2IYGcjoWB5OOQy4Tq9Uv39/947qlDcN2TLkiTzQWzvnsuYPB9TrWaNfipKQ==
+ dependencies:
+ d3-array "2"
+
+d3-voronoi@^1.1.2:
+ version "1.1.4"
+ resolved "https://registry.yarnpkg.com/d3-voronoi/-/d3-voronoi-1.1.4.tgz#dd3c78d7653d2bb359284ae478645d95944c8297"
+ integrity sha512-dArJ32hchFsrQ8uMiTBLq256MpnZjeuBtdHpaDlYuQyjU0CVzCJl/BVW+SkszaAeH95D/8gxqAhgx0ouAWAfRg==
+
damerau-levenshtein@^1.0.4:
version "1.0.6"
resolved "https://registry.yarnpkg.com/damerau-levenshtein/-/damerau-levenshtein-1.0.6.tgz#143c1641cb3d85c60c32329e26899adea8701791"
@@ -5209,6 +5251,11 @@ del@^3.0.0:
pify "^3.0.0"
rimraf "^2.2.8"
+delaunator@4:
+ version "4.0.1"
+ resolved "https://registry.yarnpkg.com/delaunator/-/delaunator-4.0.1.tgz#3d779687f57919a7a418f8ab947d3bddb6846957"
+ integrity sha512-WNPWi1IRKZfCt/qIDMfERkDp93+iZEmOxN2yy4Jg+Xhv8SLk2UTqqbe1sfiipn0and9QrE914/ihdx82Y/Giag==
+
delayed-stream@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
@@ -11708,6 +11755,17 @@ react-app-rewired@^2.1.0:
dependencies:
semver "^5.6.0"
+react-charts@^2.0.0-beta.7:
+ version "2.0.0-beta.7"
+ resolved "https://registry.yarnpkg.com/react-charts/-/react-charts-2.0.0-beta.7.tgz#cc989d78f976dc70796af3af816874274f0c75be"
+ integrity sha512-iUspg9rnx7kD0H/wsK67HNUioOgKgJ8WRXr/Tk3EGP2qcFb9Vo7pjDk4oz1jH12TC+mqL+HFxNYraMkhWd6CUw==
+ dependencies:
+ "@reach/observe-rect" "^1.1.0"
+ d3-delaunay "^5.2.1"
+ d3-scale "^3.2.1"
+ d3-shape "^1.3.7"
+ d3-voronoi "^1.1.2"
+
react-copy-to-clipboard@^5.0.1:
version "5.0.3"
resolved "https://registry.yarnpkg.com/react-copy-to-clipboard/-/react-copy-to-clipboard-5.0.3.tgz#2a0623b1115a1d8c84144e9434d3342b5af41ab4"
@@ -14460,10 +14518,8 @@ watchpack@^1.5.0:
resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-1.7.5.tgz#1267e6c55e0b9b5be44c2023aed5437a2c26c453"
integrity sha512-9P3MWk6SrKjHsGkLT2KHXdQ/9SNkyoJbabxnKOoJepsvJjJG8uYTR3yTPxPQvNDI3w4Nz1xnE0TLHK4RIVe/MQ==
dependencies:
- chokidar "^3.4.1"
graceful-fs "^4.1.2"
neo-async "^2.5.0"
- watchpack-chokidar2 "^2.0.1"
optionalDependencies:
chokidar "^3.4.1"
watchpack-chokidar2 "^2.0.1"