diff --git a/importApp/src/config/default.json b/importApp/src/config/default.json index 5e12be72..a4f3d0ad 100644 --- a/importApp/src/config/default.json +++ b/importApp/src/config/default.json @@ -1,5 +1,5 @@ { - "repoRawContent": "https://raw.githubusercontent.com/medialab/ricardo_data/", + "repoRawContent": "https://raw.githubusercontent.com/medialab/ricardo_data", "apiUri": "https://api.github.com/repos/medialab/ricardo_data/contents", "branchUri": "https://api.github.com/repos/medialab/ricardo_data/branches", "referenceUri": "https://api.github.com/repos/medialab/ricardo_data/git/refs", diff --git a/importApp/src/containers/DataPublish/DataPublish.js b/importApp/src/containers/DataPublish/DataPublish.js index d0c0ecf5..8f67e517 100644 --- a/importApp/src/containers/DataPublish/DataPublish.js +++ b/importApp/src/containers/DataPublish/DataPublish.js @@ -25,6 +25,7 @@ import {downloadFlow, downloadTable} from '../../utils/fileExporter'; import GithubAuthModal from '../../components/GithubAuthModal'; import {SOURCE_SLUG_FILENAME} from '../../constants'; +import {owner, repoName} from '../../config/default'; class DataPublish extends React.Component { @@ -45,7 +46,7 @@ class DataPublish extends React.Component { render () { const {flows, repoData, referenceTables, originalLength} = this.props; - const {selectedBranch, remoteUpdateStatus} = repoData; + const {selectedBranch, remoteUpdateStatus, lastCommit, remoteUpdateMessage} = repoData; const repoTables = repoData.tables; let updatedTables = []; @@ -68,16 +69,15 @@ class DataPublish extends React.Component { // we need to source metadata to generate source filename const sources = keyBy(referenceTables.sources, s => s.slug); const parsedFlows = csvParse(flows.data.map(d => d.join(',')).join('\n')); - console.log(sources); const groupedFlows = groupBy(parsedFlows, (item) => SOURCE_SLUG_FILENAME(sources[item['source']])); - console.log(groupedFlows); + const handleUpdateRemoteFiles= (auth) => { this.handleCloseModal(); const flowFiles = Object.keys(groupedFlows).map((file) => { return { - fileName: `${file}.csv`, + fileName: `flows/${file}.csv`, data: groupedFlows[file] } }); @@ -142,8 +142,8 @@ class DataPublish extends React.Component { - {remoteUpdateStatus === 'loading' && updating files on github, please wait...} - {remoteUpdateStatus === 'updated' && files are updated on github} + {remoteUpdateStatus === 'loading' && updating files on github: {remoteUpdateMessage}...} + {remoteUpdateStatus === 'updated' && files have been commited on github see commit details} {remoteUpdateStatus === 'fail' && fail to update files on github} (dispatch) => { const {files, branch, auth} = payload; const github = new Octokat({ - username: auth.username, - password: auth.token + token: auth.token }); dispatch(async () => { @@ -157,6 +157,25 @@ export const updateRemoteFiles = (payload) => (dispatch) => { let baseReference = await repo.git.refs(`heads/${branch}`).fetch(); let treeItems = []; for (let file of files) { + + if (!file.sha && file.fileName.includes('flows')){ + //new file flow ? + //check if file already exists + dispatch({ + type: UPDATE_REMOTE_FILES_LOG, + payload: `downloading existing flows file ${file.fileName}` + }); + const exists = await get(`${repoRawContent}/${branch}/data/${file.fileName}`,{ responseType: 'text', responseEncoding: 'utf8'}) + if (exists.status === 200) { + // append new rows at end of the existing file + file.data = csvParse(exists.data).concat(file.data) + } + // else it's a new file nothing to do + } + dispatch({ + type: UPDATE_REMOTE_FILES_LOG, + payload: `uploading ${file.fileName}` + }); let fileGit = await repo.git.blobs.create({content: Base64.encode(csvFormat(file.data)), encoding: 'base64'}); let filePath = `data/${file.fileName}`; treeItems.push({ @@ -166,11 +185,18 @@ export const updateRemoteFiles = (payload) => (dispatch) => { type: "blob" }) } - + dispatch({ + type: UPDATE_REMOTE_FILES_LOG, + payload: `creating tree` + }); let tree = await repo.git.trees.create({ tree: treeItems, base_tree: baseReference.object.sha }); + dispatch({ + type: UPDATE_REMOTE_FILES_LOG, + payload: `creating commit` + }); let commit = await repo.git.commits.create({ message: auth.message || DEFAULT_MESSAGE, tree: tree.sha, @@ -180,6 +206,7 @@ export const updateRemoteFiles = (payload) => (dispatch) => { baseReference.update({sha: commit.sha}); dispatch({ type: UPDATE_REMOTE_FILES_SUCCESS, + payload: commit.sha }); } catch(err) { console.error(err); @@ -333,12 +360,21 @@ export default function reducer(state = initialState, action){ case UPDATE_REMOTE_FILES_REQUEST: return { ...state, - remoteUpdateStatus: 'loading' + remoteUpdateStatus: 'loading', + lastCommit: null, + remoteUpdateMessage: null } + case UPDATE_REMOTE_FILES_LOG: + return { + ...state, + remoteUpdateStatus: 'loading', + remoteUpdateMessage: payload + } case UPDATE_REMOTE_FILES_SUCCESS: return { ...state, remoteUpdateStatus: "updated", + lastCommit: payload } case UPDATE_REMOTE_FILES_FAILURE: return {