diff --git a/importApp/src/config/default.json b/importApp/src/config/default.json
index 5e12be72..a4f3d0ad 100644
--- a/importApp/src/config/default.json
+++ b/importApp/src/config/default.json
@@ -1,5 +1,5 @@
{
- "repoRawContent": "https://raw.githubusercontent.com/medialab/ricardo_data/",
+ "repoRawContent": "https://raw.githubusercontent.com/medialab/ricardo_data",
"apiUri": "https://api.github.com/repos/medialab/ricardo_data/contents",
"branchUri": "https://api.github.com/repos/medialab/ricardo_data/branches",
"referenceUri": "https://api.github.com/repos/medialab/ricardo_data/git/refs",
diff --git a/importApp/src/containers/DataPublish/DataPublish.js b/importApp/src/containers/DataPublish/DataPublish.js
index d0c0ecf5..8f67e517 100644
--- a/importApp/src/containers/DataPublish/DataPublish.js
+++ b/importApp/src/containers/DataPublish/DataPublish.js
@@ -25,6 +25,7 @@ import {downloadFlow, downloadTable} from '../../utils/fileExporter';
import GithubAuthModal from '../../components/GithubAuthModal';
import {SOURCE_SLUG_FILENAME} from '../../constants';
+import {owner, repoName} from '../../config/default';
class DataPublish extends React.Component {
@@ -45,7 +46,7 @@ class DataPublish extends React.Component {
render () {
const {flows, repoData, referenceTables, originalLength} = this.props;
- const {selectedBranch, remoteUpdateStatus} = repoData;
+ const {selectedBranch, remoteUpdateStatus, lastCommit, remoteUpdateMessage} = repoData;
const repoTables = repoData.tables;
let updatedTables = [];
@@ -68,16 +69,15 @@ class DataPublish extends React.Component {
// we need to source metadata to generate source filename
const sources = keyBy(referenceTables.sources, s => s.slug);
const parsedFlows = csvParse(flows.data.map(d => d.join(',')).join('\n'));
- console.log(sources);
const groupedFlows = groupBy(parsedFlows, (item) => SOURCE_SLUG_FILENAME(sources[item['source']]));
- console.log(groupedFlows);
+
const handleUpdateRemoteFiles= (auth) => {
this.handleCloseModal();
const flowFiles = Object.keys(groupedFlows).map((file) => {
return {
- fileName: `${file}.csv`,
+ fileName: `flows/${file}.csv`,
data: groupedFlows[file]
}
});
@@ -142,8 +142,8 @@ class DataPublish extends React.Component {
- {remoteUpdateStatus === 'loading' && updating files on github, please wait...}
- {remoteUpdateStatus === 'updated' && files are updated on github}
+ {remoteUpdateStatus === 'loading' && updating files on github: {remoteUpdateMessage}...}
+ {remoteUpdateStatus === 'updated' && files have been commited on github see commit details}
{remoteUpdateStatus === 'fail' && fail to update files on github}
(dispatch) => {
const {files, branch, auth} = payload;
const github = new Octokat({
- username: auth.username,
- password: auth.token
+ token: auth.token
});
dispatch(async () => {
@@ -157,6 +157,25 @@ export const updateRemoteFiles = (payload) => (dispatch) => {
let baseReference = await repo.git.refs(`heads/${branch}`).fetch();
let treeItems = [];
for (let file of files) {
+
+ if (!file.sha && file.fileName.includes('flows')){
+ //new file flow ?
+ //check if file already exists
+ dispatch({
+ type: UPDATE_REMOTE_FILES_LOG,
+ payload: `downloading existing flows file ${file.fileName}`
+ });
+ const exists = await get(`${repoRawContent}/${branch}/data/${file.fileName}`,{ responseType: 'text', responseEncoding: 'utf8'})
+ if (exists.status === 200) {
+ // append new rows at end of the existing file
+ file.data = csvParse(exists.data).concat(file.data)
+ }
+ // else it's a new file nothing to do
+ }
+ dispatch({
+ type: UPDATE_REMOTE_FILES_LOG,
+ payload: `uploading ${file.fileName}`
+ });
let fileGit = await repo.git.blobs.create({content: Base64.encode(csvFormat(file.data)), encoding: 'base64'});
let filePath = `data/${file.fileName}`;
treeItems.push({
@@ -166,11 +185,18 @@ export const updateRemoteFiles = (payload) => (dispatch) => {
type: "blob"
})
}
-
+ dispatch({
+ type: UPDATE_REMOTE_FILES_LOG,
+ payload: `creating tree`
+ });
let tree = await repo.git.trees.create({
tree: treeItems,
base_tree: baseReference.object.sha
});
+ dispatch({
+ type: UPDATE_REMOTE_FILES_LOG,
+ payload: `creating commit`
+ });
let commit = await repo.git.commits.create({
message: auth.message || DEFAULT_MESSAGE,
tree: tree.sha,
@@ -180,6 +206,7 @@ export const updateRemoteFiles = (payload) => (dispatch) => {
baseReference.update({sha: commit.sha});
dispatch({
type: UPDATE_REMOTE_FILES_SUCCESS,
+ payload: commit.sha
});
} catch(err) {
console.error(err);
@@ -333,12 +360,21 @@ export default function reducer(state = initialState, action){
case UPDATE_REMOTE_FILES_REQUEST:
return {
...state,
- remoteUpdateStatus: 'loading'
+ remoteUpdateStatus: 'loading',
+ lastCommit: null,
+ remoteUpdateMessage: null
}
+ case UPDATE_REMOTE_FILES_LOG:
+ return {
+ ...state,
+ remoteUpdateStatus: 'loading',
+ remoteUpdateMessage: payload
+ }
case UPDATE_REMOTE_FILES_SUCCESS:
return {
...state,
remoteUpdateStatus: "updated",
+ lastCommit: payload
}
case UPDATE_REMOTE_FILES_FAILURE:
return {