diff --git a/core/bagger.go b/core/bagger.go index e1bb5c3..b25f120 100644 --- a/core/bagger.go +++ b/core/bagger.go @@ -368,10 +368,8 @@ func (b *Bagger) pathForPayloadFile(fullPath string) string { if filepath.Ext(b.OutputPath) == "" { // Bag is a directory. We don't want to duplicate the // bag name in the path because it's already there. - // We want something like this: - // /data/file.txt - // NOT this: - // /bag-name/data/file.txt + // We want something like this: /data/file.txt + // NOT this: /bag-name/data/file.txt return fmt.Sprintf("data%s", shortPath) } // Else, bag is file and we do want the bag name to diff --git a/core/upload_operation.go b/core/upload_operation.go index 344e0b1..4014865 100644 --- a/core/upload_operation.go +++ b/core/upload_operation.go @@ -14,12 +14,13 @@ import ( ) type UploadOperation struct { - Errors map[string]string `json:"errors"` - PayloadSize int64 `json:"payloadSize"` - Result *OperationResult `json:"result"` - SourceFiles []string `json:"sourceFiles"` - StorageService *StorageService `json:"storageService"` - MessageChannel chan *EventMessage `json:"-"` + Errors map[string]string `json:"errors"` + PayloadSize int64 `json:"payloadSize"` + Result *OperationResult `json:"result"` + SourceFiles []string `json:"sourceFiles"` + StorageService *StorageService `json:"storageService"` + MessageChannel chan *EventMessage `json:"-"` + ExpandedSourceList []string `json:"expandedSourceList"` } func NewUploadOperation(ss *StorageService, files []string) *UploadOperation { @@ -64,8 +65,9 @@ func (u *UploadOperation) Validate() bool { } func (u *UploadOperation) CalculatePayloadSize() error { + u.expandSourceFileList() u.PayloadSize = 0 - for _, fileOrDir := range u.SourceFiles { + for _, fileOrDir := range u.ExpandedSourceList { stat, err := os.Stat(fileOrDir) if err != nil { Dart.Log.Errorf("UploadOperation.CalculatePayloadSize - can't stat %s: %v", fileOrDir, err) @@ -90,6 +92,7 @@ func (u *UploadOperation) CalculatePayloadSize() error { } func (u *UploadOperation) DoUpload(messageChannel chan *EventMessage) bool { + u.expandSourceFileList() ok := false switch u.StorageService.Protocol { case constants.ProtocolS3: @@ -124,7 +127,7 @@ func (u *UploadOperation) sendToS3(messageChannel chan *EventMessage) bool { return false } allSucceeded := true - for _, sourceFile := range u.SourceFiles { + for _, sourceFile := range u.ExpandedSourceList { s3Key := filepath.Base(sourceFile) u.Result.RemoteURL = u.StorageService.URL(s3Key) Dart.Log.Infof("Starting S3 upload %s to %s", sourceFile, u.Result.RemoteURL) @@ -160,7 +163,7 @@ func (u *UploadOperation) sendToS3(messageChannel chan *EventMessage) bool { // no progress updates. func (u *UploadOperation) sendToSFTP(messageChannel chan *EventMessage) bool { allSucceeded := true - for _, file := range u.SourceFiles { + for _, file := range u.ExpandedSourceList { var progress *StreamProgress if messageChannel != nil { progress = NewStreamProgress(u.PayloadSize, messageChannel) @@ -179,3 +182,47 @@ func (u *UploadOperation) useSSL() bool { Dart.Log.Infof("Use SSL for upload = %t", useSSL) return useSSL } + +func (u *UploadOperation) expandSourceFileList() error { + + // START HERE + + // TODO: Set a flag describing whether this has already been done, + // so we don't do it over and over. Also, consider using util.RecursiveFileList, + // since we're going to have to stat everything anyway. + // + // Also note that when uploading loose files to S3, they all go into + // the top-level dir. They should instead mirror the local struction. + // + // SFTP uploads seem to create the necessary directories, but + // they're created as files, and then + // all of the files go into the top-level directory. Fix that. + // + // Also, the progress bar for SFTP multifile uploads is schizo. + // It's not calculating progress against the total upload size. + + u.ExpandedSourceList = make([]string, 0) + for _, filePath := range u.SourceFiles { + if util.IsDirectory(filePath) { + filesInSource, err := listDirRecursive(filePath) + if err != nil { + return err + } + u.ExpandedSourceList = append(u.ExpandedSourceList, filesInSource...) + } else { + u.ExpandedSourceList = append(u.ExpandedSourceList, filePath) + } + } + return nil +} + +func listDirRecursive(dir string) ([]string, error) { + files := make([]string, 0) + err := filepath.Walk(dir, func(filePath string, f os.FileInfo, err error) error { + if f.Mode().IsRegular() || f.IsDir() { + files = append(files, filePath) + } + return nil + }) + return files, err +} diff --git a/scripts/run.rb b/scripts/run.rb index 5d6fcee..1ff1517 100755 --- a/scripts/run.rb +++ b/scripts/run.rb @@ -116,6 +116,9 @@ def start_sftp -p 2222:22 -d #{sftp_image_name}` if $?.exitstatus == 0 puts "Started SFTP server with id #{@docker_sftp_id}" + puts "To log in and view the contents, use " + puts "sftp -P 2222 pw_user@localhost" + puts "The password is 'password' without the quotes" @sftp_started = true else puts "Error starting SFTP docker container. Is one already running?" diff --git a/server/controllers/jobs_controller.go b/server/controllers/jobs_controller.go index 03d398b..9d7e1d8 100644 --- a/server/controllers/jobs_controller.go +++ b/server/controllers/jobs_controller.go @@ -45,19 +45,6 @@ func JobIndex(c *gin.Context) { return } request.TemplateData["jobs"] = request.QueryResult.Jobs - // items := make([]JobListItem, len(jobs)) - // for i, job := range jobs { - // artifacts, err := core.ArtifactNameIDList(job.ID) - // if err != nil { - // core.Dart.Log.Warningf("Error getting artifact list for job %s: %v", job.Name(), err) - // } - // item := JobListItem{ - // Job: job, - // Artifacts: artifacts, - // } - // items[i] = item - // } - // request.TemplateData["items"] = items c.HTML(http.StatusOK, "job/list.html", request.TemplateData) } @@ -71,3 +58,14 @@ func JobNew(c *gin.Context) { } c.Redirect(http.StatusFound, fmt.Sprintf("/jobs/files/%s", job.ID)) } + +// GET /jobs/show_json/:id +func JobShowJson(c *gin.Context) { + jobID := c.Param("id") + result := core.ObjFind(jobID) + if result.Error != nil { + AbortWithErrorJSON(c, http.StatusInternalServerError, result.Error) + return + } + c.JSON(http.StatusOK, result.Job()) +} diff --git a/server/controllers/jobs_controller_test.go b/server/controllers/jobs_controller_test.go index a13920e..3901723 100644 --- a/server/controllers/jobs_controller_test.go +++ b/server/controllers/jobs_controller_test.go @@ -1,6 +1,7 @@ package controllers_test import ( + "encoding/json" "fmt" "net/http" "net/http/httptest" @@ -79,3 +80,15 @@ func TestJobDelete(t *testing.T) { require.NotNil(t, result.Error) assert.Equal(t, "sql: no rows in result set", result.Error.Error()) } + +func TestJobShowJson(t *testing.T) { + defer core.ClearDartTable() + job := loadTestJob(t) + require.NoError(t, core.ObjSave(job)) + + data, err := json.Marshal(job) + require.Nil(t, err) + expected := []string{string(data)} + + DoSimpleGetTest(t, fmt.Sprintf("/jobs/show_json/%s", job.ID), expected) +} diff --git a/server/controllers/request.go b/server/controllers/request.go index fda8136..1fb0033 100644 --- a/server/controllers/request.go +++ b/server/controllers/request.go @@ -26,7 +26,7 @@ type Request struct { // Regex to help us extract object type from handler name. // E.g. AppSettingIndex -> AppSetting, StorageServiceEdit -> StorageService. -var routeSuffix = regexp.MustCompile(`Index|New|Save|Edit|Delete|GetReport$`) +var routeSuffix = regexp.MustCompile(`Index|New|Save|Edit|Delete|GetReport|ShowJson$`) func NewRequest(c *gin.Context) *Request { pathAndQuery := c.Request.URL.Path diff --git a/server/server.go b/server/server.go index dabea25..85c32f3 100644 --- a/server/server.go +++ b/server/server.go @@ -166,6 +166,7 @@ func initRoutes(router *gin.Engine) { router.POST("/jobs/delete_file/:id", controllers.JobDeleteFile) router.GET("/jobs/summary/:id", controllers.JobRunShow) router.GET("/jobs/run/:id", controllers.JobRunExecute) + router.GET("/jobs/show_json/:id", controllers.JobShowJson) // Job Artifacts router.GET("/jobs/artifacts/list/:job_id", controllers.JobArtifactsList)