From cb46ecfcb0bb8d037770da1c2a709e8332905c9a Mon Sep 17 00:00:00 2001 From: "A. Diamond" Date: Fri, 12 Jan 2024 16:03:20 -0500 Subject: [PATCH] Working on writing untarred bags --- constants/constants.go | 14 ++++- core/bagger.go | 63 ++++++++++--------- core/bagit_profile.go | 2 +- core/bagit_profile_test.go | 2 +- core/file_system_bag_reader.go | 6 ++ core/job_runner.go | 3 +- .../controllers/job_packaging_controller.go | 31 +++++---- server/views/job/packaging.html | 37 ++++++++++- 8 files changed, 114 insertions(+), 44 deletions(-) create mode 100644 core/file_system_bag_reader.go diff --git a/constants/constants.go b/constants/constants.go index 65f19cf..65e6ab7 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -63,6 +63,8 @@ const ( ResultTypeList = "list" ResultTypeSingle = "single" ResultTypeUnitialized = "unintialized" + SerialFormatNone = "none (bag as directory)" + SerialFormatTar = "application/tar" SerializationForbidden = "forbidden" SerializationOptional = "optional" SerializationRequired = "required" @@ -97,7 +99,17 @@ var AcceptBagItVersion = []string{ } var AcceptSerialization = []string{ - "application/tar", + SerialFormatNone, + SerialFormatTar, +} + +// BagWriterTypeFor maps a BagIt serialization format to the +// type of writer that can write that format. +var BagWriterTypeFor = map[string]string{ + "": BagWriterTypeFileSystem, + ".tar": BagWriterTypeTar, + SerialFormatNone: BagWriterTypeFileSystem, + SerialFormatTar: BagWriterTypeTar, } var SerializationOptions = []string{ diff --git a/core/bagger.go b/core/bagger.go index 8106e50..47a17d4 100644 --- a/core/bagger.go +++ b/core/bagger.go @@ -18,36 +18,38 @@ import ( var bagitTxt embed.FS type Bagger struct { - Profile *BagItProfile - OutputPath string - FilesToBag []*util.ExtendedFileInfo - Errors map[string]string - MessageChannel chan *EventMessage - PayloadFiles *FileMap - PayloadManifests *FileMap - TagFiles *FileMap - TagManifests *FileMap - ManifestArtifacts map[string]string - TagFileArtifacts map[string]string - writer BagWriter - pathPrefix string - bagName string - currentFileNum int64 - totalFileCount int64 + Profile *BagItProfile + OutputPath string + FilesToBag []*util.ExtendedFileInfo + Errors map[string]string + MessageChannel chan *EventMessage + PayloadFiles *FileMap + PayloadManifests *FileMap + TagFiles *FileMap + TagManifests *FileMap + ManifestArtifacts map[string]string + TagFileArtifacts map[string]string + SerializationFormat string + writer BagWriter + pathPrefix string + bagName string + currentFileNum int64 + totalFileCount int64 } func NewBagger(outputPath string, profile *BagItProfile, filesToBag []*util.ExtendedFileInfo) *Bagger { return &Bagger{ - Profile: profile, - OutputPath: outputPath, - FilesToBag: filesToBag, - PayloadFiles: NewFileMap(constants.FileTypePayload), - PayloadManifests: NewFileMap(constants.FileTypeManifest), - TagFiles: NewFileMap(constants.FileTypeTag), - TagManifests: NewFileMap(constants.FileTypeTagManifest), - ManifestArtifacts: make(map[string]string), - TagFileArtifacts: make(map[string]string), - Errors: make(map[string]string), + Profile: profile, + OutputPath: outputPath, + FilesToBag: filesToBag, + PayloadFiles: NewFileMap(constants.FileTypePayload), + PayloadManifests: NewFileMap(constants.FileTypeManifest), + SerializationFormat: constants.SerialFormatTar, + TagFiles: NewFileMap(constants.FileTypeTag), + TagManifests: NewFileMap(constants.FileTypeTagManifest), + ManifestArtifacts: make(map[string]string), + TagFileArtifacts: make(map[string]string), + Errors: make(map[string]string), } } @@ -301,8 +303,13 @@ func (b *Bagger) initWriter() bool { // START HERE // Get the right type of bag writer, based on profile serialization. - - b.writer = NewTarredBagWriter(b.OutputPath, digestAlgs) + var err error + writerType := constants.BagWriterTypeFor[b.SerializationFormat] + b.writer, err = GetBagWriter(writerType, b.OutputPath, digestAlgs) + if err != nil { + Dart.Log.Errorf("Bagger cannot find writer for serialization type %s: %v", b.SerializationFormat, err) + return false + } b.writer.Open() return true } diff --git a/core/bagit_profile.go b/core/bagit_profile.go index e286b92..7282257 100644 --- a/core/bagit_profile.go +++ b/core/bagit_profile.go @@ -483,7 +483,7 @@ func (p *BagItProfile) ToForm() *Form { bagitVersionField.Help = "Which BagIt versions are allowed in this profile?" acceptSerializationField := form.AddMultiValueField("AcceptSerialization", "AcceptSerialization", p.AcceptSerialization, true) - acceptSerializationField.Choices = MakeMultiChoiceList(p.AcceptSerialization, p.AcceptSerialization) + acceptSerializationField.Choices = MakeMultiChoiceList(constants.AcceptSerialization, p.AcceptSerialization) acceptSerializationField.Help = "If bags using this profile can be serialized to tar, zip or other formats, enter the mime types for those formats here. E.g. application/x-tar, application/zip, etc. See https://en.wikipedia.org/wiki/List_of_archive_formats for a full list." allowFetch := form.AddField("AllowFetchTxt", "AllowFetchTxt", strconv.FormatBool(p.AllowFetchTxt), true) diff --git a/core/bagit_profile_test.go b/core/bagit_profile_test.go index 9f92c99..4edbc2b 100644 --- a/core/bagit_profile_test.go +++ b/core/bagit_profile_test.go @@ -402,7 +402,7 @@ func TestBagItProfileToForm(t *testing.T) { assert.Equal(t, constants.AcceptBagItVersion, form.Fields["AcceptBagItVersion"].Values) require.NotNil(t, form.Fields["AcceptSerialization"]) - assert.Equal(t, constants.AcceptSerialization, form.Fields["AcceptSerialization"].Values) + assert.Equal(t, profile.AcceptSerialization, form.Fields["AcceptSerialization"].Values) require.NotNil(t, form.Fields["AllowFetchTxt"]) assert.Equal(t, strconv.FormatBool(profile.AllowFetchTxt), form.Fields["AllowFetchTxt"].Value) diff --git a/core/file_system_bag_reader.go b/core/file_system_bag_reader.go new file mode 100644 index 0000000..d2c4d3a --- /dev/null +++ b/core/file_system_bag_reader.go @@ -0,0 +1,6 @@ +package core + +// START HERE + +// TODO: Define bag reader interface, similar to bag writer interface. +// Then implement this so it's interchangeable with TarredBagReader. diff --git a/core/job_runner.go b/core/job_runner.go index 76a18da..5597dc9 100644 --- a/core/job_runner.go +++ b/core/job_runner.go @@ -179,7 +179,8 @@ func (r *Runner) RunPackageOp() bool { sourceFiles = append(sourceFiles, files...) } bagger := NewBagger(op.OutputPath, r.Job.BagItProfile, sourceFiles) - bagger.MessageChannel = r.MessageChannel // Careful! This may be nil. + bagger.SerializationFormat = op.BagItSerialization // tar, loose directory, etc. + bagger.MessageChannel = r.MessageChannel // Careful! This may be nil. ok := bagger.Run() r.saveBaggingArtifacts(bagger) r.Job.ByteCount = bagger.PayloadBytes() diff --git a/server/controllers/job_packaging_controller.go b/server/controllers/job_packaging_controller.go index 81ee279..76a1cf5 100644 --- a/server/controllers/job_packaging_controller.go +++ b/server/controllers/job_packaging_controller.go @@ -25,13 +25,18 @@ func JobShowPackaging(c *gin.Context) { baggingDir = filepath.Join(core.Dart.Paths.Documents, "DART") core.Dart.Log.Warningf("Bagging Directory not set. Defaulting to %s", baggingDir) } + // The front end needs to know which profiles require serialization + // (serializationRequired) and which require a *specific* serialization + // format (autosetSerialization). + autoSetSerialization, serializationRequired := getSerlializationAutosets() data := gin.H{ - "job": job, - "form": job.ToForm(), - "pathSeparator": string(os.PathSeparator), - "baggingDir": baggingDir, - "autoSetSerialization": getSerlializationAutosets(), - "helpUrl": GetHelpUrl(c), + "job": job, + "form": job.ToForm(), + "pathSeparator": string(os.PathSeparator), + "baggingDir": baggingDir, + "autoSetSerialization": autoSetSerialization, + "serializationRequired": serializationRequired, + "helpUrl": GetHelpUrl(c), } c.HTML(http.StatusOK, "job/packaging.html", data) } @@ -107,18 +112,22 @@ func JobSavePackaging(c *gin.Context) { c.Redirect(http.StatusFound, nextPage) } -func getSerlializationAutosets() map[string]string { +func getSerlializationAutosets() (map[string]string, []string) { autosetMap := make(map[string]string) + serializationRequired := make([]string, 0) // Typical installation has 3-10 profiles. result := core.ObjList(constants.TypeBagItProfile, "obj_name", 1000, 0) if result.Error != nil { core.Dart.Log.Warningf("Could not load BagIt profiles for serialization auto-set: %s", result.Error.Error()) - return autosetMap + return autosetMap, serializationRequired } for _, profile := range result.BagItProfiles { - if profile.Serialization == constants.SerializationRequired && len(profile.AcceptSerialization) == 1 { - autosetMap[profile.ID] = profile.AcceptSerialization[0] + if profile.Serialization == constants.SerializationRequired { + serializationRequired = append(serializationRequired, profile.ID) + if len(profile.AcceptSerialization) == 1 { + autosetMap[profile.ID] = profile.AcceptSerialization[0] + } } } - return autosetMap + return autosetMap, serializationRequired } diff --git a/server/views/job/packaging.html b/server/views/job/packaging.html index f99253f..ab9e8c8 100644 --- a/server/views/job/packaging.html +++ b/server/views/job/packaging.html @@ -53,6 +53,8 @@

Packaging

let baggingDir = {{ .baggingDir }} let separator = {{ .pathSeparator }} let autoSetSerialization = {{ .autoSetSerialization }} + let serializationRequired = {{ .serializationRequired }} + let emptySerializationOption = 'none (bag as directory)' let extensionForType = { "application/x-7z-compressed": ".7z", @@ -81,10 +83,43 @@

Packaging

$('#Job_BagItSerialization').on('change', setOutputPath) function setSerialization() { - let profileId = $(this).val() + let profileId = $('#Job_BagItProfileID').val() + if (serializationRequired.includes(profileId)) { + // Don't show empty serialization option because serializion + // is required for this profile. + removeEmptySerializationOption() + } else { + addEmptySerializationOption() + } let serializationFormat = autoSetSerialization[profileId] $('#Job_BagItSerialization').val(serializationFormat) } + + function listHasEmptySerializationOption() { + var hasOption = false + $('#Job_BagItSerialization option').each(function(opt) { + if ($(this).val() == emptySerializationOption) { + hasOption = true + } + }) + return hasOption + } + function addEmptySerializationOption() { + if (!listHasEmptySerializationOption()) { + var option = `` + $(option).insertAfter($('#Job_BagItSerialization option')[0]) + } + } + function removeEmptySerializationOption() { + if (listHasEmptySerializationOption()) { + var option = $('#Job_BagItSerialization option[value="none (bag as directory)"]') + $(option).remove() + } + } + + // Run this once on page load, then attach it to the profile + // select list so we can watch for changes. + setSerialization() $('#Job_BagItProfileID').on('change', setSerialization) })